diff --git a/integration-tests/backend/.golangci.yml b/integration-tests/backend/.golangci.yml new file mode 100644 index 0000000000..1545a0878e --- /dev/null +++ b/integration-tests/backend/.golangci.yml @@ -0,0 +1,28 @@ +version: "2" +run: + go: "1.23" +linters: + enable: + - copyloopvar + - errcheck + - govet + - ineffassign + - staticcheck + - unused + exclusions: + generated: lax + presets: + - comments + - common-false-positives + - legacy + - std-error-handling + rules: + - linters: + - staticcheck + text: "QF1003:" + - linters: + - staticcheck + text: "QF1008:" +formatters: + enable: + - gofmt diff --git a/integration-tests/backend/Makefile b/integration-tests/backend/Makefile new file mode 100644 index 0000000000..435f04bbc8 --- /dev/null +++ b/integration-tests/backend/Makefile @@ -0,0 +1,17 @@ +GOLANGCI_LINT_VERSION = v2.8.0 + +##@ Development +.PHONY: prereqs +prereqs: + @echo "### Test if prerequisites are met, and installing missing dependencies" + test -f ./bin/golangci-lint-${GOLANGCI_LINT_VERSION} || ( \ + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s ${GOLANGCI_LINT_VERSION} \ + && mv ./bin/golangci-lint ./bin/golangci-lint-${GOLANGCI_LINT_VERSION}) + +fmt: ## Run go fmt against code. + go fmt ./... + +.PHONY: lint +lint: prereqs ## Run linter (golangci-lint) + @echo "### Linting code" + ./bin/golangci-lint-${GOLANGCI_LINT_VERSION} --config .golangci.yml run --timeout 15m ./... diff --git a/integration-tests/backend/OWNERS b/integration-tests/backend/OWNERS new file mode 100644 index 0000000000..2222c208f7 --- /dev/null +++ b/integration-tests/backend/OWNERS @@ -0,0 +1,13 @@ +approvers: + - memodi + - mffiedler + - Amoghrd + - oliver-smakal + - kapjain-rh + +reviewers: + - memodi + - mffiedler + - Amoghrd + - oliver-smakal + - kapjain-rh diff --git a/integration-tests/backend/alerts.go b/integration-tests/backend/alerts.go new file mode 100644 index 0000000000..b5f504879f --- /dev/null +++ b/integration-tests/backend/alerts.go @@ -0,0 +1,44 @@ +package e2etests + +import ( + "context" + "encoding/json" + "fmt" + "time" + + exutil "github.com/openshift/origin/test/extended/util" + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" + "k8s.io/apimachinery/pkg/util/wait" +) + +func getConfiguredAlertRules(oc *exutil.CLI, ruleName string, namespace string) (string, error) { + return oc.AsAdmin().WithoutNamespace().Run("get").Args("prometheusrules", ruleName, "-o=jsonpath='{.spec.groups[*].rules[*].alert}'", "-n", namespace).Output() +} + +func getAlertStatus(oc *exutil.CLI, alertName string) (map[string]interface{}, error) { + alertOut, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-monitoring", "alertmanager-main-0", "--", "amtool", "--alertmanager.url", "http://localhost:9093", "alert", "query", alertName, "-o", "json").Output() + if err != nil { + return make(map[string]interface{}), err + } + var alertStatus []interface{} + _ = json.Unmarshal([]byte(alertOut), &alertStatus) + + if len(alertStatus) == 0 { + return make(map[string]interface{}), nil + } + return alertStatus[0].(map[string]interface{}), nil +} + +func waitForAlertToBeActive(oc *exutil.CLI, alertName string) { + err := wait.PollUntilContextTimeout(context.Background(), 10*time.Second, 900*time.Second, false, func(context.Context) (done bool, err error) { + alertStatus, err := getAlertStatus(oc, alertName) + if err != nil { + return false, err + } + if len(alertStatus) == 0 { + return false, nil + } + return alertStatus["status"].(map[string]interface{})["state"] == "active", nil + }) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("%s Alert did not become active", alertName)) +} diff --git a/integration-tests/backend/aws_sts.go b/integration-tests/backend/aws_sts.go new file mode 100644 index 0000000000..6b93297877 --- /dev/null +++ b/integration-tests/backend/aws_sts.go @@ -0,0 +1,184 @@ +package e2etests + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "github.com/aws/aws-sdk-go-v2/aws" + awsConfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/iam" + "github.com/aws/aws-sdk-go-v2/service/sts" + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + exutil "github.com/openshift/origin/test/extended/util" + e2e "k8s.io/kubernetes/test/e2e/framework" +) + +// Check if credentials exist for STS clusters +func checkAWSCredentials() bool { + // set AWS_SHARED_CREDENTIALS_FILE from CLUSTER_PROFILE_DIR as the first priority" + prowConfigDir, present := os.LookupEnv("CLUSTER_PROFILE_DIR") + if present { + awsCredFile := filepath.Join(prowConfigDir, ".awscred") + if _, err := os.Stat(awsCredFile); err == nil { + err := os.Setenv("AWS_SHARED_CREDENTIALS_FILE", awsCredFile) + if err == nil { + e2e.Logf("use CLUSTER_PROFILE_DIR/.awscred") + return true + } + } + } + + // check if AWS_SHARED_CREDENTIALS_FILE exist + _, present = os.LookupEnv("AWS_SHARED_CREDENTIALS_FILE") + if present { + e2e.Logf("use Env AWS_SHARED_CREDENTIALS_FILE") + return true + } + + // check if AWS_SECRET_ACCESS_KEY exist + _, keyIDPresent := os.LookupEnv("AWS_ACCESS_KEY_ID") + _, keyPresent := os.LookupEnv("AWS_SECRET_ACCESS_KEY") + if keyIDPresent && keyPresent { + e2e.Logf("use Env AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY") + return true + } + // check if $HOME/.aws/credentials exist + home, _ := os.UserHomeDir() + if _, err := os.Stat(home + "/.aws/credentials"); err == nil { + e2e.Logf("use HOME/.aws/credentials") + return true + } + return false +} + +// get AWS Account ID +func getAwsAccount(stsClient *sts.Client) (string, string) { + result, err := stsClient.GetCallerIdentity(context.TODO(), &sts.GetCallerIdentityInput{}) + o.Expect(err).NotTo(o.HaveOccurred()) + awsAccount := aws.ToString(result.Account) + awsUserArn := aws.ToString(result.Arn) + return awsAccount, awsUserArn +} + +func readDefaultSDKExternalConfigurations(ctx context.Context, region string) aws.Config { + cfg, err := awsConfig.LoadDefaultConfig(ctx, + awsConfig.WithRegion(region), + ) + o.Expect(err).NotTo(o.HaveOccurred()) + return cfg +} + +// new AWS STS client +func newStsClient(cfg aws.Config) *sts.Client { + if !checkAWSCredentials() { + g.Skip("Skip since no AWS credetial! No Env AWS_SHARED_CREDENTIALS_FILE, Env CLUSTER_PROFILE_DIR or $HOME/.aws/credentials file") + } + return sts.NewFromConfig(cfg) +} + +// Create AWS IAM client +func newIamClient(cfg aws.Config) *iam.Client { + if !checkAWSCredentials() { + g.Skip("Skip since no AWS credetial! No Env AWS_SHARED_CREDENTIALS_FILE, Env CLUSTER_PROFILE_DIR or $HOME/.aws/credentials file") + } + return iam.NewFromConfig(cfg) +} + +// This func creates a IAM role, attaches custom trust policy and managed permission policy +func createIAMRoleOnAWS(iamClient *iam.Client, trustPolicy string, roleName string, policyArn string) string { + result, err := iamClient.CreateRole(context.TODO(), &iam.CreateRoleInput{ + AssumeRolePolicyDocument: aws.String(trustPolicy), + RoleName: aws.String(roleName), + }) + o.Expect(err).NotTo(o.HaveOccurred(), "Couldn't create role %v", roleName) + roleArn := aws.ToString(result.Role.Arn) + + // Adding managed permission policy if provided + if policyArn != "" { + _, err = iamClient.AttachRolePolicy(context.TODO(), &iam.AttachRolePolicyInput{ + PolicyArn: aws.String(policyArn), + RoleName: aws.String(roleName), + }) + o.Expect(err).NotTo(o.HaveOccurred()) + } + return roleArn +} + +// Deletes IAM role and attached policies +func deleteIAMroleonAWS(iamClient *iam.Client, roleName string) { + // List attached policies of the IAM role + listAttachedPoliciesOutput, err := iamClient.ListAttachedRolePolicies(context.TODO(), &iam.ListAttachedRolePoliciesInput{ + RoleName: aws.String(roleName), + }) + if err != nil { + e2e.Logf("Error listing attached policies of IAM role %s", roleName) + } + + if len(listAttachedPoliciesOutput.AttachedPolicies) == 0 { + e2e.Logf("No attached policies under IAM role: %s", roleName) + } + + if len(listAttachedPoliciesOutput.AttachedPolicies) != 0 { + // Detach attached policy from the IAM role + for _, policy := range listAttachedPoliciesOutput.AttachedPolicies { + _, err := iamClient.DetachRolePolicy(context.TODO(), &iam.DetachRolePolicyInput{ + RoleName: aws.String(roleName), + PolicyArn: policy.PolicyArn, + }) + if err != nil { + e2e.Logf("Error detaching policy: %v", *policy.PolicyName) + } else { + e2e.Logf("Detached policy: %v", *policy.PolicyName) + } + } + } + + // Delete the IAM role + _, err = iamClient.DeleteRole(context.TODO(), &iam.DeleteRoleInput{ + RoleName: aws.String(roleName), + }) + if err != nil { + e2e.Logf("Error deleting IAM role: %s", roleName) + } else { + e2e.Logf("IAM role deleted successfully: %s", roleName) + } +} + +// Create role_arn required for Loki deployment on STS clusters +func createIAMRoleForLokiSTSDeployment(iamClient *iam.Client, oidcName, awsAccountID, partition, lokiNamespace, lokiStackName, roleName string) string { + e2e.Logf("Running createIAMRoleForLokiSTSDeployment") + policyArn := "arn:" + partition + ":iam::aws:policy/AmazonS3FullAccess" + + lokiTrustPolicy := `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": "arn:%s:iam::%s:oidc-provider/%s" + }, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "%s:sub": [ + "system:serviceaccount:%s:%s", + "system:serviceaccount:%s:%s-ruler" + ] + } + } + } + ] + }` + lokiTrustPolicy = fmt.Sprintf(lokiTrustPolicy, partition, awsAccountID, oidcName, oidcName, lokiNamespace, lokiStackName, lokiNamespace, lokiStackName) + roleArn := createIAMRoleOnAWS(iamClient, lokiTrustPolicy, roleName, policyArn) + return roleArn +} + +// Creates Loki object storage secret on AWS STS cluster +func createObjectStorageSecretOnAWSSTSCluster(oc *exutil.CLI, region, storageSecret, bucketName, namespace string) { + err := oc.NotShowInfo().AsAdmin().WithoutNamespace().Run("create").Args("secret", "generic", storageSecret, "--from-literal=region="+region, "--from-literal=bucketnames="+bucketName, "--from-literal=audience=openshift", "-n", namespace).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) +} diff --git a/integration-tests/backend/azure_utils.go b/integration-tests/backend/azure_utils.go new file mode 100644 index 0000000000..e1b71e0e36 --- /dev/null +++ b/integration-tests/backend/azure_utils.go @@ -0,0 +1,297 @@ +package e2etests + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + azRuntime "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + azTo "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/google/uuid" + o "github.com/onsi/gomega" + exutil "github.com/openshift/origin/test/extended/util" + "github.com/tidwall/gjson" + e2e "k8s.io/kubernetes/test/e2e/framework" +) + +// To read Azure subscription json file from local disk. +// Also injects ENV vars needed to perform certain operations on Managed Identities. +func readAzureCredentials() bool { + var azureCredFile string + envDir, present := os.LookupEnv("CLUSTER_PROFILE_DIR") + if present { + azureCredFile = filepath.Join(envDir, "osServicePrincipal.json") + } else { + authFileLocation, present := os.LookupEnv("AZURE_AUTH_LOCATION") + if present { + azureCredFile = authFileLocation + } + } + if len(azureCredFile) > 0 { + fileContent, err := os.ReadFile(azureCredFile) + o.Expect(err).NotTo(o.HaveOccurred()) + + subscriptionID := gjson.Get(string(fileContent), `azure_subscription_id`).String() + if subscriptionID == "" { + subscriptionID = gjson.Get(string(fileContent), `subscriptionId`).String() + } + os.Setenv("AZURE_SUBSCRIPTION_ID", subscriptionID) + + tenantID := gjson.Get(string(fileContent), `azure_tenant_id`).String() + if tenantID == "" { + tenantID = gjson.Get(string(fileContent), `tenantId`).String() + } + os.Setenv("AZURE_TENANT_ID", tenantID) + + clientID := gjson.Get(string(fileContent), `azure_client_id`).String() + if clientID == "" { + clientID = gjson.Get(string(fileContent), `clientId`).String() + } + os.Setenv("AZURE_CLIENT_ID", clientID) + + clientSecret := gjson.Get(string(fileContent), `azure_client_secret`).String() + if clientSecret == "" { + clientSecret = gjson.Get(string(fileContent), `clientSecret`).String() + } + os.Setenv("AZURE_CLIENT_SECRET", clientSecret) + return true + } + return false +} + +// Creates a new default Azure credential +func createNewDefaultAzureCredential() *azidentity.DefaultAzureCredential { + cred, err := azidentity.NewDefaultAzureCredential(nil) + o.Expect(err).NotTo(o.HaveOccurred(), "Failed to obtain a credential") + return cred +} + +// Function to create a managed identity on Azure +func createManagedIdentityOnAzure(defaultAzureCred *azidentity.DefaultAzureCredential, azureSubscriptionID, lokiStackName, resourceGroup, region string) (string, string) { + // Create the MSI client + client, err := armmsi.NewUserAssignedIdentitiesClient(azureSubscriptionID, defaultAzureCred, nil) + o.Expect(err).NotTo(o.HaveOccurred(), "Failed to create MSI client") + + // Configure the managed identity + identity := armmsi.Identity{ + Location: ®ion, + } + + // Create the identity + result, err := client.CreateOrUpdate(context.Background(), resourceGroup, lokiStackName, identity, nil) + o.Expect(err).NotTo(o.HaveOccurred(), "Failed to create or update the identity") + return *result.Properties.ClientID, *result.Properties.PrincipalID +} + +// Function to create Federated Credentials on Azure +func createFederatedCredentialforLoki(defaultAzureCred *azidentity.DefaultAzureCredential, azureSubscriptionID, managedIdentityName, lokiServiceAccount, lokiStackNS, federatedCredentialName, serviceAccountIssuer, resourceGroup string) { + subjectName := "system:serviceaccount:" + lokiStackNS + ":" + lokiServiceAccount + + // Create the Federated Identity Credentials client + client, err := armmsi.NewFederatedIdentityCredentialsClient(azureSubscriptionID, defaultAzureCred, nil) + o.Expect(err).NotTo(o.HaveOccurred(), "Failed to create federated identity credentials client") + + // Create or update the federated identity credential + result, err := client.CreateOrUpdate( + context.Background(), + resourceGroup, + managedIdentityName, + federatedCredentialName, + armmsi.FederatedIdentityCredential{ + Properties: &armmsi.FederatedIdentityCredentialProperties{ + Issuer: &serviceAccountIssuer, + Subject: &subjectName, + Audiences: []*string{azTo.Ptr("api://AzureADTokenExchange")}, + }, + }, + nil, + ) + o.Expect(err).NotTo(o.HaveOccurred(), "Failed to create or update the federated credential: "+federatedCredentialName) + e2e.Logf("Federated credential created/updated successfully: %s\n", *result.Name) +} + +// Assigns role to a Azure Managed Identity on subscription level scope +func createRoleAssignmentForManagedIdentity(defaultAzureCred *azidentity.DefaultAzureCredential, azureSubscriptionID, identityPrincipalID string) { + clientFactory, err := armauthorization.NewClientFactory(azureSubscriptionID, defaultAzureCred, nil) + o.Expect(err).NotTo(o.HaveOccurred(), "Failed to create instance of ClientFactory") + + scope := "/subscriptions/" + azureSubscriptionID + // Below is standard role definition ID for Storage Blob Data Contributor built-in role + roleDefinitionID := scope + "/providers/Microsoft.Authorization/roleDefinitions/ba92f5b4-2d11-453d-a403-e96b0029c9fe" + + // Create or update a role assignment by scope and name + _, err = clientFactory.NewRoleAssignmentsClient().Create(context.Background(), scope, uuid.NewString(), armauthorization.RoleAssignmentCreateParameters{ + Properties: &armauthorization.RoleAssignmentProperties{ + PrincipalID: azTo.Ptr(identityPrincipalID), + PrincipalType: azTo.Ptr(armauthorization.PrincipalTypeServicePrincipal), + RoleDefinitionID: azTo.Ptr(roleDefinitionID), + }, + }, nil) + o.Expect(err).NotTo(o.HaveOccurred(), "Role Assignment operation failure....") +} + +// Creates Azure storage account +func createStorageAccountOnAzure(defaultAzureCred *azidentity.DefaultAzureCredential, azureSubscriptionID, resourceGroup, region string) string { + storageAccountName := "aosqelogging" + getRandomString() + // Create the storage account + storageClient, err := armstorage.NewAccountsClient(azureSubscriptionID, defaultAzureCred, nil) + o.Expect(err).NotTo(o.HaveOccurred()) + result, err := storageClient.BeginCreate(context.Background(), resourceGroup, storageAccountName, armstorage.AccountCreateParameters{ + Location: azTo.Ptr(region), + SKU: &armstorage.SKU{ + Name: azTo.Ptr(armstorage.SKUNameStandardLRS), + }, + Kind: azTo.Ptr(armstorage.KindStorageV2), + }, nil) + o.Expect(err).NotTo(o.HaveOccurred()) + + // Poll until the Storage account is ready + _, err = result.PollUntilDone(context.Background(), &azRuntime.PollUntilDoneOptions{ + Frequency: 10 * time.Second, + }) + o.Expect(err).NotTo(o.HaveOccurred(), "Storage account is not ready...") + os.Setenv("LOKI_OBJECT_STORAGE_STORAGE_ACCOUNT", storageAccountName) + return storageAccountName +} + +func getAzureResourceGroupFromCluster(oc *exutil.CLI) (string, error) { + resourceGroup, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("infrastructures", "cluster", "-o=jsonpath={.status.platformStatus.azure.resourceGroupName}").Output() + return resourceGroup, err +} + +// Returns the Azure environment and storage account URI suffixes +func getStorageAccountURISuffixAndEnvForAzure(oc *exutil.CLI) (string, string) { + // To return account URI suffix and env + cloudName, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("infrastructure", "cluster", "-o=jsonpath={.status.platformStatus.azure.cloudName}").Output() + storageAccountURISuffix := ".blob.core.windows.net" + environment := "AzureGlobal" + // Currently we don't have template support for STS/WIF on Azure Government + // The below code should be ok to run when support is added for WIF + if strings.ToLower(cloudName) == "azureusgovernmentcloud" { + storageAccountURISuffix = ".blob.core.usgovcloudapi.net" + environment = "AzureUSGovernment" + } + if strings.ToLower(cloudName) == "azurechinacloud" { + storageAccountURISuffix = ".blob.core.chinacloudapi.cn" + environment = "AzureChinaCloud" + } + if strings.ToLower(cloudName) == "azuregermancloud" { + environment = "AzureGermanCloud" + storageAccountURISuffix = ".blob.core.cloudapi.de" + } + return environment, storageAccountURISuffix +} + +// Creates a blob container under the provided storageAccount +func createBlobContaineronAzure(defaultAzureCred *azidentity.DefaultAzureCredential, storageAccountName, storageAccountURISuffix, containerName string) { + blobServiceClient, err := azblob.NewClient(fmt.Sprintf("https://%s%s", storageAccountName, storageAccountURISuffix), defaultAzureCred, nil) + o.Expect(err).NotTo(o.HaveOccurred()) + _, err = blobServiceClient.CreateContainer(context.Background(), containerName, nil) + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("%s container created successfully: ", containerName) +} + +// Creates Loki object storage secret required on Azure STS/WIF clusters +func createLokiObjectStorageSecretForWIF(oc *exutil.CLI, lokiStackNS, objectStorageSecretName, environment, containerName, storageAccountName string) error { + return oc.NotShowInfo().AsAdmin().WithoutNamespace().Run("create").Args("secret", "generic", "-n", lokiStackNS, objectStorageSecretName, "--from-literal=environment="+environment, "--from-literal=container="+containerName, "--from-literal=account_name="+storageAccountName).Execute() +} + +// Deletes a storage account in Microsoft Azure +func deleteAzureStorageAccount(defaultAzureCred *azidentity.DefaultAzureCredential, azureSubscriptionID, resourceGroupName, storageAccountName string) { + clientFactory, err := armstorage.NewClientFactory(azureSubscriptionID, defaultAzureCred, nil) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to create instance of ClientFactory for storage account deletion") + + _, err = clientFactory.NewAccountsClient().Delete(context.Background(), resourceGroupName, storageAccountName, nil) + if err != nil { + e2e.Logf("Error while deleting storage account. %v", err.Error()) + } else { + e2e.Logf("storage account deleted successfully..") + } +} + +// Deletes the Azure Managed identity +func deleteManagedIdentityOnAzure(defaultAzureCred *azidentity.DefaultAzureCredential, azureSubscriptionID, resourceGroupName, identityName string) { + client, err := armmsi.NewUserAssignedIdentitiesClient(azureSubscriptionID, defaultAzureCred, nil) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to create MSI client for identity deletion") + + _, err = client.Delete(context.Background(), resourceGroupName, identityName, nil) + if err != nil { + e2e.Logf("Error deleting identity. %v", err.Error()) + } else { + e2e.Logf("managed identity deleted successfully...") + } +} + +// patches CLIENT_ID, SUBSCRIPTION_ID, TENANT_ID AND REGION into Loki subscription on Azure WIF clusters +func patchLokiConfigIntoLokiSubscription(oc *exutil.CLI, azureSubscriptionID, identityClientID, region string) { + patchConfig := `{ + "spec": { + "config": { + "env": [ + { + "name": "CLIENTID", + "value": "%s" + }, + { + "name": "TENANTID", + "value": "%s" + }, + { + "name": "SUBSCRIPTIONID", + "value": "%s" + }, + { + "name": "REGION", + "value": "%s" + } + ] + } + } + }` + + err := oc.NotShowInfo().AsAdmin().WithoutNamespace().Run("patch").Args("sub", "loki-operator", "-n", loNS, "-p", fmt.Sprintf(patchConfig, identityClientID, os.Getenv("AZURE_TENANT_ID"), azureSubscriptionID, region), "--type=merge").Execute() + o.Expect(err).NotTo(o.HaveOccurred(), "Patching Loki Operator failed...") + WaitForPodsReadyWithLabel(oc, "openshift-operators-redhat", "name=loki-operator-controller-manager") +} + +// Performs creation of Managed Identity, Associated Federated credentials, Role assignment to the managed identity and object storage creation on Azure +func performManagedIdentityAndSecretSetupForAzureWIF(oc *exutil.CLI, lokistackName, lokiStackNS, azureContainerName, lokiStackStorageSecretName string) { + region, err := getAzureClusterRegion(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + serviceAccountIssuer, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("authentication.config", "cluster", "-o=jsonpath={.spec.serviceAccountIssuer}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + resourceGroup, err := getResourceGroupOnAzure(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + + azureSubscriptionID := os.Getenv("AZURE_SUBSCRIPTION_ID") + cred := createNewDefaultAzureCredential() + + identityClientID, identityPrincipalID := createManagedIdentityOnAzure(cred, azureSubscriptionID, lokistackName, resourceGroup, region) + createFederatedCredentialforLoki(cred, azureSubscriptionID, lokistackName, lokistackName, lokiStackNS, "openshift-logging-"+lokistackName, serviceAccountIssuer, resourceGroup) + createFederatedCredentialforLoki(cred, azureSubscriptionID, lokistackName, lokistackName+"-ruler", lokiStackNS, "openshift-logging-"+lokistackName+"-ruler", serviceAccountIssuer, resourceGroup) + createRoleAssignmentForManagedIdentity(cred, azureSubscriptionID, identityPrincipalID) + patchLokiConfigIntoLokiSubscription(oc, azureSubscriptionID, identityClientID, region) + storageAccountName := createStorageAccountOnAzure(cred, azureSubscriptionID, resourceGroup, region) + environment, storageAccountURISuffix := getStorageAccountURISuffixAndEnvForAzure(oc) + createBlobContaineronAzure(cred, storageAccountName, storageAccountURISuffix, azureContainerName) + err = createLokiObjectStorageSecretForWIF(oc, lokiStackNS, lokiStackStorageSecretName, environment, azureContainerName, storageAccountName) + o.Expect(err).NotTo(o.HaveOccurred()) +} + +func getResourceGroupOnAzure(oc *exutil.CLI) (string, error) { + resourceGroup, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("infrastructures", "cluster", "-o=jsonpath={.status.platformStatus.azure.resourceGroupName}").Output() + return resourceGroup, err +} + +// Get region/location of cluster running on Azure Cloud +func getAzureClusterRegion(oc *exutil.CLI) (string, error) { + return oc.AsAdmin().WithoutNamespace().Run("get").Args("node", `-ojsonpath={.items[].metadata.labels.topology\.kubernetes\.io/region}`).Output() +} diff --git a/integration-tests/backend/backend_suite_test.go b/integration-tests/backend/backend_suite_test.go new file mode 100644 index 0000000000..803aeee98d --- /dev/null +++ b/integration-tests/backend/backend_suite_test.go @@ -0,0 +1,149 @@ +package e2etests + +import ( + "flag" + "fmt" + "testing" + + g "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo/v2/types" + "github.com/onsi/gomega" + exutil "github.com/openshift/origin/test/extended/util" + e2eframework "k8s.io/kubernetes/test/e2e/framework" +) + +func init() { + // Initialize framework flags - must be done before flag.Parse() + exutil.InitStandardFlags() +} + +var _ = g.BeforeSuite(func() { + // Parse flags + flag.Parse() + + // Set up provider config after parsing flags + e2eframework.AfterReadingAllFlags(exutil.TestContext) + + // Initialize test + gomega.Expect(exutil.InitTest(false)).NotTo(gomega.HaveOccurred()) + + oc := exutil.NewCLIForMonitorTest("netobserv") + var err error + _, err = GetOCPVersion(oc) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) +}) + +func TestBackend(t *testing.T) { + exutil.WithCleanup(func() { + gomega.RegisterFailHandler(g.Fail) + + suiteConfig, reporterConfig := g.GinkgoConfiguration() + + // Apply focus filter + + if len(suiteConfig.FocusStrings) > 0 { + combinedFocus := make([]string, len(suiteConfig.FocusStrings)) + for i, userFocus := range suiteConfig.FocusStrings { + combinedFocus[i] = "sig-netobserv.*" + userFocus + } + suiteConfig.FocusStrings = combinedFocus + } else { + suiteConfig.FocusStrings = []string{"sig-netobserv"} + } + + // Configure reporter - suppress default verbose output + suiteConfig.EmitSpecProgress = true + suiteConfig.OutputInterceptorMode = "none" + reporterConfig.SilenceSkips = true // Hide the "S" characters for skipped tests + reporterConfig.NoColor = true + reporterConfig.Succinct = true + reporterConfig.Verbose = false + + // Standard Ginkgo run with custom reporting via hooks + g.RunSpecs(t, "Backend Suite", suiteConfig, reporterConfig) + }) +} + +// Custom reporting hooks +var _ = g.ReportBeforeSuite(func(report g.Report) { + fmt.Printf("Running Suite: %s - %s\n", report.SuiteDescription, report.SuitePath) + fmt.Printf("==========================================================================================================\n") + fmt.Printf("Random Seed: %d\n\n", report.SuiteConfig.RandomSeed) + fmt.Printf("Will run %d specs\n", report.PreRunStats.SpecsThatWillRun) +}) + +var _ = g.ReportAfterEach(func(report g.SpecReport) { + // Only report on specs that actually ran (not skipped on via focused filter) + if report.State == types.SpecStateSkipped && report.Failure.Message == "" && report.RunTime <= 0 { + return + } + + if report.LeafNodeType != types.NodeTypeIt { + return + } + + // Print spec progress + fmt.Printf("%s\n", report.FullText()) + fmt.Printf("%s\n", report.LeafNodeLocation.String()) + + // Print result + switch report.State { + case types.SpecStatePassed: + fmt.Printf("• PASSED [%.3f seconds]\n", report.RunTime.Seconds()) + case types.SpecStateSkipped: + fmt.Printf("• SKIPPED [%.3f seconds]\n", report.RunTime.Seconds()) + if report.Failure.Message != "" { + fmt.Printf("\n%s\n", report.Failure.Message) + fmt.Printf("%s\n", report.Failure.Location.String()) + } + case types.SpecStateFailed, types.SpecStatePanicked, types.SpecStateInvalid, types.SpecStateAborted, types.SpecStateInterrupted, types.SpecStatePending, types.SpecStateTimedout: + fmt.Printf("• FAILED [%.3f seconds]\n", report.RunTime.Seconds()) + if report.Failure.Message != "" { + fmt.Printf("\n%s\n", report.Failure.Message) + fmt.Printf("%s\n", report.Failure.Location.String()) + } + } +}) + +var _ = g.ReportAfterSuite("NetObserv Summary", func(report g.Report) { + passed := 0 + failed := 0 + skipped := 0 + ranTests := 0 + + // Get only the test specs (not setup/teardown) + specs := report.SpecReports.WithLeafNodeType(types.NodeTypeIt) + + for _, specReport := range specs { + switch specReport.State { + case types.SpecStatePassed: + passed++ + ranTests++ + case types.SpecStateFailed, types.SpecStatePanicked, types.SpecStateInvalid, types.SpecStateAborted, types.SpecStateInterrupted, types.SpecStatePending, types.SpecStateTimedout: + failed++ + ranTests++ + case types.SpecStateSkipped: + // Skip filtered-out specs: they have State==Skipped but RunTime==0 and no failure info + // Explicitly skipped specs (via Skip()) have State==Skipped but were actually evaluated + if specReport.Failure.Message != "" || specReport.RunTime > 0 { + // Explicitly skipped - test body was evaluated + skipped++ + } + } + } + + // Total specs evaluated (passed + failed + explicitly skipped) + totalEvaluated := ranTests + skipped + + fmt.Printf("------------------------------\n") + if report.SuiteSucceeded { + fmt.Printf("\nBackend Suite - %d/%d specs • SUCCESS! [%.3f seconds]\n", + totalEvaluated, totalEvaluated, report.RunTime.Seconds()) + } else { + fmt.Printf("\nBackend Suite - %d/%d specs • FAILURE! [%.3f seconds]\n", + totalEvaluated, totalEvaluated, report.RunTime.Seconds()) + } + + fmt.Printf("\nRan %d tests\n", ranTests) + fmt.Printf("Passed: %d, Failed: %d, Skipped: %d\n", passed, failed, skipped) +}) diff --git a/integration-tests/backend/custom_metrics.go b/integration-tests/backend/custom_metrics.go new file mode 100644 index 0000000000..d8b8ec1b89 --- /dev/null +++ b/integration-tests/backend/custom_metrics.go @@ -0,0 +1,109 @@ +package e2etests + +import ( + "fmt" + "os" + "reflect" + + exutil "github.com/openshift/origin/test/extended/util" + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" + "gopkg.in/yaml.v3" + e2e "k8s.io/kubernetes/test/e2e/framework" +) + +type CustomMetrics struct { + Namespace string + Template string +} + +type CustomMetricsTemplateConfig struct { + Objects []interface{} `yaml:"objects"` +} + +type CustomMetricsConfig struct { + DashboardNames []string + MetricName string + Queries []string +} + +// create flowmetrics resource from template +func (cm CustomMetrics) createCustomMetrics(oc *exutil.CLI) { + parameters := []string{"--ignore-unknown-parameters=true", "-f", cm.Template, "-p"} + cmr := reflect.ValueOf(&cm).Elem() + for i := 0; i < cmr.NumField(); i++ { + if cmr.Field(i).Interface() != "" { + if cmr.Type().Field(i).Name != "Template" { + parameters = append(parameters, fmt.Sprintf("%s=%s", cmr.Type().Field(i).Name, cmr.Field(i).Interface())) + } + } + } + compat_otp.ApplyNsResourceFromTemplate(oc, cm.Namespace, parameters...) +} + +// parse custom metrics yaml template +func (cm CustomMetrics) parseTemplate() *CustomMetricsTemplateConfig { + yamlFile, err := os.ReadFile(cm.Template) + + if err != nil { + e2e.Failf("Could not read the template file %s", cm.Template) + } + var cmc *CustomMetricsTemplateConfig + err = yaml.Unmarshal(yamlFile, &cmc) + if err != nil { + e2e.Failf("Could not Unmarshal %v", err) + } + return cmc +} + +// returns queries and dashboardNames +func getChartsConfig(chartsConfig []interface{}) ([]string, []string) { + var result []string + var dashboardNames []string + for _, conf := range chartsConfig { + chartsConf := conf.(map[string]interface{}) + for k, v := range chartsConf { + if k == "dashboardName" { + dashboardNames = append(dashboardNames, v.(string)) + } + if k == "queries" { + queries := v.([]interface{}) + for _, qConf := range queries { + queryConf := qConf.(map[string]interface{}) + for qk, qv := range queryConf { + if qk == "promQL" { + result = append(result, qv.(string)) + } + } + } + } + } + } + return result, dashboardNames +} + +// returns slice of CustomMetricsConfig +func (cm CustomMetrics) getCustomMetricConfigs() []CustomMetricsConfig { + cmc := cm.parseTemplate() + // var customMetricsConfig []map[string][]string + var cmConfigs []CustomMetricsConfig + for _, template := range cmc.Objects { + var cmConfig CustomMetricsConfig + t := template.(map[string]interface{}) + for object, v := range t { + if object == "spec" { + spec := v.(map[string]interface{}) + for config, val := range spec { + if config == "charts" { + chartsConfig := val.([]interface{}) + cmConfig.Queries, cmConfig.DashboardNames = getChartsConfig(chartsConfig) + } + if config == "metricName" { + cmConfig.MetricName = val.(string) + } + } + cmConfigs = append(cmConfigs, cmConfig) + } + } + } + return cmConfigs +} diff --git a/integration-tests/backend/flowcollector.go b/integration-tests/backend/flowcollector.go new file mode 100644 index 0000000000..a32fc13862 --- /dev/null +++ b/integration-tests/backend/flowcollector.go @@ -0,0 +1,244 @@ +package e2etests + +import ( + "context" + "fmt" + "reflect" + "strings" + "time" + + exutil "github.com/openshift/origin/test/extended/util" + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" + + "k8s.io/apimachinery/pkg/util/wait" + e2e "k8s.io/kubernetes/test/e2e/framework" +) + +// Flowcollector struct to handle Flowcollector resources +type Flowcollector struct { + Namespace string + ProcessorKind string + MultiClusterDeployment string + AddZone string + LogType string + FLPFilters string + DeploymentModel string + LokiEnable string + LokiMode string + LokiURL string + LokiTLSCertName string + LokiStatusTLSEnable string + LokiStatusURL string + LokiStatusTLSCertName string + LokiStatusTLSUserCertName string + LokiNamespace string + MonolithicLokiURL string + KafkaAddress string + KafkaTLSEnable string + KafkaClusterName string + KafkaTopic string + KafkaUser string + KafkaNamespace string + FLPMetricServerTLSType string + EBPFMetricServerTLSType string + EBPFCacheActiveTimeout string + EBPFPrivileged string + EBPFFilterEnable string + EBPFFilterRules string + Sampling string + EBPFMetrics string + EBPFeatures []string + CacheMaxFlows string + PluginEnable string + NetworkPolicyEnable string + NetworkPolicyAdditionalNamespaces []string + Exporters []string + SecondaryNetworks string + CollectionMode string + SlicesEnable string + NamespacesAllow []string + ServiceTLSType string + ServiceCASecretName string + ServiceServerCertSecretName string + ServiceClientCertSecretName string + Template string +} + +type Flowlog struct { + // Source + SrcPort int + SrcK8SType string `json:"SrcK8S_Type,omitempty"` + SrcK8SName string `json:"SrcK8S_Name,omitempty"` + SrcK8SHostName string `json:"SrcK8S_HostName,omitempty"` + SrcK8SOwnerType string `json:"SrcK8S_OwnerType,omitempty"` + SrcAddr string + SrcMac string + SrcSubnetLabel string + // Destination + DstPort int + DstK8SType string `json:"DstK8S_Type,omitempty"` + DstK8SName string `json:"DstK8S_Name,omitempty"` + DstK8SHostName string `json:"DstK8S_HostName,omitempty"` + DstK8SOwnerType string `json:"DstK8S_OwnerType,omitempty"` + DstAddr string + DstMac string + DstK8SHostIP string `json:"DstK8S_HostIP,omitempty"` + DstSubnetLabel string + // Protocol + Proto int + IcmpCode int + IcmpType int + Dscp int + Flags []string + // Time + TimeReceived int + TimeFlowEndMs int + TimeFlowStartMs int + // Interface + IfDirection int + IfDirections []int + Interfaces []string + Etype int + // Others + Packets int + Bytes int + Duplicate bool + AgentIP string + Sampling int + HashID string `json:"_HashId,omitempty"` + IsFirst bool `json:"_IsFirst,omitempty"` + RecordType string `json:"_RecordType,omitempty"` + NumFlowLogs int `json:"numFlowLogs,omitempty"` + K8SClusterName string `json:"K8S_ClusterName,omitempty"` + // Zone + SrcK8SZone string `json:"SrcK8S_Zone,omitempty"` + DstK8SZone string `json:"DstK8S_Zone,omitempty"` + // DNS + DNSLatencyMs int `json:"DnsLatencyMs,omitempty"` + DNSFlagsResponseCode string `json:"DnsFlagsResponseCode,omitempty"` + // Packet Drop + PktDropBytes int `json:"PktDropBytes,omitempty"` + PktDropPackets int `json:"PktDropPackets,omitempty"` + PktDropLatestState string `json:"PktDropLatestState,omitempty"` + PktDropLatestDropCause string `json:"PktDropLatestDropCause,omitempty"` + // RTT + TimeFlowRttNs int `json:"TimeFlowRttNs,omitempty"` + // Packet Translation + XlatDstAddr string `json:"XlatDstAddr,omitempty"` + XlatDstK8SName string `json:"XlatDstK8S_Name,omitempty"` + XlatDstK8SNamespace string `json:"XlatDstK8S_Namespace,omitempty"` + XlatDstK8SType string `json:"XlatDstK8S_Type,omitempty"` + XlatDstPort int `json:"XlatDstPort,omitempty"` + XlatSrcAddr string `json:"XlatSrcAddr,omitempty"` + XlatSrcK8SName string `json:"XlatSrcK8S_Name,omitempty"` + XlatSrcK8SNamespace string `json:"XlatSrcK8S_Namespace,omitempty"` + ZoneID int `json:"ZoneId,omitempty"` + // Network Events + NetworkEvents []NetworkEvent `json:"NetworkEvents,omitempty"` + // Secondary Network + SrcK8SNetworkName string `json:"SrcK8S_NetworkName,omitempty"` + DstK8SNetworkName string `json:"DstK8S_NetworkName,omitempty"` + // UDN + Udns []string `json:"Udns,omitempty"` + // IPSec + IPSecStatus string `json:"IPSecStatus,omitempty"` + // TLS + TLSVersion string `json:"TLSVersion,omitempty"` + TLSTypes []string `json:"TLSTypes,omitempty"` + TLSCurve string `json:"TLSCurve,omitempty"` + TLSCipherSuite string `json:"TLSCipherSuite,omitempty"` +} + +type NetworkEvent struct { + Action string `json:"Action,omitempty"` + Type string `json:"Type,omitempty"` + Name string `json:"Name,omitempty"` + Namespace string `json:"Namespace,omitempty"` + Direction string `json:"Direction,omitempty"` + Feature string `json:"Feature,omitempty"` +} + +type FlowRecord struct { + Timestamp int64 + Flowlog Flowlog +} + +type Lokilabels struct { + App string `loki:"app"` + SrcK8SNamespace string `loki:"SrcK8S_Namespace"` + DstK8SNamespace string `loki:"DstK8S_Namespace"` + RecordType string `loki:"_RecordType"` + FlowDirection string `loki:"FlowDirection"` + SrcK8SOwnerName string `loki:"SrcK8S_OwnerName"` + DstK8SOwnerName string `loki:"DstK8S_OwnerName"` + K8SClusterName string `loki:"K8S_ClusterName"` + K8SFlowLayer string `loki:"K8S_FlowLayer"` + SrcK8SType string `loki:"SrcK8S_Type"` + DstK8SType string `loki:"DstK8S_Type"` + Interfaces string `loki:"Interfaces"` +} + +// create flowcollector CRD for a given manifest file +func (flow Flowcollector) CreateFlowcollector(oc *exutil.CLI) { + parameters := []string{"--ignore-unknown-parameters=true", "-f", flow.Template, "-p"} + + flowCollector := reflect.ValueOf(&flow).Elem() + + for i := 0; i < flowCollector.NumField(); i++ { + if flowCollector.Field(i).Interface() != "" { + if flowCollector.Type().Field(i).Name != "Template" { + parameters = append(parameters, fmt.Sprintf("%s=%s", flowCollector.Type().Field(i).Name, flowCollector.Field(i).Interface())) + } + } + } + + compat_otp.ApplyNsResourceFromTemplate(oc, flow.Namespace, parameters...) + flow.WaitForFlowcollectorReady(oc) +} + +// delete flowcollector CRD from a cluster +func (flow *Flowcollector) DeleteFlowcollector(oc *exutil.CLI) error { + return oc.AsAdmin().WithoutNamespace().Run("delete").Args("flowcollector", "cluster").Execute() +} + +func (flow *Flowcollector) WaitForFlowcollectorReady(oc *exutil.CLI) { + // check FLP status + switch flow.DeploymentModel { + case "Kafka": + waitUntilDeploymentReady(oc, "flowlogs-pipeline-transformer", flow.Namespace) + case "Service": + waitUntilDeploymentReady(oc, "flowlogs-pipeline", flow.Namespace) + default: + waitUntilDaemonSetReady(oc, "flowlogs-pipeline", flow.Namespace) + } + // check ebpf-agent status + waitUntilDaemonSetReady(oc, "netobserv-ebpf-agent", flow.Namespace+"-privileged") + + // check plugin status - only wait if Loki is enabled and plugin not explicitly disabled + if flow.PluginEnable != "false" && flow.LokiEnable != "false" { + waitUntilDeploymentReady(oc, "netobserv-plugin", flow.Namespace) + } + + compat_otp.AssertAllPodsToBeReady(oc, flow.Namespace) + compat_otp.AssertAllPodsToBeReady(oc, flow.Namespace+"-privileged") + err := wait.PollUntilContextTimeout(context.Background(), 10*time.Second, 600*time.Second, false, func(context.Context) (done bool, err error) { + + status, err := oc.AsAdmin().Run("get").Args("flowcollector", "-o", "jsonpath='{.items[*].status.conditions[0].reason}'").Output() + + if err != nil { + return false, err + } + if strings.Contains(status, "Ready") { + return true, nil + } + + msg, err := oc.AsAdmin().Run("get").Args("flowcollector", "-o", "jsonpath='{.items[*].status.conditions[0].message}'").Output() + e2e.Logf("flowcollector status is %s due to %s", status, msg) + if err != nil { + return false, err + } + + return false, nil + }) + compat_otp.AssertWaitPollNoErr(err, "Flowcollector did not become Ready") +} diff --git a/integration-tests/backend/flowcollector_utils.go b/integration-tests/backend/flowcollector_utils.go new file mode 100644 index 0000000000..afbdfde1ac --- /dev/null +++ b/integration-tests/backend/flowcollector_utils.go @@ -0,0 +1,506 @@ +package e2etests + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "math" + "reflect" + "regexp" + "strconv" + "strings" + "time" + + o "github.com/onsi/gomega" + exutil "github.com/openshift/origin/test/extended/util" + "k8s.io/apimachinery/pkg/util/wait" + e2e "k8s.io/kubernetes/test/e2e/framework" +) + +type NWEvents string + +const ( + AllowRelated NWEvents = "allow-related" + Drop NWEvents = "drop" +) + +// returns ture/false if flowcollector API exists. +func isFlowCollectorAPIExists(oc *exutil.CLI) (bool, error) { + stdout, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("crd", "-o", "jsonpath='{.items[*].spec.names.kind}'").Output() + + if err != nil { + return false, err + } + return strings.Contains(stdout, "FlowCollector"), nil +} + +// Verify flow records from logs +func verifyFlowRecordFromLogs(podLog string) { + re := regexp.MustCompile("{\"AgentIP\":.*") + flowRecords := re.FindAllString(podLog, -3) + partialFlowRegex := regexp.MustCompile("DstMac\":\"00:00:00:00:00:00") + for _, flow := range flowRecords { + // skip assertions and log Partial flows + if partialFlowRegex.Match([]byte(flow)) { + e2e.Logf("Found partial flows %s", flow) + } else { + o.Expect(flow).Should(o.And( + o.MatchRegexp("Bytes.:[0-9]+"), + o.MatchRegexp("TimeFlowEndMs.:[1-9][0-9]+"), + o.MatchRegexp("TimeFlowStartMs.:[1-9][0-9]+"), + o.MatchRegexp("TimeReceived.:[1-9][0-9]+")), flow) + } + } +} + +// Get flow recrods from loki +func getFlowRecords(lokiValues [][]string) ([]FlowRecord, error) { + flowRecords := []FlowRecord{} + for _, values := range lokiValues { + timestamp, _ := strconv.ParseInt(values[0], 10, 64) + var flowlog Flowlog + err := json.Unmarshal([]byte(values[1]), &flowlog) + if err != nil { + return []FlowRecord{}, err + } + flowRecord := FlowRecord{ + Timestamp: timestamp, + Flowlog: flowlog, + } + flowRecords = append(flowRecords, flowRecord) + } + e2e.Logf("Number of flow records found %d", len(flowRecords)) + return flowRecords, nil +} + +// Get flow records from IPFIX collector HTTP API +func getIPFIXFlowRecordsFromAPI(oc *exutil.CLI, namespace, podName string) ([]FlowRecord, error) { + flowRecords := []FlowRecord{} + + // Query the collector HTTP API using kubectl exec + cmd := []string{"-n", namespace, podName, "-c", "ipfix-collector", "--", "curl", "-s", "http://localhost:8080/records?format=json"} + output, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args(cmd...).Output() + if err != nil { + return flowRecords, fmt.Errorf("failed to query collector API: %w", err) + } + + // Parse JSON response: {"flowRecords":[{"data":"key: value\n..."}, ...]} + var response struct { + FlowRecords []struct { + Data string `json:"data"` + } `json:"flowRecords"` + } + + if err := json.Unmarshal([]byte(output), &response); err != nil { + return flowRecords, fmt.Errorf("failed to parse collector response: %w", err) + } + + // Convert each flow record + for _, record := range response.FlowRecords { + // Parse the YAML-like data string into a map + dataFields := parseIPFIXDataString(record.Data) + flowlog := convertIPFIXToFlowlog(dataFields) + flowRecord := FlowRecord{ + Timestamp: time.Now().Unix(), + Flowlog: flowlog, + } + flowRecords = append(flowRecords, flowRecord) + } + + e2e.Logf("Found %d IPFIX flow records from collector API", len(flowRecords)) + return flowRecords, nil +} + +// Parse IPFIX data string format: " key: value \n key2: value2 \n ..." +func parseIPFIXDataString(data string) map[string]interface{} { + fields := make(map[string]interface{}) + lines := strings.Split(data, "\n") + + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" { + continue + } + + // Split on first colon + parts := strings.SplitN(line, ":", 2) + if len(parts) != 2 { + continue + } + + key := strings.TrimSpace(parts[0]) + valueStr := strings.TrimSpace(parts[1]) + + // Try to parse as number + if intVal, err := strconv.Atoi(valueStr); err == nil { + fields[key] = float64(intVal) + } else if floatVal, err := strconv.ParseFloat(valueStr, 64); err == nil { + fields[key] = floatVal + } else { + fields[key] = valueStr + } + } + + return fields +} + +// Convert IPFIX data fields to Flowlog struct +func convertIPFIXToFlowlog(dataFields map[string]interface{}) Flowlog { + flowlog := Flowlog{} + + // Helper to safely get string values + getString := func(key string) string { + if val, ok := dataFields[key]; ok { + if str, ok := val.(string); ok { + return str + } + } + return "" + } + + getInt := func(key string) int { + if val, ok := dataFields[key]; ok { + switch v := val.(type) { + case float64: + return int(v) + case int: + return v + case int64: + return int(v) + } + } + return 0 + } + + // Map IPFIX fields to Flowlog fields + flowlog.SrcAddr = getString("sourceIPv4Address") + if flowlog.SrcAddr == "" { + flowlog.SrcAddr = getString("sourceIPv6Address") + } + flowlog.DstAddr = getString("destinationIPv4Address") + if flowlog.DstAddr == "" { + flowlog.DstAddr = getString("destinationIPv6Address") + } + flowlog.SrcPort = getInt("sourceTransportPort") + flowlog.DstPort = getInt("destinationTransportPort") + flowlog.Proto = getInt("protocolIdentifier") + flowlog.Bytes = getInt("octetDeltaCount") + if flowlog.Bytes == 0 { + flowlog.Bytes = getInt("octetTotalCount") + } + flowlog.Packets = getInt("packetDeltaCount") + if flowlog.Packets == 0 { + flowlog.Packets = getInt("packetTotalCount") + } + flowlog.TimeFlowStartMs = getInt("flowStartMilliseconds") + flowlog.TimeFlowEndMs = getInt("flowEndMilliseconds") + flowlog.TimeReceived = int(time.Now().Unix()) + + // RFC 5477 sampling fields - the key fields for NETOBSERV-2706! + flowlog.Sampling = getInt("samplingPacketInterval") + // Also check samplingProbability and convert if present + if samplingProb, ok := dataFields["samplingProbability"].(float64); ok && samplingProb > 0 { + flowlog.Sampling = int(1.0 / samplingProb) + } + + return flowlog +} + +// Verify some key and deterministic flow recrods fields and their values +func (flowlog *Flowlog) verifyFlowRecord() { + flow := fmt.Sprintf("Flow log is: %+v\n", flowlog) + o.Expect(flowlog.AgentIP).To(o.Equal(flowlog.DstK8SHostIP), flow) + o.Expect(flowlog.Bytes).Should(o.BeNumerically(">", 0), flow) + now := time.Now() + compareTime := now.Add(time.Duration(-2) * time.Hour) + compareTimeMs := compareTime.UnixMilli() + o.Expect(flowlog.TimeFlowEndMs).Should(o.BeNumerically(">", compareTimeMs), flow) + o.Expect(flowlog.TimeFlowStartMs).Should(o.BeNumerically(">", compareTimeMs), flow) + o.Expect(flowlog.TimeReceived).Should(o.BeNumerically(">", compareTime.Unix()), flow) +} + +// Verify IPFIX-specific fields are present and valid +func (flowlog *Flowlog) verifyIPFIXFields() { + flow := fmt.Sprintf("IPFIX Flow log: %+v\n", flowlog) + + // Basic flow verification (IPFIX flows don't have AgentIP/K8S enrichment) + o.Expect(flowlog.Bytes).Should(o.BeNumerically(">", 0), flow) + now := time.Now() + compareTime := now.Add(time.Duration(-2) * time.Hour) + compareTimeMs := compareTime.UnixMilli() + o.Expect(flowlog.TimeFlowEndMs).Should(o.BeNumerically(">", compareTimeMs), flow) + o.Expect(flowlog.TimeFlowStartMs).Should(o.BeNumerically(">", compareTimeMs), flow) + o.Expect(flowlog.TimeReceived).Should(o.BeNumerically(">", compareTime.Unix()), flow) + + // Verify IPFIX standard fields are present and valid + o.Expect(flowlog.SrcAddr).NotTo(o.BeEmpty(), flow) + o.Expect(flowlog.DstAddr).NotTo(o.BeEmpty(), flow) + o.Expect(flowlog.SrcPort).Should(o.BeNumerically(">", 0), flow) + o.Expect(flowlog.DstPort).Should(o.BeNumerically(">", 0), flow) + o.Expect(flowlog.Proto).Should(o.BeNumerically(">", 0), flow) + o.Expect(flowlog.Packets).Should(o.BeNumerically(">", 0), flow) + o.Expect(flowlog.Sampling).Should(o.BeNumerically(">=", 0), flow) +} + +func (lokilabels Lokilabels) getLokiQueryLabels() string { + label := reflect.ValueOf(&lokilabels).Elem() + labelType := label.Type() + var lokiQuery = "{" + for i := 0; i < label.NumField(); i++ { + if label.Field(i).Interface() != "" { + field := labelType.Field(i) + + // Get the label name from loki tag, or use field name as fallback + labelName := field.Name + if lokiTag := field.Tag.Get("loki"); lokiTag != "" { + labelName = lokiTag + } + + // Handle FlowDirection special case: only include if value is 0, 1, or 2 + if field.Name == "FlowDirection" { + if label.Field(i).Interface() != "0" && label.Field(i).Interface() != "1" && label.Field(i).Interface() != "2" { + continue + } + } + + lokiQuery += fmt.Sprintf("%s=\"%s\", ", labelName, label.Field(i).Interface()) + } + } + lokiQuery = strings.TrimSuffix(lokiQuery, ", ") + lokiQuery += "}" + + return lokiQuery +} + +func (lokilabels Lokilabels) getLokiJSONfilterQuery(parameters ...string) string { + lokiQuery := lokilabels.getLokiQueryLabels() + if len(parameters) != 0 { + lokiQuery += " | json" + for _, p := range parameters { + if strings.Contains(p, "Flags") { + lokiQuery += fmt.Sprintf(" %s | json", p) + } else { + lokiQuery += fmt.Sprintf(" | %s", p) + } + } + } + e2e.Logf("Loki query is %s", lokiQuery) + return lokiQuery +} + +func (lokilabels Lokilabels) getLokiRegexFilterQuery(parameters ...string) string { + lokiQuery := lokilabels.getLokiQueryLabels() + if len(parameters) != 0 { + for _, p := range parameters { + lokiQuery += fmt.Sprintf(" |~ %s", p) + } + } + e2e.Logf("Loki query is %s", lokiQuery) + return lokiQuery +} + +func (lokilabels Lokilabels) getLokiQuery(filterType string, parameters ...string) string { + var lokiQuery string + switch filterType { + case "JSON": + lokiQuery = lokilabels.getLokiJSONfilterQuery(parameters...) + case "REGEX": + lokiQuery = lokilabels.getLokiRegexFilterQuery(parameters...) + default: + panic("loki filter is not supported yet") + } + return lokiQuery +} + +func (lokilabels Lokilabels) GetMonolithicLokiFlowLogs(lokiRoute string, startTime time.Time, parameters ...string) ([]FlowRecord, error) { + lc := newLokiClient(lokiRoute, startTime).retry(5) + lc.quiet = false + lc.localhost = true + lokiQuery := lokilabels.getLokiQuery("REGEX", parameters...) + flowRecords := []FlowRecord{} + var res *lokiQueryResponse + err := wait.PollUntilContextTimeout(context.Background(), 30*time.Second, 300*time.Second, false, func(context.Context) (done bool, err error) { + var qErr error + res, qErr = lc.searchLogsInLoki("", lokiQuery) + if qErr != nil { + e2e.Logf("\ngot error %v when getting logs for query: %s\n", qErr, lokiQuery) + return false, qErr + } + + // return results if no error and result is empty + // caller should add assertions to ensure len([]FlowRecord) is as they expected for given loki query + return len(res.Data.Result) > 0, nil + }) + + if err != nil { + return flowRecords, err + } + + for _, result := range res.Data.Result { + flowRecords, err = getFlowRecords(result.Values) + if err != nil { + return []FlowRecord{}, err + } + } + + return flowRecords, err +} + +// TODO: add argument for condition to be matched. +// Get flows from Loki logs +func (lokilabels Lokilabels) getLokiFlowLogs(token, lokiRoute string, startTime time.Time, parameters ...string) ([]FlowRecord, error) { + lc := newLokiClient(lokiRoute, startTime).withToken(token).retry(5) + tenantID := "network" + lokiQuery := lokilabels.getLokiQuery("JSON", parameters...) + flowRecords := []FlowRecord{} + var res *lokiQueryResponse + err := wait.PollUntilContextTimeout(context.Background(), 30*time.Second, 300*time.Second, false, func(context.Context) (done bool, err error) { + var qErr error + res, qErr = lc.searchLogsInLoki(tenantID, lokiQuery) + if qErr != nil { + e2e.Logf("\ngot error %v when getting %s logs for query: %s\n", qErr, tenantID, lokiQuery) + return false, qErr + } + + // return results if no error and result is empty + // caller should add assertions to ensure len([]FlowRecord) is as they expected for given loki query + return len(res.Data.Result) > 0, nil + }) + + if err != nil { + return flowRecords, err + } + + for _, result := range res.Data.Result { + flowRecords, err = getFlowRecords(result.Values) + if err != nil { + return []FlowRecord{}, err + } + } + + return flowRecords, err +} + +// Verify loki flow records and if it was written in the last 5 minutes +func verifyLokilogsTime(token, lokiRoute string, startTime time.Time) error { + lc := newLokiClient(lokiRoute, startTime).withToken(token).retry(5) + res, err := lc.searchLogsInLoki("network", "{app=\"netobserv-flowcollector\", FlowDirection=\"0\"}") + + if err != nil { + return err + } + if len(res.Data.Result) == 0 { + return errors.New("network logs not found") + } + flowRecords := []FlowRecord{} + + for _, result := range res.Data.Result { + flowRecords, err = getFlowRecords(result.Values) + if err != nil { + return err + } + } + + for _, r := range flowRecords { + r.Flowlog.verifyFlowRecord() + } + return nil +} + +// Verify some key and deterministic conversation record fields and their values +func (flowlog *Flowlog) verifyConversationRecord() { + conversationRecord := fmt.Sprintf("Conversation record in error: %+v\n", flowlog) + o.Expect(flowlog.Bytes).Should(o.BeNumerically(">", 0), conversationRecord) + now := time.Now() + compareTime := now.Add(time.Duration(-2) * time.Hour) + compareTimeMs := compareTime.UnixMilli() + o.Expect(flowlog.TimeFlowEndMs).Should(o.BeNumerically(">", compareTimeMs), conversationRecord) + o.Expect(flowlog.TimeFlowStartMs).Should(o.BeNumerically(">", compareTimeMs), conversationRecord) + o.Expect(flowlog.HashID).NotTo(o.BeEmpty(), conversationRecord) + o.Expect(flowlog.NumFlowLogs).Should(o.BeNumerically(">", 0), conversationRecord) +} + +// Verify loki conversation records and if it was written in the last 5 minutes +func verifyConversationRecordTime(record []FlowRecord) { + for _, r := range record { + r.Flowlog.verifyConversationRecord() + } +} + +// Verify flow correctness based on number of bytes +func verifyFlowCorrectness(objectSize string, flowRecords []FlowRecord) { + var multiplier int + switch unit := objectSize[len(objectSize)-1:]; unit { + case "K": + multiplier = 1024 + case "M": + multiplier = 1024 * 1024 + case "G": + multiplier = 1024 * 1024 * 1024 + default: + panic("invalid object size unit") + } + nObject, _ := strconv.Atoi(objectSize[0 : len(objectSize)-1]) + // minBytes is the size of the object fetched + minBytes := nObject * multiplier + // maxBytes is the minBytes +2% tolerance + maxBytes := int(float64(minBytes) + (float64(minBytes) * 0.02)) + var errFlows float64 + nflows := float64(len(flowRecords)) + + for _, r := range flowRecords { + // occurs very rarely but sometimes >= comparison can be flaky + // when eBPF-agent evicts packets sooner, + // currently it configured to be 15seconds. + if r.Flowlog.Bytes <= minBytes { + errFlows++ + } + if r.Flowlog.Bytes >= maxBytes { + errFlows++ + } + r.Flowlog.verifyFlowRecord() + } + // allow only 10% of flows to have Bytes violating minBytes and maxBytes. + tolerance := math.Ceil(nflows * 0.10) + o.Expect(errFlows).Should(o.BeNumerically("<=", tolerance)) +} + +// Verify Packet Translation feature flows +func verifyPacketTranslationFlows(nginxPodIP, nginxPodName, clientPodIP string, flowRecords []FlowRecord) { + for _, r := range flowRecords { + o.Expect(r.Flowlog.XlatDstAddr).To(o.Equal(nginxPodIP)) + o.Expect(r.Flowlog.XlatDstK8SName).To(o.Equal(nginxPodName)) + o.Expect(r.Flowlog.XlatDstK8SType).To(o.Equal("Pod")) + o.Expect(r.Flowlog.DstPort).Should(o.BeNumerically("==", 80)) + o.Expect(r.Flowlog.XlatDstPort).Should(o.BeNumerically("==", 8080)) + o.Expect(r.Flowlog.XlatSrcAddr).To(o.Equal(clientPodIP)) + o.Expect(r.Flowlog.XlatSrcK8SName).To(o.Equal("client")) + o.Expect(r.Flowlog.ZoneID).Should(o.BeNumerically(">=", 0)) + } +} + +// Verify Network Events feature flows +func verifyNetworkEvents(flowRecords []FlowRecord, action NWEvents, policytype, direction string) { + nNWEventsLogs := 0 + for _, flow := range flowRecords { + nwevent := flow.Flowlog.NetworkEvents + if len(nwevent) >= 1 { + e2e.Logf("found nwevent %v", nwevent) + // usually for our scenario we expect only one nw event + // but there could be more than 1. + o.Expect(NWEvents(nwevent[0].Action)).Should(o.Equal(action)) + o.Expect(nwevent[0].Type).Should(o.Equal(policytype)) + o.Expect(nwevent[0].Direction).Should(o.Equal(direction)) + nNWEventsLogs++ + } else { + e2e.Logf("nwevent missing %v", flow.Flowlog) + } + if action == Drop { + o.Expect(flow.Flowlog.PktDropPackets).Should(o.BeNumerically(">", 0)) + o.Expect(flow.Flowlog.PktDropLatestState).Should(o.Equal("TCP_INVALID_STATE")) + o.Expect(flow.Flowlog.PktDropLatestDropCause).Should(o.ContainSubstring("NetworkEvent_")) + } + } + o.Expect(nNWEventsLogs).Should(o.BeNumerically(">=", 1), "Found no logs with Network Events") +} diff --git a/integration-tests/backend/flowcollectorslice.go b/integration-tests/backend/flowcollectorslice.go new file mode 100644 index 0000000000..2870219a9c --- /dev/null +++ b/integration-tests/backend/flowcollectorslice.go @@ -0,0 +1,59 @@ +package e2etests + +import ( + "context" + "fmt" + "reflect" + "time" + + exutil "github.com/openshift/origin/test/extended/util" + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" + + "k8s.io/apimachinery/pkg/util/wait" + e2e "k8s.io/kubernetes/test/e2e/framework" +) + +// FlowcollectorSlice struct to handle FlowcollectorSlice resources +type FlowcollectorSlice struct { + Name string + Namespace string + Sampling string + SubnetLabels string + Template string +} + +// create flowcollector CRD for a given manifest file +func (flowSlice FlowcollectorSlice) CreateFlowcollectorSlice(oc *exutil.CLI) { + parameters := []string{"--ignore-unknown-parameters=true", "-f", flowSlice.Template, "-p"} + + flowCollector := reflect.ValueOf(&flowSlice).Elem() + + for i := 0; i < flowCollector.NumField(); i++ { + if flowCollector.Field(i).Interface() != "" { + if flowCollector.Type().Field(i).Name != "Template" { + parameters = append(parameters, fmt.Sprintf("%s=%s", flowCollector.Type().Field(i).Name, flowCollector.Field(i).Interface())) + } + } + } + + compat_otp.ApplyNsResourceFromTemplate(oc, flowSlice.Namespace, parameters...) +} + +// DeleteFlowcollectorSlice deletes FlowCollectorSlice CRD from a cluster +func (flowSlice *FlowcollectorSlice) DeleteFlowcollectorSlice(oc *exutil.CLI) error { + return oc.AsAdmin().WithoutNamespace().Run("delete").Args("flowcollectorslice", flowSlice.Name, "-n", flowSlice.Namespace).Execute() +} + +// WaitForFlowcollectorSliceReady waits for FlowCollectorSlice to be ready by checking status conditions +func (flowSlice *FlowcollectorSlice) WaitForFlowcollectorSliceReady(oc *exutil.CLI) { + err := wait.PollUntilContextTimeout(context.Background(), 10*time.Second, 600*time.Second, false, func(context.Context) (done bool, err error) { + // Check Ready condition + readyCondition, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("flowcollectorslice", flowSlice.Name, "-n", flowSlice.Namespace, "-o", "jsonpath={.status.conditions[?(@.type==\"Ready\")].status}").Output() + if err != nil || readyCondition != "True" { + e2e.Logf("Error getting Ready condition: %v", err) + return false, nil + } + return true, nil + }) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("FlowCollectorSlice %s/%s did not become Ready", flowSlice.Namespace, flowSlice.Name)) +} diff --git a/integration-tests/backend/go.mod b/integration-tests/backend/go.mod new file mode 100644 index 0000000000..1bf63a1fae --- /dev/null +++ b/integration-tests/backend/go.mod @@ -0,0 +1,375 @@ +module e2etests + +go 1.25.1 + +replace ( + bitbucket.org/ww/goautoneg => github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d + github.com/docker/docker => github.com/docker/docker v26.1.5+incompatible + github.com/jteeuwen/go-bindata => github.com/jteeuwen/go-bindata v3.0.8-0.20151023091102-a0ff2567cfb7+incompatible + github.com/onsi/ginkgo/v2 => github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20260303184444-1cc650aa0565 + github.com/opencontainers/cgroups => github.com/opencontainers/cgroups v0.0.3 + github.com/opencontainers/runc => github.com/opencontainers/runc v1.1.12 + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc => go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 + k8s.io/api => github.com/openshift/kubernetes/staging/src/k8s.io/api v0.0.0-20251017123720-96593f323733 + k8s.io/apiextensions-apiserver => github.com/openshift/kubernetes/staging/src/k8s.io/apiextensions-apiserver v0.0.0-20251017123720-96593f323733 + k8s.io/apimachinery => github.com/openshift/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20251017123720-96593f323733 + k8s.io/apiserver => github.com/openshift/kubernetes/staging/src/k8s.io/apiserver v0.0.0-20251017123720-96593f323733 + k8s.io/cli-runtime => github.com/openshift/kubernetes/staging/src/k8s.io/cli-runtime v0.0.0-20251017123720-96593f323733 + k8s.io/client-go => github.com/openshift/kubernetes/staging/src/k8s.io/client-go v0.0.0-20251017123720-96593f323733 + k8s.io/cloud-provider => github.com/openshift/kubernetes/staging/src/k8s.io/cloud-provider v0.0.0-20251017123720-96593f323733 + k8s.io/cluster-bootstrap => github.com/openshift/kubernetes/staging/src/k8s.io/cluster-bootstrap v0.0.0-20251017123720-96593f323733 + k8s.io/code-generator => github.com/openshift/kubernetes/staging/src/k8s.io/code-generator v0.0.0-20251017123720-96593f323733 + k8s.io/component-base => github.com/openshift/kubernetes/staging/src/k8s.io/component-base v0.0.0-20251017123720-96593f323733 + k8s.io/component-helpers => github.com/openshift/kubernetes/staging/src/k8s.io/component-helpers v0.0.0-20251017123720-96593f323733 + k8s.io/controller-manager => github.com/openshift/kubernetes/staging/src/k8s.io/controller-manager v0.0.0-20251017123720-96593f323733 + k8s.io/cri-api => github.com/openshift/kubernetes/staging/src/k8s.io/cri-api v0.0.0-20251017123720-96593f323733 + k8s.io/cri-client => github.com/openshift/kubernetes/staging/src/k8s.io/cri-client v0.0.0-20251017123720-96593f323733 + k8s.io/csi-translation-lib => github.com/openshift/kubernetes/staging/src/k8s.io/csi-translation-lib v0.0.0-20251017123720-96593f323733 + k8s.io/dynamic-resource-allocation => github.com/openshift/kubernetes/staging/src/k8s.io/dynamic-resource-allocation v0.0.0-20251017123720-96593f323733 + k8s.io/endpointslice => github.com/openshift/kubernetes/staging/src/k8s.io/endpointslice v0.0.0-20251017123720-96593f323733 + k8s.io/kube-aggregator => github.com/openshift/kubernetes/staging/src/k8s.io/kube-aggregator v0.0.0-20251017123720-96593f323733 + k8s.io/kube-controller-manager => github.com/openshift/kubernetes/staging/src/k8s.io/kube-controller-manager v0.0.0-20251017123720-96593f323733 + k8s.io/kube-proxy => github.com/openshift/kubernetes/staging/src/k8s.io/kube-proxy v0.0.0-20251017123720-96593f323733 + k8s.io/kube-scheduler => github.com/openshift/kubernetes/staging/src/k8s.io/kube-scheduler v0.0.0-20251017123720-96593f323733 + k8s.io/kubectl => github.com/openshift/kubernetes/staging/src/k8s.io/kubectl v0.0.0-20251017123720-96593f323733 + k8s.io/kubelet => github.com/openshift/kubernetes/staging/src/k8s.io/kubelet v0.0.0-20251017123720-96593f323733 + k8s.io/kubernetes => github.com/openshift/kubernetes v1.30.1-0.20251017123720-96593f323733 + k8s.io/legacy-cloud-providers => github.com/openshift/kubernetes/staging/src/k8s.io/legacy-cloud-providers v0.0.0-20251017123720-96593f323733 + k8s.io/metrics => github.com/openshift/kubernetes/staging/src/k8s.io/metrics v0.0.0-20251017123720-96593f323733 + k8s.io/mount-utils => github.com/openshift/kubernetes/staging/src/k8s.io/mount-utils v0.0.0-20251017123720-96593f323733 + k8s.io/pod-security-admission => github.com/openshift/kubernetes/staging/src/k8s.io/pod-security-admission v0.0.0-20251017123720-96593f323733 + k8s.io/sample-apiserver => github.com/openshift/kubernetes/staging/src/k8s.io/sample-apiserver v0.0.0-20251017123720-96593f323733 + k8s.io/sample-cli-plugin => github.com/openshift/kubernetes/staging/src/k8s.io/sample-cli-plugin v0.0.0-20251017123720-96593f323733 + k8s.io/sample-controller => github.com/openshift/kubernetes/staging/src/k8s.io/sample-controller v0.0.0-20251017123720-96593f323733 +) + +require ( + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2 v2.2.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.3.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 + github.com/aws/aws-sdk-go-v2 v1.41.7 + github.com/aws/aws-sdk-go-v2/config v1.29.14 + github.com/aws/aws-sdk-go-v2/credentials v1.17.67 + github.com/aws/aws-sdk-go-v2/service/iam v1.53.10 + github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 + github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 + github.com/google/uuid v1.6.0 + github.com/onsi/ginkgo/v2 v2.28.1 + github.com/onsi/gomega v1.39.1 + github.com/openshift/origin v1.5.0-alpha.3.0.20260403210430-c77ff4a065bf + github.com/tidwall/gjson v1.18.0 + golang.org/x/mod v0.32.0 + google.golang.org/api v0.247.0 + gopkg.in/yaml.v3 v3.0.1 + k8s.io/apimachinery v0.35.1 + k8s.io/kubernetes v1.35.1 + k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 +) + +require ( + cel.dev/expr v0.24.0 // indirect + cloud.google.com/go v0.121.6 // indirect + cloud.google.com/go/auth v0.16.5 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.8.0 // indirect + cloud.google.com/go/iam v1.5.2 // indirect + cloud.google.com/go/monitoring v1.24.2 // indirect + cloud.google.com/go/storage v1.56.0 // indirect + cyphar.com/go-pathrs v0.2.1 // indirect + github.com/Azure/azure-pipeline-go v0.2.3 // indirect + github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.4.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armfeatures v1.2.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 // indirect + github.com/Azure/azure-storage-blob-go v0.15.0 // indirect + github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect + github.com/Azure/go-autorest v14.2.0+incompatible // indirect + github.com/Azure/go-autorest/autorest v0.11.29 // indirect + github.com/Azure/go-autorest/autorest/adal v0.9.24 // indirect + github.com/Azure/go-autorest/autorest/azure/auth v0.5.13 // indirect + github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 // indirect + github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect + github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect + github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect + github.com/Azure/go-autorest/logger v0.2.1 // indirect + github.com/Azure/go-autorest/tracing v0.6.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 // indirect + github.com/IBM-Cloud/power-go-client v1.12.0 // indirect + github.com/IBM/go-sdk-core/v5 v5.21.0 // indirect + github.com/IBM/platform-services-go-sdk v0.81.0 // indirect + github.com/IBM/vpc-go-sdk v0.70.1 // indirect + github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab // indirect + github.com/MakeNowJust/heredoc v1.0.0 // indirect + github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/Microsoft/hcsshim v0.14.1 // indirect + github.com/Microsoft/hnslib v0.1.2 // indirect + github.com/NYTimes/gziphandler v1.1.1 // indirect + github.com/ProtonMail/go-crypto v1.1.6 // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect + github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect + github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2 // indirect + github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect + github.com/aws/aws-sdk-go v1.55.8 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 // indirect + github.com/aws/smithy-go v1.25.1 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cenkalti/backoff/v5 v5.0.2 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/chai2010/gettext-go v1.0.2 // indirect + github.com/cloudflare/circl v1.6.0 // indirect + github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 // indirect + github.com/container-storage-interface/spec v1.9.0 // indirect + github.com/containerd/containerd v1.7.31 // indirect + github.com/containerd/containerd/api v1.9.0 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/ttrpc v1.2.7 // indirect + github.com/containerd/typeurl/v2 v2.2.3 // indirect + github.com/coreos/go-semver v0.3.1 // indirect + github.com/coreos/go-systemd/v22 v22.5.0 // indirect + github.com/coreos/stream-metadata-go v0.4.9 // indirect + github.com/cyphar/filepath-securejoin v0.6.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/dimchansky/utfbom v1.1.1 // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/docker v28.5.2+incompatible // indirect + github.com/docker/go-connections v0.5.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/emicklei/go-restful/v3 v3.13.0 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect + github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect + github.com/euank/go-kmsg-parser v2.0.0+incompatible // indirect + github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect + github.com/fatih/camelcase v1.0.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/fsouza/go-dockerclient v1.12.0 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.9 // indirect + github.com/gebn/bmc v0.0.0-20250519231546-bf709e03fe3c // indirect + github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 // indirect + github.com/go-errors/errors v1.4.2 // indirect + github.com/go-jose/go-jose/v4 v4.1.1 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-openapi/analysis v0.23.0 // indirect + github.com/go-openapi/errors v0.22.1 // indirect + github.com/go-openapi/jsonpointer v0.21.2 // indirect + github.com/go-openapi/jsonreference v0.21.0 // indirect + github.com/go-openapi/loads v0.22.0 // indirect + github.com/go-openapi/runtime v0.28.0 // indirect + github.com/go-openapi/spec v0.21.0 // indirect + github.com/go-openapi/strfmt v0.23.0 // indirect + github.com/go-openapi/swag v0.23.1 // indirect + github.com/go-openapi/validate v0.24.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.26.0 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/godbus/dbus/v5 v5.1.0 // indirect + github.com/gofrs/uuid v4.4.0+incompatible // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt/v4 v4.5.2 // indirect + github.com/golang-jwt/jwt/v5 v5.3.0 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/btree v1.1.3 // indirect + github.com/google/cadvisor v0.53.0 // indirect + github.com/google/cel-go v0.26.1 // indirect + github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/gopacket v1.1.19 // indirect + github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect + github.com/googleapis/gax-go/v2 v2.15.0 // indirect + github.com/gophercloud/gophercloud v1.14.1 // indirect + github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect + github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect + github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.0 // indirect + github.com/hashicorp/go-checkpoint v0.5.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-retryablehttp v0.7.7 // indirect + github.com/hashicorp/go-uuid v1.0.3 // indirect + github.com/hashicorp/go-version v1.7.0 // indirect + github.com/hashicorp/hc-install v0.9.2 // indirect + github.com/hashicorp/terraform-exec v0.23.0 // indirect + github.com/hashicorp/terraform-json v0.25.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/karrick/godirwalk v1.17.0 // indirect + github.com/klauspost/compress v1.18.2 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect + github.com/mailru/easyjson v0.9.0 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-ieproxy v0.0.11 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/microsoft/kiota-abstractions-go v1.9.3 // indirect + github.com/microsoft/kiota-authentication-azure-go v1.3.0 // indirect + github.com/microsoft/kiota-http-go v1.5.2 // indirect + github.com/microsoft/kiota-serialization-form-go v1.1.2 // indirect + github.com/microsoft/kiota-serialization-json-go v1.1.2 // indirect + github.com/microsoft/kiota-serialization-multipart-go v1.1.2 // indirect + github.com/microsoft/kiota-serialization-text-go v1.1.2 // indirect + github.com/microsoftgraph/msgraph-sdk-go v1.81.0 // indirect + github.com/microsoftgraph/msgraph-sdk-go-core v1.3.2 // indirect + github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/go-wordwrap v1.0.1 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/patternmatcher v0.6.0 // indirect + github.com/moby/spdystream v0.5.1 // indirect + github.com/moby/sys/mountinfo v0.7.2 // indirect + github.com/moby/sys/sequential v0.6.0 // indirect + github.com/moby/sys/user v0.4.0 // indirect + github.com/moby/sys/userns v0.1.0 // indirect + github.com/moby/term v0.5.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect + github.com/morikuni/aec v1.0.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect + github.com/oklog/ulid v1.3.1 // indirect + github.com/opencontainers/cgroups v0.0.3 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect + github.com/opencontainers/runc v1.3.0 // indirect + github.com/opencontainers/runtime-spec v1.2.1 // indirect + github.com/opencontainers/selinux v1.13.1 // indirect + github.com/openshift/api v0.0.0-20260327065519-582dc3d316b7 // indirect + github.com/openshift/client-go v0.0.0-20260330134249-7e1499aaacd7 // indirect + github.com/openshift/library-go v0.0.0-20260303171201-5d9eb6295ff6 // indirect + github.com/opentracing/opentracing-go v1.2.0 // indirect + github.com/pborman/uuid v1.2.0 // indirect + github.com/peterbourgon/diskv v2.0.1+incompatible // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/client_golang v1.23.2 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/procfs v0.17.0 // indirect + github.com/rs/zerolog v1.34.0 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/spf13/cobra v1.10.1 // indirect + github.com/spf13/pflag v1.0.9 // indirect + github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect + github.com/std-uritemplate/std-uritemplate/go/v2 v2.0.3 // indirect + github.com/stoewer/go-strcase v1.3.1 // indirect + github.com/stretchr/objx v0.5.2 // indirect + github.com/stretchr/testify v1.11.1 // indirect + github.com/tecbiz-ch/nutanix-go-sdk v0.1.15 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.1 // indirect + github.com/vmware/govmomi v0.51.0 // indirect + github.com/x448/float16 v0.8.4 // indirect + github.com/xlab/treeprint v1.2.0 // indirect + github.com/zclconf/go-cty v1.16.2 // indirect + github.com/zeebo/errs v1.4.0 // indirect + go.etcd.io/etcd/api/v3 v3.6.5 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.6.5 // indirect + go.etcd.io/etcd/client/v3 v3.6.5 // indirect + go.mongodb.org/mongo-driver v1.17.3 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.36.0 // indirect + go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.44.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect + go.opentelemetry.io/otel v1.37.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.36.0 // indirect + go.opentelemetry.io/otel/metric v1.37.0 // indirect + go.opentelemetry.io/otel/sdk v1.37.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.37.0 // indirect + go.opentelemetry.io/otel/trace v1.37.0 // indirect + go.opentelemetry.io/proto/otlp v1.7.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/crypto v0.47.0 // indirect + golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect + golang.org/x/net v0.49.0 // indirect + golang.org/x/oauth2 v0.30.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/sys v0.40.0 // indirect + golang.org/x/term v0.39.0 // indirect + golang.org/x/text v0.33.0 // indirect + golang.org/x/time v0.12.0 // indirect + golang.org/x/tools v0.41.0 // indirect + google.golang.org/genproto v0.0.0-20250603155806-513f23925822 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250818200422-3122310a409c // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250826171959-ef028d996bc1 // indirect + google.golang.org/grpc v1.75.1 // indirect + google.golang.org/protobuf v1.36.8 // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect + gopkg.in/go-jose/go-jose.v2 v2.6.3 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + k8s.io/api v0.35.1 // indirect + k8s.io/apiextensions-apiserver v0.35.1 // indirect + k8s.io/apiserver v0.35.1 // indirect + k8s.io/cli-runtime v0.33.4 // indirect + k8s.io/client-go v0.35.1 // indirect + k8s.io/cloud-provider v0.31.1 // indirect + k8s.io/cluster-bootstrap v0.0.0 // indirect + k8s.io/component-base v0.35.1 // indirect + k8s.io/component-helpers v0.35.1 // indirect + k8s.io/controller-manager v0.32.1 // indirect + k8s.io/cri-api v0.27.1 // indirect + k8s.io/cri-client v0.0.0 // indirect + k8s.io/csi-translation-lib v0.0.0 // indirect + k8s.io/dynamic-resource-allocation v0.35.1 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kms v0.35.1 // indirect + k8s.io/kube-aggregator v0.35.1 // indirect + k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 // indirect + k8s.io/kube-scheduler v0.0.0 // indirect + k8s.io/kubectl v0.35.1 // indirect + k8s.io/kubelet v0.31.1 // indirect + k8s.io/mount-utils v0.0.0 // indirect + k8s.io/pod-security-admission v0.35.1 // indirect + k8s.io/sample-apiserver v0.0.0 // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.33.0 // indirect + sigs.k8s.io/gateway-api v1.4.0 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect + sigs.k8s.io/kube-storage-version-migrator v0.0.6-0.20230721195810-5c8923c5ff96 // indirect + sigs.k8s.io/kustomize/api v0.20.1 // indirect + sigs.k8s.io/kustomize/kyaml v0.20.1 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect +) diff --git a/integration-tests/backend/go.sum b/integration-tests/backend/go.sum new file mode 100644 index 0000000000..58a3afe3bd --- /dev/null +++ b/integration-tests/backend/go.sum @@ -0,0 +1,920 @@ +cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY= +cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= +cloud.google.com/go v0.121.6 h1:waZiuajrI28iAf40cWgycWNgaXPO06dupuS+sgibK6c= +cloud.google.com/go v0.121.6/go.mod h1:coChdst4Ea5vUpiALcYKXEpR1S9ZgXbhEzzMcMR66vI= +cloud.google.com/go/auth v0.16.5 h1:mFWNQ2FEVWAliEQWpAdH80omXFokmrnbDhUS9cBywsI= +cloud.google.com/go/auth v0.16.5/go.mod h1:utzRfHMP+Vv0mpOkTRQoWD2q3BatTOoWbA7gCc2dUhQ= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.8.0 h1:HxMRIbao8w17ZX6wBnjhcDkW6lTFpgcaobyVfZWqRLA= +cloud.google.com/go/compute/metadata v0.8.0/go.mod h1:sYOGTp851OV9bOFJ9CH7elVvyzopvWQFNNghtDQ/Biw= +cloud.google.com/go/iam v1.5.2 h1:qgFRAGEmd8z6dJ/qyEchAuL9jpswyODjA2lS+w234g8= +cloud.google.com/go/iam v1.5.2/go.mod h1:SE1vg0N81zQqLzQEwxL2WI6yhetBdbNQuTvIKCSkUHE= +cloud.google.com/go/logging v1.13.0 h1:7j0HgAp0B94o1YRDqiqm26w4q1rDMH7XNRU34lJXHYc= +cloud.google.com/go/logging v1.13.0/go.mod h1:36CoKh6KA/M0PbhPKMq6/qety2DCAErbhXT62TuXALA= +cloud.google.com/go/longrunning v0.6.7 h1:IGtfDWHhQCgCjwQjV9iiLnUta9LBCo8R9QmAFsS/PrE= +cloud.google.com/go/longrunning v0.6.7/go.mod h1:EAFV3IZAKmM56TyiE6VAP3VoTzhZzySwI/YI1s/nRsY= +cloud.google.com/go/monitoring v1.24.2 h1:5OTsoJ1dXYIiMiuL+sYscLc9BumrL3CarVLL7dd7lHM= +cloud.google.com/go/monitoring v1.24.2/go.mod h1:x7yzPWcgDRnPEv3sI+jJGBkwl5qINf+6qY4eq0I9B4U= +cloud.google.com/go/storage v1.56.0 h1:iixmq2Fse2tqxMbWhLWC9HfBj1qdxqAmiK8/eqtsLxI= +cloud.google.com/go/storage v1.56.0/go.mod h1:Tpuj6t4NweCLzlNbw9Z9iwxEkrSem20AetIeH/shgVU= +cloud.google.com/go/trace v1.11.6 h1:2O2zjPzqPYAHrn3OKl029qlqG6W8ZdYaOWRyr8NgMT4= +cloud.google.com/go/trace v1.11.6/go.mod h1:GA855OeDEBiBMzcckLPE2kDunIpC72N+Pq8WFieFjnI= +cyphar.com/go-pathrs v0.2.1 h1:9nx1vOgwVvX1mNBWDu93+vaceedpbsDqo+XuBGL40b8= +cyphar.com/go-pathrs v0.2.1/go.mod h1:y8f1EMG7r+hCuFf/rXsKqMJrJAUoADZGNh5/vZPKcGc= +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= +github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= +github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 h1:JXg2dwJUmPB9JmtVmdEB16APJ7jurfbY5jnfXpJoRMc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0/go.mod h1:YD5h/ldMsG0XiIw7PdyNhLxaM317eFh5yNLccNfGdyw= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2 v2.2.0 h1:Hp+EScFOu9HeCbeW8WU2yQPJd4gGwhMgKxWe+G6jNzw= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2 v2.2.0/go.mod h1:/pz8dyNQe+Ey3yBp/XuYz7oqX8YDNWVpPB0hH3XWfbc= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0 h1:LkHbJbgF3YyvC53aqYGR+wWQDn2Rdp9AQdGndf9QvY4= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0/go.mod h1:QyiQdW4f4/BIfB8ZutZ2s+28RAgfa/pT+zS++ZHyM1I= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFGRSlMKCQelWwxUyYVEUqseBJVemLyqWJjvMyt0do= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0/go.mod h1:LRr2FzBTQlONPPa5HREE5+RjSCTXl7BwOvYOaWTqCaI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.0 h1:2qsIIvxVT+uE6yrNldntJKlLRgxGbZ85kgtz5SNBhMw= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.0/go.mod h1:AW8VEadnhw9xox+VaVd9sP7NjzOAnaZBLRH6Tq3cJ38= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.4.0 h1:HlZMUZW8S4P9oob1nCHxCCKrytxyLc+24nUJGssoEto= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.4.0/go.mod h1:StGsLbuJh06Bd8IBfnAlIFV3fLb+gkczONWf15hpX2E= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.0.0 h1:pPvTJ1dY0sA35JOeFq6TsY2xj6Z85Yo23Pj4wCCvu4o= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.0.0/go.mod h1:mLfWfj8v3jfWKsL9G4eoBoXVcsqcIUTapmdKy7uGOp0= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.3.0 h1:L7G3dExHBgUxsO3qpTGhk/P2dgnYyW48yn7AO33Tbek= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.3.0/go.mod h1:Ms6gYEy0+A2knfKrwdatsggTXYA2+ICKug8w7STorFw= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armfeatures v1.2.0 h1:wIDqH4WA5uJ6irRqjzodeSw6Pmp0tu3oIbwzBZEdMfQ= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armfeatures v1.2.0/go.mod h1:g8mnARUMaYRsg80mxm3PxjF7+oUotB/lneDbwYbGNxg= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 h1:Dd+RhdJn0OTtVGaeDLZpcumkIVCtA/3/Fo42+eoYvVM= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0/go.mod h1:5kakwfW5CjC9KK+Q4wjXAg+ShuIm2mBMua0ZFj2C8PE= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBADtcH2rRqPxYB1Ljwms5gFA2LqrM= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4/go.mod h1:8mwH4klAm9DUgR2EEHyEEAQlRDvLPyg5fQry3y+cDew= +github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= +github.com/Azure/azure-storage-blob-go v0.15.0/go.mod h1:vbjsVbX0dlxnRc4FFMPsS9BsJWPcne7GB7onqlPvz58= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.28/go.mod h1:MrkzG3Y3AH668QyF9KRk5neJnGgmhQ6krbhR8Q5eMvA= +github.com/Azure/go-autorest/autorest v0.11.29 h1:I4+HL/JDvErx2LjyzaVxllw2lRDB5/BT2Bm4g20iqYw= +github.com/Azure/go-autorest/autorest v0.11.29/go.mod h1:ZtEzC4Jy2JDrZLxvWs8LrBWEBycl1hbT1eknI8MtfAs= +github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= +github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= +github.com/Azure/go-autorest/autorest/adal v0.9.22/go.mod h1:XuAbAEUv2Tta//+voMI038TrJBqjKam0me7qR+L8Cmk= +github.com/Azure/go-autorest/autorest/adal v0.9.24 h1:BHZfgGsGwdkHDyZdtQRQk1WeUdW0m2WPAwuHZwUi5i4= +github.com/Azure/go-autorest/autorest/adal v0.9.24/go.mod h1:7T1+g0PYFmACYW5LlG2fcoPiPlFHjClyRGL7dRlP5c8= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.13 h1:Ov8avRZi2vmrE2JcXw+tu5K/yB41r7xK9GZDiBF7NdM= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.13/go.mod h1:5BAVfWLWXihP47vYrPuBKKf4cS0bXI+KM9Qx6ETDJYo= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 h1:w77/uPk80ZET2F+AfQExZyEWtn+0Rk/uw17m9fv5Ajc= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.6/go.mod h1:piCfgPho7BiIDdEQ1+g4VmKyD5y+p/XtSNqE6Hc4QD0= +github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.2 h1:PGN4EDXnuQbojHbU0UWoNvmu9AGVwYHG9/fkDYhtAfw= +github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU= +github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= +github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= +github.com/Azure/go-autorest/autorest/validation v0.3.1 h1:AgyqjAd94fwNAoTjl/WQXg4VvFeRFpO+UhNyRXqF1ac= +github.com/Azure/go-autorest/autorest/validation v0.3.1/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E= +github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= +github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs= +github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 h1:UQUsRi8WTzhZntp5313l+CHIAT95ojUI2lpP/ExlZa4= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0/go.mod h1:Cz6ft6Dkn3Et6l2v2a9/RpN7epQ1GtDlO6lj8bEcOvw= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 h1:owcC2UnmsZycprQ5RfRgjydWhuoxg71LUfyiQdijZuM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0/go.mod h1:ZPpqegjbE99EPKsu3iUWV22A04wzGPcAY/ziSIQEEgs= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.53.0 h1:4LP6hvB4I5ouTbGgWtixJhgED6xdf67twf9PoY96Tbg= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.53.0/go.mod h1:jUZ5LYlw40WMd07qxcQJD5M40aUxrfwqQX1g7zxYnrQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 h1:Ron4zCA/yk6U7WOBXhTJcDpsUBG9npumK6xw2auFltQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0/go.mod h1:cSgYe11MCNYunTnRXrKiR/tHc0eoKjICUuWpNZoVCOo= +github.com/IBM-Cloud/power-go-client v1.12.0 h1:tF9Mq5GLYHebpzQT6IYB89lIxEST1E9teuchjxSAaw0= +github.com/IBM-Cloud/power-go-client v1.12.0/go.mod h1:SpTK1ttW8bfMNUVQS8qOEuWn2KOkzaCLyzfze8MG1JE= +github.com/IBM/go-sdk-core/v5 v5.21.0 h1:DUnYhvC4SoC8T84rx5omnhY3+xcQg/Whyoa3mDPIMkk= +github.com/IBM/go-sdk-core/v5 v5.21.0/go.mod h1:Q3BYO6iDA2zweQPDGbNTtqft5tDcEpm6RTuqMlPcvbw= +github.com/IBM/platform-services-go-sdk v0.81.0 h1:jyVQYobEeC+l770YmKVtETFwPpD8DoapZfKY005f/jQ= +github.com/IBM/platform-services-go-sdk v0.81.0/go.mod h1:XOowH+JnIih3FA7uilLVM/9VH7XgCmJ4T/i6eZi7gkw= +github.com/IBM/vpc-go-sdk v0.70.1 h1:6NsbRkiA5gDNxe7cjNx8Pi1j9s0PlhwNQj29wsKZxAo= +github.com/IBM/vpc-go-sdk v0.70.1/go.mod h1:K3vVlje72PYE3ZRt1iouE+jSIq+vCyYzT1HiFC06hUA= +github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab h1:UKkYhof1njT1/xq4SEg5z+VpTgjmNeHwPGRQl7takDI= +github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab/go.mod h1:3VYc5hodBMJ5+l/7J4xAyMeuM2PNuepvHlGs8yilUCA= +github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= +github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/Microsoft/hcsshim v0.14.1 h1:CMuB3fqQVfPdhyXhUqYdUmPUIOhJkmghCx3dJet8Cqs= +github.com/Microsoft/hcsshim v0.14.1/go.mod h1:VnzvPLyWUhxiPVsJ31P6XadxCcTogTguBFDy/1GR/OM= +github.com/Microsoft/hnslib v0.1.2 h1:CshjwTQsNx1o7BIA1XO8HtgDsiCqn+b6kGjL/tIxXQQ= +github.com/Microsoft/hnslib v0.1.2/go.mod h1:5vTyBey4N/VI2ZTNh2gdWhkPMefSbCFYjpvVwye+qtI= +github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= +github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= +github.com/ProtonMail/go-crypto v1.1.6 h1:ZcV+Ropw6Qn0AX9brlQLAUXfqLBc7Bl+f/DmNxpLfdw= +github.com/ProtonMail/go-crypto v1.1.6/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= +github.com/apparentlymart/go-cidr v1.1.0 h1:2mAhrMoF+nhXqxTzSZMUzDHkLjmIHC+Zzn4tdgBZjnU= +github.com/apparentlymart/go-cidr v1.1.0/go.mod h1:EBcsNrHc3zQeuaeCeCtQruQm+n9/YjEn/vI25Lg7Gwc= +github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= +github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= +github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2 h1:7Ip0wMmLHLRJdrloDxZfhMm0xrLXZS8+COSu2bXmEQs= +github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ= +github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk= +github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= +github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= +github.com/aws/aws-sdk-go-v2/config v1.29.14 h1:f+eEi/2cKCg9pqKBoAIwRGzVb70MRKqWX4dg1BDcSJM= +github.com/aws/aws-sdk-go-v2/config v1.29.14/go.mod h1:wVPHWcIFv3WO89w0rE10gzf17ZYy+UVS1Geq8Iei34g= +github.com/aws/aws-sdk-go-v2/credentials v1.17.67 h1:9KxtdcIA/5xPNQyZRgUSpYOE6j9Bc4+D7nZua0KGYOM= +github.com/aws/aws-sdk-go-v2/credentials v1.17.67/go.mod h1:p3C44m+cfnbv763s52gCqrjaqyPikj9Sg47kUVaNZQQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE= +github.com/aws/aws-sdk-go-v2/service/iam v1.53.10 h1:kcN3I3llO7VwIY5w3Pc5FmEonpsr23Ou7Cwk4qf7dik= +github.com/aws/aws-sdk-go-v2/service/iam v1.53.10/go.mod h1:1vkJzjCYC3byO0kIrBqLPzvZpuvYhPXkuyARs6E7tM4= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 h1:etqBTKY581iwLL/H/S2sVgk3C9lAsTJFeXWFDsDcWOU= +github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4= +github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 h1:1Gw+9ajCV1jogloEv1RRnvfRFia2cL6c9cuKV2Ps+G8= +github.com/aws/aws-sdk-go-v2/service/sso v1.25.3/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 h1:hXmVKytPfTy5axZ+fYbR5d0cFmC3JvwLm5kM83luako= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 h1:1XuUZ8mYJw9B6lzAkXhqHlJd/XvaX32evhproijJEZY= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.19/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4= +github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= +github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v5 v5.0.2 h1:rIfFVxEf1QsI7E1ZHfp/B4DF/6QBAUhmgkxc0H7Zss8= +github.com/cenkalti/backoff/v5 v5.0.2/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk= +github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA= +github.com/cloudflare/circl v1.6.0 h1:cr5JKic4HI+LkINy2lg3W2jF8sHCVTBncJr5gIIq7qk= +github.com/cloudflare/circl v1.6.0/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= +github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 h1:aQ3y1lwWyqYPiWZThqv1aFbZMiM9vblcSArJRf2Irls= +github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= +github.com/container-storage-interface/spec v1.9.0 h1:zKtX4STsq31Knz3gciCYCi1SXtO2HJDecIjDVboYavY= +github.com/container-storage-interface/spec v1.9.0/go.mod h1:ZfDu+3ZRyeVqxZM0Ds19MVLkN2d1XJ5MAfi1L3VjlT0= +github.com/containerd/containerd v1.7.31 h1:jn3IMuTV4Bb1Uwb0MFPW2ASJAD3W1lh6QqqZHIZwDh4= +github.com/containerd/containerd v1.7.31/go.mod h1:jdwD6s/BhV4XVJGrvtziNPVA+83n66TwptVaPKprq4E= +github.com/containerd/containerd/api v1.9.0 h1:HZ/licowTRazus+wt9fM6r/9BQO7S0vD5lMcWspGIg0= +github.com/containerd/containerd/api v1.9.0/go.mod h1:GhghKFmTR3hNtyznBoQ0EMWr9ju5AqHjcZPsSpTKutI= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/ttrpc v1.2.7 h1:qIrroQvuOL9HQ1X6KHe2ohc7p+HP/0VE6XPU7elJRqQ= +github.com/containerd/ttrpc v1.2.7/go.mod h1:YCXHsb32f+Sq5/72xHubdiJRQY9inL4a4ZQrAbN1q9o= +github.com/containerd/typeurl/v2 v2.2.3 h1:yNA/94zxWdvYACdYO8zofhrTVuQY73fFU1y++dYSw40= +github.com/containerd/typeurl/v2 v2.2.3/go.mod h1:95ljDnPfD3bAbDJRugOiShd/DlAAsxGtUBhJxIn7SCk= +github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= +github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= +github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/stream-metadata-go v0.4.9 h1:7EHsEYr0/oEJZumWc4b7+2KxD5PXy43esipOII2+JVk= +github.com/coreos/stream-metadata-go v0.4.9/go.mod h1:fMObQqQm8Ku91G04btKzEH3AsdP1mrAb986z9aaK0tE= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= +github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/cyphar/filepath-securejoin v0.6.0 h1:BtGB77njd6SVO6VztOHfPxKitJvd/VPT+OFBFMOi1Is= +github.com/cyphar/filepath-securejoin v0.6.0/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= +github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v26.1.5+incompatible h1:NEAxTwEjxV6VbBMBoGG3zPqbiJosIApZjxlbrG9q3/g= +github.com/docker/docker v26.1.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= +github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= +github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M= +github.com/envoyproxy/go-control-plane v0.13.4/go.mod h1:kDfuBlDVsSj2MjrLEtRWtHlsWIFcGyB2RMO44Dc5GZA= +github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A= +github.com/envoyproxy/go-control-plane/envoy v1.32.4/go.mod h1:Gzjc5k8JcJswLjAx1Zm+wSYE20UrLtt7JZMWiWQXQEw= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= +github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= +github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= +github.com/euank/go-kmsg-parser v2.0.0+incompatible h1:cHD53+PLQuuQyLZeriD1V/esuG4MuU0Pjs5y6iknohY= +github.com/euank/go-kmsg-parser v2.0.0+incompatible/go.mod h1:MhmAMZ8V4CYH4ybgdRwPr2TU5ThnS43puaKEMpja1uw= +github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f h1:Wl78ApPPB2Wvf/TIe2xdyJxTlb6obmF18d8QdkxNDu4= +github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSYXu++VVOHnXeitef/D8n/6y4QV8uLHSFXX4NeXMGc= +github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8= +github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fsouza/go-dockerclient v1.12.0 h1:S2f2crEUbBNCFiF06kR/GvioEB8EMsb3Td/bpawD+aU= +github.com/fsouza/go-dockerclient v1.12.0/go.mod h1:YWUtjg8japrqD/80L98nTtCoxQFp5B5wrSsnyeB5lFo= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/gabriel-vasile/mimetype v1.4.9 h1:5k+WDwEsD9eTLL8Tz3L0VnmVh9QxGjRmjBvAG7U/oYY= +github.com/gabriel-vasile/mimetype v1.4.9/go.mod h1:WnSQhFKJuBlRyLiKohA/2DtIlPFAbguNaG7QCHcyGok= +github.com/gebn/bmc v0.0.0-20250519231546-bf709e03fe3c h1:EIa2nx4hmRi1tayvnbw7++VVKN/pIIirmdvSh6DGR68= +github.com/gebn/bmc v0.0.0-20250519231546-bf709e03fe3c/go.mod h1:7FfuX+OqHI+MyF1eUL5/HBoDhzHBfp6qfKMb87PHPtQ= +github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 h1:Mn26/9ZMNWSw9C9ERFA1PUxfmGpolnw2v0bKOREu5ew= +github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32/go.mod h1:GIjDIg/heH5DOkXY3YJ/wNhfHsQHoXGjl8G8amsYQ1I= +github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= +github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= +github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= +github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= +github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= +github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= +github.com/go-git/go-billy/v5 v5.6.2 h1:6Q86EsPXMa7c3YZ3aLAQsMA0VlWmy43r6FHqa/UNbRM= +github.com/go-git/go-billy/v5 v5.6.2/go.mod h1:rcFC2rAsp/erv7CMz9GczHcuD0D32fWzH+MJAU+jaUU= +github.com/go-git/go-git/v5 v5.14.0 h1:/MD3lCrGjCen5WfEAzKg00MJJffKhC8gzS80ycmCi60= +github.com/go-git/go-git/v5 v5.14.0/go.mod h1:Z5Xhoia5PcWA3NF8vRLURn9E5FRhSl7dGj9ItW3Wk5k= +github.com/go-jose/go-jose/v4 v4.1.1 h1:JYhSgy4mXXzAdF3nUx3ygx347LRXJRrpgyU3adRmkAI= +github.com/go-jose/go-jose/v4 v4.1.1/go.mod h1:BdsZGqgdO3b6tTc6LSE56wcDbMMLuPsw5d4ZD5f94kA= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/analysis v0.23.0 h1:aGday7OWupfMs+LbmLZG4k0MYXIANxcuBTYUC03zFCU= +github.com/go-openapi/analysis v0.23.0/go.mod h1:9mz9ZWaSlV8TvjQHLl2mUW2PbZtemkE8yA5v22ohupo= +github.com/go-openapi/errors v0.22.1 h1:kslMRRnK7NCb/CvR1q1VWuEQCEIsBGn5GgKD9e+HYhU= +github.com/go-openapi/errors v0.22.1/go.mod h1:+n/5UdIqdVnLIJ6Q9Se8HNGUXYaY6CN8ImWzfi/Gzp0= +github.com/go-openapi/jsonpointer v0.21.2 h1:AqQaNADVwq/VnkCmQg6ogE+M3FOsKTytwges0JdwVuA= +github.com/go-openapi/jsonpointer v0.21.2/go.mod h1:50I1STOfbY1ycR8jGz8DaMeLCdXiI6aDteEdRNNzpdk= +github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= +github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= +github.com/go-openapi/loads v0.22.0 h1:ECPGd4jX1U6NApCGG1We+uEozOAvXvJSF4nnwHZ8Aco= +github.com/go-openapi/loads v0.22.0/go.mod h1:yLsaTCS92mnSAZX5WWoxszLj0u+Ojl+Zs5Stn1oF+rs= +github.com/go-openapi/runtime v0.28.0 h1:gpPPmWSNGo214l6n8hzdXYhPuJcGtziTOgUpvsFWGIQ= +github.com/go-openapi/runtime v0.28.0/go.mod h1:QN7OzcS+XuYmkQLw05akXk0jRH/eZ3kb18+1KwW9gyc= +github.com/go-openapi/spec v0.21.0 h1:LTVzPc3p/RzRnkQqLRndbAzjY0d0BCL72A6j3CdL9ZY= +github.com/go-openapi/spec v0.21.0/go.mod h1:78u6VdPw81XU44qEWGhtr982gJ5BWg2c0I5XwVMotYk= +github.com/go-openapi/strfmt v0.23.0 h1:nlUS6BCqcnAk0pyhi9Y+kdDVZdZMHfEKQiS4HaMgO/c= +github.com/go-openapi/strfmt v0.23.0/go.mod h1:NrtIpfKtWIygRkKVsxh7XQMDQW5HKQl6S5ik2elW+K4= +github.com/go-openapi/swag v0.23.1 h1:lpsStH0n2ittzTnbaSloVZLuB5+fvSY/+hnagBjSNZU= +github.com/go-openapi/swag v0.23.1/go.mod h1:STZs8TbRvEQQKUA+JZNAm3EWlgaOBGpyFDqQnDHMef0= +github.com/go-openapi/validate v0.24.0 h1:LdfDKwNbpB6Vn40xhTdNZAnfLECL81w+VX3BumrGD58= +github.com/go-openapi/validate v0.24.0/go.mod h1:iyeX1sEufmv3nPbBdX3ieNviWnOZaJ1+zquzJEf2BAQ= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k= +github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= +github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= +github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA= +github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= +github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= +github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/cadvisor v0.53.0 h1:pmveUw2VBlr/T2SBE9Fsp8gdLhKWyOBkECGbaas9mcI= +github.com/google/cadvisor v0.53.0/go.mod h1:Tz3zf/exzFfdWd1T/U/9eNst0ZR2C6CIV62LJATj5tg= +github.com/google/cel-go v0.26.1 h1:iPbVVEdkhTX++hpe3lzSk7D3G3QSYqLGoHOcEio+UXQ= +github.com/google/cel-go v0.26.1/go.mod h1:A9O8OU9rdvrK5MQyrqfIxo1a0u4g3sF8KB6PUIaryMM= +github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= +github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= +github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= +github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 h1:z2ogiKUYzX5Is6zr/vP9vJGqPwcdqsWjOt+V8J7+bTc= +github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4= +github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= +github.com/googleapis/gax-go/v2 v2.15.0 h1:SyjDc1mGgZU5LncH8gimWo9lW1DtIfPibOG81vgd/bo= +github.com/googleapis/gax-go/v2 v2.15.0/go.mod h1:zVVkkxAQHa1RQpg9z2AUCMnKhi0Qld9rcmyfL1OZhoc= +github.com/gophercloud/gophercloud v1.14.1 h1:DTCNaTVGl8/cFu58O1JwWgis9gtISAFONqpMKNg/Vpw= +github.com/gophercloud/gophercloud v1.14.1/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= +github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA= +github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.0 h1:FbSCl+KggFl+Ocym490i/EyXF4lPgLoUtcSWquBM0Rs= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.0/go.mod h1:qOchhhIlmRcqk/O9uCo/puJlyo07YINaIqdZfZG3Jkc= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.0 h1:+epNPbD5EqgpEMm5wrl4Hqts3jZt8+kYaqUisuuIGTk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.0/go.mod h1:Zanoh4+gvIgluNqcfMVTJueD4wSS5hT7zTt4Mrutd90= +github.com/hashicorp/go-checkpoint v0.5.0 h1:MFYpPZCnQqQTE18jFwSII6eUQrD/oxMFp3mlgcqk5mU= +github.com/hashicorp/go-checkpoint v0.5.0/go.mod h1:7nfLNL10NsxqO4iWuW6tWW0HjZuDrwkBuEQsVcpCOgg= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= +github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= +github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= +github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/hc-install v0.9.2 h1:v80EtNX4fCVHqzL9Lg/2xkp62bbvQMnvPQ0G+OmtO24= +github.com/hashicorp/hc-install v0.9.2/go.mod h1:XUqBQNnuT4RsxoxiM9ZaUk0NX8hi2h+Lb6/c0OZnC/I= +github.com/hashicorp/terraform-exec v0.23.0 h1:MUiBM1s0CNlRFsCLJuM5wXZrzA3MnPYEsiXmzATMW/I= +github.com/hashicorp/terraform-exec v0.23.0/go.mod h1:mA+qnx1R8eePycfwKkCRk3Wy65mwInvlpAeOwmA7vlY= +github.com/hashicorp/terraform-json v0.25.0 h1:rmNqc/CIfcWawGiwXmRuiXJKEiJu1ntGoxseG1hLhoQ= +github.com/hashicorp/terraform-json v0.25.0/go.mod h1:sMKS8fiRDX4rVlR6EJUMudg1WcanxCMoWwTLkgZP/vc= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I= +github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= +github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= +github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/karrick/godirwalk v1.17.0 h1:b4kY7nqDdioR/6qnbHQyDvmA17u5G1cZ6J+CZXwSWoI= +github.com/karrick/godirwalk v1.17.0/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= +github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= +github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= +github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/libopenstorage/openstorage v1.0.0 h1:GLPam7/0mpdP8ZZtKjbfcXJBTIA/T1O6CBErVEFEyIM= +github.com/libopenstorage/openstorage v1.0.0/go.mod h1:Sp1sIObHjat1BeXhfMqLZ14wnOzEhNx2YQedreMcUyc= +github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= +github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= +github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY= +github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= +github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= +github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= +github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= +github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= +github.com/mattn/go-ieproxy v0.0.11 h1:MQ/5BuGSgDAHZOJe6YY80IF2UVCfGkwfo6AeD7HtHYo= +github.com/mattn/go-ieproxy v0.0.11/go.mod h1:/NsJd+kxZBmjMc5hrJCKMbP57B84rvq9BiDRbtO9AS0= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= +github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= +github.com/microsoft/kiota-abstractions-go v1.9.3 h1:cqhbqro+VynJ7kObmo7850h3WN2SbvoyhypPn8uJ1SE= +github.com/microsoft/kiota-abstractions-go v1.9.3/go.mod h1:f06pl3qSyvUHEfVNkiRpXPkafx7khZqQEb71hN/pmuU= +github.com/microsoft/kiota-authentication-azure-go v1.3.0 h1:PWH6PgtzhJjnmvR6N1CFjriwX09Kv7S5K3vL6VbPVrg= +github.com/microsoft/kiota-authentication-azure-go v1.3.0/go.mod h1:l/MPGUVvD7xfQ+MYSdZaFPv0CsLDqgSOp8mXwVgArIs= +github.com/microsoft/kiota-http-go v1.5.2 h1:xqvo4ssWwSvCJw2yuRocKFTxm3Y1iN+a4rrhuTYtBWg= +github.com/microsoft/kiota-http-go v1.5.2/go.mod h1:L+5Ri+SzwELnUcNA0cpbFKp/pBbvypLh3Cd1PR6sjx0= +github.com/microsoft/kiota-serialization-form-go v1.1.2 h1:SD6MATqNw+Dc5beILlsb/D87C36HKC/Zw7l+N9+HY2A= +github.com/microsoft/kiota-serialization-form-go v1.1.2/go.mod h1:m4tY2JT42jAZmgbqFwPy3zGDF+NPJACuyzmjNXeuHio= +github.com/microsoft/kiota-serialization-json-go v1.1.2 h1:eJrPWeQ665nbjO0gsHWJ0Bw6V/ZHHU1OfFPaYfRG39k= +github.com/microsoft/kiota-serialization-json-go v1.1.2/go.mod h1:deaGt7fjZarywyp7TOTiRsjfYiyWxwJJPQZytXwYQn8= +github.com/microsoft/kiota-serialization-multipart-go v1.1.2 h1:1pUyA1QgIeKslQwbk7/ox1TehjlCUUT3r1f8cNlkvn4= +github.com/microsoft/kiota-serialization-multipart-go v1.1.2/go.mod h1:j2K7ZyYErloDu7Kuuk993DsvfoP7LPWvAo7rfDpdPio= +github.com/microsoft/kiota-serialization-text-go v1.1.2 h1:7OfKFlzdjpPygca/+OtqafkEqCWR7+94efUFGC28cLw= +github.com/microsoft/kiota-serialization-text-go v1.1.2/go.mod h1:QNTcswkBPFY3QVBFmzfk00UMNViKQtV0AQKCrRw5ibM= +github.com/microsoftgraph/msgraph-sdk-go v1.81.0 h1:TZ+YbXGCOyRU2A5IWJLOIIKMECMyeRQBr6mcExLne80= +github.com/microsoftgraph/msgraph-sdk-go v1.81.0/go.mod h1:1V9jKcRL+Czs3u8gI2XjUn7xJCAWRKGizA7l14Bg9zQ= +github.com/microsoftgraph/msgraph-sdk-go-core v1.3.2 h1:5jCUSosTKaINzPPQXsz7wsHWwknyBmJSu8+ZWxx3kdQ= +github.com/microsoftgraph/msgraph-sdk-go-core v1.3.2/go.mod h1:iD75MK3LX8EuwjDYCmh0hkojKXK6VKME33u4daCo3cE= +github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible h1:aKW/4cBs+yK6gpqU3K/oIwk9Q/XICqd3zOX/UFuvqmk= +github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= +github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= +github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/spdystream v0.5.1 h1:9sNYeYZUcci9R6/w7KDaFWEWeV4LStVG78Mpyq/Zm/Y= +github.com/moby/spdystream v0.5.1/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= +github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= +github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= +github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= +github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= +github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= +github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/gomega v1.39.1 h1:1IJLAad4zjPn2PsnhH70V4DKRFlrCzGBNrNaru+Vf28= +github.com/onsi/gomega v1.39.1/go.mod h1:hL6yVALoTOxeWudERyfppUcZXjMwIMLnuSfruD2lcfg= +github.com/opencontainers/cgroups v0.0.3 h1:Jc9dWh/0YLGjdy6J/9Ln8NM5BfTA4W2BY0GMozy3aDU= +github.com/opencontainers/cgroups v0.0.3/go.mod h1:s8lktyhlGUqM7OSRL5P7eAW6Wb+kWPNvt4qvVfzA5vs= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/opencontainers/runc v1.1.12 h1:BOIssBaW1La0/qbNZHXOOa71dZfZEQOzW7dqQf3phss= +github.com/opencontainers/runc v1.1.12/go.mod h1:S+lQwSfncpBha7XTy/5lBwWgm5+y5Ma/O44Ekby9FK8= +github.com/opencontainers/runtime-spec v1.2.1 h1:S4k4ryNgEpxW1dzyqffOmhI1BHYcjzU8lpJfSlR0xww= +github.com/opencontainers/runtime-spec v1.2.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/selinux v1.13.1 h1:A8nNeceYngH9Ow++M+VVEwJVpdFmrlxsN22F+ISDCJE= +github.com/opencontainers/selinux v1.13.1/go.mod h1:S10WXZ/osk2kWOYKy1x2f/eXF5ZHJoUs8UU/2caNRbg= +github.com/openshift-eng/openshift-tests-extension v0.0.0-20260127124016-0fed2b824818 h1:jJLE/aCAqDf8U4wc3bE1IEKgIxbb0ICjCNVFA49x/8s= +github.com/openshift-eng/openshift-tests-extension v0.0.0-20260127124016-0fed2b824818/go.mod h1:6gkP5f2HL0meusT0Aim8icAspcD1cG055xxBZ9yC68M= +github.com/openshift/api v0.0.0-20260327065519-582dc3d316b7 h1:7AmoMSqTryaZu65nij6EACe8+DmlMlmR1giaUx5S5sQ= +github.com/openshift/api v0.0.0-20260327065519-582dc3d316b7/go.mod h1:pyVjK0nZ4sRs4fuQVQ4rubsJdahI1PB94LnQ8sGdvxo= +github.com/openshift/client-go v0.0.0-20260330134249-7e1499aaacd7 h1:5GSoQlywIwYsRCw3qN+ZDmN6HrXTMZfI33bdRNm2jRQ= +github.com/openshift/client-go v0.0.0-20260330134249-7e1499aaacd7/go.mod h1:HhXTUIMhgzxR3Ln/zEkr4QjTL0NN7A+t9Py/we9j2ug= +github.com/openshift/kubernetes v1.30.1-0.20251017123720-96593f323733 h1:Mpab1CmJPLVWGB0CNGoWnup/NScvv55MVPe94c8JgUk= +github.com/openshift/kubernetes v1.30.1-0.20251017123720-96593f323733/go.mod h1:w3+IfrXNp5RosdDXg3LB55yijJqR/FwouvVntYHQf0o= +github.com/openshift/kubernetes/staging/src/k8s.io/api v0.0.0-20251017123720-96593f323733 h1:42lm41QwjG8JoSicx4FHcuIG2kxHxlUnz6c+ftg2e0E= +github.com/openshift/kubernetes/staging/src/k8s.io/api v0.0.0-20251017123720-96593f323733/go.mod h1:sRDdfB9W3pU52PnpjJ9RuMVsg/UQ5iLNlVfbRpb250o= +github.com/openshift/kubernetes/staging/src/k8s.io/apiextensions-apiserver v0.0.0-20251017123720-96593f323733 h1:8NT55In5sAdZVLSDm4jyZ7Q7Gi/DTw/Tns5OQtN4i1w= +github.com/openshift/kubernetes/staging/src/k8s.io/apiextensions-apiserver v0.0.0-20251017123720-96593f323733/go.mod h1:ZxysqjDkqvJUamd823zDHJXKD4X19Q1HFmkLG63o9eU= +github.com/openshift/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20251017123720-96593f323733 h1:f/lXWnFFn8f5CKE0obK8PRC4l7fDzmncfvKVxJLBdoU= +github.com/openshift/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20251017123720-96593f323733/go.mod h1:c6W+CrhzWKfUpUBjoSx/88x7wmaGRznQEcR6jN1H3Tg= +github.com/openshift/kubernetes/staging/src/k8s.io/apiserver v0.0.0-20251017123720-96593f323733 h1:tUxTpKWhjvFJey3guoLabrkNjNKfrBVl7VFraLon91Q= +github.com/openshift/kubernetes/staging/src/k8s.io/apiserver v0.0.0-20251017123720-96593f323733/go.mod h1:TRSSqgXggJaDK5vtVtlQ9wEYOk32Pl+9tf0ROf3ljiM= +github.com/openshift/kubernetes/staging/src/k8s.io/cli-runtime v0.0.0-20251017123720-96593f323733 h1:Qttpp28LaO9FG9Db7wl8/UETsexFjRqZbiQCqg3PM8k= +github.com/openshift/kubernetes/staging/src/k8s.io/cli-runtime v0.0.0-20251017123720-96593f323733/go.mod h1:Y/3QBingpsDLPy704y1u6CCThOBJTNxITcOAjVWZcKg= +github.com/openshift/kubernetes/staging/src/k8s.io/client-go v0.0.0-20251017123720-96593f323733 h1:62i8XkBwvTM7d9P+1la2JVsuuLMxtJCCd2jR3xkjAj0= +github.com/openshift/kubernetes/staging/src/k8s.io/client-go v0.0.0-20251017123720-96593f323733/go.mod h1:pqajivnjOqvKyXx5bPYITDe/uBLBA+Tk6f8E01CGcA4= +github.com/openshift/kubernetes/staging/src/k8s.io/cloud-provider v0.0.0-20251017123720-96593f323733 h1:M3wl3m7qduIVzMNYvlXcy+S7dWmD3LZjn/7sbDaxgUM= +github.com/openshift/kubernetes/staging/src/k8s.io/cloud-provider v0.0.0-20251017123720-96593f323733/go.mod h1:46jYZR2jZ3bmcRXZPZzHfJLFD7qR44/AcZ72oiAGVsQ= +github.com/openshift/kubernetes/staging/src/k8s.io/cluster-bootstrap v0.0.0-20251017123720-96593f323733 h1:UUDc/DqmTcT2gU86x2cnGXCxtGyNbZ+uPzVU+l0MwEc= +github.com/openshift/kubernetes/staging/src/k8s.io/cluster-bootstrap v0.0.0-20251017123720-96593f323733/go.mod h1:3jCpXJTFASgtHmv0Ax+rFB6BVpe3DOxJKe/v8wTf/iE= +github.com/openshift/kubernetes/staging/src/k8s.io/component-base v0.0.0-20251017123720-96593f323733 h1:fY15tmTbBFYtxIiv3LldWyJHlNWOqUWdWxz523Z3dF4= +github.com/openshift/kubernetes/staging/src/k8s.io/component-base v0.0.0-20251017123720-96593f323733/go.mod h1:TYThr4NC8GXH90tsn+yCMH6LiXHj7pGNijDwBN6ZsG0= +github.com/openshift/kubernetes/staging/src/k8s.io/component-helpers v0.0.0-20251017123720-96593f323733 h1:q11kjR6cnzaAh57kaH81Obl5hHFqnVwkD1WOUnvj+Go= +github.com/openshift/kubernetes/staging/src/k8s.io/component-helpers v0.0.0-20251017123720-96593f323733/go.mod h1:/yyEEP5EdBUI2dmEyMKzS9XDXrKQBD1Q3G/UFGyBIy0= +github.com/openshift/kubernetes/staging/src/k8s.io/controller-manager v0.0.0-20251017123720-96593f323733 h1:cK+a41pyUZ7FsJZAiExYXVJc4X8hV4TIgeE/lSRwMWQ= +github.com/openshift/kubernetes/staging/src/k8s.io/controller-manager v0.0.0-20251017123720-96593f323733/go.mod h1:uIPPF88dUOgzUajix3EMCWGA4YChCoOo8ikkPyhwDnI= +github.com/openshift/kubernetes/staging/src/k8s.io/cri-api v0.0.0-20251017123720-96593f323733 h1:fSYRAWS1LindhtDYmUjZhoC9lyvHi/H2UF3ammAd4Mc= +github.com/openshift/kubernetes/staging/src/k8s.io/cri-api v0.0.0-20251017123720-96593f323733/go.mod h1:SrD2bRkLK0Fra2C8qzzuRWciVrAkVq6qKgQZqY+psvs= +github.com/openshift/kubernetes/staging/src/k8s.io/cri-client v0.0.0-20251017123720-96593f323733 h1:EU9R8OFlDHiROJ3MTMXtYM4yrNlhqxo9x7fXECXopAo= +github.com/openshift/kubernetes/staging/src/k8s.io/cri-client v0.0.0-20251017123720-96593f323733/go.mod h1:oiryEAfmSayRHtdki0nmpAjQfku0aP4Y+0NIqaqRn3E= +github.com/openshift/kubernetes/staging/src/k8s.io/csi-translation-lib v0.0.0-20251017123720-96593f323733 h1:uFbdQh5m7QGyjpxoiRIgubC3NKlfG/IK7vFTmxgwIEE= +github.com/openshift/kubernetes/staging/src/k8s.io/csi-translation-lib v0.0.0-20251017123720-96593f323733/go.mod h1:jYAZWKz2s5rQTVDh35tpx466iVAoZO+JvuNkt8h2um4= +github.com/openshift/kubernetes/staging/src/k8s.io/dynamic-resource-allocation v0.0.0-20251017123720-96593f323733 h1:VG4UthsljFwvUiDbcpMtw5XOrelJrxU5sVVxBJwLzHY= +github.com/openshift/kubernetes/staging/src/k8s.io/dynamic-resource-allocation v0.0.0-20251017123720-96593f323733/go.mod h1:DdewGEPN49xRm+9KnI5T8nFsDKjSVAfyWtLO7H6Mlsc= +github.com/openshift/kubernetes/staging/src/k8s.io/kube-aggregator v0.0.0-20251017123720-96593f323733 h1:+BXNmgsApDG2ptHVl9RiN8LmjhTBh7234iKKVWemrOs= +github.com/openshift/kubernetes/staging/src/k8s.io/kube-aggregator v0.0.0-20251017123720-96593f323733/go.mod h1:e9QMt2iRFn39p0C6B/6qirIs9hj8p3y4DaGrdEGXuY8= +github.com/openshift/kubernetes/staging/src/k8s.io/kube-scheduler v0.0.0-20251017123720-96593f323733 h1:YXrEzhBTClZ195q3eQl6LUtKjnyBGMso6K4HgxLyj1w= +github.com/openshift/kubernetes/staging/src/k8s.io/kube-scheduler v0.0.0-20251017123720-96593f323733/go.mod h1:pTSelVB5l12qziWSDxi77oc4P2t5N0dxYhhj4uxjxiM= +github.com/openshift/kubernetes/staging/src/k8s.io/kubectl v0.0.0-20251017123720-96593f323733 h1:qRJFpBOLD0wpDvUczHdBZgG7pDe07O5QJiGjbzKEM0k= +github.com/openshift/kubernetes/staging/src/k8s.io/kubectl v0.0.0-20251017123720-96593f323733/go.mod h1:EU/sHfUc/w62dGZ1VmEysozxDAFvARFrIcQmHEmObaY= +github.com/openshift/kubernetes/staging/src/k8s.io/kubelet v0.0.0-20251017123720-96593f323733 h1:Us43/HbC/zmntiPVmZF38Y37Vnk5SAqUcV1Z89hSUUM= +github.com/openshift/kubernetes/staging/src/k8s.io/kubelet v0.0.0-20251017123720-96593f323733/go.mod h1:+bTwPbT5dZB8j6eKQHBZRMfNmY6MEryba1wQljr9VWw= +github.com/openshift/kubernetes/staging/src/k8s.io/mount-utils v0.0.0-20251017123720-96593f323733 h1:J8bbx4ZSR4AKl9MYuX9xG66d8Ehjre/v7pxKLKU5y7c= +github.com/openshift/kubernetes/staging/src/k8s.io/mount-utils v0.0.0-20251017123720-96593f323733/go.mod h1:T7oGB72dQtHfSIJWZmeNU4Xo5QvIpzIuJ8X20TWu628= +github.com/openshift/kubernetes/staging/src/k8s.io/pod-security-admission v0.0.0-20251017123720-96593f323733 h1:2vQPmqKwQU+jpqm7Iv3EU3k8DYYNqZwN/A1AdydMYpc= +github.com/openshift/kubernetes/staging/src/k8s.io/pod-security-admission v0.0.0-20251017123720-96593f323733/go.mod h1:yuCdx9wLndqpNhmsYZh48wtbgrqc8ql1191ke9zIOfg= +github.com/openshift/kubernetes/staging/src/k8s.io/sample-apiserver v0.0.0-20251017123720-96593f323733 h1:BGNp5XlBh6O6GGOzo2698VK5dCVUL58+pKNMb0DB98o= +github.com/openshift/kubernetes/staging/src/k8s.io/sample-apiserver v0.0.0-20251017123720-96593f323733/go.mod h1:7JLAj6I7UWR3Akqvb3hwGRBdV3dgTASNQJhMqdowC0s= +github.com/openshift/library-go v0.0.0-20260303171201-5d9eb6295ff6 h1:xjqy0OolrFdJ+ofI/aD0+2k9+MSk5anP5dXifFt539Q= +github.com/openshift/library-go v0.0.0-20260303171201-5d9eb6295ff6/go.mod h1:D797O/ssKTNglbrGchjIguFq+DbyRYdeds5w4/VTrKM= +github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20260303184444-1cc650aa0565 h1:3/q8qM4HbFa+Een8wgzpwO8W6mO7Po+MwY6uxiXi/ac= +github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20260303184444-1cc650aa0565/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= +github.com/openshift/origin v1.5.0-alpha.3.0.20260403210430-c77ff4a065bf h1:qv31miXP2++Ge08NCfV/f2F/IR3YAdPnDT+LukZ0/gY= +github.com/openshift/origin v1.5.0-alpha.3.0.20260403210430-c77ff4a065bf/go.mod h1:msmaHac7Uiux1ECL98P70F4lNrkxjzngP/2+DvA8q0Q= +github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= +github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pjbgf/sha1cd v0.3.2 h1:a9wb0bp1oC2TGwStyn0Umc/IGKQnEgF0vVaZ8QF8eo4= +github.com/pjbgf/sha1cd v0.3.2/go.mod h1:zQWigSxVmsHEZow5qaLtPYxpcKMMQpa09ixqBxuCS6A= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0= +github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= +github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnBY8= +github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY= +github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= +github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= +github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s= +github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spiffe/go-spiffe/v2 v2.5.0 h1:N2I01KCUkv1FAjZXJMwh95KK1ZIQLYbPfhaxw8WS0hE= +github.com/spiffe/go-spiffe/v2 v2.5.0/go.mod h1:P+NxobPc6wXhVtINNtFjNWGBTreew1GBUCwT2wPmb7g= +github.com/std-uritemplate/std-uritemplate/go/v2 v2.0.3 h1:7hth9376EoQEd1hH4lAp3vnaLP2UMyxuMMghLKzDHyU= +github.com/std-uritemplate/std-uritemplate/go/v2 v2.0.3/go.mod h1:Z5KcoM0YLC7INlNhEezeIZ0TZNYf7WSNO0Lvah4DSeQ= +github.com/stoewer/go-strcase v1.3.1 h1:iS0MdW+kVTxgMoE1LAZyMiYJFKlOzLooE4MxjirtkAs= +github.com/stoewer/go-strcase v1.3.1/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tecbiz-ch/nutanix-go-sdk v0.1.15 h1:ZT5I6OFGswvMceujUE10ZXPNnT5UQIW9gAX4FEFK6Ds= +github.com/tecbiz-ch/nutanix-go-sdk v0.1.15/go.mod h1:wpqz3KCR/I3t/IGzZiFOy6ZRcz1IcR0hzAVbj0UJ388= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= +github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= +github.com/vmware/govmomi v0.51.0 h1:n3RLS9aw/irTOKbiIyJzAb6rOat4YOVv/uDoRsNTSQI= +github.com/vmware/govmomi v0.51.0/go.mod h1:3ywivawGRfMP2SDCeyKqxTl2xNIHTXF0ilvp72dot5A= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= +github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= +github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 h1:S2dVYn90KE98chqDkyE9Z4N61UnQd+KOfgp5Iu53llk= +github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= +github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/zclconf/go-cty v1.16.2 h1:LAJSwc3v81IRBZyUVQDUdZ7hs3SYs9jv0eZJDWHD/70= +github.com/zclconf/go-cty v1.16.2/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= +github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM= +github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= +go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= +go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= +go.etcd.io/etcd/api/v3 v3.6.5 h1:pMMc42276sgR1j1raO/Qv3QI9Af/AuyQUW6CBAWuntA= +go.etcd.io/etcd/api/v3 v3.6.5/go.mod h1:ob0/oWA/UQQlT1BmaEkWQzI0sJ1M0Et0mMpaABxguOQ= +go.etcd.io/etcd/client/pkg/v3 v3.6.5 h1:Duz9fAzIZFhYWgRjp/FgNq2gO1jId9Yae/rLn3RrBP8= +go.etcd.io/etcd/client/pkg/v3 v3.6.5/go.mod h1:8Wx3eGRPiy0qOFMZT/hfvdos+DjEaPxdIDiCDUv/FQk= +go.etcd.io/etcd/client/v3 v3.6.5 h1:yRwZNFBx/35VKHTcLDeO7XVLbCBFbPi+XV4OC3QJf2U= +go.etcd.io/etcd/client/v3 v3.6.5/go.mod h1:ZqwG/7TAFZ0BJ0jXRPoJjKQJtbFo/9NIY8uoFFKcCyo= +go.etcd.io/etcd/pkg/v3 v3.6.5 h1:byxWB4AqIKI4SBmquZUG1WGtvMfMaorXFoCcFbVeoxM= +go.etcd.io/etcd/pkg/v3 v3.6.5/go.mod h1:uqrXrzmMIJDEy5j00bCqhVLzR5jEJIwDp5wTlLwPGOU= +go.etcd.io/etcd/server/v3 v3.6.5 h1:4RbUb1Bd4y1WkBHmuF+cZII83JNQMuNXzyjwigQ06y0= +go.etcd.io/etcd/server/v3 v3.6.5/go.mod h1:PLuhyVXz8WWRhzXDsl3A3zv/+aK9e4A9lpQkqawIaH0= +go.etcd.io/raft/v3 v3.6.0 h1:5NtvbDVYpnfZWcIHgGRk9DyzkBIXOi8j+DDp1IcnUWQ= +go.etcd.io/raft/v3 v3.6.0/go.mod h1:nLvLevg6+xrVtHUmVaTcTz603gQPHfh7kUAwV6YpfGo= +go.mongodb.org/mongo-driver v1.17.3 h1:TQyXhnsWfWtgAhMtOgtYHMTkZIfBTpMTsMnd9ZBeHxQ= +go.mongodb.org/mongo-driver v1.17.3/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/detectors/gcp v1.36.0 h1:F7q2tNlCaHY9nMKHR6XH9/qkp8FktLnIcy6jJNyOCQw= +go.opentelemetry.io/contrib/detectors/gcp v1.36.0/go.mod h1:IbBN8uAIIx734PTonTPxAxnjc2pQTxWNkwfstZ+6H2k= +go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.44.0 h1:KemlMZlVwBSEGaO91WKgp41BBFsnWqqj9sKRwmOqC40= +go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.44.0/go.mod h1:uq8DrRaen3suIWTpdR/JNHCGpurSvMv9D5Nr5CU5TXc= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 h1:9G6E0TXzGFVfTnawRzrPl83iHOAV7L8NJiR8RSGYV1g= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0/go.mod h1:azvtTADFQJA8mX80jIH/akaE7h+dbm/sVuaHqN13w74= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= +go.opentelemetry.io/contrib/propagators/b3 v1.19.0 h1:ulz44cpm6V5oAeg5Aw9HyqGFMS6XM7untlMEhD7YzzA= +go.opentelemetry.io/contrib/propagators/b3 v1.19.0/go.mod h1:OzCmE2IVS+asTI+odXQstRGVfXQ4bXv9nMBRK0nNyqQ= +go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= +go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 h1:dNzwXjZKpMpE2JhmO+9HsPl42NIXFIFSUSSs0fiqra0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0/go.mod h1:90PoxvaEB5n6AOdZvi+yWJQoE95U8Dhhw2bSyRqnTD0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.36.0 h1:JgtbA0xkWHnTmYk7YusopJFX6uleBmAuZ8n05NEh8nQ= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.36.0/go.mod h1:179AK5aar5R3eS9FucPy6rggvU0g52cvKId8pv4+v0c= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0 h1:rixTyDGXFxRy1xzhKrotaHy3/KXdPhlWARrCgK+eqUY= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0/go.mod h1:dowW6UsM9MKbJq5JTz2AMVp3/5iW5I/TStsk8S+CfHw= +go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= +go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= +go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= +go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= +go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= +go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= +go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz09os= +go.opentelemetry.io/proto/otlp v1.7.0/go.mod h1:fSKjH6YJ7HDlwzltzyMj036AJ3ejJLCgCSHGj4efDDo= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8= +golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A= +golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o= +golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c= +golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= +golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= +golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= +golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= +golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY= +golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= +golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= +golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= +golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc= +golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/api v0.247.0 h1:tSd/e0QrUlLsrwMKmkbQhYVa109qIintOls2Wh6bngc= +google.golang.org/api v0.247.0/go.mod h1:r1qZOPmxXffXg6xS5uhx16Fa/UFY8QU/K4bfKrnvovM= +google.golang.org/genproto v0.0.0-20250603155806-513f23925822 h1:rHWScKit0gvAPuOnu87KpaYtjK5zBMLcULh7gxkCXu4= +google.golang.org/genproto v0.0.0-20250603155806-513f23925822/go.mod h1:HubltRL7rMh0LfnQPkMH4NPDFEWp0jw3vixw7jEM53s= +google.golang.org/genproto/googleapis/api v0.0.0-20250818200422-3122310a409c h1:AtEkQdl5b6zsybXcbz00j1LwNodDuH6hVifIaNqk7NQ= +google.golang.org/genproto/googleapis/api v0.0.0-20250818200422-3122310a409c/go.mod h1:ea2MjsO70ssTfCjiwHgI0ZFqcw45Ksuk2ckf9G468GA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250826171959-ef028d996bc1 h1:pmJpJEvT846VzausCQ5d7KreSROcDqmO388w5YbnltA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250826171959-ef028d996bc1/go.mod h1:GmFNa4BdJZ2a8G+wCe9Bg3wwThLrJun751XstdJt5Og= +google.golang.org/grpc v1.75.1 h1:/ODCNEuf9VghjgO3rqLcfg8fiOP0nSluljWFlDxELLI= +google.golang.org/grpc v1.75.1/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/go-jose/go-jose.v2 v2.6.3 h1:nt80fvSDlhKWQgSWyHyy5CfmlQr+asih51R8PTWNKKs= +gopkg.in/go-jose/go-jose.v2 v2.6.3/go.mod h1:zzZDPkNNw/c9IE7Z9jr11mBZQhKQTMzoEEIoEdZlFBI= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.0 h1:Ljk6PdHdOhAb5aDMWXjDLMMhph+BpztA4v1QdqEW2eY= +gotest.tools/v3 v3.5.0/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kms v0.35.1 h1:kjv2r9g1mY7uL+l1RhyAZvWVZIA/4qIfBHXyjFGLRhU= +k8s.io/kms v0.35.1/go.mod h1:VT+4ekZAdrZDMgShK37vvlyHUVhwI9t/9tvh0AyCWmQ= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 h1:Y3gxNAuB0OBLImH611+UDZcmKS3g6CthxToOb37KgwE= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzkbzn+gDM4X9T4Ck= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.33.0 h1:qPrZsv1cwQiFeieFlRqT627fVZ+tyfou/+S5S0H5ua0= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.33.0/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= +sigs.k8s.io/gateway-api v1.4.0 h1:ZwlNM6zOHq0h3WUX2gfByPs2yAEsy/EenYJB78jpQfQ= +sigs.k8s.io/gateway-api v1.4.0/go.mod h1:AR5RSqciWP98OPckEjOjh2XJhAe2Na4LHyXD2FUY7Qk= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/kube-storage-version-migrator v0.0.6-0.20230721195810-5c8923c5ff96 h1:PFWFSkpArPNJxFX4ZKWAk9NSeRoZaXschn+ULa4xVek= +sigs.k8s.io/kube-storage-version-migrator v0.0.6-0.20230721195810-5c8923c5ff96/go.mod h1:EOBQyBowOUsd7U4CJnMHNE0ri+zCXyouGdLwC/jZU+I= +sigs.k8s.io/kustomize/api v0.20.1 h1:iWP1Ydh3/lmldBnH/S5RXgT98vWYMaTUL1ADcr+Sv7I= +sigs.k8s.io/kustomize/api v0.20.1/go.mod h1:t6hUFxO+Ph0VxIk1sKp1WS0dOjbPCtLJ4p8aADLwqjM= +sigs.k8s.io/kustomize/kyaml v0.20.1 h1:PCMnA2mrVbRP3NIB6v9kYCAc38uvFLVs8j/CD567A78= +sigs.k8s.io/kustomize/kyaml v0.20.1/go.mod h1:0EmkQHRUsJxY8Ug9Niig1pUMSCGHxQ5RklbpV/Ri6po= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/integration-tests/backend/ip_utils.go b/integration-tests/backend/ip_utils.go new file mode 100644 index 0000000000..1726177b51 --- /dev/null +++ b/integration-tests/backend/ip_utils.go @@ -0,0 +1,70 @@ +package e2etests + +import ( + "net" + "strings" + + o "github.com/onsi/gomega" + exutil "github.com/openshift/origin/test/extended/util" + e2e "k8s.io/kubernetes/test/e2e/framework" + e2eoutput "k8s.io/kubernetes/test/e2e/framework/pod/output" + netutils "k8s.io/utils/net" +) + +func checkIPStackType(oc *exutil.CLI) string { + svcNetwork, err := oc.WithoutNamespace().AsAdmin().Run("get").Args("network.operator", "cluster", "-o=jsonpath={.spec.serviceNetwork}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if strings.Count(svcNetwork, ":") >= 2 && strings.Count(svcNetwork, ".") >= 2 { + return "dualstack" + } else if strings.Count(svcNetwork, ":") >= 2 { + return "ipv6single" + } else if strings.Count(svcNetwork, ".") >= 2 { + return "ipv4single" + } + return "" +} + +func getServiceIPv4(oc *exutil.CLI, namespace, serviceName string) string { + serviceIPv4, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", namespace, serviceName, "-o=jsonpath={.spec.clusterIP}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The service %s IP in namespace %s is %q", serviceName, namespace, serviceIPv4) + return serviceIPv4 +} + +// getPodIP returns IPv6 and IPv4 in vars in order on dual stack respectively and main IP in case of single stack (v4 or v6) in 1st var, and nil in 2nd var +func getPodIP(oc *exutil.CLI, namespace, podName, ipStack string) (string, string) { + if (ipStack == "ipv6single") || (ipStack == "ipv4single") { + podIP, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", "-n", namespace, podName, "-o=jsonpath={.status.podIPs[0].ip}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The pod %s IP in namespace %s is %q", podName, namespace, podIP) + return podIP, "" + } else if ipStack == "dualstack" { + podIP1, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", "-n", namespace, podName, "-o=jsonpath={.status.podIPs[1].ip}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The pod's %s 1st IP in namespace %s is %q", podName, namespace, podIP1) + podIP2, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", "-n", namespace, podName, "-o=jsonpath={.status.podIPs[0].ip}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The pod's %s 2nd IP in namespace %s is %q", podName, namespace, podIP2) + if netutils.IsIPv6String(podIP1) { + e2e.Logf("This is IPv4 primary dual stack cluster with IP %s", podIP1) + return podIP1, podIP2 + } + e2e.Logf("This is IPv6 primary dual stack cluster with IP %s", podIP2) + return podIP2, podIP1 + } + return "", "" +} + +// CurlPod2PodFail ensures no connectivity from a pod to pod regardless of network addressing type on cluster +func CurlPod2PodFail(oc *exutil.CLI, namespaceSrc, podNameSrc, namespaceDst, podNameDst, ipStackType string) { + podIP1, podIP2 := getPodIP(oc, namespaceDst, podNameDst, ipStackType) + if podIP2 != "" { + _, err := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl --connect-timeout 5 -s "+net.JoinHostPort(podIP1, "8080")) + o.Expect(err).To(o.HaveOccurred()) + _, err = e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl --connect-timeout 5 -s "+net.JoinHostPort(podIP2, "8080")) + o.Expect(err).To(o.HaveOccurred()) + } else { + _, err := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl --connect-timeout 5 -s "+net.JoinHostPort(podIP1, "8080")) + o.Expect(err).To(o.HaveOccurred()) + } +} diff --git a/integration-tests/backend/kafka.go b/integration-tests/backend/kafka.go new file mode 100644 index 0000000000..ac2c601837 --- /dev/null +++ b/integration-tests/backend/kafka.go @@ -0,0 +1,187 @@ +package e2etests + +import ( + "context" + "fmt" + "strings" + "time" + + o "github.com/onsi/gomega" + exutil "github.com/openshift/origin/test/extended/util" + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" + + "k8s.io/apimachinery/pkg/util/wait" + e2e "k8s.io/kubernetes/test/e2e/framework" +) + +// Kafka struct to handle default Kafka installation +type Kafka struct { + Name string + Namespace string + Template string +} + +// KafkaMetrics struct to handle kafka metrics config deployment +type KafkaMetrics struct { + Namespace string + Template string +} + +// KafkaNodePool struct handles creation of kafka node pool +type KafkaNodePool struct { + Namespace string + NodePoolName string + Name string + Template string +} + +// KafkaTopic struct handles creation of kafka topic +type KafkaTopic struct { + Namespace string + TopicName string + Name string + Template string +} + +type KafkaUser struct { + Namespace string + UserName string + Name string + Template string +} + +// deploys default Kafka +func (kafka *Kafka) deployKafka(oc *exutil.CLI) { + e2e.Logf("Deploy Default Kafka") + parameters := []string{"--ignore-unknown-parameters=true", "-f", kafka.Template, "-p", "NAMESPACE=" + kafka.Namespace} + + if kafka.Name != "" { + parameters = append(parameters, "NAME="+kafka.Name) + } + + compat_otp.ApplyNsResourceFromTemplate(oc, kafka.Namespace, parameters...) +} + +// deploys Kafka Metrics +func (kafkaMetrics *KafkaMetrics) deployKafkaMetrics(oc *exutil.CLI) { + e2e.Logf("Deploy Kafka metrics") + parameters := []string{"--ignore-unknown-parameters=true", "-f", kafkaMetrics.Template, "-p", "NAMESPACE=" + kafkaMetrics.Namespace} + + compat_otp.ApplyNsResourceFromTemplate(oc, kafkaMetrics.Namespace, parameters...) +} + +// creates a Kafka topic +func (kafkaTopic *KafkaTopic) deployKafkaTopic(oc *exutil.CLI) { + e2e.Logf("Create Kafka topic") + parameters := []string{"--ignore-unknown-parameters=true", "-f", kafkaTopic.Template, "-p", "NAMESPACE=" + kafkaTopic.Namespace} + + if kafkaTopic.Name != "" { + parameters = append(parameters, "NAME="+kafkaTopic.Name) + } + + if kafkaTopic.TopicName != "" { + parameters = append(parameters, "TOPIC="+kafkaTopic.TopicName) + } + + compat_otp.ApplyNsResourceFromTemplate(oc, kafkaTopic.Namespace, parameters...) +} + +// creates a Kafka nodePool +func (kafkaNodePool *KafkaNodePool) deployKafkaNodePool(oc *exutil.CLI) { + e2e.Logf("Create Kafka nodePool") + parameters := []string{"--ignore-unknown-parameters=true", "-f", kafkaNodePool.Template, "-p", "NAMESPACE=" + kafkaNodePool.Namespace} + + if kafkaNodePool.Name != "" { + parameters = append(parameters, "NAME="+kafkaNodePool.Name) + } + + if kafkaNodePool.NodePoolName != "" { + parameters = append(parameters, "NODEPOOL="+kafkaNodePool.NodePoolName) + } + + compat_otp.ApplyNsResourceFromTemplate(oc, kafkaNodePool.Namespace, parameters...) +} + +// deploys KafkaUser +func (kafkaUser *KafkaUser) deployKafkaUser(oc *exutil.CLI) { + e2e.Logf("Create Kafka User") + parameters := []string{"--ignore-unknown-parameters=true", "-f", kafkaUser.Template, "-p", "NAMESPACE=" + kafkaUser.Namespace} + + if kafkaUser.UserName != "" { + parameters = append(parameters, "USER_NAME="+kafkaUser.UserName) + } + + if kafkaUser.Name != "" { + parameters = append(parameters, "NAME="+kafkaUser.Name) + } + + compat_otp.ApplyNsResourceFromTemplate(oc, kafkaUser.Namespace, parameters...) +} + +// deletes kafkaUser +func (k *KafkaUser) deleteKafkaUser(oc *exutil.CLI) { + e2e.Logf("Deleting Kafka user") + command := []string{"kafkauser", k.UserName, "-n", k.Namespace} + _, err := oc.AsAdmin().WithoutNamespace().Run("delete").Args(command...).Output() + o.Expect(err).NotTo(o.HaveOccurred()) +} + +// deletes kafkaTopic +func (kafkaTopic *KafkaTopic) deleteKafkaTopic(oc *exutil.CLI) { + e2e.Logf("Deleting Kafka topic") + command := []string{"kafkatopic", kafkaTopic.TopicName, "-n", kafkaTopic.Namespace} + _, err := oc.AsAdmin().WithoutNamespace().Run("delete").Args(command...).Output() + o.Expect(err).NotTo(o.HaveOccurred()) +} + +// deletes kafkaNodePool +func (kafkaNodePool *KafkaNodePool) deleteKafkaNodePool(oc *exutil.CLI) { + e2e.Logf("Deleting KafkaNodePool") + command := []string{"kafkaNodePool", kafkaNodePool.NodePoolName, "-n", kafkaNodePool.Namespace} + _, err := oc.AsAdmin().WithoutNamespace().Run("delete").Args(command...).Output() + o.Expect(err).NotTo(o.HaveOccurred()) +} + +// deletes kafka +func (kafka *Kafka) deleteKafka(oc *exutil.CLI) { + e2e.Logf("Deleting Kafka") + command := []string{"kafka", kafka.Name, "-n", kafka.Namespace} + _, err := oc.AsAdmin().WithoutNamespace().Run("delete").Args(command...).Output() + o.Expect(err).NotTo(o.HaveOccurred()) +} + +// Poll to wait for kafka to be ready +func waitForKafkaReady(oc *exutil.CLI, kafkaName string, kafkaNS string) { + err := wait.PollUntilContextTimeout(context.Background(), 6*time.Second, 360*time.Second, false, func(context.Context) (done bool, err error) { + command := []string{"kafka", kafkaName, "-n", kafkaNS, `-o=jsonpath={.status.conditions[*].type}`} + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args(command...).Output() + if err != nil { + e2e.Logf("kafka status ready error: %v", err) + return false, err + } + if strings.Contains(output, "Ready") { + return true, nil + } + return false, nil + }) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("resource kafka/%s did not appear", kafkaName)) +} + +// Poll to wait for kafka Topic to be ready +func waitForKafkaTopicReady(oc *exutil.CLI, kafkaTopicName string, kafkaTopicNS string) { + err := wait.PollUntilContextTimeout(context.Background(), 6*time.Second, 360*time.Second, false, func(context.Context) (done bool, err error) { + command := []string{"kafkaTopic", kafkaTopicName, "-n", kafkaTopicNS, `-o=jsonpath='{.status.conditions[*].type}'`} + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args(command...).Output() + if err != nil { + e2e.Logf("kafka Topic status ready error: %v", err) + return false, err + } + status := strings.Replace(output, "'", "", 2) + e2e.Logf("Waiting for kafka status %s", status) + if status == "Ready" { + return true, nil + } + return false, nil + }) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("resource kafkaTopic/%s did not appear", kafkaTopicName)) +} diff --git a/integration-tests/backend/loki.go b/integration-tests/backend/loki.go new file mode 100644 index 0000000000..ee8a5f63f5 --- /dev/null +++ b/integration-tests/backend/loki.go @@ -0,0 +1,114 @@ +package e2etests + +import ( + "fmt" + "reflect" + "strings" + + o "github.com/onsi/gomega" + exutil "github.com/openshift/origin/test/extended/util" + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" +) + +// lokiStack contains the configurations of loki stack +type lokiStack struct { + Name string // lokiStack name + Namespace string // lokiStack namespace + TSize string // size + StorageType string // the backend storage type, currently support s3, gcs, azure, swift, ODF and minIO + StorageSecret string // the secret name for loki to use to connect to backend storage + StorageClass string // storage class name + BucketName string // the butcket or the container name where loki stores it's data in + Tenant string // Loki tenant name + Template string // the file used to create the loki stack + Route string // lokistack-gateway-http route to be initialized after lokistack is up. + EnableIPV6 string // enable IPV6 +} + +// LokiPersistentVolumeClaim struct to handle Loki PVC resources +type LokiPersistentVolumeClaim struct { + Namespace string + Template string +} + +// LokiStorage struct to handle LokiStorage resources +type LokiStorage struct { + Namespace string + Template string +} + +// DeployLokiStack creates the lokiStack CR with basic settings: name, namespace, size, storage.secret.name, storage.secret.type, storageClassName +// optionalParameters is designed for adding parameters to deploy lokiStack with different tenants or some other settings +func (l lokiStack) deployLokiStack(oc *exutil.CLI) error { + parameters := []string{"--ignore-unknown-parameters=true", "-f", l.Template, "-p"} + + lokistack := reflect.ValueOf(&l).Elem() + + for i := 0; i < lokistack.NumField(); i++ { + if lokistack.Field(i).Interface() != "" { + if lokistack.Type().Field(i).Name == "StorageType" { + if lokistack.Field(i).Interface() == "odf" || lokistack.Field(i).Interface() == "minio" { + parameters = append(parameters, fmt.Sprintf("%s=%s", lokistack.Type().Field(i).Name, "s3")) + } else { + parameters = append(parameters, fmt.Sprintf("%s=%s", lokistack.Type().Field(i).Name, lokistack.Field(i).Interface())) + } + } else { + if lokistack.Type().Field(i).Name == "Template" { + continue + } else { + parameters = append(parameters, fmt.Sprintf("%s=%s", lokistack.Type().Field(i).Name, lokistack.Field(i).Interface())) + } + } + } + } + + file, err := processTemplate(oc, parameters...) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("Can not process %v", parameters)) + err = oc.AsAdmin().WithoutNamespace().Run("apply").Args("-f", file, "-n", l.Namespace).Execute() + return err +} + +func (l lokiStack) waitForLokiStackToBeReady(oc *exutil.CLI) error { + var err error + for _, deploy := range []string{l.Name + "-distributor", l.Name + "-gateway", l.Name + "-querier", l.Name + "-query-frontend"} { + err = waitForDeploymentPodsToBeReady(oc, l.Namespace, deploy) + } + for _, ss := range []string{l.Name + "-compactor", l.Name + "-index-gateway", l.Name + "-ingester"} { + err = waitForStatefulsetReady(oc, l.Namespace, ss) + } + if compat_otp.IsWorkloadIdentityCluster(oc) { + currentPlatform := compat_otp.CheckPlatform(oc) + switch currentPlatform { + case "aws", "azure", "gcp": + validateCredentialsRequestGenerationOnSTS(oc, l.Name, l.Namespace) + } + } + return err +} + +// To check CredentialsRequest is generated by Loki Operator on STS clusters for CCO flow +func validateCredentialsRequestGenerationOnSTS(oc *exutil.CLI, lokiStackName, lokiNamespace string) { + compat_otp.By("Validate that Loki Operator creates a CredentialsRequest object") + err := oc.AsAdmin().WithoutNamespace().Run("get").Args("CredentialsRequest", lokiStackName, "-n", lokiNamespace).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + cloudTokenPath, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("CredentialsRequest", lokiStackName, "-n", lokiNamespace, `-o=jsonpath={.spec.cloudTokenPath}`).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(cloudTokenPath).Should(o.Equal("/var/run/secrets/storage/serviceaccount/token")) + serviceAccountNames, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("CredentialsRequest", lokiStackName, "-n", lokiNamespace, `-o=jsonpath={.spec.serviceAccountNames}`).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(serviceAccountNames).Should(o.Equal(fmt.Sprintf(`["%s","%s-ruler"]`, lokiStackName, lokiStackName))) +} + +func (l lokiStack) removeLokiStack(oc *exutil.CLI) { + _ = Resource{"lokistack", l.Name, l.Namespace}.clear(oc) + _ = oc.AsAdmin().WithoutNamespace().Run("delete").Args("pvc", "-n", l.Namespace, "-l", "app.kubernetes.io/instance="+l.Name).Execute() +} + +// Get OIDC provider for the cluster +func getOIDC(oc *exutil.CLI) (string, error) { + oidc, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("authentication.config", "cluster", "-o=jsonpath={.spec.serviceAccountIssuer}").Output() + if err != nil { + return "", err + } + return strings.TrimPrefix(oidc, "https://"), nil +} diff --git a/integration-tests/backend/loki_client.go b/integration-tests/backend/loki_client.go new file mode 100644 index 0000000000..ba99ad44d4 --- /dev/null +++ b/integration-tests/backend/loki_client.go @@ -0,0 +1,355 @@ +package e2etests + +import ( + "context" + "crypto/tls" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path" + "strconv" + "strings" + "time" + + o "github.com/onsi/gomega" + exutil "github.com/openshift/origin/test/extended/util" + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" + apierrors "k8s.io/apimachinery/pkg/api/errors" + k8sresource "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + e2e "k8s.io/kubernetes/test/e2e/framework" +) + +type Resource struct { + Kind string + Name string + Namespace string +} + +// CompareClusterResources compares the remaning resource with the requested resource provide by user +func compareClusterResources(oc *exutil.CLI, cpu, memory string) bool { + nodes, err := compat_otp.GetSchedulableLinuxWorkerNodes(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + var remainingCPU, remainingMemory int64 + re := compat_otp.GetRemainingResourcesNodesMap(oc, nodes) + for _, node := range nodes { + remainingCPU += re[node.Name].CPU + remainingMemory += re[node.Name].Memory + } + + requiredCPU, _ := k8sresource.ParseQuantity(cpu) + requiredMemory, _ := k8sresource.ParseQuantity(memory) + e2e.Logf("the required cpu is: %d, and the required memory is: %d", requiredCPU.MilliValue(), requiredMemory.MilliValue()) + e2e.Logf("the remaining cpu is: %d, and the remaning memory is: %d", remainingCPU, remainingMemory) + return remainingCPU > requiredCPU.MilliValue() && remainingMemory > requiredMemory.MilliValue() +} + +// ValidateInfraAndResourcesForLoki checks cluster remaning resources and platform type +// supportedPlatforms the platform types which the case can be executed on, if it's empty, then skip this check +func validateInfraAndResourcesForLoki(oc *exutil.CLI, reqMemory, reqCPU string, supportedPlatforms ...string) bool { + currentPlatform := compat_otp.CheckPlatform(oc) + if currentPlatform == "aws" { + // skip the case on aws sts clusters + _, err := oc.AdminKubeClient().CoreV1().Secrets("kube-system").Get(context.Background(), "aws-creds", metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return false + } + } + if len(supportedPlatforms) > 0 { + return contain(supportedPlatforms, currentPlatform) && compareClusterResources(oc, reqCPU, reqMemory) + } + return compareClusterResources(oc, reqCPU, reqMemory) +} + +type lokiClient struct { + username string // Username for HTTP basic auth. + password string // Password for HTTP basic auth + address string // Server address. + orgID string // adds X-Scope-OrgID to API requests for representing tenant ID. Useful for requesting tenant data when bypassing an auth gateway. + bearerToken string // adds the Authorization header to API requests for authentication purposes. + bearerTokenFile string // adds the Authorization header to API requests for authentication purposes. + retries int // How many times to retry each query when getting an error response from Loki. + queryTags string // adds X-Query-Tags header to API requests. + quiet bool // Suppress query metadata. + startTime time.Time // Start time for reading logs + localhost bool // whether loki is port-forwarded to localhost, useful for monolithic loki +} + +type lokiQueryResponse struct { + Status string `json:"status"` + Data struct { + ResultType string `json:"resultType"` + Result []struct { + Stream struct { + App string `json:"app"` + DstK8SNamespace string `json:"DstK8S_Namespace"` + FlowDirection string `json:"FlowDirection"` + SrcK8SNamespace string `json:"SrcK8S_Namespace"` + SrcK8SOwnerName string `json:"SrcK8S_OwnerName"` + DstK8SOwnerName string `json:"kubernetes_pod_name"` + } `json:"stream"` + Values [][]string `json:"values"` + } `json:"result"` + Stats struct { + Summary struct { + BytesProcessedPerSecond int `json:"bytesProcessedPerSecond"` + LinesProcessedPerSecond int `json:"linesProcessedPerSecond"` + TotalBytesProcessed int `json:"totalBytesProcessed"` + TotalLinesProcessed int `json:"totalLinesProcessed"` + ExecTime float32 `json:"execTime"` + } `json:"summary"` + Store struct { + TotalChunksRef int `json:"totalChunksRef"` + TotalChunksDownloaded int `json:"totalChunksDownloaded"` + ChunksDownloadTime int `json:"chunksDownloadTime"` + HeadChunkBytes int `json:"headChunkBytes"` + HeadChunkLines int `json:"headChunkLines"` + DecompressedBytes int `json:"decompressedBytes"` + DecompressedLines int `json:"decompressedLines"` + CompressedBytes int `json:"compressedBytes"` + TotalDuplicates int `json:"totalDuplicates"` + } `json:"store"` + Ingester struct { + TotalReached int `json:"totalReached"` + TotalChunksMatched int `json:"totalChunksMatched"` + TotalBatches int `json:"totalBatches"` + TotalLinesSent int `json:"totalLinesSent"` + HeadChunkBytes int `json:"headChunkBytes"` + HeadChunkLines int `json:"headChunkLines"` + DecompressedBytes int `json:"decompressedBytes"` + DecompressedLines int `json:"decompressedLines"` + CompressedBytes int `json:"compressedBytes"` + TotalDuplicates int `json:"totalDuplicates"` + } `json:"ingester"` + } `json:"stats"` + } `json:"data"` +} + +// newLokiClient initializes a lokiClient with server address +func newLokiClient(routeAddress string, time time.Time) *lokiClient { + client := &lokiClient{} + client.address = routeAddress + client.retries = 5 + client.quiet = false + client.startTime = time + client.localhost = false + return client +} + +// retry sets how many times to retry each query +func (c *lokiClient) retry(retry int) *lokiClient { + nc := *c + nc.retries = retry + return &nc +} + +// withToken sets the token used to do query +func (c *lokiClient) withToken(bearerToken string) *lokiClient { + nc := *c + nc.bearerToken = bearerToken + return &nc +} + +// buildURL concats a url `http://foo/bar` with a path `/buzz`. +func buildURL(u, p, q string) (string, error) { + url, err := url.Parse(u) + if err != nil { + return "", err + } + url.Path = path.Join(url.Path, p) + url.RawQuery = q + return url.String(), nil +} + +type queryStringBuilder struct { + values url.Values +} + +func newQueryStringBuilder() *queryStringBuilder { + return &queryStringBuilder{ + values: url.Values{}, + } +} + +// encode returns the URL-encoded query string based on key-value +// parameters added to the builder calling Set functions. +func (b *queryStringBuilder) encode() string { + return b.values.Encode() +} + +func (b *queryStringBuilder) setString(name, value string) { + b.values.Set(name, value) +} + +func (b *queryStringBuilder) setInt(name string, value int64) { + b.setString(name, strconv.FormatInt(value, 10)) +} + +func (b *queryStringBuilder) setInt32(name string, value int) { + b.setString(name, strconv.Itoa(value)) +} + +func (c *lokiClient) getHTTPRequestHeader() (http.Header, error) { + h := make(http.Header) + if c.username != "" && c.password != "" { + h.Set( + "Authorization", + "Basic "+base64.StdEncoding.EncodeToString([]byte(c.username+":"+c.password)), + ) + } + h.Set("User-Agent", "loki-logcli") + + if c.orgID != "" { + h.Set("X-Scope-OrgID", c.orgID) + } + + if c.queryTags != "" { + h.Set("X-Query-Tags", c.queryTags) + } + + if (c.username != "" || c.password != "") && (len(c.bearerToken) > 0 || len(c.bearerTokenFile) > 0) { + return nil, fmt.Errorf("at most one of HTTP basic auth (username/password), bearer-token & bearer-token-file is allowed to be configured") + } + + if len(c.bearerToken) > 0 && len(c.bearerTokenFile) > 0 { + return nil, fmt.Errorf("at most one of the options bearer-token & bearer-token-file is allowed to be configured") + } + + if c.bearerToken != "" { + h.Set("Authorization", "Bearer "+c.bearerToken) + } + + if c.bearerTokenFile != "" { + b, err := os.ReadFile(c.bearerTokenFile) + if err != nil { + return nil, fmt.Errorf("unable to read authorization credentials file %s: %w", c.bearerTokenFile, err) + } + bearerToken := strings.TrimSpace(string(b)) + h.Set("Authorization", "Bearer "+bearerToken) + } + return h, nil +} + +func (c *lokiClient) doRequest(path, query string, quiet bool, out interface{}) error { + us, err := buildURL(c.address, path, query) + if err != nil { + return err + } + if !quiet { + e2e.Logf("%s", us) + } + + req, err := http.NewRequest("GET", us, nil) + if err != nil { + return err + } + + h, err := c.getHTTPRequestHeader() + if err != nil { + return err + } + req.Header = h + + var tr *http.Transport + proxy := getProxyFromEnv() + + // don't use proxy if svc/loki is port-forwarded to localhost + if !c.localhost && len(proxy) > 0 { + proxyURL, err := url.Parse(proxy) + o.Expect(err).NotTo(o.HaveOccurred()) + tr = &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + Proxy: http.ProxyURL(proxyURL), + } + } else { + tr = &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + } + + client := &http.Client{Transport: tr} + + var resp *http.Response + attempts := c.retries + 1 + success := false + + for attempts > 0 { + attempts-- + resp, err = client.Do(req) + if err != nil { + e2e.Logf("error sending request %v", err) + continue + } + if resp.StatusCode/100 != 2 { + buf, _ := io.ReadAll(resp.Body) // nolint + e2e.Logf("Error response from server: %s (%v) attempts remaining: %d", string(buf), err, attempts) + if err := resp.Body.Close(); err != nil { + e2e.Logf("error closing body: %v", err) + } + continue + } + success = true + break + } + if !success { + return fmt.Errorf("run out of attempts while querying the server") + } + + defer func() { + if err := resp.Body.Close(); err != nil { + e2e.Logf("error closing body: %v", err) + } + }() + return json.NewDecoder(resp.Body).Decode(out) +} + +func (c *lokiClient) doQuery(path string, query string, quiet bool) (*lokiQueryResponse, error) { + var err error + var r lokiQueryResponse + + if err = c.doRequest(path, query, quiet, &r); err != nil { + return nil, err + } + + return &r, nil +} + +// queryRange uses the /api/v1/query_range endpoint to execute a range query +// logType: application, infrastructure, audit +// queryStr: string to filter logs, for example: "{kubernetes_namespace_name="test"}" +// limit: max log count +// start: Start looking for logs at this absolute time(inclusive), e.g.: time.Now().Add(time.Duration(-1)*time.Hour) means 1 hour ago +// end: Stop looking for logs at this absolute time (exclusive) +// forward: true means scan forwards through logs, false means scan backwards through logs +func (c *lokiClient) queryRange(logType string, queryStr string, limit int, start, end time.Time, forward bool) (*lokiQueryResponse, error) { + direction := func() string { + if forward { + return "FORWARD" + } + return "BACKWARD" + } + params := newQueryStringBuilder() + params.setString("query", queryStr) + params.setInt32("limit", limit) + params.setInt("start", start.UnixNano()) + params.setInt("end", end.UnixNano()) + params.setString("direction", direction()) + logPath := "" + if len(logType) > 0 { + logPath = apiPath + logType + queryRangePath + } else { + logPath = queryRangePath + } + + return c.doQuery(logPath, params.encode(), c.quiet) +} + +func (c *lokiClient) searchLogsInLoki(logType, query string) (*lokiQueryResponse, error) { + e2e.Logf("Loki query is %s", query) + res, err := c.queryRange(logType, query, 50, c.startTime, time.Now(), false) + return res, err +} diff --git a/integration-tests/backend/loki_storage.go b/integration-tests/backend/loki_storage.go new file mode 100644 index 0000000000..ba7450a9c9 --- /dev/null +++ b/integration-tests/backend/loki_storage.go @@ -0,0 +1,1003 @@ +package e2etests + +import ( + "context" + "crypto/tls" + "fmt" + "net/http" + "net/url" + "os" + filePath "path/filepath" + "strconv" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + exutil "github.com/openshift/origin/test/extended/util" + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" + "google.golang.org/api/cloudresourcemanager/v1" + "google.golang.org/api/iam/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + e2e "k8s.io/kubernetes/test/e2e/framework" +) + +const ( + minioNS = "minio-aosqe" + minioSecret = "minio-creds" + apiPath = "/api/logs/v1/" + queryRangePath = "/loki/api/v1/query_range" + loNS = "openshift-operators-redhat" +) + +// s3Credential defines the s3 credentials +type s3Credential struct { + Region string + AccessKeyID string + SecretAccessKey string + Endpoint string // the endpoint of s3 service +} + +func getAWSCredentialFromCluster(oc *exutil.CLI) s3Credential { + region, err := compat_otp.GetAWSClusterRegion(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + + dirname := "/tmp/" + oc.Namespace() + "-creds" + defer os.RemoveAll(dirname) + err = os.MkdirAll(dirname, 0777) + o.Expect(err).NotTo(o.HaveOccurred()) + + _, err = oc.AsAdmin().WithoutNamespace().Run("extract").Args("secret/aws-creds", "-n", "kube-system", "--confirm", "--to="+dirname).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + accessKeyID, err := os.ReadFile(dirname + "/aws_access_key_id") + o.Expect(err).NotTo(o.HaveOccurred()) + secretAccessKey, err := os.ReadFile(dirname + "/aws_secret_access_key") + o.Expect(err).NotTo(o.HaveOccurred()) + + cred := s3Credential{Region: region, AccessKeyID: string(accessKeyID), SecretAccessKey: string(secretAccessKey)} + return cred +} + +func getMinIOCreds(oc *exutil.CLI, ns string) s3Credential { + dirname := "/tmp/" + oc.Namespace() + "-creds" + defer os.RemoveAll(dirname) + err := os.MkdirAll(dirname, 0777) + o.Expect(err).NotTo(o.HaveOccurred()) + + _, err = oc.AsAdmin().WithoutNamespace().Run("extract").Args("secret/"+minioSecret, "-n", ns, "--confirm", "--to="+dirname).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + accessKeyID, err := os.ReadFile(dirname + "/access_key_id") + o.Expect(err).NotTo(o.HaveOccurred()) + secretAccessKey, err := os.ReadFile(dirname + "/secret_access_key") + o.Expect(err).NotTo(o.HaveOccurred()) + + endpoint := "http://" + getRouteAddress(oc, ns, "minio") + return s3Credential{Endpoint: endpoint, AccessKeyID: string(accessKeyID), SecretAccessKey: string(secretAccessKey)} +} + +func generateS3Config(cred s3Credential) aws.Config { + var err error + var cfg aws.Config + if len(cred.Endpoint) > 0 { + customResolver := aws.EndpointResolverWithOptionsFunc(func(_, _ string, _ ...interface{}) (aws.Endpoint, error) { + return aws.Endpoint{ + URL: cred.Endpoint, + HostnameImmutable: true, + Source: aws.EndpointSourceCustom, + }, nil + }) + // For ODF and Minio, they're deployed in OCP clusters + // In some clusters, we can't connect it without proxy, here add proxy settings to s3 client when there has http_proxy or https_proxy in the env var + httpClient := awshttp.NewBuildableClient().WithTransportOptions(func(tr *http.Transport) { + proxy := getProxyFromEnv() + if len(proxy) > 0 { + proxyURL, err := url.Parse(proxy) + o.Expect(err).NotTo(o.HaveOccurred()) + tr.Proxy = http.ProxyURL(proxyURL) + } + tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + }) + cfg, err = config.LoadDefaultConfig(context.TODO(), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(cred.AccessKeyID, cred.SecretAccessKey, "")), + config.WithEndpointResolverWithOptions(customResolver), + config.WithHTTPClient(httpClient), + config.WithRegion("auto")) + } else { + // aws s3 + cfg, err = config.LoadDefaultConfig(context.TODO(), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(cred.AccessKeyID, cred.SecretAccessKey, "")), + config.WithRegion(cred.Region)) + } + o.Expect(err).NotTo(o.HaveOccurred()) + return cfg +} + +func createS3Bucket(client *s3.Client, bucketName, region string) error { + // check if the bucket exists or not + // if exists, clear all the objects in the bucket + // if not, create the bucket + exist := false + buckets, err := client.ListBuckets(context.TODO(), &s3.ListBucketsInput{}) + o.Expect(err).NotTo(o.HaveOccurred()) + for _, bu := range buckets.Buckets { + if *bu.Name == bucketName { + exist = true + break + } + } + // clear all the objects in the bucket + if exist { + return emptyS3Bucket(client, bucketName) + } + + /* + Per https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateBucket.html#API_CreateBucket_RequestBody, + us-east-1 is the default region and it's not a valid value of LocationConstraint, + using `LocationConstraint: types.BucketLocationConstraint("us-east-1")` gets error `InvalidLocationConstraint`. + Here remove the configration when the region is us-east-1 + */ + if len(region) == 0 || region == "us-east-1" { + _, err = client.CreateBucket(context.TODO(), &s3.CreateBucketInput{Bucket: &bucketName}) + return err + } + _, err = client.CreateBucket(context.TODO(), &s3.CreateBucketInput{Bucket: &bucketName, CreateBucketConfiguration: &types.CreateBucketConfiguration{LocationConstraint: types.BucketLocationConstraint(region)}}) + return err +} + +func deleteS3Bucket(client *s3.Client, bucketName string) error { + // empty bucket + err := emptyS3Bucket(client, bucketName) + if err != nil { + return err + } + // delete bucket + _, err = client.DeleteBucket(context.TODO(), &s3.DeleteBucketInput{Bucket: &bucketName}) + return err +} + +func emptyS3Bucket(client *s3.Client, bucketName string) error { + // List objects in the bucket + objects, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{ + Bucket: &bucketName, + }) + if err != nil { + return err + } + + // Delete objects in the bucket + if len(objects.Contents) > 0 { + objectIdentifiers := make([]types.ObjectIdentifier, len(objects.Contents)) + for i, object := range objects.Contents { + objectIdentifiers[i] = types.ObjectIdentifier{Key: object.Key} + } + + quiet := true + _, err = client.DeleteObjects(context.TODO(), &s3.DeleteObjectsInput{ + Bucket: &bucketName, + Delete: &types.Delete{ + Objects: objectIdentifiers, + Quiet: &quiet, + }, + }) + if err != nil { + return err + } + } + + // Check if there are more objects to delete and handle pagination + if *objects.IsTruncated { + return emptyS3Bucket(client, bucketName) + } + + return nil +} + +// createSecretForAWSS3Bucket creates a secret for Loki to connect to s3 bucket +func createSecretForAWSS3Bucket(oc *exutil.CLI, bucketName, secretName, ns string, cred s3Credential) error { + if len(secretName) == 0 { + return fmt.Errorf("secret name shouldn't be empty") + } + + endpoint := "https://s3." + cred.Region + ".amazonaws.com" + return oc.NotShowInfo().AsAdmin().WithoutNamespace().Run("create").Args("secret", "generic", secretName, "--from-literal=access_key_id="+cred.AccessKeyID, "--from-literal=access_key_secret="+cred.SecretAccessKey, "--from-literal=region="+cred.Region, "--from-literal=bucketnames="+bucketName, "--from-literal=endpoint="+endpoint, "-n", ns).Execute() +} + +func createSecretForODFBucket(oc *exutil.CLI, bucketName, secretName, ns string) error { + if len(secretName) == 0 { + return fmt.Errorf("secret name shouldn't be empty") + } + dirname := "/tmp/" + oc.Namespace() + "-creds" + err := os.MkdirAll(dirname, 0777) + o.Expect(err).NotTo(o.HaveOccurred()) + defer os.RemoveAll(dirname) + _, err = oc.AsAdmin().WithoutNamespace().Run("extract").Args("secret/noobaa-admin", "-n", "openshift-storage", "--confirm", "--to="+dirname).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + endpoint := "http://s3.openshift-storage.svc:80" + return oc.AsAdmin().WithoutNamespace().Run("create").Args("secret", "generic", secretName, "--from-file=access_key_id="+dirname+"/AWS_ACCESS_KEY_ID", "--from-file=access_key_secret="+dirname+"/AWS_SECRET_ACCESS_KEY", "--from-literal=bucketnames="+bucketName, "--from-literal=endpoint="+endpoint, "-n", ns).Execute() +} + +func createSecretForMinIOBucket(oc *exutil.CLI, bucketName, secretName, ns string, cred s3Credential) error { + if len(secretName) == 0 { + return fmt.Errorf("secret name shouldn't be empty") + } + return oc.NotShowInfo().AsAdmin().WithoutNamespace().Run("create").Args("secret", "generic", secretName, "--from-literal=access_key_id="+cred.AccessKeyID, "--from-literal=access_key_secret="+cred.SecretAccessKey, "--from-literal=bucketnames="+bucketName, "--from-literal=endpoint="+cred.Endpoint, "-n", ns).Execute() +} + +func getGCPProjectNumber(projectID string) (string, error) { + crmService, err := cloudresourcemanager.NewService(context.Background()) + if err != nil { + return "", err + } + + project, err := crmService.Projects.Get(projectID).Do() + if err != nil { + return "", err + } + return strconv.FormatInt(project.ProjectNumber, 10), nil +} + +func generateServiceAccountNameForGCS(clusterName string) string { + // Service Account should be between 6-30 characters long + name := clusterName + getRandomString() + if len(name) > 30 { + return (name[0:30]) + } + return name +} + +func createServiceAccountOnGCP(projectID, name string) (*iam.ServiceAccount, error) { + ctx := context.Background() + service, err := iam.NewService(ctx) + if err != nil { + return nil, fmt.Errorf("iam.NewService: %w", err) + } + + request := &iam.CreateServiceAccountRequest{ + AccountId: name, + ServiceAccount: &iam.ServiceAccount{ + DisplayName: "Service Account for " + name, + }, + } + account, err := service.Projects.ServiceAccounts.Create("projects/"+projectID, request).Do() + if err != nil { + return nil, fmt.Errorf("failed to create serviceaccount: %w", err) + } + e2e.Logf("create serviceaccount: %s successfully", account.Name) + return account, nil +} + +// ref: https://github.com/GoogleCloudPlatform/golang-samples/blob/main/iam/quickstart/quickstart.go +func addBinding(projectID, member, role string) error { + crmService, err := cloudresourcemanager.NewService(context.Background()) + if err != nil { + return fmt.Errorf("cloudresourcemanager.NewService: %w", err) + } + + err = wait.ExponentialBackoffWithContext(context.Background(), wait.Backoff{Steps: 5, Factor: 2, Duration: 5 * time.Second}, func(context.Context) (done bool, err error) { + policy, err := getPolicy(crmService, projectID) + if err != nil { + return false, fmt.Errorf("error getting policy: %w", err) + } + // Find the policy binding for role. Only one binding can have the role. + var binding *cloudresourcemanager.Binding + for _, b := range policy.Bindings { + if b.Role == role { + binding = b + break + } + } + if binding != nil { + // If the binding exists, adds the member to the binding + binding.Members = append(binding.Members, member) + } else { + // If the binding does not exist, adds a new binding to the policy + binding = &cloudresourcemanager.Binding{ + Role: role, + Members: []string{member}, + } + policy.Bindings = append(policy.Bindings, binding) + } + err = setPolicy(crmService, projectID, policy) + if err == nil { + return true, nil + } + /* + According to https://github.com/hashicorp/terraform-provider-google/issues/8280, deleting another serviceaccount can make 400 error happen, so retry this step when 400 error happens + */ + if strings.Contains(err.Error(), `googleapi: Error 409: There were concurrent policy changes. Please retry the whole read-modify-write with exponential backoff.`) || + (strings.Contains(err.Error(), "googleapi: Error 400: Service account") && strings.Contains(err.Error(), "does not exist., badRequest")) { + e2e.Logf("Hit error: %v, retry the request", err) + return false, nil + } + e2e.Logf("Failed to update polilcy: %v", err) + return false, err + }) + if err != nil { + return fmt.Errorf("failed to add role %s to %s", role, member) + } + return nil +} + +// removeMember removes the member from the project's IAM policy +func removeMember(projectID, member, role string) error { + crmService, err := cloudresourcemanager.NewService(context.Background()) + if err != nil { + return fmt.Errorf("cloudresourcemanager.NewService: %w", err) + } + err = wait.ExponentialBackoffWithContext(context.Background(), wait.Backoff{Steps: 5, Factor: 2, Duration: 5 * time.Second}, func(context.Context) (done bool, err error) { + policy, err := getPolicy(crmService, projectID) + if err != nil { + return false, fmt.Errorf("error getting policy: %w", err) + } + // Find the policy binding for role. Only one binding can have the role. + var binding *cloudresourcemanager.Binding + var bindingIndex int + for i, b := range policy.Bindings { + if b.Role == role { + binding = b + bindingIndex = i + break + } + } + + if len(binding.Members) == 1 && binding.Members[0] == member { + // If the member is the only member in the binding, removes the binding + last := len(policy.Bindings) - 1 + policy.Bindings[bindingIndex] = policy.Bindings[last] + policy.Bindings = policy.Bindings[:last] + } else { + // If there is more than one member in the binding, removes the member + var memberIndex int + var exist bool + for i, mm := range binding.Members { + if mm == member { + memberIndex = i + exist = true + break + } + } + if exist { + last := len(policy.Bindings[bindingIndex].Members) - 1 + binding.Members[memberIndex] = binding.Members[last] + binding.Members = binding.Members[:last] + } + } + + err = setPolicy(crmService, projectID, policy) + if err == nil { + return true, nil + } + if strings.Contains(err.Error(), `googleapi: Error 409: There were concurrent policy changes. Please retry the whole read-modify-write with exponential backoff.`) || + (strings.Contains(err.Error(), "googleapi: Error 400: Service account") && strings.Contains(err.Error(), "does not exist., badRequest")) { + e2e.Logf("Hit error: %v, retry the request", err) + return false, nil + } + e2e.Logf("Failed to update polilcy: %v", err) + return false, err + }) + if err != nil { + return fmt.Errorf("failed to remove %s", member) + } + return nil +} + +// getPolicy gets the project's IAM policy +func getPolicy(crmService *cloudresourcemanager.Service, projectID string) (*cloudresourcemanager.Policy, error) { + request := new(cloudresourcemanager.GetIamPolicyRequest) + policy, err := crmService.Projects.GetIamPolicy(projectID, request).Do() + if err != nil { + return nil, err + } + return policy, nil +} + +// setPolicy sets the project's IAM policy +func setPolicy(crmService *cloudresourcemanager.Service, projectID string, policy *cloudresourcemanager.Policy) error { + request := new(cloudresourcemanager.SetIamPolicyRequest) + request.Policy = policy + _, err := crmService.Projects.SetIamPolicy(projectID, request).Do() + return err +} + +func grantPermissionsToGCPServiceAccount(poolID, projectID, projectNumber, lokiNS, lokiStackName, serviceAccountEmail string) error { + gcsRoles := []string{ + "roles/iam.workloadIdentityUser", + "roles/storage.objectAdmin", + } + subjects := []string{ + "system:serviceaccount:" + lokiNS + ":" + lokiStackName, + "system:serviceaccount:" + lokiNS + ":" + lokiStackName + "-ruler", + } + + for _, role := range gcsRoles { + err := addBinding(projectID, "serviceAccount:"+serviceAccountEmail, role) + if err != nil { + return fmt.Errorf("error adding role %s to %s: %w", role, serviceAccountEmail, err) + } + for _, sub := range subjects { + err := addBinding(projectID, "principal://iam.googleapis.com/projects/"+projectNumber+"/locations/global/workloadIdentityPools/"+poolID+"/subject/"+sub, role) + if err != nil { + return fmt.Errorf("error adding role %s to %s: %w", role, sub, err) + } + } + } + return nil +} + +func removePermissionsFromGCPServiceAccount(poolID, projectID, projectNumber, lokiNS, lokiStackName, serviceAccountEmail string) error { + gcsRoles := []string{ + "roles/iam.workloadIdentityUser", + "roles/storage.objectAdmin", + } + subjects := []string{ + "system:serviceaccount:" + lokiNS + ":" + lokiStackName, + "system:serviceaccount:" + lokiNS + ":" + lokiStackName + "-ruler", + } + + for _, role := range gcsRoles { + err := removeMember(projectID, "serviceAccount:"+serviceAccountEmail, role) + if err != nil { + return fmt.Errorf("error removing role %s from %s: %w", role, serviceAccountEmail, err) + } + for _, sub := range subjects { + err := removeMember(projectID, "principal://iam.googleapis.com/projects/"+projectNumber+"/locations/global/workloadIdentityPools/"+poolID+"/subject/"+sub, role) + if err != nil { + return fmt.Errorf("error removing role %s from %s: %w", role, sub, err) + } + } + } + return nil +} + +func removeServiceAccountFromGCP(name string) error { + ctx := context.Background() + service, err := iam.NewService(ctx) + if err != nil { + return fmt.Errorf("iam.NewService: %w", err) + } + _, err = service.Projects.ServiceAccounts.Delete(name).Do() + if err != nil { + return fmt.Errorf("can't remove service account: %w", err) + } + return nil +} + +func createSecretForGCSBucketWithSTS(oc *exutil.CLI, namespace, secretName, bucketName string) error { + return oc.NotShowInfo().AsAdmin().WithoutNamespace().Run("create").Args("secret", "generic", "-n", namespace, secretName, "--from-literal=bucketname="+bucketName).Execute() +} + +// creates a secret for Loki to connect to gcs bucket +func createSecretForGCSBucket(oc *exutil.CLI, bucketName, secretName, ns string) error { + if len(secretName) == 0 { + return fmt.Errorf("secret name shouldn't be empty") + } + + // get gcp-credentials from env var GOOGLE_APPLICATION_CREDENTIALS + gcsCred := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") + return oc.AsAdmin().WithoutNamespace().Run("create").Args("secret", "generic", secretName, "-n", ns, "--from-literal=bucketname="+bucketName, "--from-file=key.json="+gcsCred).Execute() +} + +// creates a secret for Loki to connect to azure container +func createSecretForAzureContainer(oc *exutil.CLI, bucketName, secretName, ns string) error { + environment := "AzureGlobal" + cloudName, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("infrastructure", "cluster", "-o=jsonpath={.status.platformStatus.azure.cloudName}").Output() + if err != nil { + return fmt.Errorf("can't get azure cluster type %w", err) + } + if strings.ToLower(cloudName) == "azureusgovernmentcloud" { + environment = "AzureUSGovernment" + } + if strings.ToLower(cloudName) == "azurechinacloud" { + environment = "AzureChinaCloud" + } + if strings.ToLower(cloudName) == "azuregermancloud" { + environment = "AzureGermanCloud" + } + + accountName, accountKey, err1 := compat_otp.GetAzureStorageAccountFromCluster(oc) + if err1 != nil { + return fmt.Errorf("can't get azure storage account from cluster: %w", err1) + } + return oc.NotShowInfo().AsAdmin().WithoutNamespace().Run("create").Args("secret", "generic", "-n", ns, secretName, "--from-literal=environment="+environment, "--from-literal=container="+bucketName, "--from-literal=account_name="+accountName, "--from-literal=account_key="+accountKey).Execute() +} + +func createSecretForSwiftContainer(oc *exutil.CLI, containerName, secretName, ns string, cred *compat_otp.OpenstackCredentials) error { + userID, domainID := compat_otp.GetOpenStackUserIDAndDomainID(cred) + err := oc.NotShowInfo().AsAdmin().WithoutNamespace().Run("create").Args("secret", "generic", "-n", ns, secretName, + "--from-literal=auth_url="+cred.Clouds.Openstack.Auth.AuthURL, + "--from-literal=username="+cred.Clouds.Openstack.Auth.Username, + "--from-literal=user_domain_name="+cred.Clouds.Openstack.Auth.UserDomainName, + "--from-literal=user_domain_id="+domainID, + "--from-literal=user_id="+userID, + "--from-literal=password="+cred.Clouds.Openstack.Auth.Password, + "--from-literal=domain_id="+domainID, + "--from-literal=domain_name="+cred.Clouds.Openstack.Auth.UserDomainName, + "--from-literal=container_name="+containerName, + "--from-literal=project_id="+cred.Clouds.Openstack.Auth.ProjectID, + "--from-literal=project_name="+cred.Clouds.Openstack.Auth.ProjectName, + "--from-literal=project_domain_id="+domainID, + "--from-literal=project_domain_name="+cred.Clouds.Openstack.Auth.UserDomainName).Execute() + return err +} + +// checkODF check if the ODF is installed in the cluster or not +// here only checks the sc/ocs-storagecluster-ceph-rbd and svc/s3 +func checkODF(oc *exutil.CLI) bool { + svcFound := false + expectedSC := []string{"openshift-storage.noobaa.io"} + var scInCluster []string + scs, err := oc.AdminKubeClient().StorageV1().StorageClasses().List(context.Background(), metav1.ListOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + for _, sc := range scs.Items { + scInCluster = append(scInCluster, sc.Name) + } + + for _, s := range expectedSC { + if !contain(scInCluster, s) { + return false + } + } + + _, err = oc.AdminKubeClient().CoreV1().Services("openshift-storage").Get(context.Background(), "s3", metav1.GetOptions{}) + if err == nil { + svcFound = true + } + return svcFound +} + +func createObjectBucketClaim(oc *exutil.CLI, ns, name string) error { + template, _ := filePath.Abs("testdata/logging/odf/objectBucketClaim.yaml") + obc := Resource{"objectbucketclaims", name, ns} + + err := obc.applyFromTemplate(oc, "-f", template, "-n", ns, "-p", "NAME="+name, "NAMESPACE="+ns) + if err != nil { + return err + } + _ = obc.WaitForResourceToAppear(oc) + _ = Resource{"objectbuckets", "obc-" + ns + "-" + name, ns}.WaitForResourceToAppear(oc) + assertResourceStatus(oc, "objectbucketclaims", name, ns, "{.status.phase}", "Bound") + return nil +} + +func deleteObjectBucketClaim(oc *exutil.CLI, ns, name string) error { + obc := Resource{"objectbucketclaims", name, ns} + err := obc.clear(oc) + if err != nil { + return err + } + return obc.WaitUntilResourceIsGone(oc) +} + +// checkMinIO +func checkMinIO(oc *exutil.CLI, ns string) (bool, error) { + podReady, svcFound := false, false + pod, err := oc.AdminKubeClient().CoreV1().Pods(ns).List(context.Background(), metav1.ListOptions{LabelSelector: "app=minio"}) + if err != nil { + return false, err + } + if len(pod.Items) > 0 && pod.Items[0].Status.Phase == "Running" { + podReady = true + } + _, err = oc.AdminKubeClient().CoreV1().Services(ns).Get(context.Background(), "minio", metav1.GetOptions{}) + if err == nil { + svcFound = true + } + return podReady && svcFound, err +} + +func useExtraObjectStorage(oc *exutil.CLI) string { + if checkODF(oc) { + e2e.Logf("use the existing ODF storage service") + return "odf" + } + ready, err := checkMinIO(oc, minioNS) + if ready { + e2e.Logf("use existing MinIO storage service") + return "minio" + } + if strings.Contains(err.Error(), "No resources found") || strings.Contains(err.Error(), "not found") { + e2e.Logf("deploy MinIO and use this MinIO as storage service") + deployMinIO(oc) + return "minio" + } + return "" +} + +func patchLokiOperatorWithAWSRoleArn(oc *exutil.CLI, subNamespace, roleArn string) { + roleArnPatchConfig := `{ + "spec": { + "config": { + "env": [ + { + "name": "ROLEARN", + "value": "%s" + } + ] + } + } + }` + + subName, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("sub", "-n", subNamespace, `-ojsonpath={.items[?(@.spec.name=="loki-operator")].metadata.name}`).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(subName).ShouldNot(o.BeEmpty()) + err = oc.NotShowInfo().AsAdmin().WithoutNamespace().Run("patch").Args("sub", subName, "-n", subNamespace, "-p", fmt.Sprintf(roleArnPatchConfig, roleArn), "--type=merge").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + WaitForPodsReadyWithLabel(oc, loNS, "name=loki-operator-controller-manager") +} + +// return the storage type per different platform +func getStorageType(oc *exutil.CLI) string { + platform := compat_otp.CheckPlatform(oc) + switch platform { + case "aws": + { + return "s3" + } + case "gcp": + { + return "gcs" + } + case "azure": + { + return "azure" + } + case "openstack": + { + return "swift" + } + default: + { + return useExtraObjectStorage(oc) + } + } +} + +// initialize a s3 client with credential +func newS3Client(cfg aws.Config) *s3.Client { + return s3.NewFromConfig(cfg) +} + +func getStorageClassName(oc *exutil.CLI) (string, error) { + scs, err := oc.AdminKubeClient().StorageV1().StorageClasses().List(context.Background(), metav1.ListOptions{}) + if err != nil { + return "", err + } + if len(scs.Items) == 0 { + return "", fmt.Errorf("there is no storageclass in the cluster") + } + for _, sc := range scs.Items { + if sc.ObjectMeta.Annotations["storageclass.kubernetes.io/is-default-class"] == "true" { + return sc.Name, nil + } + } + return scs.Items[0].Name, nil +} + +// prepareResourcesForLokiStack creates buckets/containers in backend storage provider, and creates the secret for Loki to use +func (l lokiStack) prepareResourcesForLokiStack(oc *exutil.CLI) error { + var err error + if len(l.BucketName) == 0 { + return fmt.Errorf("the bucketName should not be empty") + } + switch l.StorageType { + case "s3": + { + var cfg aws.Config + region, err := compat_otp.GetAWSClusterRegion(oc) + if err != nil { + return err + } + if compat_otp.IsWorkloadIdentityCluster(oc) { + if !checkAWSCredentials() { + g.Skip("Skip since no AWS credetial! No Env AWS_SHARED_CREDENTIALS_FILE, Env CLUSTER_PROFILE_DIR or $HOME/.aws/credentials file") + } + partition := "aws" + if strings.HasPrefix(region, "us-gov") { + partition = "aws-us-gov" + } + cfg = readDefaultSDKExternalConfigurations(context.TODO(), region) + iamClient := newIamClient(cfg) + stsClient := newStsClient(cfg) + awsAccountID, _ := getAwsAccount(stsClient) + oidcName, err := getOIDC(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + lokiIAMRoleName := l.Name + "-" + compat_otp.GetRandomString() + roleArn := createIAMRoleForLokiSTSDeployment(iamClient, oidcName, awsAccountID, partition, l.Namespace, l.Name, lokiIAMRoleName) + os.Setenv("LOKI_ROLE_NAME_ON_STS", lokiIAMRoleName) + patchLokiOperatorWithAWSRoleArn(oc, loNS, roleArn) + createObjectStorageSecretOnAWSSTSCluster(oc, region, l.StorageSecret, l.BucketName, l.Namespace) + } else { + cred := getAWSCredentialFromCluster(oc) + cfg = generateS3Config(cred) + err = createSecretForAWSS3Bucket(oc, l.BucketName, l.StorageSecret, l.Namespace, cred) + o.Expect(err).NotTo(o.HaveOccurred()) + } + client := newS3Client(cfg) + err = createS3Bucket(client, l.BucketName, region) + if err != nil { + return err + } + } + case "azure": + { + if compat_otp.IsWorkloadIdentityCluster(oc) { + if !readAzureCredentials() { + g.Skip("Azure Credentials not found. Skip case!") + } else { + performManagedIdentityAndSecretSetupForAzureWIF(oc, l.Name, l.Namespace, l.BucketName, l.StorageSecret) + } + } else { + accountName, accountKey, err1 := compat_otp.GetAzureStorageAccountFromCluster(oc) + if err1 != nil { + return fmt.Errorf("can't get azure storage account from cluster: %w", err1) + } + client, err2 := compat_otp.NewAzureContainerClient(oc, accountName, accountKey, l.BucketName) + if err2 != nil { + return err2 + } + err = compat_otp.CreateAzureStorageBlobContainer(client) + if err != nil { + return err + } + err = createSecretForAzureContainer(oc, l.BucketName, l.StorageSecret, l.Namespace) + } + } + case "gcs": + { + projectID, errGetID := compat_otp.GetGcpProjectID(oc) + o.Expect(errGetID).NotTo(o.HaveOccurred()) + err = compat_otp.CreateGCSBucket(projectID, l.BucketName) + if err != nil { + return err + } + if compat_otp.IsWorkloadIdentityCluster(oc) { + clusterName := getInfrastructureName(oc) + gcsSAName := generateServiceAccountNameForGCS(clusterName) + os.Setenv("LOGGING_GCS_SERVICE_ACCOUNT_NAME", gcsSAName) + projectNumber, err1 := getGCPProjectNumber(projectID) + if err1 != nil { + return fmt.Errorf("can't get GCP project number: %w", err1) + } + poolID, err2 := getPoolID(oc) + if err2 != nil { + return fmt.Errorf("can't get pool ID: %w", err2) + } + sa, err3 := createServiceAccountOnGCP(projectID, gcsSAName) + if err3 != nil { + return fmt.Errorf("can't create service account: %w", err3) + } + os.Setenv("LOGGING_GCS_SERVICE_ACCOUNT_EMAIL", sa.Email) + err4 := grantPermissionsToGCPServiceAccount(poolID, projectID, projectNumber, l.Namespace, l.Name, sa.Email) + if err4 != nil { + return fmt.Errorf("can't add roles to the serviceaccount: %w", err4) + } + + patchLokiOperatorOnGCPSTSforCCO(oc, loNS, projectNumber, poolID, sa.Email) + + err = createSecretForGCSBucketWithSTS(oc, l.Namespace, l.StorageSecret, l.BucketName) + } else { + err = createSecretForGCSBucket(oc, l.BucketName, l.StorageSecret, l.Namespace) + } + } + case "swift": + { + cred, err1 := compat_otp.GetOpenStackCredentials(oc) + o.Expect(err1).NotTo(o.HaveOccurred()) + client := compat_otp.NewOpenStackClient(cred, "object-store") + err = compat_otp.CreateOpenStackContainer(client, l.BucketName) + if err != nil { + return err + } + err = createSecretForSwiftContainer(oc, l.BucketName, l.StorageSecret, l.Namespace, cred) + } + case "odf": + { + err = createObjectBucketClaim(oc, l.Namespace, l.BucketName) + if err != nil { + return err + } + err = createSecretForODFBucket(oc, l.BucketName, l.StorageSecret, l.Namespace) + } + case "minio": + { + cred := getMinIOCreds(oc, minioNS) + cfg := generateS3Config(cred) + client := newS3Client(cfg) + err = createS3Bucket(client, l.BucketName, "") + if err != nil { + return err + } + err = createSecretForMinIOBucket(oc, l.BucketName, l.StorageSecret, l.Namespace, cred) + } + } + return err +} + +func (l lokiStack) removeObjectStorage(oc *exutil.CLI) { + e2e.Logf("Remove Object Storage") + _ = Resource{"secret", l.StorageSecret, l.Namespace}.clear(oc) + var err error + switch l.StorageType { + case "s3": + { + var cfg aws.Config + if compat_otp.IsWorkloadIdentityCluster(oc) { + region, err := compat_otp.GetAWSClusterRegion(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + cfg = readDefaultSDKExternalConfigurations(context.TODO(), region) + iamClient := newIamClient(cfg) + deleteIAMroleonAWS(iamClient, os.Getenv("LOKI_ROLE_NAME_ON_STS")) + os.Unsetenv("LOKI_ROLE_NAME_ON_STS") + subName, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("sub", "-n", loNS, `-ojsonpath={.items[?(@.spec.name=="loki-operator")].metadata.name}`).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(subName).ShouldNot(o.BeEmpty()) + err = oc.AsAdmin().WithoutNamespace().Run("patch").Args("sub", subName, "-n", loNS, "-p", `[{"op": "remove", "path": "/spec/config"}]`, "--type=json").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + WaitForPodsReadyWithLabel(oc, loNS, "name=loki-operator-controller-manager") + } else { + cred := getAWSCredentialFromCluster(oc) + cfg = generateS3Config(cred) + } + client := newS3Client(cfg) + err = deleteS3Bucket(client, l.BucketName) + } + case "azure": + { + if compat_otp.IsWorkloadIdentityCluster(oc) { + resourceGroup, err := getAzureResourceGroupFromCluster(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + azureSubscriptionID := os.Getenv("AZURE_SUBSCRIPTION_ID") + cred := createNewDefaultAzureCredential() + deleteManagedIdentityOnAzure(cred, azureSubscriptionID, resourceGroup, l.Name) + deleteAzureStorageAccount(cred, azureSubscriptionID, resourceGroup, os.Getenv("LOKI_OBJECT_STORAGE_STORAGE_ACCOUNT")) + os.Unsetenv("LOKI_OBJECT_STORAGE_STORAGE_ACCOUNT") + } else { + accountName, accountKey, err1 := compat_otp.GetAzureStorageAccountFromCluster(oc) + o.Expect(err1).NotTo(o.HaveOccurred()) + client, err2 := compat_otp.NewAzureContainerClient(oc, accountName, accountKey, l.BucketName) + o.Expect(err2).NotTo(o.HaveOccurred()) + err = compat_otp.DeleteAzureStorageBlobContainer(client) + } + } + case "gcs": + { + if compat_otp.IsWorkloadIdentityCluster(oc) { + sa := os.Getenv("LOGGING_GCS_SERVICE_ACCOUNT_NAME") + if sa == "" { + e2e.Logf("LOGGING_GCS_SERVICE_ACCOUNT_NAME is not set, no need to delete the serviceaccount") + } else { + os.Unsetenv("LOGGING_GCS_SERVICE_ACCOUNT_NAME") + email := os.Getenv("LOGGING_GCS_SERVICE_ACCOUNT_EMAIL") + if email == "" { + e2e.Logf("LOGGING_GCS_SERVICE_ACCOUNT_EMAIL is not set, no need to delete the policies") + } else { + os.Unsetenv("LOGGING_GCS_SERVICE_ACCOUNT_EMAIL") + projectID, errGetID := compat_otp.GetGcpProjectID(oc) + o.Expect(errGetID).NotTo(o.HaveOccurred()) + projectNumber, _ := getGCPProjectNumber(projectID) + poolID, _ := getPoolID(oc) + err = removePermissionsFromGCPServiceAccount(poolID, projectID, projectNumber, l.Namespace, l.Name, email) + o.Expect(err).NotTo(o.HaveOccurred()) + err = removeServiceAccountFromGCP("projects/" + projectID + "/serviceAccounts/" + email) + o.Expect(err).NotTo(o.HaveOccurred()) + } + } + } + err = compat_otp.DeleteGCSBucket(l.BucketName) + } + case "swift": + { + cred, err1 := compat_otp.GetOpenStackCredentials(oc) + o.Expect(err1).NotTo(o.HaveOccurred()) + client := compat_otp.NewOpenStackClient(cred, "object-store") + err = compat_otp.DeleteOpenStackContainer(client, l.BucketName) + } + case "odf": + { + err = deleteObjectBucketClaim(oc, l.Namespace, l.BucketName) + } + case "minio": + { + cred := getMinIOCreds(oc, minioNS) + cfg := generateS3Config(cred) + client := newS3Client(cfg) + err = deleteS3Bucket(client, l.BucketName) + } + } + o.Expect(err).NotTo(o.HaveOccurred()) +} + +func deployMinIO(oc *exutil.CLI) { + // create namespace + _, err := oc.AdminKubeClient().CoreV1().Namespaces().Get(context.Background(), minioNS, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + err = oc.AsAdmin().WithoutNamespace().Run("create").Args("namespace", minioNS).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + } + // create secret + _, err = oc.AdminKubeClient().CoreV1().Secrets(minioNS).Get(context.Background(), minioSecret, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + err = oc.AsAdmin().WithoutNamespace().Run("create").Args("secret", "generic", minioSecret, "-n", minioNS, "--from-literal=access_key_id="+getRandomString(), "--from-literal=secret_access_key=passwOOrd"+getRandomString()).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + } + // deploy minIO + deployTemplate, _ := filePath.Abs("testdata/logging/minIO/deploy.yaml") + deployFile, err := processTemplate(oc, "-n", minioNS, "-f", deployTemplate, "-p", "NAMESPACE="+minioNS, "NAME=minio", "SECRET_NAME="+minioSecret) + defer os.Remove(deployFile) + o.Expect(err).NotTo(o.HaveOccurred()) + err = oc.AsAdmin().Run("apply").Args("-f", deployFile, "-n", minioNS).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + // wait for minio to be ready + for _, rs := range []string{"deployment", "svc", "route"} { + _ = Resource{rs, "minio", minioNS}.WaitForResourceToAppear(oc) + } + WaitForDeploymentPodsToBeReady(oc, minioNS, "minio") +} + +func getPoolID(oc *exutil.CLI) (string, error) { + // pool_id="$(oc get authentication cluster -o json | jq -r .spec.serviceAccountIssuer | sed 's/.*\/\([^\/]*\)-oidc/\1/')" + issuer, err := getOIDC(oc) + if err != nil { + return "", err + } + + return strings.Split(strings.Split(issuer, "/")[1], "-oidc")[0], nil +} + +// delete the objects in the cluster +func (r Resource) clear(oc *exutil.CLI) error { + msg, err := oc.AsAdmin().WithoutNamespace().Run("delete").Args("-n", r.Namespace, r.Kind, r.Name).Output() + if err != nil { + errstring := fmt.Sprintf("%v", msg) + if strings.Contains(errstring, "NotFound") || strings.Contains(errstring, "the server doesn't have a resource type") { + return nil + } + return err + } + err = r.WaitUntilResourceIsGone(oc) + return err +} + +// Patches Loki Operator running on a GCP WIF cluster. Operator is deployed with CCO mode after patching. +func patchLokiOperatorOnGCPSTSforCCO(oc *exutil.CLI, namespace string, projectNumber string, poolID string, serviceAccount string) { + patchConfig := `{ + "spec": { + "config": { + "env": [ + { + "name": "PROJECT_NUMBER", + "value": "%s" + }, + { + "name": "POOL_ID", + "value": "%s" + }, + { + "name": "PROVIDER_ID", + "value": "%s" + }, + { + "name": "SERVICE_ACCOUNT_EMAIL", + "value": "%s" + } + ] + } + } + }` + + err := oc.NotShowInfo().AsAdmin().WithoutNamespace().Run("patch").Args("sub", "loki-operator", "-n", namespace, "-p", fmt.Sprintf(patchConfig, projectNumber, poolID, poolID, serviceAccount), "--type=merge").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + WaitForPodsReadyWithLabel(oc, loNS, "name=loki-operator-controller-manager") +} diff --git a/integration-tests/backend/metrics.go b/integration-tests/backend/metrics.go new file mode 100644 index 0000000000..bbc97554ec --- /dev/null +++ b/integration-tests/backend/metrics.go @@ -0,0 +1,176 @@ +package e2etests + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strconv" + "time" + + o "github.com/onsi/gomega" + exutil "github.com/openshift/origin/test/extended/util" + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" + "k8s.io/apimachinery/pkg/util/wait" + e2e "k8s.io/kubernetes/test/e2e/framework" +) + +// prometheusQueryResult the response of querying prometheus APIs +type prometheusQueryResult struct { + Data struct { + Result []metric `json:"result"` + ResultType string `json:"resultType"` + } `json:"data"` + Status string `json:"status"` +} + +// metric the prometheus metric +type metric struct { + Metric struct { + Name string `json:"__name__"` + Cluster string `json:"cluster,omitempty"` + Container string `json:"container,omitempty"` + ContainerName string `json:"containername,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + Instance string `json:"instance,omitempty"` + Job string `json:"job,omitempty"` + Namespace string `json:"namespace,omitempty"` + Path string `json:"path,omitempty"` + Pod string `json:"pod,omitempty"` + PodName string `json:"podname,omitempty"` + Service string `json:"service,omitempty"` + } `json:"metric"` + Value []interface{} `json:"value"` +} + +func getMetric(oc *exutil.CLI, query string) ([]metric, error) { + bearerToken := getSAToken(oc, "prometheus-k8s", "openshift-monitoring") + promRoute := "https://" + getRouteAddress(oc, "openshift-monitoring", "prometheus-k8s") + res, err := queryPrometheus(promRoute, query, bearerToken) + if err != nil { + return []metric{}, err + } + attempts := 10 + for len(res.Data.Result) == 0 && attempts > 0 { + time.Sleep(5 * time.Second) + res, err = queryPrometheus(promRoute, query, bearerToken) + if err != nil { + return []metric{}, err + } + attempts-- + } + errMsg := fmt.Sprintf("0 results returned for query %s", query) + o.Expect(len(res.Data.Result)).Should(o.BeNumerically(">=", 1), errMsg) + return res.Data.Result, nil +} + +// queryPrometheus returns the promtheus metrics which match the query string +// path: the api path, for example: /api/v1/query? +// query: the metric or alert you want to search +// action: it can be "GET", "get", "Get", "POST", "post", "Post" +func queryPrometheus(promRoute string, query string, bearerToken string) (*prometheusQueryResult, error) { + path := "/api/v1/query" + action := "GET" + + h := make(http.Header) + h.Add("Content-Type", "application/json") + h.Add("Authorization", "Bearer "+bearerToken) + + params := url.Values{} + if len(query) > 0 { + params.Add("query", query) + } + + var p prometheusQueryResult + resp, err := doHTTPRequest(h, promRoute, path, params.Encode(), action, false, 5, nil, 200) + if err != nil { + return nil, err + } + err = json.Unmarshal(resp, &p) + if err != nil { + return nil, err + } + return &p, nil +} + +// return the first metric value +func popMetricValue(metrics []metric) float64 { + valInterface := metrics[0].Value[1] + val, _ := valInterface.(string) + value, err := strconv.ParseFloat(val, 64) + o.Expect(err).NotTo(o.HaveOccurred()) + return value +} + +// polls any prometheus metrics +func pollMetrics(oc *exutil.CLI, promQuery string) float64 { + var metricsVal float64 + e2e.Logf("Query is %s", promQuery) + err := wait.PollUntilContextTimeout(context.Background(), 60*time.Second, 600*time.Second, false, func(context.Context) (bool, error) { + metrics, err := getMetric(oc, promQuery) + if err != nil { + return false, err + } + metricsVal = popMetricValue(metrics) + if metricsVal <= 0 { + e2e.Logf("%s did not return metrics value > 0, will try again", promQuery) + } + return metricsVal > 0, nil + }) + + msg := fmt.Sprintf("%s did not return valid metrics in 600 seconds", promQuery) + compat_otp.AssertWaitPollNoErr(err, msg) + return metricsVal +} + +// verify FLP metrics +func verifyFLPMetrics(oc *exutil.CLI) { + query := "sum(netobserv_ingest_flows_processed)" + pollMetrics(oc, query) + query = "sum(netobserv_loki_sent_entries_total)" + pollMetrics(oc, query) +} + +// verify eBPF metrics +func verifyEBPFMetrics(oc *exutil.CLI) { + query := "sum(netobserv_agent_exported_batch_total)" + pollMetrics(oc, query) + query = "sum(netobserv_agent_evictions_total)" + pollMetrics(oc, query) +} + +// verify eBPF filter metrics +func verifyEBPFFilterMetrics(oc *exutil.CLI, reason string) { + query := fmt.Sprintf(`100 * sum(rate(netobserv_agent_filtered_flows_total{reason="%s"}[1m])) / sum(rate(netobserv_agent_filtered_flows_total[1m]))`, reason) + metrics := pollMetrics(oc, query) + switch reason { + case "FilterAccept": + // Expected to be around 1 + o.Expect(metrics).Should(o.BeNumerically("~", 0.01, 2), "Accept metrics are beyond threshold values") + case "FilterNoMatch": + // Expected to be around 100 + o.Expect(metrics).Should(o.BeNumerically("~", 98, 100), "NoMatch metrics are beyond threshold values") + case "FilterReject": + // Expected to be around 2 + o.Expect(metrics).Should(o.BeNumerically("~", 1, 5), "Reject metrics are beyond threshold values") + } +} + +// verify eBPF feature metrics +func verifyEBPFFeatureMetrics(oc *exutil.CLI, feature string) { + query := fmt.Sprintf(`sum(rate(netobserv_agent_buffer_size{name="%s"}[1m]))`, feature) + metrics := pollMetrics(oc, query) + // making sure it's simply greater than 0 because we don't know the deteministic value to expect. + o.Expect(metrics).Should(o.BeNumerically(">", 0), fmt.Sprintf("%s metrics is 0", feature)) +} + +func getMetricsScheme(oc *exutil.CLI, servicemonitor string, namespace string) (string, error) { + out, err := oc.AsAdmin().Run("get").Args("servicemonitor", servicemonitor, "-n", namespace, "-o", "jsonpath='{.spec.endpoints[].scheme}'").Output() + return out, err +} + +func getMetricsServerName(oc *exutil.CLI, servicemonitor string, namespace string) (string, error) { + out, err := oc.AsAdmin().Run("get").Args("servicemonitor", servicemonitor, "-n", namespace, "-o", "jsonpath='{.spec.endpoints[].tlsConfig.serverName}'").Output() + return out, err +} diff --git a/integration-tests/backend/multitenants.go b/integration-tests/backend/multitenants.go new file mode 100644 index 0000000000..f9b70a9af5 --- /dev/null +++ b/integration-tests/backend/multitenants.go @@ -0,0 +1,167 @@ +package e2etests + +import ( + "context" + "fmt" + "os" + "os/exec" + filePath "path/filepath" + "reflect" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + exutil "github.com/openshift/origin/test/extended/util" + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" + "k8s.io/apimachinery/pkg/util/wait" + e2e "k8s.io/kubernetes/test/e2e/framework" +) + +type User struct { + Username string + Password string +} + +func getCoStatus(oc *exutil.CLI, coName string, statusToCompare map[string]string) map[string]string { + newStatusToCompare := make(map[string]string) + for key := range statusToCompare { + args := fmt.Sprintf(`-o=jsonpath={.status.conditions[?(.type == '%s')].status}`, key) + status, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("co", args, coName).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + newStatusToCompare[key] = status + } + return newStatusToCompare +} + +func waitCoBecomes(oc *exutil.CLI, coName string, waitTime int, expectedStatus map[string]string) error { + errCo := wait.PollUntilContextTimeout(context.Background(), 10*time.Second, time.Duration(waitTime)*time.Second, false, func(context.Context) (bool, error) { + gottenStatus := getCoStatus(oc, coName, expectedStatus) + eq := reflect.DeepEqual(expectedStatus, gottenStatus) + if eq { + eq := reflect.DeepEqual(expectedStatus, map[string]string{"Available": "True", "Progressing": "False", "Degraded": "False"}) + if eq { + // For True False False, we want to wait some bit more time and double check, to ensure it is stably healthy + time.Sleep(25 * time.Second) + gottenStatus := getCoStatus(oc, coName, expectedStatus) + eq := reflect.DeepEqual(expectedStatus, gottenStatus) + if eq { + e2e.Logf("Given operator %s becomes available/non-progressing/non-degraded +%v", coName, gottenStatus) + return true, nil + } + } else { + e2e.Logf("Given operator %s becomes %s", coName, gottenStatus) + return true, nil + } + } + return false, nil + }) + if errCo != nil { + err := oc.AsAdmin().WithoutNamespace().Run("get").Args("co").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + } + return errCo +} + +func generateUsersHtpasswd(passwdFile *string, users []*User) error { + for i := 0; i < len(users); i++ { + // Generate new username and password + username := fmt.Sprintf("testuser-%v-%v", i, compat_otp.GetRandomString()) + password := compat_otp.GetRandomString() + users[i] = &User{Username: username, Password: password} + + // Add new user to htpasswd file in the temp directory + cmd := fmt.Sprintf("htpasswd -b %v %v %v", *passwdFile, users[i].Username, users[i].Password) + err := exec.Command("bash", "-c", cmd).Run() + if err != nil { + return err + } + } + return nil +} + +func getNewUser(oc *exutil.CLI, count int) ([]*User, string, string) { + usersDirPath := "/tmp/" + compat_otp.GetRandomString() + usersHTpassFile := usersDirPath + "/htpasswd" + err := os.MkdirAll(usersDirPath, 0o755) + o.Expect(err).NotTo(o.HaveOccurred()) + + htPassSecret, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("oauth/cluster", "-o", "jsonpath={.spec.identityProviders[0].htpasswd.fileData.name}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + users := make([]*User, count) + if htPassSecret == "" { + htPassSecret = "htpass-secret" + _, _ = os.Create(usersHTpassFile) + err = generateUsersHtpasswd(&usersHTpassFile, users) + o.Expect(err).NotTo(o.HaveOccurred()) + err = oc.AsAdmin().WithoutNamespace().Run("create").Args("-n", "openshift-config", "secret", "generic", htPassSecret, "--from-file", "htpasswd="+usersHTpassFile).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + err := oc.AsAdmin().WithoutNamespace().Run("patch").Args("--type=json", "-p", `[{"op": "add", "path": "/spec/identityProviders", "value": [{"htpasswd": {"fileData": {"name": "htpass-secret"}}, "mappingMethod": "claim", "name": "htpasswd", "type": "HTPasswd"}]}]`, "oauth/cluster").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + } else { + err = oc.AsAdmin().WithoutNamespace().Run("extract").Args("-n", "openshift-config", "secret/"+htPassSecret, "--to", usersDirPath, "--confirm").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + err = generateUsersHtpasswd(&usersHTpassFile, users) + o.Expect(err).NotTo(o.HaveOccurred()) + // Update htpass-secret with the modified htpasswd file + err = oc.AsAdmin().WithoutNamespace().Run("set").Args("-n", "openshift-config", "data", "secret/"+htPassSecret, "--from-file", "htpasswd="+usersHTpassFile).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + } + + g.By("Checking authentication operator should be in Progressing in 180 seconds") + err = waitCoBecomes(oc, "authentication", 180, map[string]string{"Progressing": "True"}) + compat_otp.AssertWaitPollNoErr(err, "authentication operator did not start progressing in 180 seconds") + e2e.Logf("Checking authentication operator should be Available in 600 seconds") + err = waitCoBecomes(oc, "authentication", 600, map[string]string{"Available": "True", "Progressing": "False", "Degraded": "False"}) + compat_otp.AssertWaitPollNoErr(err, "authentication operator did not become available in 600 seconds") + + return users, usersHTpassFile, htPassSecret +} + +func userCleanup(oc *exutil.CLI, users []*User, usersHTpassFile string, htPassSecret string) { + defer os.RemoveAll(usersHTpassFile) + for i := range users { + // Add new user to htpasswd file in the temp directory + cmd := fmt.Sprintf("htpasswd -D %v %v", usersHTpassFile, users[i].Username) + err := exec.Command("bash", "-c", cmd).Run() + o.Expect(err).NotTo(o.HaveOccurred()) + } + + // Update htpass-secret with the modified htpasswd file + err := oc.AsAdmin().WithoutNamespace().Run("set").Args("-n", "openshift-config", "data", "secret/"+htPassSecret, "--from-file", "htpasswd="+usersHTpassFile).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Checking authentication operator should be in Progressing in 180 seconds") + err = waitCoBecomes(oc, "authentication", 180, map[string]string{"Progressing": "True"}) + compat_otp.AssertWaitPollNoErr(err, "authentication operator did not start progressing in 180 seconds") + e2e.Logf("Checking authentication operator should be Available in 600 seconds") + err = waitCoBecomes(oc, "authentication", 600, map[string]string{"Available": "True", "Progressing": "False", "Degraded": "False"}) + compat_otp.AssertWaitPollNoErr(err, "authentication operator did not become available in 600 seconds") +} + +func addUserAsReader(oc *exutil.CLI, username string) { + baseDir, _ := filePath.Abs("testdata") + readerCRBPath := filePath.Join(baseDir, "netobserv-loki-reader-multitenant-crb.yaml") + parameters := []string{"-f", readerCRBPath, "-p", "USERNAME=" + username} + compat_otp.CreateClusterResourceFromTemplate(oc, parameters...) +} + +func removeUserAsReader(oc *exutil.CLI, username string) { + err := oc.AsAdmin().WithoutNamespace().Run("adm").Args("policy", "remove-cluster-role-from-user", "netobserv-loki-reader", username).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) +} + +func addTemplatePermissions(oc *exutil.CLI, username string) { + baseDir, _ := filePath.Abs("testdata") + readerCRBPath := filePath.Join(baseDir, "testuser-template-crb.yaml") + parameters := []string{"-f", readerCRBPath, "-p", "USERNAME=" + username} + compat_otp.CreateClusterResourceFromTemplate(oc, parameters...) +} + +func removeTemplatePermissions(oc *exutil.CLI, username string) { + baseDir, _ := filePath.Abs("testdata") + readerCRBPath := filePath.Join(baseDir, "testuser-template-crb.yaml") + parameters := []string{"-f", readerCRBPath, "-p", "USERNAME=" + username} + configFile := compat_otp.ProcessTemplate(oc, parameters...) + err := oc.AsAdmin().WithoutNamespace().Run("delete").Args("-f", configFile).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) +} diff --git a/integration-tests/backend/operator.go b/integration-tests/backend/operator.go new file mode 100644 index 0000000000..f4f8db5119 --- /dev/null +++ b/integration-tests/backend/operator.go @@ -0,0 +1,571 @@ +package e2etests + +import ( + "context" + "fmt" + filePath "path/filepath" + "regexp" + "strings" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + exutil "github.com/openshift/origin/test/extended/util" + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + e2e "k8s.io/kubernetes/test/e2e/framework" +) + +const ( + netobservNS = "openshift-netobserv-operator" + NOPackageName = "netobserv-operator" +) + +// SubscriptionObjects objects are used to create operators via OLM +type SubscriptionObjects struct { + OperatorName string + Namespace string + OperatorGroup string // the file used to create operator group + Subscription string // the file used to create subscription + PackageName string + CatalogSource *CatalogSourceObjects `json:",omitempty"` + OperatorPodLabel string +} + +// CatalogSourceObjects defines the source used to subscribe an operator +type CatalogSourceObjects struct { + Channel string `json:",omitempty"` + SourceName string `json:",omitempty"` + SourceNamespace string `json:",omitempty"` +} + +// OperatorNamespace struct to handle creation of namespace +type OperatorNamespace struct { + Name string + NamespaceTemplate string +} + +type subscriptionResource struct { + name string + namespace string + operatorName string + channel string + catalog string + catalogNamespace string + template string +} + +type operatorGroupResource struct { + name string + namespace string + targetNamespaces string + template string +} + +// waitForPackagemanifestAppear waits for the packagemanifest to appear in the cluster +// chSource: bool value, true means the packagemanifests' source name must match the so.CatalogSource.SourceName, e.g.: oc get packagemanifests xxxx -l catalog=$source-name +func (so *SubscriptionObjects) waitForPackagemanifestAppear(oc *exutil.CLI, chSource bool) { + args := []string{"-n", so.CatalogSource.SourceNamespace, "packagemanifests"} + if chSource { + args = append(args, "-l", "catalog="+so.CatalogSource.SourceName) + } else { + args = append(args, so.PackageName) + } + err := wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 180*time.Second, false, func(context.Context) (done bool, err error) { + packages, err := oc.AsAdmin().WithoutNamespace().Run("get").Args(args...).Output() + if err != nil { + msg := fmt.Sprintf("%v", err) + if strings.Contains(msg, "No resources found") || strings.Contains(msg, "NotFound") { + return false, nil + } + return false, err + } + if strings.Contains(packages, so.PackageName) { + return true, nil + } + e2e.Logf("Waiting for packagemanifest/%s to appear", so.PackageName) + return false, nil + }) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("Packagemanifest %s is not availabile", so.PackageName)) +} + +// setCatalogSourceObjects set the default values of channel, source namespace and source name if they're not specified +func (so *SubscriptionObjects) setCatalogSourceObjects(oc *exutil.CLI) { + // set channel + if so.CatalogSource.Channel == "" { + so.CatalogSource.Channel = "stable" + } + + // set source namespace + if so.CatalogSource.SourceNamespace == "" { + so.CatalogSource.SourceNamespace = "openshift-marketplace" + } + + // set source and check if the packagemanifest exists or not + if so.CatalogSource.SourceName != "" { + so.waitForPackagemanifestAppear(oc, true) + } else { + catsrc, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("catsrc", "-n", so.CatalogSource.SourceNamespace, "qe-app-registry").Output() + if catsrc != "" && !(strings.Contains(catsrc, "NotFound")) { + so.CatalogSource.SourceName = "qe-app-registry" + so.waitForPackagemanifestAppear(oc, true) + } else { + so.waitForPackagemanifestAppear(oc, false) + source, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("packagemanifests", so.PackageName, "-o", "jsonpath={.status.catalogSource}").Output() + if err != nil { + e2e.Logf("error getting catalog source name: %v", err) + } + so.CatalogSource.SourceName = source + } + } +} + +// SubscribeOperator is used to subcribe the CLO and EO +func (so *SubscriptionObjects) SubscribeOperator(oc *exutil.CLI) { + // check if the namespace exists, if it doesn't exist, create the namespace + _, err := oc.AdminKubeClient().CoreV1().Namespaces().Get(context.Background(), so.Namespace, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + e2e.Logf("The project %s is not found, create it now...", so.Namespace) + namespaceTemplate, _ := filePath.Abs("testdata/logging/subscription/namespace.yaml") + namespaceFile, err := processTemplate(oc, "-f", namespaceTemplate, "-p", "NAMESPACE_NAME="+so.Namespace) + o.Expect(err).NotTo(o.HaveOccurred()) + err = wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 120*time.Second, false, func(context.Context) (done bool, err error) { + output, err := oc.AsAdmin().Run("apply").Args("-f", namespaceFile).Output() + if err != nil { + if strings.Contains(output, "AlreadyExists") { + return true, nil + } + return false, err + } + return true, nil + }) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("can't create project %s", so.Namespace)) + } + } + + // check the operator group, if no object found, then create an operator group in the project + og, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("-n", so.Namespace, "og").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + msg := fmt.Sprintf("%v", og) + if strings.Contains(msg, "No resources found") { + // create operator group + ogFile, err := processTemplate(oc, "-n", so.Namespace, "-f", so.OperatorGroup, "-p", "OG_NAME="+so.Namespace, "NAMESPACE="+so.Namespace) + o.Expect(err).NotTo(o.HaveOccurred()) + err = wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 120*time.Second, false, func(context.Context) (done bool, err error) { + output, err := oc.AsAdmin().Run("apply").Args("-f", ogFile, "-n", so.Namespace).Output() + if err != nil { + if strings.Contains(output, "AlreadyExists") { + return true, nil + } + return false, err + } + return true, nil + }) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("can't create operatorgroup %s in %s project", so.Namespace, so.Namespace)) + } + + // check subscription, if there is no subscription objets, then create one + sub, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("sub", "-n", so.Namespace, so.PackageName).Output() + if err != nil { + msg := fmt.Sprint("v%", sub) + if strings.Contains(msg, "NotFound") { + so.setCatalogSourceObjects(oc) + // create subscription object + subscriptionFile, err := processTemplate(oc, "-n", so.Namespace, "-f", so.Subscription, "-p", "PACKAGE_NAME="+so.PackageName, "NAMESPACE="+so.Namespace, "CHANNEL="+so.CatalogSource.Channel, "SOURCE="+so.CatalogSource.SourceName, "SOURCE_NAMESPACE="+so.CatalogSource.SourceNamespace) + o.Expect(err).NotTo(o.HaveOccurred()) + err = wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 120*time.Second, false, func(context.Context) (done bool, err error) { + output, err := oc.AsAdmin().Run("apply").Args("-f", subscriptionFile, "-n", so.Namespace).Output() + if err != nil { + if strings.Contains(output, "AlreadyExists") { + return true, nil + } + return false, err + } + return true, nil + }) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("can't create subscription %s in %s project", so.PackageName, so.Namespace)) + } + } + //WaitForDeploymentPodsToBeReady(oc, so.Namespace, so.OperatorName) +} + +func deleteNamespace(oc *exutil.CLI, ns string) { + err := oc.AdminKubeClient().CoreV1().Namespaces().Delete(context.Background(), ns, metav1.DeleteOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + err = nil + } + } + o.Expect(err).NotTo(o.HaveOccurred()) + err = wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 180*time.Second, false, func(context.Context) (bool, error) { + _, err := oc.AdminKubeClient().CoreV1().Namespaces().Get(context.Background(), ns, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + return false, err + } + return false, nil + }) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("Namespace %s is not deleted in 3 minutes", ns)) +} + +func (so *SubscriptionObjects) uninstallOperator(oc *exutil.CLI) { + _ = Resource{"subscription", so.PackageName, so.Namespace}.clear(oc) + _ = oc.AsAdmin().WithoutNamespace().Run("delete").Args("-n", so.Namespace, "csv", "-l", "operators.coreos.com/"+so.PackageName+"."+so.Namespace+"=").Execute() + // do not remove namespace openshift-logging and openshift-operators-redhat, and preserve the operatorgroup as there may have several operators deployed in one namespace + // for example: loki-operator and elasticsearch-operator + if so.Namespace != "openshift-logging" && so.Namespace != "openshift-operators-redhat" && so.Namespace != "openshift-operators" && so.Namespace != "openshift-netobserv-operator" && !strings.HasPrefix(so.Namespace, "e2e-test-") { + deleteNamespace(oc, so.Namespace) + } +} + +func checkOperatorChannel(oc *exutil.CLI, operatorNamespace string, operatorName string) (string, error) { + channelName, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("sub", operatorName, "-n", operatorNamespace, "-o=jsonpath={.spec.channel}").Output() + if err != nil { + return "", err + } + return channelName, nil +} + +func CheckOperatorStatus(oc *exutil.CLI, operatorNamespace string, operatorName string) (bool, error) { + err := oc.AsAdmin().WithoutNamespace().Run("get").Args("namespace", operatorNamespace).Execute() + if err == nil { + err1 := oc.AsAdmin().WithoutNamespace().Run("get").Args("sub", operatorName, "-n", operatorNamespace).Execute() + if err1 == nil { + csvName, err2 := oc.AsAdmin().WithoutNamespace().Run("get").Args("sub", operatorName, "-n", operatorNamespace, "-o=jsonpath={.status.installedCSV}").Output() + o.Expect(err2).NotTo(o.HaveOccurred()) + o.Expect(csvName).NotTo(o.BeEmpty()) + err = wait.PollUntilContextTimeout(context.Background(), 10*time.Second, 360*time.Second, false, func(context.Context) (bool, error) { + csvState, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("csv", csvName, "-n", operatorNamespace, "-o=jsonpath={.status.phase}").Output() + if err != nil { + return false, err + } + e2e.Logf("CSV %s state: %s", csvName, csvState) + return csvState == "Succeeded", nil + }) + if err != nil { + return false, err + } + return true, nil + } + } + e2e.Logf("%s operator will be created by tests", operatorName) + return false, nil +} + +func (ns *OperatorNamespace) DeployOperatorNamespace(oc *exutil.CLI) { + e2e.Logf("Creating %s operator namespace", ns.Name) + nsParameters := []string{"--ignore-unknown-parameters=true", "-f", ns.NamespaceTemplate, "-p", "NAMESPACE_NAME=" + ns.Name} + compat_otp.ApplyClusterResourceFromTemplate(oc, nsParameters...) +} + +func generateTemplateAbsolutePath(fileName string) string { + testDataDir, _ := filePath.Abs("testdata/networking/nmstate") + return filePath.Join(testDataDir, fileName) +} + +func operatorInstall(oc *exutil.CLI, sub subscriptionResource, ns OperatorNamespace, og operatorGroupResource) (status bool) { + //Installing Operator + g.By("INSTALLING Operator in the namespace") + + //Applying the config of necessary yaml files from templates to create metallb operator + g.By("Applying namespace template") + err0 := applyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", ns.NamespaceTemplate, "-p", "NAME="+ns.Name) + if err0 != nil { + e2e.Logf("Error creating namespace %v", err0) + } + + g.By("Applying operatorgroup yaml") + err0 = applyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", og.template, "-p", "NAME="+og.name, "NAMESPACE="+og.namespace, "TARGETNAMESPACES="+og.targetNamespaces) + if err0 != nil { + e2e.Logf("Error creating operator group %v", err0) + } + + g.By("Creating subscription YAML from template") + // no need to check for an existing subscription + err0 = applyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", sub.template, "-p", "OPERATORNAME="+sub.operatorName, "SUBSCRIPTIONNAME="+sub.name, "NAMESPACE="+sub.namespace, "CHANNEL="+sub.channel, + "CATALOGSOURCE="+sub.catalog, "CATALOGSOURCENAMESPACE="+sub.catalogNamespace) + if err0 != nil { + e2e.Logf("Error creating subscription %v", err0) + } + + //confirming operator install + g.By("Verify the operator finished subscribing") + errCheck := wait.PollUntilContextTimeout(context.Background(), 10*time.Second, 360*time.Second, false, func(context.Context) (bool, error) { + subState, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("sub", sub.name, "-n", sub.namespace, "-o=jsonpath={.status.state}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if strings.Compare(subState, "AtLatestKnown") == 0 { + return true, nil + } + // log full status of sub for installation failure debugging + subState, _ = oc.AsAdmin().WithoutNamespace().Run("get").Args("sub", sub.name, "-n", sub.namespace, "-o=jsonpath={.status}").Output() + e2e.Logf("Status of subscription: %v", subState) + return false, nil + }) + compat_otp.AssertWaitPollNoErr(errCheck, fmt.Sprintf("Subscription %s in namespace %v does not have expected status", sub.name, sub.namespace)) + + g.By("Get csvName") + csvName, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("sub", sub.name, "-n", sub.namespace, "-o=jsonpath={.status.installedCSV}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(csvName).NotTo(o.BeEmpty()) + errCheck = wait.PollUntilContextTimeout(context.Background(), 10*time.Second, 360*time.Second, false, func(context.Context) (bool, error) { + csvState, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("csv", csvName, "-n", sub.namespace, "-o=jsonpath={.status.phase}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if strings.Compare(csvState, "Succeeded") == 0 { + e2e.Logf("CSV check complete!!!") + return true, nil + + } + return false, nil + }) + compat_otp.AssertWaitPollNoErr(errCheck, fmt.Sprintf("CSV %v in %v namespace does not have expected status", csvName, sub.namespace)) + return true +} + +func getOpenshiftVersion(oc *exutil.CLI) string { + version, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("clusterversion/version", "-ojsonpath={.status.desired.version}").Output() + if err != nil { + return "" + } + re := regexp.MustCompile(`^(\d+\.\d+)`) + matches := re.FindStringSubmatch(version) + if len(matches) < 2 { + return "" + } + return matches[1] +} + +func createImageDigestMirrorSet(oc *exutil.CLI, imagedigestmirrorsetname string, imageDigestMirrorSetFile string) error { + pollInterval := 10 * time.Second + waitTimeout := 120 * time.Second + err := oc.AsAdmin().WithoutNamespace().Run("create").Args("-f", imageDigestMirrorSetFile).Execute() + if err != nil { + return fmt.Errorf("error applying image digest mirror set: %w", err) + } + return wait.PollUntilContextTimeout(context.Background(), pollInterval, waitTimeout, false, func(_ context.Context) (bool, error) { + err := oc.AsAdmin().WithoutNamespace(). + Run("get").Args("imagedigestmirrorset", imagedigestmirrorsetname).Execute() + return err == nil, nil + }) +} + +func createCatalogSource(oc *exutil.CLI, operatorName string, catalogSourceName string, catalogNamespace string, catalogSourceTemplateFile string) error { + pollInterval := 10 * time.Second + waitTimeout := 120 * time.Second + openshiftVersion := getOpenshiftVersion(oc) + if openshiftVersion == "" { + return fmt.Errorf("failed to get OpenShift version") + } + image := "quay.io/redhat-user-workloads/ocp-art-tenant/art-fbc:ocp__" + openshiftVersion + "__" + operatorName + "-rhel9-operator" + e2e.Logf("Creating catalog source with name '%s' in namespace '%s'", catalogSourceName, catalogNamespace) + err := applyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", catalogSourceTemplateFile, "-p", "CATALOGSOURCENAME="+catalogSourceName, "CATALOGNAMESPACE="+catalogNamespace, "IMAGE="+image) + if err != nil { + return fmt.Errorf("error applying catalog source: %w", err) + } + + // Wait for CatalogSource to exist and be ready + return wait.PollUntilContextTimeout(context.Background(), pollInterval, waitTimeout, false, func(_ context.Context) (bool, error) { + // Check if CatalogSource exists + err := oc.AsAdmin().WithoutNamespace().Run("get").Args("catalogsource", catalogSourceName, "-n", catalogNamespace).Execute() + if err != nil { + e2e.Logf("CatalogSource not found yet: %v", err) + return false, nil + } + + // Check if CatalogSource connection state is READY + connectionState, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("catalogsource", catalogSourceName, + "-n", catalogNamespace, "-o=jsonpath={.status.connectionState.lastObservedState}").Output() + if err != nil { + e2e.Logf("Failed to get connection state: %v", err) + return false, nil + } + + if string(connectionState) != "READY" { + e2e.Logf("CatalogSource connection state is '%s', waiting for 'READY'", string(connectionState)) + return false, nil + } + + // Check if registry pod is running and ready + podName, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pods", "-n", catalogNamespace, + "-l", "olm.catalogSource="+catalogSourceName, + "-o=jsonpath={.items[0].metadata.name}").Output() + if err != nil || len(podName) == 0 { + e2e.Logf("Registry pod not found yet: %v", err) + return false, nil + } + + // Check pod ready condition + podReady, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", string(podName), "-n", catalogNamespace, + "-o=jsonpath={.status.conditions[?(@.type=='Ready')].status}").Output() + if err != nil { + e2e.Logf("Failed to get pod ready status: %v", err) + return false, nil + } + + if string(podReady) != "True" { + e2e.Logf("Registry pod '%s' is not ready yet: %s", string(podName), string(podReady)) + return false, nil + } + e2e.Logf("CatalogSource '%s' is ready with pod '%s'", catalogSourceName, string(podName)) + return true, nil + }) +} + +func getOperatorCatalogSource(oc *exutil.CLI, catalog string, namespace string) string { + if isBaselineCapsSet(oc) && !(isEnabledCapability(oc, "OperatorLifecycleManager")) { + g.Skip("Skipping the test as baselinecaps have been set and OperatorLifecycleManager capability is not enabled!") + } + catalogSourceNames, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("catalogsource", "-n", namespace, "-o=jsonpath={.items[*].metadata.name}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if strings.Contains(catalogSourceNames, catalog) { + return catalog + } + return "" +} + +func getImageDigestMirrorSet(oc *exutil.CLI, imagedigestmirrorsetname string) string { + imageDigestMirrorSetNames, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("imagedigestmirrorset", "-o=jsonpath={.items[*].metadata.name}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if strings.Contains(imageDigestMirrorSetNames, imagedigestmirrorsetname) { + return imagedigestmirrorsetname + } + return "" +} + +func installNMstateOperator(oc *exutil.CLI) { + var ( + opNamespace = "openshift-nmstate" + opName = "kubernetes-nmstate-operator" + catalogNamespace = "openshift-marketplace" + catalogSourceName = "kubernetes-nmstate-operator-fbc-catalog" + imageDigestMirrorSetName = "kubernetes-nmstate-images-mirror-set" + ) + + e2e.Logf("Check catalogsource and install nmstate operator.") + namespaceTemplate := generateTemplateAbsolutePath("namespace-template.yaml") + operatorGroupTemplate := generateTemplateAbsolutePath("operatorgroup-template.yaml") + subscriptionTemplate := generateTemplateAbsolutePath("subscription-template.yaml") + catalogSourceTemplate := generateTemplateAbsolutePath("catalogsource-template.yaml") + imageDigestMirrorSetFile := generateTemplateAbsolutePath("image-digest-mirrorset.yaml") + sub := subscriptionResource{ + name: "nmstate-operator-sub", + namespace: opNamespace, + operatorName: opName, + channel: "stable", + catalog: catalogSourceName, + catalogNamespace: catalogNamespace, + template: subscriptionTemplate, + } + compat_otp.By("Check the image digest mirror set and catalog source") + imageDigestMirrorSet := getImageDigestMirrorSet(oc, imageDigestMirrorSetName) + if imageDigestMirrorSet == "" { + compat_otp.By("Creating image digest mirror set") + o.Expect(createImageDigestMirrorSet(oc, imageDigestMirrorSetName, imageDigestMirrorSetFile)).NotTo(o.HaveOccurred()) + } + catalogSource := getOperatorCatalogSource(oc, catalogSourceName, catalogNamespace) + if catalogSource == "" { + compat_otp.By("Creating catalog source") + o.Expect(createCatalogSource(oc, "kubernetes-nmstate", catalogSourceName, catalogNamespace, catalogSourceTemplate)).NotTo(o.HaveOccurred()) + } + //sub.catalog = catalogSource + ns := OperatorNamespace{ + Name: opNamespace, + NamespaceTemplate: namespaceTemplate, + } + og := operatorGroupResource{ + name: opName, + namespace: opNamespace, + targetNamespaces: opNamespace, + template: operatorGroupTemplate, + } + + operatorInstall(oc, sub, ns, og) + e2e.Logf("SUCCESS - NMState operator installed") +} + +// setupCatalogSource deploys the catalog source and image digest mirror set +func setupCatalogSource(oc *exutil.CLI, catSrc Resource, catSrcTemplate, imageDigest, catalogSource string, isHypershift bool, NOSource *CatalogSourceObjects, NO *SubscriptionObjects) (bool, error) { + g.By("Deploy konflux FBC and ImageDigestMirrorSet") + upstreamCatalogSource := "quay.io/netobserv/network-observability-operator-catalog:v0.0.0-sha-main" + deployedUpstreamCatalogSource := false + var catsrcErr error + + if catalogSource != "" { + e2e.Logf("Using %s catalog", catalogSource) + catsrcErr = catSrc.applyFromTemplate(oc, "-n", catSrc.Namespace, "-f", catSrcTemplate, "-p", "NAMESPACE="+catSrc.Namespace, "IMAGE="+catalogSource) + } else if isHypershift { + e2e.Logf("Using v0.0.0-sha-main catalog for hypershift") + catsrcErr = catSrc.applyFromTemplate(oc, "-n", catSrc.Namespace, "-f", catSrcTemplate, "-p", "NAMESPACE="+catSrc.Namespace, "IMAGE="+upstreamCatalogSource) + NOSource.Channel = "latest" + NO.CatalogSource = NOSource + deployedUpstreamCatalogSource = true + } else { + e2e.Logf("Using default ystream catalog") + catsrcErr = catSrc.applyFromTemplate(oc, "-n", catSrc.Namespace, "-f", catSrcTemplate, "-p", "NAMESPACE="+catSrc.Namespace) + } + catSrc.WaitUntilCatSrcReady(oc) + + if !isHypershift { + ApplyResourceFromFile(oc, catSrc.Namespace, imageDigest) + } + return deployedUpstreamCatalogSource, catsrcErr +} + +// ensureOperatorDeployed checks and deploys an operator if not already present +func ensureOperatorDeployed(oc *exutil.CLI, operator SubscriptionObjects, operatorSource CatalogSourceObjects, podLabel string) { + g.By(fmt.Sprintf("Subscribe %s operator to %s channel", operator.OperatorName, operatorSource.Channel)) + operatorExisting, err := CheckOperatorStatus(oc, operator.Namespace, operator.PackageName) + o.Expect(err).NotTo(o.HaveOccurred()) + + if !operatorExisting { + e2e.Logf("%s operator not found, subscribing to operator", operator.OperatorName) + operator.SubscribeOperator(oc) + + // Wait for operator pods to be ready + if podLabel != "" { + WaitForPodsReadyWithLabel(oc, operator.Namespace, podLabel) + } + + // Verify operator status + operatorStatus, err := CheckOperatorStatus(oc, operator.Namespace, operator.PackageName) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(operatorStatus).To(o.BeTrue()) + + e2e.Logf("%s operator deployed successfully", operator.OperatorName) + } else { + e2e.Logf("%s operator already exists, skipping deployment", operator.OperatorName) + } +} + +// ensureNetObservOperatorDeployed checks and deploys the NetObserv operator with specific configurations +func ensureNetObservOperatorDeployed(oc *exutil.CLI, NO SubscriptionObjects, NOSource CatalogSourceObjects, deployedUpstreamCatalogSource bool) { + ensureOperatorDeployed(oc, NO, NOSource, "app="+NO.OperatorName) + + // NetObserv-specific checks only if operator was just deployed + NOexisting, err := CheckOperatorStatus(oc, NO.Namespace, NO.PackageName) + o.Expect(err).NotTo(o.HaveOccurred()) + + if NOexisting { + // Verify FlowCollector API exists + flowcollectorAPIExists, err := isFlowCollectorAPIExists(oc) + o.Expect(flowcollectorAPIExists).To(o.BeTrue()) + o.Expect(err).NotTo(o.HaveOccurred()) + + // Patch upstream catalog source if needed + if deployedUpstreamCatalogSource { + _, err := oc.AsAdmin().WithoutNamespace().Run("patch").Args("csv", "netobserv-operator.v0.0.0-sha-main", "-n", NO.Namespace, + "--type=json", "--patch", "[{\"op\": \"replace\",\"path\": \"/spec/install/spec/deployments/0/spec/template/spec/containers/0/env/4/value\", \"value\": \"true\"}]").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + } + } +} + +func getOperatorChannel(oc *exutil.CLI, catalog string, packageName string) (operatorChannel string, err error) { + channels, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("packagemanifests", "-l", "catalog="+catalog, "-n", "openshift-marketplace", "-o=jsonpath={.items[?(@.metadata.name==\""+packageName+"\")].status.channels[*].name}").Output() + channelArr := strings.Split(channels, " ") + return channelArr[len(channelArr)-1], err +} diff --git a/integration-tests/backend/sctp.go b/integration-tests/backend/sctp.go new file mode 100644 index 0000000000..ffdc1c68b4 --- /dev/null +++ b/integration-tests/backend/sctp.go @@ -0,0 +1,80 @@ +package e2etests + +import ( + "context" + "fmt" + "strings" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + exutil "github.com/openshift/origin/test/extended/util" + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" + e2e "k8s.io/kubernetes/test/e2e/framework" + e2enode "k8s.io/kubernetes/test/e2e/framework/node" +) + +// enableSCTPModuleOnNode Manual way to enable sctp in a cluster +func enableSCTPModuleOnNode(oc *exutil.CLI, nodeName, role string) { + e2e.Logf("This is %s worker node: %s", role, nodeName) + checkSCTPCmd := "cat /sys/module/sctp/initstate" + output, err := compat_otp.DebugNodeWithChroot(oc, nodeName, "bash", "-c", checkSCTPCmd) + var installCmd string + if err != nil || !strings.Contains(output, "live") { + e2e.Logf("No sctp module installed, will enable sctp module!!!") + installCmd = "modprobe sctp" + + // Try 3 times to enable sctp + o.Eventually(func() error { + _, installErr := compat_otp.DebugNodeWithChroot(oc, nodeName, "bash", "-c", installCmd) + if installErr != nil && strings.EqualFold(role, "rhel") { + e2e.Logf("%v", installErr) + g.Skip("Yum insall to enable sctp cannot work in a disconnected cluster, skip the test!!!") + } + return installErr + }, "15s", "5s").ShouldNot(o.HaveOccurred(), fmt.Sprintf("Failed to install sctp module on node %s", nodeName)) + + // Wait for sctp applied + o.Eventually(func() string { + output, err := compat_otp.DebugNodeWithChroot(oc, nodeName, "bash", "-c", checkSCTPCmd) + if err != nil { + e2e.Logf("Wait for sctp applied, %v", err) + } + return output + }, "60s", "10s").Should(o.ContainSubstring("live"), fmt.Sprintf("Failed to load sctp module on node %s", nodeName)) + } else { + e2e.Logf("sctp module is loaded on node %s\n%s", nodeName, output) + } +} + +func prepareSCTPModule(oc *exutil.CLI) { + nodesOutput, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("node").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if strings.Contains(nodesOutput, "SchedulingDisabled") || strings.Contains(nodesOutput, "NotReady") { + g.Skip("There are already some nodes in NotReady or SchedulingDisabled status in cluster, skip the test!!! ") + } + + workerNodeList, err := e2enode.GetReadySchedulableNodes(context.TODO(), oc.KubeFramework().ClientSet) + if err != nil || len(workerNodeList.Items) == 0 { + g.Skip("Can not find any woker nodes in the cluster") + } + + // Will enable sctp by command + rhelWorkers, err := compat_otp.GetAllWorkerNodesByOSID(oc, "rhel") + o.Expect(err).NotTo(o.HaveOccurred()) + if len(rhelWorkers) > 0 { + e2e.Logf("There are %v number rhel workers in this cluster, will use manual way to load sctp module.", len(rhelWorkers)) + for _, worker := range rhelWorkers { + enableSCTPModuleOnNode(oc, worker, "rhel") + } + + } + + rhcosWorkers, err := compat_otp.GetAllWorkerNodesByOSID(oc, "rhcos") + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("%v", rhcosWorkers) + if len(rhcosWorkers) > 0 { + for _, worker := range rhcosWorkers { + enableSCTPModuleOnNode(oc, worker, "rhcos") + } + } +} diff --git a/integration-tests/backend/test_exporters.go b/integration-tests/backend/test_exporters.go new file mode 100644 index 0000000000..b60ca2f57f --- /dev/null +++ b/integration-tests/backend/test_exporters.go @@ -0,0 +1,269 @@ +package e2etests + +import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "strconv" + + filePath "path/filepath" + "strings" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" +) + +var _ = g.Describe("[sig-netobserv] Network_Observability", func() { + + defer g.GinkgoRecover() + var ( + oc = compat_otp.NewCLI("netobserv", compat_otp.KubeConfigPath()) + // NetObserv Operator variables + NOcatSrc = Resource{"catsrc", "netobserv-konflux-fbc", netobservNS} + NOSource = CatalogSourceObjects{"stable", NOcatSrc.Name, NOcatSrc.Namespace} + + // Template directories + baseDir, _ = filePath.Abs("testdata") + subscriptionDir = filePath.Join(baseDir, "subscription") + flowFixturePath = filePath.Join(baseDir, "flowcollector_v1beta2_template.yaml") + + // Operator namespace object + OperatorNS = OperatorNamespace{ + Name: netobservNS, + NamespaceTemplate: filePath.Join(subscriptionDir, "namespace.yaml"), + } + NO = SubscriptionObjects{ + OperatorName: "netobserv-operator", + Namespace: netobservNS, + PackageName: NOPackageName, + Subscription: filePath.Join(subscriptionDir, "sub-template.yaml"), + OperatorGroup: filePath.Join(subscriptionDir, "allnamespace-og.yaml"), + CatalogSource: &NOSource, + } + imageDigest = filePath.Join(subscriptionDir, "image-digest-mirror-set.yaml") + catSrcTemplate = filePath.Join(subscriptionDir, "catalog-source.yaml") + catalogSource = os.Getenv("MULTISTAGE_PARAM_OVERRIDE_NETOBSERV_CS_IMAGE") + + OtelNS = OperatorNamespace{ + Name: "openshift-opentelemetry-operator", + NamespaceTemplate: filePath.Join(subscriptionDir, "namespace.yaml"), + } + + OTELSource = CatalogSourceObjects{"stable", "redhat-operators", "openshift-marketplace"} + + OTEL = SubscriptionObjects{ + OperatorName: "opentelemetry-operator", + Namespace: OtelNS.Name, + PackageName: "opentelemetry-product", + Subscription: filePath.Join(subscriptionDir, "sub-template.yaml"), + OperatorGroup: filePath.Join(subscriptionDir, "allnamespace-og.yaml"), + CatalogSource: &OTELSource, + } + namespace string + ) + + g.BeforeEach(func() { + namespace = oc.Namespace() + + if strings.Contains(os.Getenv("E2E_RUN_TAGS"), "disconnected") { + g.Skip("Skipping tests for disconnected profiles") + } + + OperatorNS.DeployOperatorNamespace(oc) + deployedUpstreamCatalogSource, catSrcErr := setupCatalogSource(oc, NOcatSrc, catSrcTemplate, imageDigest, catalogSource, false, &NOSource, &NO) + o.Expect(catSrcErr).NotTo(o.HaveOccurred()) + ensureNetObservOperatorDeployed(oc, NO, NOSource, deployedUpstreamCatalogSource) + }) + + g.It("Author:aramesha-High-64156-Verify IPFIX-exporter [Serial]", func() { + SkipIfOCPBelow("v4.10") + clusterArch, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("nodes", "-o=jsonpath={.items[0].status.nodeInfo.architecture}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if !strings.Contains(clusterArch, "amd64") { + g.Skip("IPFIX collector image only supports amd64 architecture. Skip this test!") + } + + g.By("Create IPFIX namespace") + ipfixCollectorTemplatePath := filePath.Join(baseDir, "exporters", "ipfix-collector.yaml") + IPFIXns := "ipfix" + defer oc.DeleteSpecifiedNamespaceAsAdmin(IPFIXns) + oc.CreateSpecifiedNamespaceAsAdmin(IPFIXns) + _ = compat_otp.SetNamespacePrivileged(oc, IPFIXns) + + g.By("Deploy IPFIX collector") + createResourceFromFile(oc, IPFIXns, ipfixCollectorTemplatePath) + WaitForPodsReadyWithLabel(oc, IPFIXns, "app=ipfix-collector") + + g.By("Wait for IPFIX collector TCP listener to initialize") + time.Sleep(10 * time.Second) + + IPFIXconfig := map[string]interface{}{ + "ipfix": map[string]interface{}{ + "targetHost": "ipfix-collector.ipfix.svc.cluster.local", + "targetPort": 2055, + "transport": "TCP", + "enterpriseID": 0}, + "type": "IPFIX", + } + + config, err := json.Marshal(IPFIXconfig) + o.Expect(err).ToNot(o.HaveOccurred()) + IPFIXexporter := string(config) + additionalNamespaces := fmt.Sprintf("\"%s\"", IPFIXns) + samplingValue := 3 + + g.By("Deploy FlowCollector with IPFIX exporter and sampling") + flow := Flowcollector{ + Namespace: namespace, + Template: flowFixturePath, + LokiEnable: "false", + LokiNamespace: namespace, + Exporters: []string{IPFIXexporter}, + NetworkPolicyAdditionalNamespaces: []string{additionalNamespaces}, + Sampling: strconv.Itoa(samplingValue), + } + + defer func() { _ = flow.DeleteFlowcollector(oc) }() + flow.CreateFlowcollector(oc) + + g.By("Verify flowcollector is deployed with IPFIX exporter") + flowPatch, err := oc.AsAdmin().Run("get").Args("flowcollector", "cluster", "-n", namespace, "-o", "jsonpath='{.spec.exporters[0].type}'").Output() + o.Expect(err).ToNot(o.HaveOccurred()) + o.Expect(flowPatch).To(o.Equal(`'IPFIX'`)) + + g.By("Get IPFIX collector pod") + collectorPod, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pods", "-n", IPFIXns, "-l", "app=ipfix-collector", "-o=jsonpath={.items[0].metadata.name}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Wait for IPFIX flows to be collected") + time.Sleep(60 * time.Second) + + g.By("Retrieve and parse IPFIX flow records from collector API") + flowRecords, err := getIPFIXFlowRecordsFromAPI(oc, IPFIXns, collectorPod) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "No IPFIX flow records found in collector") + + g.By("Verify all IPFIX fields are present and valid") + for _, record := range flowRecords { + record.Flowlog.verifyIPFIXFields() + } + + g.By("Verify sampling value matches FlowCollector configuration (NETOBSERV-2706)") + for _, record := range flowRecords { + o.Expect(record.Flowlog.Sampling).Should(o.BeNumerically("==", samplingValue), + fmt.Sprintf("Expected Sampling=%d, got %d", samplingValue, record.Flowlog.Sampling)) + } + }) + + g.It("Author:memodi-High-74977-Verify OTEL exporter with TLS [Serial]", func() { + SkipIfOCPBelow("v4.13") + // don't delete the OTEL Operator at the end of the test + g.By("Subscribe to OTEL Operator") + OtelNS.DeployOperatorNamespace(oc) + OTEL.SubscribeOperator(oc) + WaitForPodsReadyWithLabel(oc, OTEL.Namespace, "app.kubernetes.io/name="+OTEL.OperatorName) + OTELStatus, err := CheckOperatorStatus(oc, OTEL.Namespace, OTEL.PackageName) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect((OTELStatus)).To(o.BeTrue()) + + g.By("Create OTEL Collector with TLS enabled") + otelCollectorTemplatePath := filePath.Join(baseDir, "exporters", "otel-collector-tls.yaml") + otlpEndpoint := 4317 + promEndpoint := "8889" + collectorname := "otel" + compat_otp.ApplyNsResourceFromTemplate(oc, namespace, "-f", otelCollectorTemplatePath, "-p", "NAME="+collectorname, "OTLP_GRPC_ENDPOINT="+strconv.Itoa(otlpEndpoint), "OTLP_PROM_PORT="+promEndpoint) + otelPodLabel := "app.kubernetes.io/component=opentelemetry-collector" + defer func() { + _ = oc.AsAdmin().WithoutNamespace().Run("delete").Args("opentelemetrycollector", collectorname, "-n", namespace).Execute() + _ = oc.AsAdmin().WithoutNamespace().Run("delete").Args("service", collectorname+"-collector", "-n", namespace).Execute() + _ = oc.AsAdmin().WithoutNamespace().Run("delete").Args("configmap", "service-ca", "-n", namespace).Execute() + }() + WaitForPodsReadyWithLabel(oc, namespace, otelPodLabel) + + g.By("Wait for service-ca configmap to be injected with CA bundle") + waitForConfigMapDataInjection(oc, namespace, "service-ca", "service-ca.crt") + + targetHost := fmt.Sprintf("%s-collector.%s.svc", collectorname, namespace) + otel_config := map[string]interface{}{ + "openTelemetry": map[string]interface{}{ + "logs": map[string]bool{"enable": true}, + "metrics": map[string]interface{}{"enable": true, + "pushTimeInterval": "20s"}, + "targetHost": targetHost, + "targetPort": otlpEndpoint, + "protocol": "grpc", + "tls": map[string]interface{}{ + "enable": true, + "insecureSkipVerify": false, + "caCert": map[string]interface{}{ + "type": "configmap", + "name": "service-ca", + "certFile": "service-ca.crt", + }, + }, + }, + "type": "OpenTelemetry", + } + config, err := json.Marshal(otel_config) + o.Expect(err).NotTo(o.HaveOccurred()) + config_str := string(config) + + g.By("Deploy FlowCollector with OTEL TLS exporter and Loki disabled") + flow := Flowcollector{ + Namespace: namespace, + Template: flowFixturePath, + LokiEnable: "false", + LokiNamespace: namespace, + Exporters: []string{config_str}, + } + + defer func() { _ = flow.DeleteFlowcollector(oc) }() + flow.CreateFlowcollector(oc) + + g.By("Verify OTEL collector is receiving TLS-encrypted flows") + otelCollectorPod, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pods", "-n", namespace, "-l", otelPodLabel, "-o=jsonpath={.items[0].metadata.name}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + // wait for 60 seconds to ensure we collected enough logs to grep from + time.Sleep(60 * time.Second) + + g.By("Verify OTEL flowlogs are seen in collector pod logs") + textToExist := "Attributes:" + textToNotExist := "INVALID" + + podLogs, err := getPodLogs(oc, namespace, otelCollectorPod) + o.Expect(err).ToNot(o.HaveOccurred()) + + grepCmd := fmt.Sprintf("grep %s %s", textToExist, podLogs) + textToExistLogs, err := exec.Command("bash", "-c", grepCmd).Output() + + o.Expect(err).ToNot(o.HaveOccurred()) + o.Expect(len(textToExistLogs)).To(o.BeNumerically(">", 0)) + + grepCmd = fmt.Sprintf("grep %s %s || true", textToNotExist, podLogs) + textToNotExistLogs, err := exec.Command("bash", "-c", grepCmd).Output() + o.Expect(err).ToNot(o.HaveOccurred()) + o.Expect(len(textToNotExistLogs)).To(o.BeNumerically("==", 0), string(textToNotExistLogs)) + + g.By("Verify OTEL prometheus has metrics") + // Get the service IP for the service with label operator.opentelemetry.io/collector-service-type=base + svcIP, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("svc", "-n", namespace, "-l", "operator.opentelemetry.io/collector-service-type=base", "-o=jsonpath={.items[0].spec.clusterIP}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + // Get one of the flowlogs-pipeline pods + flowlogsPipelinePod, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pods", "-n", namespace, "-l", "app=flowlogs-pipeline", "-o=jsonpath={.items[0].metadata.name}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + // Use the flowlogs-pipeline pod to curl the metrics endpoint of the otel collector service + command := fmt.Sprintf("curl -s http://%s:%s/metrics | grep 'netobserv_workload_flows_total{' | head -1 | awk '{print $2}'", svcIP, promEndpoint) + cmd := []string{"-n", namespace, flowlogsPipelinePod, "--", "/bin/sh", "-c", command} + count, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args(cmd...).Output() + o.Expect(err).ToNot(o.HaveOccurred()) + nCount, err := strconv.Atoi(strings.Trim(count, "\n")) + o.Expect(err).ToNot(o.HaveOccurred()) + o.Expect(nCount).To(o.BeNumerically(">", 0)) + }) +}) diff --git a/integration-tests/backend/test_flowcollector.go b/integration-tests/backend/test_flowcollector.go new file mode 100644 index 0000000000..660961810c --- /dev/null +++ b/integration-tests/backend/test_flowcollector.go @@ -0,0 +1,3586 @@ +package e2etests + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + + filePath "path/filepath" + "strings" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" + "k8s.io/apimachinery/pkg/util/wait" + e2e "k8s.io/kubernetes/test/e2e/framework" + e2eoutput "k8s.io/kubernetes/test/e2e/framework/pod/output" +) + +var _ = g.Describe("[sig-netobserv] Network_Observability", func() { + + defer g.GinkgoRecover() + var ( + oc = compat_otp.NewCLI("netobserv", compat_otp.KubeConfigPath()) + // NetObserv Operator variables + NOcatSrc = Resource{"catsrc", "netobserv-konflux-fbc", netobservNS} + NOSource = CatalogSourceObjects{"stable", NOcatSrc.Name, NOcatSrc.Namespace} + + // Template directories + baseDir, _ = filePath.Abs("testdata") + networkingDir = filePath.Join(baseDir, "networking") + subscriptionDir = filePath.Join(baseDir, "subscription") + flowFixturePath = filePath.Join(baseDir, "flowcollector_v1beta2_template.yaml") + + // Operator namespace object + OperatorNS = OperatorNamespace{ + Name: netobservNS, + NamespaceTemplate: filePath.Join(subscriptionDir, "namespace.yaml"), + } + NO = SubscriptionObjects{ + OperatorName: "netobserv-operator", + Namespace: netobservNS, + PackageName: NOPackageName, + Subscription: filePath.Join(subscriptionDir, "sub-template.yaml"), + OperatorGroup: filePath.Join(subscriptionDir, "allnamespace-og.yaml"), + CatalogSource: &NOSource, + } + imageDigest = filePath.Join(subscriptionDir, "image-digest-mirror-set.yaml") + catSrcTemplate = filePath.Join(subscriptionDir, "catalog-source.yaml") + catalogSource = os.Getenv("MULTISTAGE_PARAM_OVERRIDE_NETOBSERV_CS_IMAGE") + + kubeadminToken string + namespace string + ) + + g.BeforeEach(func() { + if strings.Contains(os.Getenv("E2E_RUN_TAGS"), "disconnected") { + g.Skip("Skipping tests for disconnected profiles") + } + namespace = oc.Namespace() + + g.By("Get kubeadmin token") + kubeAdminPasswd := os.Getenv("QE_KUBEADMIN_PASSWORD") + if kubeAdminPasswd == "" { + g.Skip("no kubeAdminPasswd is provided in this profile, set QE_KUBEADMIN_PASSWORD env var") + } + serverURL, serverURLErr := oc.AsAdmin().WithoutNamespace().Run("whoami").Args("--show-server").Output() + o.Expect(serverURLErr).NotTo(o.HaveOccurred()) + currentContext, currentContextErr := oc.WithoutNamespace().Run("config").Args("current-context").Output() + o.Expect(currentContextErr).NotTo(o.HaveOccurred()) + defer func() { + rollbackCtxErr := oc.WithoutNamespace().Run("config").Args("set", "current-context", currentContext).Execute() + o.Expect(rollbackCtxErr).NotTo(o.HaveOccurred()) + }() + + kubeadminToken = getKubeAdminToken(oc, kubeAdminPasswd, serverURL, currentContext) + o.Expect(kubeadminToken).NotTo(o.BeEmpty()) + + isHypershift := compat_otp.IsHypershiftHostedCluster(oc) + + OperatorNS.DeployOperatorNamespace(oc) + deployedUpstreamCatalogSource, catSrcErr := setupCatalogSource(oc, NOcatSrc, catSrcTemplate, imageDigest, catalogSource, isHypershift, &NOSource, &NO) + o.Expect(catSrcErr).NotTo(o.HaveOccurred()) + ensureNetObservOperatorDeployed(oc, NO, NOSource, deployedUpstreamCatalogSource) + }) + + g.It("Author:memodi-NonPreRelease-Longduration-Medium-60664-Medium-61482-Alerts-with-NetObserv [Serial][Slow]", func() { + SkipIfOCPBelow("v4.10") + flpAlertRuleName := "flowlogs-pipeline-alert" + ebpfAlertRuleName := "ebpf-agent-prom-alert" + + flow := Flowcollector{ + Namespace: namespace, + Template: flowFixturePath, + LokiEnable: "false", + } + defer func() { _ = flow.DeleteFlowcollector(oc) }() + flow.CreateFlowcollector(oc) + + // verify configured alerts for flp + g.By("Get FLP Alert name and Alert Rules") + rules, err := getConfiguredAlertRules(oc, flpAlertRuleName, namespace) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(rules).To(o.ContainSubstring("NetObservNoFlows")) + o.Expect(rules).To(o.ContainSubstring("NetObservLokiError")) + + // verify configured alerts for ebpf-agent + g.By("Get EBPF Alert name and Alert Rules") + ebpfRules, err := getConfiguredAlertRules(oc, ebpfAlertRuleName, namespace+"-privileged") + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(ebpfRules).To(o.ContainSubstring("NetObservDroppedFlows")) + + // verify disable alerts feature + g.By("Verify alerts can be disabled") + gen, err := getResourceGeneration(oc, "prometheusRule", flpAlertRuleName, namespace) + o.Expect(err).NotTo(o.HaveOccurred()) + disableAlertPatchTemp := `[{"op": "$op", "path": "/spec/processor/metrics/disableAlerts", "value": ["NetObservLokiError"]}]` + disableAlertPatch := strings.Replace(disableAlertPatchTemp, "$op", "add", 1) + out, err := oc.AsAdmin().WithoutNamespace().Run("patch").Args("flowcollector", "cluster", "--type=json", "-p", disableAlertPatch).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(out).To(o.ContainSubstring("patched")) + + waitForResourceGenerationUpdate(oc, "prometheusRule", flpAlertRuleName, "generation", gen, namespace) + rules, err = getConfiguredAlertRules(oc, flpAlertRuleName, namespace) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(rules).To(o.ContainSubstring("NetObservNoFlows")) + o.Expect(rules).ToNot(o.ContainSubstring("NetObservLokiError")) + + gen, err = getResourceGeneration(oc, "prometheusRule", flpAlertRuleName, namespace) + o.Expect(err).NotTo(o.HaveOccurred()) + disableAlertPatch = strings.Replace(disableAlertPatchTemp, "$op", "remove", 1) + out, err = oc.AsAdmin().WithoutNamespace().Run("patch").Args("flowcollector", "cluster", "--type=json", "-p", disableAlertPatch).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(out).To(o.ContainSubstring("patched")) + waitForResourceGenerationUpdate(oc, "prometheusRule", flpAlertRuleName, "generation", gen, namespace) + rules, err = getConfiguredAlertRules(oc, flpAlertRuleName, namespace) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(rules).To(o.ContainSubstring("NetObservNoFlows")) + o.Expect(rules).To(o.ContainSubstring("NetObservLokiError")) + + g.By("delete flowcollector") + _ = flow.DeleteFlowcollector(oc) + + // verify alert firing. + // configure flowcollector with incorrect loki URL + // configure very low CacheMaxFlows to have ebpf alert fired. + flow = Flowcollector{ + Namespace: namespace, + Template: flowFixturePath, + CacheMaxFlows: "100", + LokiMode: "Monolithic", + MonolithicLokiURL: "http://loki.no-ns.svc:3100", + } + g.By("Deploy flowcollector with incorrect loki URL and lower cacheMaxFlows value") + flow.CreateFlowcollector(oc) + + g.By("Wait for alerts to be active") + waitForAlertToBeActive(oc, "NetObservLokiError") + }) + + g.It("Author:memodi-Medium-63185-Verify NetOberv must-gather plugin [Serial]", func() { + SkipIfOCPBelow("v4.10") + mustgatherDir := "/tmp/must-gather-63185" + mustgatherImage := "quay.io/netobserv/must-gather" + + g.By("Deploy FlowCollector") + flow := Flowcollector{ + Namespace: namespace, + Template: flowFixturePath, + LokiEnable: "false", + } + + defer func() { _ = flow.DeleteFlowcollector(oc) }() + flow.CreateFlowcollector(oc) + + g.By("Run must-gather command") + defer func() { _, _ = exec.Command("bash", "-c", "rm -rf "+mustgatherDir).Output() }() + output, err := oc.AsAdmin().WithoutNamespace().Run("adm").Args("must-gather", "--image", mustgatherImage, "--dest-dir="+mustgatherDir).Output() + o.Expect(err).NotTo(o.HaveOccurred(), "must-gather command failed") + o.Expect(output).NotTo(o.ContainSubstring("error")) + + g.By("Wait for must-gather directory to be populated") + var mustgatherLogsDir string + err = wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 2*time.Minute, false, func(context.Context) (bool, error) { + matches, globErr := filePath.Glob(mustgatherDir + "/quay-io-netobserv-must-gather-*") + if globErr != nil || len(matches) == 0 { + e2e.Logf("Waiting for must-gather directory to be created...") + return false, nil + } + mustgatherLogsDir = matches[0] + // Check if at least one expected file exists to confirm completion + checkPattern := fmt.Sprintf("%s/namespaces/*/pods/*", mustgatherLogsDir) + checkMatches, _ := filePath.Glob(checkPattern) + if len(checkMatches) == 0 { + e2e.Logf("Must-gather directory exists but waiting for pod data to be collected...") + return false, nil + } + e2e.Logf("Must-gather data collection completed") + return true, nil + }) + compat_otp.AssertWaitPollNoErr(err, "must-gather data not populated within timeout") + + g.By("Verify operator namespace logs are scraped") + operatorLogsPattern := fmt.Sprintf("%s/namespaces/openshift-netobserv-operator/pods/netobserv-controller-manager-*/manager/manager/logs/current.log", mustgatherLogsDir) + operatorlogs, err := filePath.Glob(operatorLogsPattern) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(operatorlogs)).Should(o.BeNumerically(">", 0), "No logs were saved to: "+operatorLogsPattern) + _, err = os.Stat(operatorlogs[0]) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Verify flowlogs-pipeline pod logs are scraped") + pods, err := compat_otp.GetAllPods(oc, namespace) + o.Expect(err).NotTo(o.HaveOccurred()) + flpLogsPattern := fmt.Sprintf("%s/namespaces/%s/pods/%s/flowlogs-pipeline/flowlogs-pipeline/logs/current.log", mustgatherLogsDir, namespace, pods[0]) + podlogs, err := filePath.Glob(flpLogsPattern) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(podlogs)).Should(o.BeNumerically(">", 0), "No logs were saved to: "+flpLogsPattern) + _, err = os.Stat(podlogs[0]) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Verify eBPF agent pod logs are scraped") + ebpfPods, err := compat_otp.GetAllPods(oc, namespace+"-privileged") + o.Expect(err).NotTo(o.HaveOccurred()) + ebpfLogsPattern := fmt.Sprintf("%s/namespaces/%s/pods/%s/netobserv-ebpf-agent/netobserv-ebpf-agent/logs/current.log", mustgatherLogsDir, namespace+"-privileged", ebpfPods[0]) + ebpfLogs, err := filePath.Glob(ebpfLogsPattern) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(ebpfLogs)).Should(o.BeNumerically(">", 0), "No logs were saved to: "+ebpfLogsPattern) + _, err = os.Stat(ebpfLogs[0]) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Verify FlowCollector CR is dumped") + fcPattern := fmt.Sprintf("%s/cluster-scoped-resources/flows.netobserv.io/flowcollectors/cluster.yaml", mustgatherLogsDir) + fcDump, err := filePath.Glob(fcPattern) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(fcDump)).Should(o.BeNumerically(">", 0), "FlowCollector CR not dumped to: "+fcPattern) + _, err = os.Stat(fcDump[0]) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Verify FlowCollector CRD definition is dumped") + crdPattern := fmt.Sprintf("%s/cluster-scoped-resources/apiextensions.k8s.io/customresourcedefinitions/flowcollectors.flows.netobserv.io.yaml", mustgatherLogsDir) + crdDump, err := filePath.Glob(crdPattern) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(crdDump)).Should(o.BeNumerically(">", 0), "FlowCollector CRD not dumped to: "+crdPattern) + _, err = os.Stat(crdDump[0]) + o.Expect(err).NotTo(o.HaveOccurred()) + }) + + g.It("Author:aramesha-NonPreRelease-Medium-72875-Verify nodeSelector and tolerations with netobserv components [Serial]", func() { + SkipIfOCPBelow("v4.12") + + // verify tolerations + g.By("Get worker node of the cluster") + workerNode, err := compat_otp.GetFirstWorkerNode(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Taint worker node") + defer func() { + err := oc.AsAdmin().WithoutNamespace().Run("adm").Args("taint", "node", workerNode, "netobserv-agent-", "--overwrite").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + }() + err = oc.AsAdmin().WithoutNamespace().Run("adm").Args("taint", "node", workerNode, "netobserv-agent=true:NoSchedule", "--overwrite").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Deploy FlowCollector") + flow := Flowcollector{ + Namespace: namespace, + Template: flowFixturePath, + LokiEnable: "false", + } + + defer func() { _ = flow.DeleteFlowcollector(oc) }() + flow.CreateFlowcollector(oc) + + g.By("Add wrong toleration for eBPF spec for the taint netobserv-agent=false:NoSchedule") + patchValue := `{"scheduling":{"tolerations":[{"effect": "NoSchedule", "key": "netobserv-agent", "value": "false", "operator": "Equal"}]}}` + _, _ = oc.AsAdmin().WithoutNamespace().Run("patch").Args("flowcollector", "cluster", "-p", `[{"op": "replace", "path": "/spec/agent/ebpf/advanced", "value": `+patchValue+`}]`, "--type=json").Output() + + g.By("Ensure flowcollector is ready") + flow.WaitForFlowcollectorReady(oc) + + g.By(fmt.Sprintf("Verify eBPF pod is not scheduled on the %s", workerNode)) + eBPFPod, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("-n", flow.Namespace+"-privileged", "pods", "--field-selector", "spec.nodeName="+workerNode+"", "-o", "name").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(eBPFPod).Should(o.BeEmpty()) + + g.By("Add correct toleration for eBPF spec for the taint netobserv-agent=true:NoSchedule") + _ = flow.DeleteFlowcollector(oc) + flow.CreateFlowcollector(oc) + patchValue = `{"scheduling":{"tolerations":[{"effect": "NoSchedule", "key": "netobserv-agent", "value": "true", "operator": "Equal"}]}}` + _, _ = oc.AsAdmin().WithoutNamespace().Run("patch").Args("flowcollector", "cluster", "-p", `[{"op": "replace", "path": "/spec/agent/ebpf/advanced", "value": `+patchValue+`}]`, "--type=json").Output() + + g.By("Ensure flowcollector is ready") + flow.WaitForFlowcollectorReady(oc) + + g.By(fmt.Sprintf("Verify eBPF pod is scheduled on the node %s after applying toleration for taint netobserv-agent=true:NoSchedule", workerNode)) + eBPFPod, err = oc.AsAdmin().WithoutNamespace().Run("get").Args("-n", flow.Namespace+"-privileged", "pods", "--field-selector", "spec.nodeName="+workerNode+"", "-o", "name").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(eBPFPod).NotTo(o.BeEmpty()) + + // verify nodeSelector + g.By("Add netobserv label to above worker node") + defer func() { _, _ = compat_otp.DeleteLabelFromNode(oc, workerNode, "test") }() + _, _ = compat_otp.AddLabelToNode(oc, workerNode, "netobserv-agent", "true") + + g.By("Patch flowcollector with nodeSelector for eBPF pods") + _ = flow.DeleteFlowcollector(oc) + flow.CreateFlowcollector(oc) + patchValue = `{"scheduling":{"nodeSelector":{"netobserv-agent": "true"}}}` + _, _ = oc.AsAdmin().WithoutNamespace().Run("patch").Args("flowcollector", "cluster", "-p", `[{"op": "replace", "path": "/spec/agent/ebpf/advanced", "value": `+patchValue+`}]`, "--type=json").Output() + + g.By("Ensure flowcollector is ready") + flow.WaitForFlowcollectorReady(oc) + + g.By("Verify all eBPF pods are deployed on the above worker node") + eBPFpods, err := compat_otp.GetAllPodsWithLabel(oc, flow.Namespace+"-privileged", "app=netobserv-ebpf-agent") + o.Expect(err).NotTo(o.HaveOccurred()) + for _, pod := range eBPFpods { + nodeName, err := compat_otp.GetPodNodeName(oc, flow.Namespace+"-privileged", pod) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(nodeName).To(o.Equal(workerNode)) + } + }) + + g.Context("with Loki", func() { + var ( + lokiDir, _ = filePath.Abs("testdata/loki") + // Loki Operator variables + lokiPackageName = "loki-operator" + lokiSource CatalogSourceObjects + lokiCatalog = "redhat-operators" + ls *lokiStack + Lokiexisting = false + lokiStackNS = "netobserv-loki" + LO = SubscriptionObjects{ + OperatorName: "loki-operator-controller-manager", + Namespace: loNS, + PackageName: lokiPackageName, + Subscription: filePath.Join(subscriptionDir, "sub-template.yaml"), + OperatorGroup: filePath.Join(subscriptionDir, "allnamespace-og.yaml"), + CatalogSource: &lokiSource, + } + // LokiStack variables + ipStackType string + lokiStackTemplate = filePath.Join(lokiDir, "lokistack-simple.yaml") + lokiTenant = "openshift-network" + ) + + g.BeforeEach(func() { + ipStackType = checkIPStackType(oc) + + g.By("Deploy loki operator") + if !validateInfraAndResourcesForLoki(oc, "10Gi", "6") { + g.Skip("Current platform does not have enough resources available for this test!") + } + + // check if Loki Operator exists + var err error + Lokiexisting, err = CheckOperatorStatus(oc, LO.Namespace, LO.PackageName) + o.Expect(err).NotTo(o.HaveOccurred()) + + lokiChannel, err := getOperatorChannel(oc, lokiCatalog, lokiPackageName) + if err != nil || lokiChannel == "" { + g.Skip("Loki channel not found, skip this case") + } + lokiSource = CatalogSourceObjects{lokiChannel, lokiCatalog, "openshift-marketplace"} + + // Don't delete if Loki Operator existed already before NetObserv + // unless it is not using the 'stable' operator + // If Loki Operator was installed by NetObserv tests, + // it will install and uninstall after each spec/test. + if !Lokiexisting { + ensureOperatorDeployed(oc, LO, lokiSource, "name="+LO.OperatorName) + } else { + channelName, err := checkOperatorChannel(oc, LO.Namespace, LO.PackageName) + o.Expect(err).NotTo(o.HaveOccurred()) + if channelName != lokiChannel { + e2e.Logf("found %s channel for loki operator, removing and reinstalling with %s channel instead", channelName, lokiSource.Channel) + LO.uninstallOperator(oc) + ensureOperatorDeployed(oc, LO, lokiSource, "name="+LO.OperatorName) + Lokiexisting = false + } + } + + g.By("Deploy lokiStack") + // get storageClass Name + sc, err := getStorageClassName(oc) + if err != nil || len(sc) == 0 { + g.Skip("StorageClass not found in cluster, skip this case") + } + + objectStorageType := getStorageType(oc) + if len(objectStorageType) == 0 && ipStackType != "ipv6single" { + g.Skip("Current cluster doesn't have a proper object storage for this test!") + } + oc.CreateSpecifiedNamespaceAsAdmin(lokiStackNS) + + ls = &lokiStack{ + Name: "lokistack", + Namespace: lokiStackNS, + TSize: "1x.demo", + StorageType: objectStorageType, + StorageSecret: "objectstore-secret", + StorageClass: sc, + BucketName: "netobserv-loki-" + getInfrastructureName(oc), + Tenant: lokiTenant, + Template: lokiStackTemplate, + } + + if ipStackType == "ipv6single" { + e2e.Logf("running IPv6 test") + ls.EnableIPV6 = "true" + } + + err = ls.prepareResourcesForLokiStack(oc) + if err != nil { + g.Skip("Skipping test since LokiStack resources were not deployed") + } + + err = ls.deployLokiStack(oc) + if err != nil { + g.Skip("Skipping test since LokiStack was not deployed") + } + + lokiStackResource := Resource{"lokistack", ls.Name, ls.Namespace} + err = lokiStackResource.WaitForResourceToAppear(oc) + if err != nil { + g.Skip("Skipping test since LokiStack did not become ready") + } + + err = ls.waitForLokiStackToBeReady(oc) + if err != nil { + g.Skip("Skipping test since LokiStack is not ready") + } + ls.Route = "https://" + getRouteAddress(oc, ls.Namespace, ls.Name) + }) + + g.AfterEach(func() { + ls.removeLokiStack(oc) + ls.removeObjectStorage(oc) + if !Lokiexisting { + LO.uninstallOperator(oc) + } + oc.DeleteSpecifiedNamespaceAsAdmin(lokiStackNS) + }) + + g.Context("FLP, eBPF and Console metrics:", func() { + g.When("processor.metrics.TLS == Disabled and agent.ebpf.metrics.TLS == Disabled", func() { + g.It("Author:aramesha-Critical-50504-Critical-72959-Verify flowlogs-pipeline and eBPF metrics and health [Serial]", func() { + SkipIfOCPBelow("v4.12") + var ( + flpPromSM = "flowlogs-pipeline-monitor" + namespace = oc.Namespace() + eBPFPromSM = "ebpf-agent-svc-monitor" + curlLive = "http://localhost:8080/live" + ) + + g.By("Deploy flowcollector") + flow := Flowcollector{ + Namespace: namespace, + Template: flowFixturePath, + LokiNamespace: lokiStackNS, + FLPMetricServerTLSType: "Disabled", + } + + defer func() { _ = flow.DeleteFlowcollector(oc) }() + flow.CreateFlowcollector(oc) + + g.By("Verify flowlogs-pipeline metrics") + FLPpods, err := compat_otp.GetAllPodsWithLabel(oc, namespace, "app=flowlogs-pipeline") + o.Expect(err).NotTo(o.HaveOccurred()) + + for _, pod := range FLPpods { + command := []string{"exec", "-n", namespace, pod, "--", "curl", "-s", curlLive} + output, err := oc.AsAdmin().WithoutNamespace().Run(command...).Args().Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.Equal("{}")) + } + + FLPtlsScheme, err := getMetricsScheme(oc, flpPromSM, flow.Namespace) + o.Expect(err).NotTo(o.HaveOccurred()) + FLPtlsScheme = strings.Trim(FLPtlsScheme, "'") + o.Expect(FLPtlsScheme).To(o.Equal("http")) + + g.By("Wait for a min before scraping metrics") + time.Sleep(60 * time.Second) + + g.By("Verify prometheus is able to scrape FLP metrics") + verifyFLPMetrics(oc) + + g.By("Verify eBPF agent metrics") + eBPFpods, err := compat_otp.GetAllPodsWithLabel(oc, namespace, "app=netobserv-ebpf-agent") + o.Expect(err).NotTo(o.HaveOccurred()) + + for _, pod := range eBPFpods { + command := []string{"exec", "-n", namespace, pod, "--", "curl", "-s", curlLive} + output, err := oc.AsAdmin().WithoutNamespace().Run(command...).Args().Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.Equal("{}")) + } + + eBPFtlsScheme, err := getMetricsScheme(oc, eBPFPromSM, flow.Namespace+"-privileged") + o.Expect(err).NotTo(o.HaveOccurred()) + eBPFtlsScheme = strings.Trim(eBPFtlsScheme, "'") + o.Expect(eBPFtlsScheme).To(o.Equal("http")) + + g.By("Wait for a min before scraping metrics") + time.Sleep(60 * time.Second) + + g.By("Verify prometheus is able to scrape eBPF metrics") + verifyEBPFMetrics(oc) + }) + }) + + g.When("processor.metrics.TLS == Auto and ebpf.agent.metrics.TLS == Auto", func() { + g.It("Author:aramesha-Critical-54043-Critical-66031-Critical-72959-Verify flowlogs-pipeline, eBPF and Console metrics [Serial]", func() { + SkipIfOCPBelow("v4.12") + var ( + flpPromSM = "flowlogs-pipeline-monitor" + flpPromSA = "flowlogs-pipeline-prom" + eBPFPromSM = "ebpf-agent-svc-monitor" + eBPFPromSA = "ebpf-agent-svc-prom" + namespace = oc.Namespace() + ) + + flow := Flowcollector{ + Namespace: namespace, + Template: flowFixturePath, + LokiNamespace: lokiStackNS, + EBPFMetricServerTLSType: "Auto", + } + + defer func() { _ = flow.DeleteFlowcollector(oc) }() + flow.CreateFlowcollector(oc) + + g.By("Verify flowlogs-pipeline metrics") + FLPtlsScheme, err := getMetricsScheme(oc, flpPromSM, flow.Namespace) + o.Expect(err).NotTo(o.HaveOccurred()) + FLPtlsScheme = strings.Trim(FLPtlsScheme, "'") + o.Expect(FLPtlsScheme).To(o.Equal("https")) + + FLPserverName, err := getMetricsServerName(oc, flpPromSM, flow.Namespace) + FLPserverName = strings.Trim(FLPserverName, "'") + o.Expect(err).NotTo(o.HaveOccurred()) + FLPexpectedServerName := fmt.Sprintf("%s.%s.svc", flpPromSA, namespace) + o.Expect(FLPserverName).To(o.Equal(FLPexpectedServerName)) + + g.By("Wait for a min before scraping metrics") + time.Sleep(60 * time.Second) + + g.By("Verify prometheus is able to scrape FLP and Console metrics") + verifyFLPMetrics(oc) + query := fmt.Sprintf("process_start_time_seconds{namespace=\"%s\", job=\"netobserv-plugin-metrics\"}", namespace) + metrics, err := getMetric(oc, query) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(popMetricValue(metrics)).Should(o.BeNumerically(">", 0)) + + g.By("Verify eBPF metrics") + eBPFtlsScheme, err := getMetricsScheme(oc, eBPFPromSM, flow.Namespace+"-privileged") + o.Expect(err).NotTo(o.HaveOccurred()) + eBPFtlsScheme = strings.Trim(eBPFtlsScheme, "'") + o.Expect(eBPFtlsScheme).To(o.Equal("https")) + + eBPFserverName, err := getMetricsServerName(oc, eBPFPromSM, flow.Namespace+"-privileged") + eBPFserverName = strings.Trim(eBPFserverName, "'") + o.Expect(err).NotTo(o.HaveOccurred()) + eBPFexpectedServerName := fmt.Sprintf("%s.%s.svc", eBPFPromSA, namespace+"-privileged") + o.Expect(eBPFserverName).To(o.Equal(eBPFexpectedServerName)) + + g.By("Verify prometheus is able to scrape eBPF agent metrics") + verifyEBPFMetrics(oc) + }) + }) + }) + + g.It("Author:memodi-High-53595-High-49107-High-45304-High-54929-High-54840-High-68310-Verify flow correctness and metrics [Serial]", func() { + SkipIfOCPBelow("v4.11") + g.By("Deploying test server and client pods") + serverTemplatePath := filePath.Join(baseDir, "test-nginx-server_template.yaml") + testServer := TestServerTemplate{ + ServerNS: "test-server-54929", + Template: serverTemplatePath, + } + defer oc.DeleteSpecifiedNamespaceAsAdmin(testServer.ServerNS) + err := testServer.createServer(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + compat_otp.AssertAllPodsToBeReady(oc, testServer.ServerNS) + + clientTemplatePath := filePath.Join(baseDir, "test-nginx-client_template.yaml") + testClient := TestClientTemplate{ + ServerNS: testServer.ServerNS, + ClientNS: "test-client-54929", + ObjectSize: "100K", + Template: clientTemplatePath, + } + + defer oc.DeleteSpecifiedNamespaceAsAdmin(testClient.ClientNS) + err = testClient.createClient(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + compat_otp.AssertAllPodsToBeReady(oc, testClient.ClientNS) + + startTime := time.Now() + + g.By("Deploy FlowCollector") + flow := Flowcollector{ + Namespace: namespace, + Template: flowFixturePath, + LokiNamespace: lokiStackNS, + } + + defer func() { _ = flow.DeleteFlowcollector(oc) }() + flow.CreateFlowcollector(oc) + + g.By("get flowlogs from loki") + lokilabels := Lokilabels{ + App: "netobserv-flowcollector", + SrcK8SNamespace: testServer.ServerNS, + DstK8SNamespace: testClient.ClientNS, + SrcK8SOwnerName: "nginx-service", + FlowDirection: "0", + } + + g.By("Wait for 2 mins before logs gets collected and written to loki") + time.Sleep(120 * time.Second) + + flowRecords, err := lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of flowRecords > 0") + + // verify flow correctness + verifyFlowCorrectness(testClient.ObjectSize, flowRecords) + + // verify inner metrics + query := fmt.Sprintf(`sum(rate(netobserv_workload_ingress_bytes_total{SrcK8S_Namespace="%s"}[1m]))`, testClient.ClientNS) + metrics := pollMetrics(oc, query) + + // verfy metric is between 270 and 330 + o.Expect(metrics).Should(o.BeNumerically("~", 330, 270)) + }) + + g.It("Author:aramesha-NonPreRelease-Longduration-High-60701-Verify connection tracking [Serial]", func() { + SkipIfOCPBelow("v4.10") + startTime := time.Now() + + g.By("Deploying test server and client pods") + serverTemplate := filePath.Join(baseDir, "test-nginx-server_template.yaml") + testServerTemplate := TestServerTemplate{ + ServerNS: "test-server-60701", + Template: serverTemplate, + } + + defer oc.DeleteSpecifiedNamespaceAsAdmin(testServerTemplate.ServerNS) + err := testServerTemplate.createServer(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + compat_otp.AssertAllPodsToBeReady(oc, testServerTemplate.ServerNS) + + clientTemplate := filePath.Join(baseDir, "test-nginx-client_template.yaml") + + testClientTemplate := TestClientTemplate{ + ServerNS: testServerTemplate.ServerNS, + ClientNS: "test-client-60701", + Template: clientTemplate, + } + + defer oc.DeleteSpecifiedNamespaceAsAdmin(testClientTemplate.ClientNS) + err = testClientTemplate.createClient(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + compat_otp.AssertAllPodsToBeReady(oc, testClientTemplate.ClientNS) + + g.By("Deploy FlowCollector with endConversations LogType") + flow := Flowcollector{ + Namespace: namespace, + Template: flowFixturePath, + LogType: "EndedConversations", + LokiNamespace: lokiStackNS, + } + + defer func() { _ = flow.DeleteFlowcollector(oc) }() + flow.CreateFlowcollector(oc) + + // verify logs + g.By("Wait for a min before logs gets collected and written to loki") + time.Sleep(60 * time.Second) + + lokilabels := Lokilabels{ + App: "netobserv-flowcollector", + SrcK8SNamespace: testClientTemplate.ClientNS, + DstK8SNamespace: testClientTemplate.ServerNS, + RecordType: "endConnection", + DstK8SOwnerName: "nginx-service", + } + + g.By("Verify endConnection Records from loki") + endConnectionRecords, err := lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(endConnectionRecords)).Should(o.BeNumerically(">", 0), "expected number of endConnectionRecords > 0") + verifyConversationRecordTime(endConnectionRecords) + + g.By("Deploy FlowCollector with Conversations LogType") + _ = flow.DeleteFlowcollector(oc) + + flow.LogType = "Conversations" + flow.CreateFlowcollector(oc) + + g.By("Wait for a min before logs gets collected and written to loki") + startTime = time.Now() + time.Sleep(60 * time.Second) + + g.By("Verify NewConnection Records from loki") + lokilabels.RecordType = "newConnection" + + newConnectionRecords, err := lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(newConnectionRecords)).Should(o.BeNumerically(">", 0), "expected number of newConnectionRecords > 0") + verifyConversationRecordTime(newConnectionRecords) + + g.By("Verify HeartbeatConnection Records from loki") + lokilabels.RecordType = "heartbeat" + heartbeatConnectionRecords, err := lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(heartbeatConnectionRecords)).Should(o.BeNumerically(">", 0), "expected number of heartbeatConnectionRecords > 0") + verifyConversationRecordTime(heartbeatConnectionRecords) + + g.By("Verify EndConnection Records from loki") + lokilabels.RecordType = "endConnection" + endConnectionRecords, err = lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(endConnectionRecords)).Should(o.BeNumerically(">", 0), "expected number of endConnectionRecords > 0") + verifyConversationRecordTime(endConnectionRecords) + }) + + g.It("Author:memodi-NonPreRelease-Longduration-High-63839-Verify-multi-tenancy [Disruptive][Slow]", func() { + SkipIfOCPBelow("v4.15") + users, usersHTpassFile, htPassSecret := getNewUser(oc, 2) + defer userCleanup(oc, users, usersHTpassFile, htPassSecret) + + g.By("Creating client server template and template CRBs for testusers") + // create templates for testuser to be used later + testUserstemplate := filePath.Join(baseDir, "testuser-client-server_template.yaml") + stdout, stderr, err := oc.AsAdmin().Run("apply").Args("-f", testUserstemplate).Outputs() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(stderr).To(o.BeEmpty()) + templateResource := strings.Split(stdout, " ")[0] + templateName := strings.Split(templateResource, "/")[1] + defer removeTemplatePermissions(oc, users[0].Username) + addTemplatePermissions(oc, users[0].Username) + + g.By("Deploy FlowCollector") + flow := Flowcollector{ + Namespace: namespace, + Template: flowFixturePath, + LokiNamespace: lokiStackNS, + } + + defer func() { _ = flow.DeleteFlowcollector(oc) }() + flow.CreateFlowcollector(oc) + + g.By("Deploying test server and client pods") + serverTemplate := filePath.Join(baseDir, "test-nginx-server_template.yaml") + testServerTemplate := TestServerTemplate{ + ServerNS: "test-server-63839", + Template: serverTemplate, + } + defer oc.DeleteSpecifiedNamespaceAsAdmin(testServerTemplate.ServerNS) + err = testServerTemplate.createServer(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + compat_otp.AssertAllPodsToBeReady(oc, testServerTemplate.ServerNS) + + clientTemplate := filePath.Join(baseDir, "test-nginx-client_template.yaml") + + testClientTemplate := TestClientTemplate{ + ServerNS: testServerTemplate.ServerNS, + ClientNS: "test-client-63839", + Template: clientTemplate, + } + + defer oc.DeleteSpecifiedNamespaceAsAdmin(testClientTemplate.ClientNS) + err = testClientTemplate.createClient(oc) + compat_otp.AssertAllPodsToBeReady(oc, testClientTemplate.ClientNS) + o.Expect(err).NotTo(o.HaveOccurred()) + + // save original context + origContxt, contxtErr := oc.AsAdmin().WithoutNamespace().Run("config").Args("current-context").Output() + o.Expect(contxtErr).NotTo(o.HaveOccurred()) + e2e.Logf("orginal context is %v", origContxt) + defer removeUserAsReader(oc, users[0].Username) + addUserAsReader(oc, users[0].Username) + origUser := oc.Username() + + e2e.Logf("current user is %s", origUser) + defer func() { _ = oc.AsAdmin().WithoutNamespace().Run("config").Args("use-context", origContxt).Execute() }() + defer oc.ChangeUser(origUser) + oc.ChangeUser(users[0].Username) + + curUser := oc.Username() + e2e.Logf("current user is %s", curUser) + + o.Expect(err).NotTo(o.HaveOccurred()) + user0Contxt, contxtErr := oc.WithoutNamespace().Run("config").Args("current-context").Output() + o.Expect(contxtErr).NotTo(o.HaveOccurred()) + + e2e.Logf("user0 context is %v", user0Contxt) + + g.By("Deploying test server and client pods as user0") + var ( + testUserServerNS = fmt.Sprintf("%s-server", users[0].Username) + testUserClientNS = fmt.Sprintf("%s-client", users[0].Username) + ) + + defer oc.DeleteSpecifiedNamespaceAsAdmin(testUserClientNS) + defer oc.DeleteSpecifiedNamespaceAsAdmin(testUserServerNS) + configFile := compat_otp.ProcessTemplate(oc, "--ignore-unknown-parameters=true", templateName, "-p", "SERVER_NS="+testUserServerNS, "-p", "CLIENT_NS="+testUserClientNS) + err = oc.WithoutNamespace().Run("create").Args("-f", configFile).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + // only required to getFlowLogs + lokilabels := Lokilabels{ + App: "netobserv-flowcollector", + SrcK8SNamespace: testUserServerNS, + DstK8SNamespace: testUserClientNS, + SrcK8SOwnerName: "nginx-service", + FlowDirection: "0", + } + + user0token, err := oc.WithoutNamespace().Run("whoami").Args("-t").Output() + e2e.Logf("token is %s", user0token) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Wait for a min before logs gets collected and written to loki") + startTime := time.Now() + time.Sleep(60 * time.Second) + + g.By("get flowlogs from loki") + flowRecords, err := lokilabels.getLokiFlowLogs(user0token, ls.Route, startTime) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of flowRecords > 0") + + g.By("verify no logs are fetched from an NS that user is not admin for") + lokilabels = Lokilabels{ + App: "netobserv-flowcollector", + SrcK8SNamespace: testClientTemplate.ServerNS, + DstK8SNamespace: testClientTemplate.ClientNS, + SrcK8SOwnerName: "nginx-service", + FlowDirection: "0", + } + flowRecords, err = lokilabels.getLokiFlowLogs(user0token, ls.Route, startTime) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).NotTo(o.BeNumerically(">", 0), "expected number of flowRecords to be equal to 0") + }) + + g.It("Author:aramesha-NonPreRelease-Critical-59746-NetObserv upgrade testing [Serial]", func() { + SkipIfOCPBelow("v4.10") + g.By("Uninstall operator deployed by BeforeEach and delete operator NS") + NO.uninstallOperator(oc) + oc.DeleteSpecifiedNamespaceAsAdmin(netobservNS) + err := Resource{"namespace", netobservNS, ""}.WaitUntilResourceIsGone(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Deploy older version of netobserv operator") + NOcatSrc = Resource{"catsrc", "redhat-operators", "openshift-marketplace"} + NOSource = CatalogSourceObjects{"stable", NOcatSrc.Name, NOcatSrc.Namespace} + + NO.CatalogSource = &NOSource + + g.By(fmt.Sprintf("Subscribe operators to %s channel", NOSource.Channel)) + OperatorNS.DeployOperatorNamespace(oc) + NO.SubscribeOperator(oc) + // check if NO operator is deployed + WaitForPodsReadyWithLabel(oc, netobservNS, "app="+NO.OperatorName) + NOStatus, err := CheckOperatorStatus(oc, netobservNS, NOPackageName) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("found err %v", err)) + o.Expect((NOStatus)).To(o.BeTrue()) + + // check if flowcollector API exists + flowcollectorAPIExists, err := isFlowCollectorAPIExists(oc) + o.Expect((flowcollectorAPIExists)).To(o.BeTrue()) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Deploy FlowCollector") + flow := Flowcollector{ + Namespace: namespace, + Template: flowFixturePath, + LokiNamespace: lokiStackNS, + } + + defer func() { _ = flow.DeleteFlowcollector(oc) }() + flow.CreateFlowcollector(oc) + + g.By("Get NetObserv and components versions") + NOCSV, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pods", "-l", "app=netobserv-operator", "-n", netobservNS, "-o=jsonpath={.items[*].spec.containers[0].env[?(@.name=='OPERATOR_CONDITION_NAME')].value}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + preUpgradeNOVersion := strings.Split(NOCSV, ".v")[1] + preUpgradeEBPFVersion, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pods", "-l", "app=netobserv-operator", "-n", netobservNS, "-o=jsonpath={.items[*].spec.containers[0].env[0].value}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + preUpgradeEBPFVersion = strings.Split(preUpgradeEBPFVersion, ":")[1] + preUpgradeFLPVersion, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pods", "-l", "app=netobserv-operator", "-n", netobservNS, "-o=jsonpath={.items[*].spec.containers[0].env[1].value}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + preUpgradeFLPVersion = strings.Split(preUpgradeFLPVersion, ":")[1] + preUpgradePluginVersion, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pods", "-l", "app=netobserv-operator", "-n", netobservNS, "-o=jsonpath={.items[*].spec.containers[0].env[2].value}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + preUpgradePluginVersion = strings.Split(preUpgradePluginVersion, ":")[1] + + g.By("Deploy latest catalog and upgrade to latest version") + NOcatSrc.Name = "netobserv-konflux-fbc" + NOcatSrc.Namespace = OperatorNS.Name + var catsrcErr error + if catalogSource != "" { + e2e.Logf("Using %s catalog", catalogSource) + catsrcErr = NOcatSrc.applyFromTemplate(oc, "-n", NOcatSrc.Namespace, "-f", catSrcTemplate, "-p", "NAMESPACE="+NOcatSrc.Namespace, "IMAGE="+catalogSource) + } else { + e2e.Logf("Using default ystream catalog") + catsrcErr = NOcatSrc.applyFromTemplate(oc, "-n", NOcatSrc.Namespace, "-f", catSrcTemplate, "-p", "NAMESPACE="+NOcatSrc.Namespace) + } + o.Expect(catsrcErr).NotTo(o.HaveOccurred()) + _, _ = oc.AsAdmin().WithoutNamespace().Run("patch").Args("subscription", "netobserv-operator", "-n", netobservNS, "-p", `[{"op": "replace", "path": "/spec/source", "value": `+NOcatSrc.Name+`}, {"op": "replace", "path": "/spec/sourceNamespace", "value": `+NOcatSrc.Namespace+`}]`, "--type=json").Output() + + g.By("Wait for a min for operator upgrade") + time.Sleep(60 * time.Second) + + WaitForPodsReadyWithLabel(oc, netobservNS, "app=netobserv-operator") + NOStatus, err = CheckOperatorStatus(oc, netobservNS, NOPackageName) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("found err %v", err)) + o.Expect((NOStatus)).To(o.BeTrue()) + + g.By("Get NetObserv operator and components versions") + NOCSV, err = oc.AsAdmin().WithoutNamespace().Run("get").Args("pods", "-l", "app=netobserv-operator", "-n", netobservNS, "-o=jsonpath={.items[*].spec.containers[0].env[?(@.name=='OPERATOR_CONDITION_NAME')].value}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + postUpgradeNOVersion := strings.Split(NOCSV, ".v")[1] + postUpgradeEBPFVersion, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pods", "-l", "app=netobserv-operator", "-n", netobservNS, "-o=jsonpath={.items[*].spec.containers[0].env[0].value}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + postUpgradeEBPFVersion = strings.Split(postUpgradeEBPFVersion, ":")[1] + postUpgradeFLPVersion, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pods", "-l", "app=netobserv-operator", "-n", netobservNS, "-o=jsonpath={.items[*].spec.containers[0].env[1].value}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + postUpgradeFLPVersion = strings.Split(postUpgradeFLPVersion, ":")[1] + postUpgradePluginVersion, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pods", "-l", "app=netobserv-operator", "-n", netobservNS, "-o=jsonpath={.items[*].spec.containers[0].env[2].value}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + postUpgradePluginVersion = strings.Split(postUpgradePluginVersion, ":")[1] + + g.By("Verify versions are updated") + o.Expect(preUpgradeNOVersion).NotTo(o.Equal(postUpgradeNOVersion)) + o.Expect(preUpgradeEBPFVersion).NotTo(o.Equal(postUpgradeEBPFVersion)) + o.Expect(preUpgradeFLPVersion).NotTo(o.Equal(postUpgradeFLPVersion)) + o.Expect(preUpgradePluginVersion).NotTo(o.Equal(postUpgradePluginVersion)) + + // verify logs + g.By("Wait for a min before logs gets collected and written to loki") + startTime := time.Now() + time.Sleep(60 * time.Second) + + g.By("Get flowlogs from loki") + err = verifyLokilogsTime(kubeadminToken, ls.Route, startTime) + o.Expect(err).NotTo(o.HaveOccurred()) + }) + + g.It("Author:aramesha-NonPreRelease-High-62989-Verify SCTP, ICMP, ICMPv6 traffic is observed [Disruptive]", func() { + SkipIfOCPBelow("v4.10") + var ( + sctpClientPodTemplatePath = filePath.Join(networkingDir, "sctpclient.yaml") + sctpServerPodTemplatePath = filePath.Join(networkingDir, "sctpserver.yaml") + sctpServerPodname = "sctpserver" + sctpClientPodname = "sctpclient" + ) + + g.By("install load-sctp-module in all workers") + prepareSCTPModule(oc) + + g.By("Create netobserv-sctp NS") + SCTPns := "netobserv-sctp-62989" + defer oc.DeleteSpecifiedNamespaceAsAdmin(SCTPns) + oc.CreateSpecifiedNamespaceAsAdmin(SCTPns) + _ = compat_otp.SetNamespacePrivileged(oc, SCTPns) + + g.By("create sctpClientPod") + createResourceFromFile(oc, SCTPns, sctpClientPodTemplatePath) + WaitForPodsReadyWithLabel(oc, SCTPns, "name=sctpclient") + + g.By("create sctpServerPod") + createResourceFromFile(oc, SCTPns, sctpServerPodTemplatePath) + WaitForPodsReadyWithLabel(oc, SCTPns, "name=sctpserver") + + g.By("Deploy FlowCollector") + flow := Flowcollector{ + Namespace: namespace, + Template: flowFixturePath, + LokiNamespace: lokiStackNS, + } + + defer func() { _ = flow.DeleteFlowcollector(oc) }() + flow.CreateFlowcollector(oc) + + g.By("get primary IP address of sctpServerPod") + sctpServerPodIP, _ := getPodIP(oc, SCTPns, sctpServerPodname, ipStackType) + + g.By("sctpserver pod start to wait for sctp traffic") + cmd, _, _, _ := oc.AsAdmin().Run("exec").Args("-n", SCTPns, sctpServerPodname, "--", "/usr/bin/ncat", "-l", "30102", "--sctp").Background() + defer func() { _ = cmd.Process.Kill() }() + time.Sleep(5 * time.Second) + + g.By("check sctp process enabled in the sctp server pod") + msg, err := e2eoutput.RunHostCmd(SCTPns, sctpServerPodname, "ps aux | grep sctp") + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(strings.Contains(msg, "/usr/bin/ncat -l 30102 --sctp")).To(o.BeTrue()) + + g.By("sctpclient pod start to send sctp traffic") + startTime := time.Now() + _, _ = e2eoutput.RunHostCmd(SCTPns, sctpClientPodname, "echo 'Test traffic using sctp port from sctpclient to sctpserver' | { ncat -v "+sctpServerPodIP+" 30102 --sctp; }") + + g.By("server sctp process will end after get sctp traffic from sctp client") + time.Sleep(5 * time.Second) + msg1, err1 := e2eoutput.RunHostCmd(SCTPns, sctpServerPodname, "ps aux | grep sctp") + o.Expect(err1).NotTo(o.HaveOccurred()) + o.Expect(msg1).NotTo(o.ContainSubstring("/usr/bin/ncat -l 30102 --sctp")) + + // verify logs + g.By("Wait for a min before logs gets collected and written to loki") + time.Sleep(60 * time.Second) + + // Scenario1: Verify SCTP traffic + lokilabels := Lokilabels{ + App: "netobserv-flowcollector", + SrcK8SNamespace: SCTPns, + DstK8SNamespace: SCTPns, + } + + g.By("Verify SCTP flows are seen on loki") + parameters := []string{"Proto=\"132\"", "DstPort=\"30102\""} + + SCTPflows, err := lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime, parameters...) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(SCTPflows)).Should(o.BeNumerically(">", 0), "expected number of SCTP flows > 0") + + // Scenario2: Verify ICMP traffic + g.By("sctpclient ping sctpserver") + _, _ = e2eoutput.RunHostCmd(SCTPns, sctpClientPodname, "ping -c 10 "+sctpServerPodIP) + ICMPEchoReq := 8 + ICMPEchoRes := 0 + if ipStackType == "ipv4single" { + parameters = []string{"Proto=\"1\""} + } + g.By("test ipv6 in ipv6 cluster or dualstack cluster") + if ipStackType == "ipv6single" || ipStackType == "dualstack" { + parameters = []string{"Proto=\"58\""} + ICMPEchoReq = 128 + ICMPEchoRes = 129 + } + + g.By("Wait for a min before logs gets collected and written to loki") + time.Sleep(60 * time.Second) + + ICMPflows, err := lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime, parameters...) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(ICMPflows)).Should(o.BeNumerically(">", 0), "expected number of ICMP flows > 0") + + nICMPFlows := 0 + for _, r := range ICMPflows { + if r.Flowlog.IcmpType == ICMPEchoReq || r.Flowlog.IcmpType == ICMPEchoRes { + nICMPFlows++ + } + } + o.Expect(nICMPFlows).Should(o.BeNumerically(">", 0), "expected number of ICMP flows of type 8/128 or 0/129 (echo request or reply) > 0") + }) + + g.It("Author:aramesha-NonPreRelease-High-68125-Verify DSCP with NetObserv [Serial]", func() { + SkipIfOCPBelow("v4.14") + g.By("Deploying test server and client pods") + serverTemplate := filePath.Join(baseDir, "test-nginx-server_template.yaml") + testServerTemplate := TestServerTemplate{ + ServerNS: "test-server-68125", + Template: serverTemplate, + } + defer oc.DeleteSpecifiedNamespaceAsAdmin(testServerTemplate.ServerNS) + err := testServerTemplate.createServer(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + compat_otp.AssertAllPodsToBeReady(oc, testServerTemplate.ServerNS) + + clientTemplate := filePath.Join(baseDir, "test-nginx-client_template.yaml") + testClientTemplate := TestClientTemplate{ + ServerNS: testServerTemplate.ServerNS, + ClientNS: "test-client-68125", + Template: clientTemplate, + } + defer oc.DeleteSpecifiedNamespaceAsAdmin(testClientTemplate.ClientNS) + err = testClientTemplate.createClient(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + compat_otp.AssertAllPodsToBeReady(oc, testClientTemplate.ClientNS) + + compat_otp.By("Check cluster network type") + networkType := compat_otp.CheckNetworkType(oc) + o.Expect(networkType).NotTo(o.BeEmpty()) + if networkType == "ovnkubernetes" { + g.By("Deploy egressQoS for OVN CNI") + clientDSCPPath := filePath.Join(networkingDir, "test-client-DSCP.yaml") + egressQoSPath := filePath.Join(networkingDir, "egressQoS.yaml") + g.By("Deploy nginx client pod and egressQoS") + createResourceFromFile(oc, testClientTemplate.ClientNS, clientDSCPPath) + createResourceFromFile(oc, testClientTemplate.ClientNS, egressQoSPath) + } + + g.By("Deploy FlowCollector") + flow := Flowcollector{ + Namespace: namespace, + Template: flowFixturePath, + LokiNamespace: lokiStackNS, + } + + defer func() { _ = flow.DeleteFlowcollector(oc) }() + flow.CreateFlowcollector(oc) + + // verify logs + g.By("Wait for a min before logs gets collected and written to loki") + startTime := time.Now() + time.Sleep(60 * time.Second) + + // Scenario1: Verify default DSCP value=0 + lokilabels := Lokilabels{ + App: "netobserv-flowcollector", + SrcK8SNamespace: testClientTemplate.ClientNS, + DstK8SNamespace: testClientTemplate.ServerNS, + } + parameters := []string{"SrcK8S_Name=\"client\""} + + g.By("Verify DSCP value=0") + flowRecords, err := lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime, parameters...) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of flows > 0") + for _, r := range flowRecords { + o.Expect(r.Flowlog.Dscp).To(o.Equal(0)) + } + + // Scenario2: Verify egress QoS feature for OVN CNI + if networkType == "ovnkubernetes" { + parameters = []string{"SrcK8S_Name=\"client-dscp\", Dscp=\"59\""} + + g.By("Wait for a min before logs gets collected and written to loki") + time.Sleep(60 * time.Second) + + g.By("Verify DSCP value=59 for flows from DSCP client pod") + flowRecords, err = lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime, parameters...) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of flows with DSCP value 59 should be > 0") + + g.By("Verify DSCP value=0 for flows from pods other than DSCP client pod in test-client namespace") + parameters = []string{"SrcK8S_Name=\"client\", Dscp=\"0\""} + + flowRecords, err = lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime, parameters...) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of flows with DSCP value 0 should be > 0") + } + + // Scenario3: Explicitly passing QoS value in ping command + var destinationIP string + switch ipStackType { + case "ipv4single": + destinationIP = "1.1.1.1" + case "ipv6single": + destinationIP = "::1" + default: + destinationIP = "1.1.1.1" + } + + g.By("Ping loopback address with custom QoS from client pod") + startTime = time.Now() + _, _ = e2eoutput.RunHostCmd(testClientTemplate.ClientNS, "client", "ping -c 10 -Q 0x80 "+destinationIP) + + lokilabels = Lokilabels{ + App: "netobserv-flowcollector", + SrcK8SNamespace: testClientTemplate.ClientNS, + } + parameters = []string{"Dscp=\"32\", DstAddr=\"" + destinationIP + "\""} + + g.By("Wait for a min before logs gets collected and written to loki") + time.Sleep(60 * time.Second) + + g.By("Verify DSCP value=32") + flowRecords, err = lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime, parameters...) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of flows with DSCP value 32 > 0") + }) + + g.It("Author:aramesha-NonPreRelease-High-69218-High-71291-Verify cluster ID and zone in multiCluster deployment [Serial]", func() { + SkipIfOCPBelow("v4.11") + g.By("Get clusterID of the cluster") + clusterID, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("clusterversion", "-o=jsonpath={.items[].spec.clusterID}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("Cluster ID is %s", clusterID) + + g.By("Deploy FlowCollector with multiCluster and addZone enabled") + flow := Flowcollector{ + Namespace: namespace, + MultiClusterDeployment: "true", + AddZone: "true", + Template: flowFixturePath, + LokiNamespace: lokiStackNS, + } + + defer func() { _ = flow.DeleteFlowcollector(oc) }() + flow.CreateFlowcollector(oc) + + // verify logs + g.By("Wait for a min before logs gets collected and written to loki") + startTime := time.Now() + time.Sleep(60 * time.Second) + + g.By("Verify K8SClusterName = Cluster ID") + clusteridlabels := Lokilabels{ + App: "netobserv-flowcollector", + K8SClusterName: clusterID, + } + clusterIDFlowRecords, err := clusteridlabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(clusterIDFlowRecords)).Should(o.BeNumerically(">", 0), "expected number of flows > 0") + + g.By("Verify SrcK8S_Zone and DstK8S_Zone are present and have expected values") + zonelabels := Lokilabels{ + App: "netobserv-flowcollector", + SrcK8SType: "Node", + DstK8SType: "Node", + } + + zoneFlowRecords, err := zonelabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime) + o.Expect(err).NotTo(o.HaveOccurred()) + for _, r := range zoneFlowRecords { + expectedSrcK8SZone, err := compat_otp.GetResourceSpecificLabelValue(oc, "node/"+r.Flowlog.SrcK8SHostName, "", `topology.kubernetes.io/zone`) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(r.Flowlog.SrcK8SZone).To(o.Equal(expectedSrcK8SZone)) + + expectedDstK8SZone, err := compat_otp.GetResourceSpecificLabelValue(oc, "node/"+r.Flowlog.DstK8SHostName, "", `topology.kubernetes.io/zone`) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(r.Flowlog.DstK8SZone).To(o.Equal(expectedDstK8SZone)) + } + }) + + g.It("Author:aramesha-NonPreRelease-Longduration-High-73175-Verify eBPF agent filtering [Serial]", func() { + SkipIfOCPBelow("v4.14") + g.By("Deploy test server and client pods") + serverTemplate := filePath.Join(baseDir, "test-nginx-server_template.yaml") + testServerTemplate := TestServerTemplate{ + ServerNS: "test-server-73175", + Template: serverTemplate, + } + defer oc.DeleteSpecifiedNamespaceAsAdmin(testServerTemplate.ServerNS) + err := testServerTemplate.createServer(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + compat_otp.AssertAllPodsToBeReady(oc, testServerTemplate.ServerNS) + + clientTemplate := filePath.Join(baseDir, "test-nginx-client_template.yaml") + testClientTemplate := TestClientTemplate{ + ServerNS: testServerTemplate.ServerNS, + ClientNS: "test-client-73175", + Template: clientTemplate, + } + defer oc.DeleteSpecifiedNamespaceAsAdmin(testClientTemplate.ClientNS) + err = testClientTemplate.createClient(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + compat_otp.AssertAllPodsToBeReady(oc, testClientTemplate.ClientNS) + + clientServiceInfo, err := getClientServerInfo(oc, testClientTemplate.ServerNS, testClientTemplate.ClientNS, ipStackType) + o.Expect(err).NotTo(o.HaveOccurred()) + + // Scenario 1: + // Accept TCP flows between client pod and nginx-service + // Accept ICMP flows between client and nginx pod + // Default Reject all other flows + g.By("Deploy FlowCollector with eBPF filter") + filterRulesConfig := []map[string]interface{}{ + { + "action": "Accept", + "cidr": clientServiceInfo["service"]["ip"] + "/32", + "peerIP": clientServiceInfo["client"]["ip"], + "protocol": "TCP", + "ports": "80", + "sampling": 2, + }, + { + "action": "Accept", + "cidr": clientServiceInfo["client"]["ip"] + "/32", + "peerCIDR": clientServiceInfo["server"]["ip"] + "/32", + "protocol": "ICMP", + "icmpType": 8, + "sampling": 3, + }, + } + + config, err := json.Marshal(filterRulesConfig) + o.Expect(err).ToNot(o.HaveOccurred()) + filter := string(config) + + flow := Flowcollector{ + Namespace: namespace, + Template: flowFixturePath, + LokiNamespace: lokiStackNS, + EBPFFilterRules: filter, + } + + defer func() { _ = flow.DeleteFlowcollector(oc) }() + flow.CreateFlowcollector(oc) + + g.By("Ping nginx pod from client pod") + startTime := time.Now() + _, _ = e2eoutput.RunHostCmd(testClientTemplate.ClientNS, clientServiceInfo["client"]["name"], "ping -c 10 "+clientServiceInfo["server"]["ip"]) + + g.By("Wait for a min before logs gets collected and written to loki") + time.Sleep(60 * time.Second) + + lokilabels := Lokilabels{ + App: "netobserv-flowcollector", + } + + g.By("Verify number of flows with on UDP Protcol with SrcPort 53 = 0") + lokiParams := []string{"Proto=\"17\"", "SrcPort=\"53\""} + flowRecords, err := lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime, lokiParams...) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically("==", 0), "expected number of flows on UDP with SrcPort 53 = 0") + + g.By("Verify flows from client pod to nginx pod > 0") + lokilabels.SrcK8SNamespace = testClientTemplate.ClientNS + lokilabels.DstK8SNamespace = testClientTemplate.ServerNS + lokiParams = []string{"SrcAddr=" + "\"" + clientServiceInfo["client"]["ip"] + "\"", "DstAddr=" + "\"" + clientServiceInfo["server"]["ip"] + "\""} + + flowRecords, err = lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime, lokiParams...) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of flows from client pod to nginx pod > 0") + + for _, r := range flowRecords { + o.Expect(r.Flowlog.Proto).Should(o.BeNumerically("==", 1)) + o.Expect(r.Flowlog.IcmpType).Should(o.BeNumerically("==", 8)) + o.Expect(r.Flowlog.Sampling).Should(o.BeNumerically("==", 3)) + } + + g.By("Verify flows from client pod to nginx-service > 0") + lokilabels.DstK8SType = "Service" + + flowRecords, err = lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of flows from client pod to nginx-service > 0") + + for _, r := range flowRecords { + o.Expect(r.Flowlog.Proto).Should(o.BeNumerically("==", 6)) + o.Expect(r.Flowlog.Sampling).Should(o.BeNumerically("==", 2)) + } + + g.By("Verify prometheus is able to scrape eBPF metrics") + verifyEBPFFilterMetrics(oc, "FilterAccept") + verifyEBPFFilterMetrics(oc, "FilterNoMatch") + + // Scenario2: + // Accept only flows with drops + g.By("Deploy flowcollector with eBPF filter for flows with drops") + filterRulesConfig = []map[string]interface{}{ + { + "action": "Accept", + "cidr": "172.30.0.0/16", + "pktDrops": true, + }, + } + + config, err = json.Marshal(filterRulesConfig) + o.Expect(err).ToNot(o.HaveOccurred()) + filter = string(config) + + _ = flow.DeleteFlowcollector(oc) + flow.EBPFPrivileged = "true" + flow.EBPFeatures = []string{"\"PacketDrop\""} + flow.EBPFFilterRules = filter + flow.CreateFlowcollector(oc) + + g.By("Wait for a min before logs gets collected and written to loki") + startTime = time.Now() + time.Sleep(60 * time.Second) + + lokilabels = Lokilabels{ + App: "netobserv-flowcollector", + } + lokiParams = []string{"Proto=\"6\""} + + flowRecords, err = lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime, lokiParams...) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of flows with drops > 0") + + for _, r := range flowRecords { + o.Expect(r.Flowlog.PktDropBytes).Should(o.BeNumerically(">", 0)) + o.Expect(r.Flowlog.PktDropPackets).Should(o.BeNumerically(">", 0)) + } + }) + + g.It("Author:memodi-Critical-53844-Sanity Test NetObserv [Serial]", func() { + SkipIfOCPBelow("v4.11") + g.By("Deploy FlowCollector") + flow := Flowcollector{ + Namespace: namespace, + Template: flowFixturePath, + LokiNamespace: lokiStackNS, + } + + defer func() { _ = flow.DeleteFlowcollector(oc) }() + flow.CreateFlowcollector(oc) + + g.By("Wait for a min before logs gets collected and written to loki") + startTime := time.Now() + time.Sleep(60 * time.Second) + + lokilabels := Lokilabels{ + App: "netobserv-flowcollector", + } + + g.By("Verify flows are written to loki") + flowRecords, err := lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of flows written to loki > 0") + }) + + g.It("Author:aramesha-High-67782-Verify large volume downloads [Serial]", func() { + SkipIfOCPBelow("v4.11") + g.By("Deploy FlowCollector") + flow := Flowcollector{ + Namespace: namespace, + Template: flowFixturePath, + LokiNamespace: lokiStackNS, + EBPFCacheActiveTimeout: "30s", + } + + defer func() { _ = flow.DeleteFlowcollector(oc) }() + flow.CreateFlowcollector(oc) + + g.By("Deploy test server and client pods") + serverTemplate := filePath.Join(baseDir, "test-nginx-server_template.yaml") + testServerTemplate := TestServerTemplate{ + ServerNS: "test-server-67782", + Template: serverTemplate, + LargeBlob: "yes", + } + defer oc.DeleteSpecifiedNamespaceAsAdmin(testServerTemplate.ServerNS) + err := testServerTemplate.createServer(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + compat_otp.AssertAllPodsToBeReady(oc, testServerTemplate.ServerNS) + + clientTemplate := filePath.Join(baseDir, "test-nginx-client_template.yaml") + testClientTemplate := TestClientTemplate{ + ServerNS: testServerTemplate.ServerNS, + ClientNS: "test-client-67782", + ObjectSize: "100M", + Template: clientTemplate, + } + defer oc.DeleteSpecifiedNamespaceAsAdmin(testClientTemplate.ClientNS) + err = testClientTemplate.createClient(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + compat_otp.AssertAllPodsToBeReady(oc, testClientTemplate.ClientNS) + + g.By("Wait for 2 mins before logs gets collected and written to loki") + startTime := time.Now() + time.Sleep(120 * time.Second) + + lokilabels := Lokilabels{ + App: "netobserv-flowcollector", + SrcK8SNamespace: testClientTemplate.ServerNS, + DstK8SNamespace: testClientTemplate.ClientNS, + SrcK8SOwnerName: "nginx-service", + FlowDirection: "0", + } + + g.By("Verify flows are written to loki") + flowRecords, err := lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of flows written to loki > 0") + + g.By("Verify flow correctness") + verifyFlowCorrectness(testClientTemplate.ObjectSize, flowRecords) + }) + + g.It("Author:aramesha-High-75656-Verify TCP flags [Disruptive]", func() { + SkipIfOCPBelow("v4.13") + SYNFloodMetricsPath := filePath.Join(baseDir, "SYN_flood_metrics_template.yaml") + SYNFloodAlertsPath := filePath.Join(baseDir, "SYN_flood_alert_template.yaml") + + g.By("Deploy flowcollector with eBPF filter to Reject flows with tcpFlags SYN-ACK and TCP Protocol") + filterRulesConfig := []map[string]string{ + { + "action": "Reject", + "cidr": "0.0.0.0/0", + "protocol": "TCP", + "tcpFlags": "SYN-ACK", + }, + } + + config, err := json.Marshal(filterRulesConfig) + o.Expect(err).ToNot(o.HaveOccurred()) + filter := string(config) + + flow := Flowcollector{ + Namespace: namespace, + Template: flowFixturePath, + LokiNamespace: lokiStackNS, + EBPFFilterRules: filter, + } + + defer func() { _ = flow.DeleteFlowcollector(oc) }() + flow.CreateFlowcollector(oc) + + g.By("Ensure flowcollector is ready with Reject flowFilter") + flowPatch, err := oc.AsAdmin().Run("get").Args("flowcollector", "cluster", "-o", "jsonpath='{.spec.agent.ebpf.flowFilter.rules[0].action}'").Output() + o.Expect(err).ToNot(o.HaveOccurred()) + o.Expect(flowPatch).To(o.Equal(`'Reject'`)) + + g.By("Deploy custom metrics to detect SYN flooding") + customMetrics := CustomMetrics{ + Namespace: namespace, + Template: SYNFloodMetricsPath, + } + + curv, err := getResourceVersion(oc, "cm", "flowlogs-pipeline-config-dynamic", namespace) + o.Expect(err).NotTo(o.HaveOccurred()) + customMetrics.createCustomMetrics(oc) + waitForResourceGenerationUpdate(oc, "cm", "flowlogs-pipeline-config-dynamic", "resourceVersion", curv, namespace) + + g.By("Deploy SYN flooding alert rule") + defer oc.AsAdmin().WithoutNamespace().Run("delete").Args("alertingrule.monitoring.openshift.io", "netobserv-syn-alerts", "-n", "openshift-monitoring") + configFile := compat_otp.ProcessTemplate(oc, "--ignore-unknown-parameters=true", "-f", SYNFloodAlertsPath, "-p", "Namespace=openshift-monitoring") + err = oc.AsAdmin().WithoutNamespace().Run("create").Args("-f", configFile).Execute() + o.Expect(err).ToNot(o.HaveOccurred()) + + g.By("Deploy test client pod to induce SYN flooding") + template := filePath.Join(baseDir, "test-SYN-flood-client_template.yaml") + testTemplate := TestClientTemplate{ + ClientNS: "test-client-75656", + Template: template, + } + + defer oc.DeleteSpecifiedNamespaceAsAdmin(testTemplate.ClientNS) + configFile = compat_otp.ProcessTemplate(oc, "--ignore-unknown-parameters=true", "-f", testTemplate.Template, "-p", "CLIENT_NS="+testTemplate.ClientNS) + err = oc.AsAdmin().WithoutNamespace().Run("create").Args("-f", configFile).Execute() + o.Expect(err).ToNot(o.HaveOccurred()) + + g.By("Wait for a min before logs gets collected and written to loki") + startTime := time.Now() + time.Sleep(60 * time.Second) + + lokilabels := Lokilabels{ + App: "netobserv-flowcollector", + } + + g.By("Verify no flows with SYN_ACK TCP flag") + parameters := []string{"Flags=\"SYN_ACK\""} + + flowRecords, err := lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime, parameters...) + o.Expect(err).NotTo(o.HaveOccurred()) + // Loop needed since even flows with flags SYN, ACK are matched + count := 0 + for _, r := range flowRecords { + for _, f := range r.Flowlog.Flags { + o.Expect(f).ToNot(o.Equal("SYN_ACK")) + } + } + o.Expect(count).Should(o.BeNumerically("==", 0), "expected number of flows with SYN_ACK TCPFlag = 0") + verifyEBPFFilterMetrics(oc, "FilterReject") + + g.By("Verify SYN flooding flows") + parameters = []string{"Flags=\"SYN\"", "DstAddr=\"192.168.1.159\""} + + flowRecords, err = lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime, parameters...) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of SYN flows > 0") + for _, r := range flowRecords { + o.Expect(r.Flowlog.Bytes).Should(o.BeNumerically("==", 54)) + } + + g.By("Wait for alerts to be active") + waitForAlertToBeActive(oc, "NetObserv-SYNFlood-out") + waitForAlertToBeActive(oc, "NetObserv-SYNFlood-in") + }) + + g.It("Author:aramesha-NonPreRelease-Longduration-Medium-78480-NetObserv with sampling 50 [Serial][Slow]", func() { + SkipIfOCPBelow("v4.14") + g.By("Deploy DNS pods") + DNSTemplate := filePath.Join(baseDir, "DNS-pods.yaml") + DNSNamespace := "dns-traffic" + defer oc.DeleteSpecifiedNamespaceAsAdmin(DNSNamespace) + ApplyResourceFromFile(oc, DNSNamespace, DNSTemplate) + compat_otp.AssertAllPodsToBeReady(oc, DNSNamespace) + + g.By("Deploy test server and client pods") + servertemplate := filePath.Join(baseDir, "test-nginx-server_template.yaml") + testServerTemplate := TestServerTemplate{ + ServerNS: "test-server-78480", + Template: servertemplate, + } + defer oc.DeleteSpecifiedNamespaceAsAdmin(testServerTemplate.ServerNS) + err := testServerTemplate.createServer(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + compat_otp.AssertAllPodsToBeReady(oc, testServerTemplate.ServerNS) + + clientTemplate := filePath.Join(baseDir, "test-nginx-client_template.yaml") + testClientTemplate := TestClientTemplate{ + ServerNS: testServerTemplate.ServerNS, + ClientNS: "test-client-78480", + Template: clientTemplate, + } + defer oc.DeleteSpecifiedNamespaceAsAdmin(testClientTemplate.ClientNS) + err = testClientTemplate.createClient(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + compat_otp.AssertAllPodsToBeReady(oc, testClientTemplate.ClientNS) + + g.By("Deploy FlowCollector with all features enabled with sampling 50") + flow := Flowcollector{ + Namespace: namespace, + EBPFPrivileged: "true", + EBPFeatures: []string{"\"DNSTracking\", \"PacketDrop\", \"FlowRTT\", \"PacketTranslation\""}, + Sampling: "50", + LokiNamespace: lokiStackNS, + Template: flowFixturePath, + } + + defer func() { _ = flow.DeleteFlowcollector(oc) }() + flow.CreateFlowcollector(oc) + + g.By("Wait for 4 mins before logs gets collected and written to loki") + startTime := time.Now() + time.Sleep(240 * time.Second) + + lokilabels := Lokilabels{ + App: "netobserv-flowcollector", + } + + g.By("Verify Packet Drop flows") + lokiParams := []string{"PktDropLatestState=\"TCP_INVALID_STATE\"", "Proto=\"6\""} + flowRecords, err := lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime, lokiParams...) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of TCP Invalid State flows > 0") + for _, r := range flowRecords { + o.Expect(r.Flowlog.PktDropLatestDropCause).NotTo(o.BeEmpty()) + o.Expect(r.Flowlog.PktDropBytes).Should(o.BeNumerically(">", 0)) + o.Expect(r.Flowlog.PktDropPackets).Should(o.BeNumerically(">", 0)) + } + + lokiParams = []string{"PktDropLatestDropCause=\"SKB_DROP_REASON_NO_SOCKET\"", "Proto=\"6\""} + flowRecords, err = lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime, lokiParams...) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of No Socket TCP flows > 0") + for _, r := range flowRecords { + o.Expect(r.Flowlog.PktDropLatestState).NotTo(o.BeEmpty()) + o.Expect(r.Flowlog.PktDropBytes).Should(o.BeNumerically(">", 0)) + o.Expect(r.Flowlog.PktDropPackets).Should(o.BeNumerically(">", 0)) + } + + g.By("Verify flowRTT flows") + lokiParams = []string{"Proto=\"6\""} + flowRecords, err = lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime, lokiParams...) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of TCP flows > 0") + for _, r := range flowRecords { + o.Expect(r.Flowlog.TimeFlowRttNs).Should(o.BeNumerically(">=", 0)) + } + + g.By("Verify TCP DNS flows") + lokilabels.DstK8SNamespace = DNSNamespace + lokiParams = []string{"DnsFlagsResponseCode=\"NoError\"", "SrcPort=\"53\"", "DstK8S_Name=\"dnsutils1\"", "Proto=\"6\""} + flowRecords, err = lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime, lokiParams...) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of TCP DNS flows > 0") + for _, r := range flowRecords { + o.Expect(r.Flowlog.DNSLatencyMs).Should(o.BeNumerically(">=", 0)) + } + + g.By("Verify UDP DNS flows") + lokiParams = []string{"DnsFlagsResponseCode=\"NoError\"", "SrcPort=\"53\"", "Proto=\"17\""} + flowRecords, err = lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime, lokiParams...) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of UDP DNS flows > 0") + for _, r := range flowRecords { + o.Expect(r.Flowlog.DNSLatencyMs).Should(o.BeNumerically(">=", 0)) + } + + g.By("Verify Packet Translation flows") + lokilabels = Lokilabels{ + App: "netobserv-flowcollector", + DstK8SType: "Service", + DstK8SNamespace: testClientTemplate.ServerNS, + SrcK8SNamespace: testClientTemplate.ClientNS, + } + lokiParams = []string{"ZoneId>0"} + + g.By("Verify PacketTranslation flows") + flowRecords, err = lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime, lokiParams...) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of PacketTranslation flows > 0") + + clientServiceInfo, err := getClientServerInfo(oc, testClientTemplate.ServerNS, testClientTemplate.ClientNS, ipStackType) + verifyPacketTranslationFlows(clientServiceInfo["server"]["ip"], clientServiceInfo["server"]["name"], clientServiceInfo["client"]["ip"], flowRecords) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Verify eBPF feature metrics") + verifyEBPFFeatureMetrics(oc, "pktdropsmap") + verifyEBPFFeatureMetrics(oc, "additionalmap") // for RTT/IPSec map size + verifyEBPFFeatureMetrics(oc, "dnsmap") + verifyEBPFFeatureMetrics(oc, "xlatmap") + }) + + g.It("Author:aramesha-NonPreRelease-High-79015-Verify PacketTranslation feature [Serial]", func() { + SkipIfOCPBelow("v4.14") + g.By("Deploy test server and client pods") + servertemplate := filePath.Join(baseDir, "test-nginx-server_template.yaml") + testServerTemplate := TestServerTemplate{ + ServerNS: "test-server-79015", + ServiceType: "ClusterIP", + Template: servertemplate, + } + defer oc.DeleteSpecifiedNamespaceAsAdmin(testServerTemplate.ServerNS) + err := testServerTemplate.createServer(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + compat_otp.AssertAllPodsToBeReady(oc, testServerTemplate.ServerNS) + + clientTemplate := filePath.Join(baseDir, "test-nginx-client_template.yaml") + testClientTemplate := TestClientTemplate{ + ServerNS: testServerTemplate.ServerNS, + ClientNS: "test-client-79015", + Template: clientTemplate, + } + defer oc.DeleteSpecifiedNamespaceAsAdmin(testClientTemplate.ClientNS) + err = testClientTemplate.createClient(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + compat_otp.AssertAllPodsToBeReady(oc, testClientTemplate.ClientNS) + + g.By("Deploy FlowCollector with PacketTranslation feature enabled") + flow := Flowcollector{ + Namespace: namespace, + EBPFeatures: []string{"\"PacketTranslation\""}, + LokiNamespace: lokiStackNS, + Template: flowFixturePath, + } + + defer func() { _ = flow.DeleteFlowcollector(oc) }() + flow.CreateFlowcollector(oc) + + g.By("Wait for 2 mins before logs gets collected and written to loki") + startTime := time.Now() + time.Sleep(120 * time.Second) + + lokilabels := Lokilabels{ + App: "netobserv-flowcollector", + DstK8SType: "Service", + DstK8SNamespace: testClientTemplate.ServerNS, + SrcK8SNamespace: testClientTemplate.ClientNS, + } + lokiParams := []string{"ZoneId>0"} + + g.By("Verify PacketTranslation flows") + flowRecords, err := lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime, lokiParams...) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of PacketTranslation flows > 0") + + clientServiceInfo, err := getClientServerInfo(oc, testClientTemplate.ServerNS, testClientTemplate.ClientNS, ipStackType) + o.Expect(err).NotTo(o.HaveOccurred()) + verifyPacketTranslationFlows(clientServiceInfo["server"]["ip"], clientServiceInfo["server"]["name"], clientServiceInfo["client"]["ip"], flowRecords) + }) + + // NetworkEvents ebpf hook only supported for OCP >= 4.19 + g.It("Author:memodi-NonPreRelease-Medium-77894-TechPreview Network Policies Correlation [Serial]", func() { + SkipIfOCPBelow("v4.19") + if !compat_otp.IsTechPreviewNoUpgrade(oc) { + g.Skip("Skipping because the TechPreviewNoUpgrade is not enabled on the cluster.") + } + + g.By("Deploy client-server pods in 2 client NS and one Server NS") + serverTemplate := filePath.Join(baseDir, "test-nginx-server_template.yaml") + testServerTemplate := TestServerTemplate{ + ServerNS: "test-server-77894", + Template: serverTemplate, + } + defer oc.DeleteSpecifiedNamespaceAsAdmin(testServerTemplate.ServerNS) + err := testServerTemplate.createServer(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + compat_otp.AssertAllPodsToBeReady(oc, testServerTemplate.ServerNS) + + client1Template := filePath.Join(baseDir, "test-nginx-client_template.yaml") + testClient1Template := TestClientTemplate{ + ServerNS: testServerTemplate.ServerNS, + ClientNS: "test-client1-77894", + Template: client1Template, + } + defer oc.DeleteSpecifiedNamespaceAsAdmin(testClient1Template.ClientNS) + err = testClient1Template.createClient(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + compat_otp.AssertAllPodsToBeReady(oc, testClient1Template.ClientNS) + + testClient2Template := TestClientTemplate{ + ServerNS: testServerTemplate.ServerNS, + ClientNS: "test-client2-77894", + Template: client1Template, + } + defer oc.DeleteSpecifiedNamespaceAsAdmin(testClient2Template.ClientNS) + err = testClient2Template.createClient(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + compat_otp.AssertAllPodsToBeReady(oc, testClient2Template.ClientNS) + + // create flowcollector with NWEvents. + flow := Flowcollector{ + Namespace: namespace, + Template: flowFixturePath, + LokiNamespace: lokiStackNS, + EBPFeatures: []string{"\"NetworkEvents\""}, + EBPFPrivileged: "true", + } + defer func() { _ = flow.DeleteFlowcollector(oc) }() + flow.CreateFlowcollector(oc) + + g.By("Wait for 60 secs before logs gets collected and written to loki") + time.Sleep(60 * time.Second) + + g.By("get flowlogs from loki") + lokilabels := Lokilabels{ + App: "netobserv-flowcollector", + DstK8SNamespace: testClient1Template.ServerNS, + DstK8SType: "Pod", + SrcK8SType: "Pod", + } + lokiParams := []string{"FlowDirection!=1"} + lokilabels.SrcK8SNamespace = testClient1Template.ClientNS + flowRecords, err := lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, time.Now().Add(-2*time.Minute), lokiParams...) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of flowRecords with 'flowDirection != 1' > 0") + + g.By("deploy BANP policy") + banpTemplate := filePath.Join(baseDir, "networking", "baselineadminnetworkPolicy.yaml") + banpParameters := []string{"--ignore-unknown-parameters=true", "-p", "SERVER_NS=" + testClient1Template.ServerNS, "CLIENT1_NS=" + testClient1Template.ClientNS, "CLIENT2_NS=" + testClient2Template.ClientNS, "-f", banpTemplate} + + // banp is a cluster scoped resource so passing empty string for NS arg. + defer deleteResource(oc, "banp", "default", "") + err = compat_otp.ApplyClusterResourceFromTemplateWithError(oc, banpParameters...) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Wait for 60 secs before logs gets collected and written to loki") + time.Sleep(60 * time.Second) + + g.By("check flows have NW Events") + flowRecords, err = lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, time.Now().Add(-45*time.Second), lokiParams...) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of flowRecords with 'flowDirection != 1' > 0") + verifyNetworkEvents(flowRecords, Drop, "BaselineAdminNetworkPolicy", "Ingress") + + g.By("deploy NetworkPolicy") + netpolTemplate := filePath.Join(baseDir, "networking", "networkPolicy.yaml") + netpolName := "allow-ingress" + netPolParameters := []string{"--ignore-unknown-parameters=true", "-p", "NAME=" + netpolName, "SERVER_NS=" + testClient1Template.ServerNS, "ALLOW_NS=" + testClient1Template.ClientNS, "-f", netpolTemplate} + defer deleteResource(oc, "netpol", netpolName, testClient1Template.ServerNS) + err = compat_otp.ApplyClusterResourceFromTemplateWithError(oc, netPolParameters...) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Wait for 60 secs before logs gets collected and written to loki") + time.Sleep(60 * time.Second) + + g.By("check flows from server to client1") + flowRecords, err = lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, time.Now().Add(-1*time.Minute), lokiParams...) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of flowRecords with 'flowDirection != 1' > 0") + verifyNetworkEvents(flowRecords, AllowRelated, "NetworkPolicy", "Ingress") + + g.By("check flows from server to client2") + lokilabels.SrcK8SNamespace = testClient2Template.ClientNS + flowRecords, err = lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, time.Now().Add(-1*time.Minute), lokiParams...) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of flowRecords with 'flowDirection != 1' > 0") + verifyNetworkEvents(flowRecords, Drop, "NetpolNamespace", "Ingress") + + g.By("deploy ANP policy") + anpTemplate := filePath.Join(baseDir, "networking", "adminnetworkPolicy.yaml") + anpName := "server-ns" + anpParameters := []string{"--ignore-unknown-parameters=true", "-p", "NAM=" + anpName, "SERVER_NS=" + testClient1Template.ServerNS, "ALLOW_NS=" + testClient2Template.ClientNS, "DENY_NS=" + testClient1Template.ClientNS, "-f", anpTemplate} + defer deleteResource(oc, "anp", anpName, "") + err = compat_otp.ApplyClusterResourceFromTemplateWithError(oc, anpParameters...) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Wait for 60 secs before logs gets collected and written to loki") + time.Sleep(60 * time.Second) + + g.By("check flows from server to client2") + flowRecords, err = lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, time.Now().Add(-1*time.Minute), lokiParams...) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of flowRecords with 'flowDirection != 1' > 0") + verifyNetworkEvents(flowRecords, AllowRelated, "AdminNetworkPolicy", "Ingress") + + g.By("check flows from server to client1") + lokilabels.SrcK8SNamespace = testClient1Template.ClientNS + flowRecords, err = lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, time.Now().Add(-1*time.Minute), lokiParams...) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of flowRecords with 'flowDirection != 1' > 0") + verifyNetworkEvents(flowRecords, Drop, "AdminNetworkPolicy", "Ingress") + }) + + g.It("Author:aramesha-NonPreRelease-High-80090-Verify FLP tail-based filtering [Serial]", func() { + SkipIfOCPBelow("v4.15") + // Accept flows with Source Namespace = < namespace > and + // Source Name containing 'flowlogs-pipeline-' and + // NOT Source Port 9401 and + // having field TimeFlowRttNs + g.By("Deploy FlowCollector with FLP tail-based filter and FlowRTT enabled") + FLPFiltersConfig := []map[string]any{ + { + "query": fmt.Sprintf(`SrcK8S_Namespace="%s" and SrcK8S_Name=~"flowlogs-pipeline-*" and SrcPort!=9401 and with(TimeFlowRttNs)`, namespace), + "outputTarget": "Loki", + "sampling": 2, + }, + } + + config, err := json.Marshal(FLPFiltersConfig) + o.Expect(err).ToNot(o.HaveOccurred()) + FLPFilter := string(config) + + flow := Flowcollector{ + Namespace: namespace, + Template: flowFixturePath, + Sampling: "2", + LokiNamespace: lokiStackNS, + EBPFeatures: []string{`"FlowRTT"`}, + FLPFilters: FLPFilter, + } + + defer func() { _ = flow.DeleteFlowcollector(oc) }() + flow.CreateFlowcollector(oc) + + // verify logs + g.By("Wait for a min before logs gets collected and written to loki") + startTime := time.Now() + time.Sleep(60 * time.Second) + + lokilabels := Lokilabels{ + App: "netobserv-flowcollector", + SrcK8SNamespace: namespace, + } + + g.By("Verify number of flows > 0") + flowRecords, err := lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of flows > 0") + + for _, r := range flowRecords { + o.Expect(r.Flowlog.SrcK8SName).Should(o.ContainSubstring("flowlogs-pipeline-")) + o.Expect(r.Flowlog.SrcPort).ShouldNot(o.BeNumerically("==", 9401)) + o.Expect(r.Flowlog.TimeFlowRttNs).Should(o.BeNumerically(">", 0)) + o.Expect(r.Flowlog.Sampling).Should(o.BeNumerically("==", 4)) + } + }) + + g.It("Author:aramesha-High-81677-Validate UDN with NetObserv [Serial]", func() { + SkipIfOCPBelow("v4.18") + var ( + namespace = oc.Namespace() + networkingUDNDir, _ = filePath.Abs("testdata/networking/udn") + udnPodTemplate = filePath.Join(networkingUDNDir, "udn_test_pod_template.yaml") + matchLabelKey = "test.io" + matchValue = "netobserv-cudn-" + getRandomString() + cudnName = "cudn-network-81677" + udnName = "udn-network-81677" + cudnNS = []string{"netobserv-cudn1-81677", "netobserv-cudn2-81677"} + udnNS = "netobserv-udn-81677" + ) + + g.By("Create three namespaces, 2 for CUDN, 1 for UDN") + defer deleteNamespace(oc, cudnNS[0]) + defer deleteNamespace(oc, cudnNS[1]) + oc.CreateSpecificNamespaceUDN(cudnNS[0]) + oc.CreateSpecificNamespaceUDN(cudnNS[1]) + for _, ns := range cudnNS { + defer func() { + _ = oc.AsAdmin().WithoutNamespace().Run("label").Args("ns", ns, fmt.Sprintf("%s-", matchLabelKey)).Execute() + }() + err := oc.AsAdmin().WithoutNamespace().Run("label").Args("ns", ns, fmt.Sprintf("%s=%s", matchLabelKey, matchValue)).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + } + + defer deleteNamespace(oc, udnNS) + oc.CreateSpecificNamespaceUDN(udnNS) + + g.By("Deploy CUDN in CUDNns") + var cidr, ipv4cidr, ipv6cidr string + if ipStackType == "ipv4single" { + cidr = "10.150.0.0/16" + } else { + if ipStackType == "ipv6single" { + cidr = "2010:100:200::0/60" + } else { + ipv4cidr = "10.150.0.0/16" + ipv6cidr = "2010:100:200::0/60" + } + } + defer removeResource(oc, true, true, "clusteruserdefinednetwork", cudnName) + _, err := applyCUDNtoMatchLabelNS(oc, matchLabelKey, matchValue, cudnName, ipv4cidr, ipv6cidr, cidr, "layer3") + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Deploy UDN in UDNns") + if ipStackType == "ipv4single" { + cidr = "10.151.0.0/16" + } else { + if ipStackType == "ipv6single" { + cidr = "2011:100:200::0/48" + } else { + ipv4cidr = "10.151.0.0/16" + ipv6cidr = "2011:100:200::0/48" + } + } + createGeneralUDNCRD(oc, udnNS, udnName, ipv4cidr, ipv6cidr, cidr, "layer2") + + g.By("Deploy a pod in each CUDN namespace") + CUDNpods := make([]udnPodResource, 2) + for i, ns := range cudnNS { + CUDNpods[i] = udnPodResource{ + name: "hello-pod-" + ns, + namespace: ns, + label: "hello-pod", + template: udnPodTemplate, + } + defer removeResource(oc, true, true, "pod", CUDNpods[i].name, "-n", CUDNpods[i].namespace) + CUDNpods[i].createUdnPod(oc) + compat_otp.AssertAllPodsToBeReady(oc, CUDNpods[i].namespace) + } + + g.By("Deploy 2 pods in UDN namespace") + UDNpods := make([]udnPodResource, 2) + for j := range UDNpods { + UDNpods[j] = udnPodResource{ + name: fmt.Sprintf("hello-pod-%s-%d", udnNS, j), + namespace: udnNS, + label: "hello-pod", + template: udnPodTemplate, + } + defer removeResource(oc, true, true, "pod", UDNpods[j].name, "-n", UDNpods[j].namespace) + UDNpods[j].createUdnPod(oc) + } + compat_otp.AssertAllPodsToBeReady(oc, udnNS) + + g.By("Deploy FlowCollector with UDNMapping feature enabled with eBPF in privileged mode") + flow := Flowcollector{ + Namespace: namespace, + EBPFPrivileged: "true", + EBPFeatures: []string{"\"UDNMapping\""}, + LokiNamespace: lokiStackNS, + Template: flowFixturePath, + } + + defer func() { _ = flow.DeleteFlowcollector(oc) }() + flow.CreateFlowcollector(oc) + + startTime := time.Now() + + g.By("Validate isolation from an UDN pod to a CUDN pod") + CurlPod2PodFailUDN(oc, udnNS, UDNpods[1].name, CUDNpods[0].namespace, CUDNpods[0].name) + //default network connectivity is isolated + CurlPod2PodFail(oc, udnNS, UDNpods[1].name, CUDNpods[0].namespace, CUDNpods[0].name, ipStackType) + + g.By("Validate isolation from a CUDN pod to an UDN pod") + CurlPod2PodFailUDN(oc, CUDNpods[1].namespace, CUDNpods[1].name, udnNS, UDNpods[1].name) + //default network connectivity is isolated + CurlPod2PodFail(oc, CUDNpods[1].namespace, CUDNpods[1].name, udnNS, UDNpods[1].name, ipStackType) + + g.By("Validate connection among CUDN pods") + CurlPod2PodPassUDN(oc, CUDNpods[0].namespace, CUDNpods[0].name, CUDNpods[1].namespace, CUDNpods[1].name) + //default network connectivity is isolated + CurlPod2PodFail(oc, CUDNpods[0].namespace, CUDNpods[0].name, CUDNpods[1].namespace, CUDNpods[1].name, ipStackType) + + g.By("Validate connection among UDN pods") + CurlPod2PodPassUDN(oc, udnNS, UDNpods[0].name, udnNS, UDNpods[1].name) + //default network connectivity is isolated + CurlPod2PodFail(oc, udnNS, UDNpods[1].name, udnNS, UDNpods[0].name, ipStackType) + + g.By("Wait for 3 mins before logs gets collected and written to loki") + time.Sleep(180 * time.Second) + + g.By("Verify CUDN flows") + lokilabels := Lokilabels{ + App: "netobserv-flowcollector", + DstK8SNamespace: cudnNS[1], + DstK8SOwnerName: CUDNpods[1].name, + SrcK8SNamespace: cudnNS[0], + SrcK8SOwnerName: CUDNpods[0].name, + } + + flowRecords, err := lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of CUDN flows > 0") + for _, r := range flowRecords { + o.Expect(r.Flowlog.Udns).Should(o.ContainElement(cudnName)) + o.Expect(r.Flowlog.DstK8SNetworkName).Should(o.ContainSubstring(cudnName)) + o.Expect(r.Flowlog.SrcK8SNetworkName).Should(o.ContainSubstring(cudnName)) + } + + g.By("Verify UDN flows") + lokilabels = Lokilabels{ + App: "netobserv-flowcollector", + DstK8SNamespace: udnNS, + DstK8SOwnerName: UDNpods[1].name, + SrcK8SNamespace: udnNS, + SrcK8SOwnerName: UDNpods[0].name, + } + + flowRecords, err = lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of UDN flows > 0") + for _, r := range flowRecords { + o.Expect(r.Flowlog.Udns).Should(o.ContainElement(fmt.Sprintf("%s/%s", udnNS, udnName))) + o.Expect(r.Flowlog.DstK8SNetworkName).Should(o.ContainSubstring(fmt.Sprintf("%s/%s", udnNS, udnName))) + o.Expect(r.Flowlog.SrcK8SNetworkName).Should(o.ContainSubstring(fmt.Sprintf("%s/%s", udnNS, udnName))) + } + }) + + g.It("Author:aramesha-High-83022-Validate CUDN with Localnet [Serial]", func() { + SkipIfOCPBelow("v4.19") + var ( + namespace = oc.Namespace() + opNamespace = "openshift-nmstate" + buildPruningBaseDir, _ = filePath.Abs("testdata/networking/nmstate") + testDataDirUDN, _ = filePath.Abs("testdata/networking/udn") + nmstateCRTemplate = filePath.Join(buildPruningBaseDir, "nmstate-cr-template.yaml") + ovnMappingPolicyTemplate = filePath.Join(buildPruningBaseDir, "ovn-mapping-policy-template.yaml") + matchLabelKey = "test.io" + matchValue = "cudn-network-" + getRandomString() + secondaryCUDNName = "secondary-localnet-83022" + nodeSelectLabel = "node-role.kubernetes.io/worker" + udnStatefulSetTemplate = filePath.Join(testDataDirUDN, "udn_statefulset_template.yaml") + cudnNS = []string{"netobserv-cudn1-83022", "netobserv-cudn2-83022"} + ) + + g.By("Check the platform and network plugin type if it is suitable for running the test") + networkType := checkNetworkType(oc) + if !(isPlatformSuitableForNMState(oc)) || !strings.Contains(networkType, "ovn") { + g.Skip("Skipping for unsupported platform or non-OVN network plugin type!") + } + installNMstateOperator(oc) + + workerNode, getNodeErr := compat_otp.GetFirstWorkerNode(oc) + o.Expect(getNodeErr).NotTo(o.HaveOccurred()) + o.Expect(workerNode).NotTo(o.BeEmpty()) + + compat_otp.By("Create NMState CR") + nmstateCR := nmstateCRResource{ + name: "nmstate", + template: nmstateCRTemplate, + } + defer deleteNMStateCR(oc, nmstateCR) + result, crErr := createNMStateCR(oc, nmstateCR, opNamespace) + compat_otp.AssertWaitPollNoErr(crErr, "create nmstate cr failed") + o.Expect(result).To(o.BeTrue()) + e2e.Logf("SUCCESS - NMState CR Created") + + compat_otp.By("Configure NNCP for creating OvnMapping NMstate Feature") + ovnMappingPolicy := ovnMappingPolicyResource{ + name: "bridge-mapping-83022", + nodelabel: nodeSelectLabel, + labelvalue: "", + localnet1: "mylocalnet", + bridge1: "br-ex", + template: ovnMappingPolicyTemplate, + } + defer deleteNNCP(oc, ovnMappingPolicy.name) + defer func() { + ovnmapping, deferErr := compat_otp.DebugNodeWithChroot(oc, workerNode, "ovs-vsctl", "get", "Open_vSwitch", ".", "external_ids:ovn-bridge-mappings") + o.Expect(deferErr).NotTo(o.HaveOccurred()) + if strings.Contains(ovnmapping, ovnMappingPolicy.localnet1) { + // ovs-vsctl can only use "set" to reserve some fields + _, err := compat_otp.DebugNodeWithChroot(oc, workerNode, "ovs-vsctl", "set", "Open_vSwitch", ".", "external_ids:ovn-bridge-mappings=\"physnet:br-ex\"") + o.Expect(err).NotTo(o.HaveOccurred()) + } + }() + configErr3 := ovnMappingPolicy.configNNCP(oc) + o.Expect(configErr3).NotTo(o.HaveOccurred()) + nncpErr3 := checkNNCPStatus(oc, ovnMappingPolicy.name, "Available") + compat_otp.AssertWaitPollNoErr(nncpErr3, fmt.Sprintf("%s policy applied failed", ovnMappingPolicy.name)) + + compat_otp.By("Create two namespaces and label them") + for _, ns := range cudnNS { + defer oc.DeleteSpecifiedNamespaceAsAdmin(ns) + oc.CreateSpecifiedNamespaceAsAdmin(ns) + err := oc.AsAdmin().WithoutNamespace().Run("label").Args("ns", ns, fmt.Sprintf("%s=%s", matchLabelKey, matchValue)).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + } + + compat_otp.By("Create secondary localnet CUDN") + defer removeResource(oc, true, true, "clusteruserdefinednetwork", secondaryCUDNName) + _, err := applyLocalnetCUDNtoMatchLabelNS(oc, matchLabelKey, matchValue, secondaryCUDNName, "mylocalnet", "192.168.100.0/24", "192.168.100.1/32", false) + o.Expect(err).NotTo(o.HaveOccurred()) + + compat_otp.By("Deploy statefulset in both cudnNS") + for _, ns := range cudnNS { + defer removeResource(oc, true, true, "statefulset", "hello", "-n", ns) + compat_otp.ApplyNsResourceFromTemplate(oc, ns, "-f", udnStatefulSetTemplate, "NETWORK_NAME="+secondaryCUDNName) + compat_otp.AssertAllPodsToBeReady(oc, ns) + } + + g.By("Deploy FlowCollector with UDNMapping feature enabled with eBPF in privileged mode") + flow := Flowcollector{ + Namespace: namespace, + EBPFPrivileged: "true", + EBPFeatures: []string{"\"UDNMapping\""}, + LokiNamespace: lokiStackNS, + Template: flowFixturePath, + } + + defer func() { _ = flow.DeleteFlowcollector(oc) }() + flow.CreateFlowcollector(oc) + + g.By("Validate connection among CUDN pods") + cudn1Pods, err := compat_otp.GetAllPods(oc, cudnNS[0]) + o.Expect(err).NotTo(o.HaveOccurred()) + cudn2Pods, err := compat_otp.GetAllPods(oc, cudnNS[1]) + o.Expect(err).NotTo(o.HaveOccurred()) + startTime := time.Now() + CurlPod2PodPassUDN(oc, cudnNS[0], cudn1Pods[0], cudnNS[1], cudn2Pods[0]) + + g.By("Wait for 2 mins before logs gets collected and written to loki") + time.Sleep(120 * time.Second) + + g.By("Verify CUDN Localnet flows") + lokilabels := Lokilabels{ + App: "netobserv-flowcollector", + DstK8SNamespace: cudnNS[1], + DstK8SOwnerName: "hello", + SrcK8SNamespace: cudnNS[0], + SrcK8SOwnerName: "hello", + } + + flowRecords, err := lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of CUDN Localnet flows > 0") + for _, r := range flowRecords { + o.Expect(r.Flowlog.Udns).Should(o.ContainElement(secondaryCUDNName)) + o.Expect(r.Flowlog.DstK8SNetworkName).Should(o.ContainSubstring(secondaryCUDNName)) + o.Expect(r.Flowlog.SrcK8SNetworkName).Should(o.ContainSubstring(secondaryCUDNName)) + } + }) + + g.It("Author:aramesha-NonPreRelease-Longduration-Medium-81410-NetObserv with eBPF manager [Serial][Slow]", func() { + SkipIfOCPBelow("v4.18") + g.By("Deploy eBPF manager operator") + // eBPF manager operator variables + bpfDir, _ := filePath.Abs("testdata/bpfman") + bpfIDMS := filePath.Join(bpfDir, "image-digest-mirror-set.yaml") + bpfCatSrcTemplate := filePath.Join(bpfDir, "catalog-source.yaml") + + bpfNS := OperatorNamespace{ + Name: "bpfman", + NamespaceTemplate: filePath.Join(bpfDir, "namespace.yaml"), + } + bpfCatSrc := Resource{"catsrc", "bpfman-konflux-fbc", bpfNS.Name} + bpfSource := CatalogSourceObjects{"stable", bpfCatSrc.Name, bpfNS.Name} + + g.By("Deploy bpfman konflux FBC and ImageDigestMirrorSet") + bpfNS.DeployOperatorNamespace(oc) + catsrcErr := bpfCatSrc.applyFromTemplate(oc, "-n", bpfNS.Name, "-f", bpfCatSrcTemplate, "-p", "NAMESPACE="+bpfNS.Name) + o.Expect(catsrcErr).NotTo(o.HaveOccurred()) + bpfCatSrc.WaitUntilCatSrcReady(oc) + ApplyResourceFromFile(oc, bpfNS.Name, bpfIDMS) + + BPF := SubscriptionObjects{ + OperatorName: "bpfman-operator", + Namespace: "bpfman", + PackageName: "bpfman-operator", + Subscription: filePath.Join(subscriptionDir, "sub-template.yaml"), + OperatorGroup: filePath.Join(subscriptionDir, "allnamespace-og.yaml"), + CatalogSource: &bpfSource, + } + + bpfExisting, err := CheckOperatorStatus(oc, BPF.Namespace, BPF.PackageName) + o.Expect(err).NotTo(o.HaveOccurred()) + // Deploy eBPF manager operator if not present + if !bpfExisting { + ensureOperatorDeployed(oc, BPF, bpfSource, "name=bpfman-daemon") + } + + g.By("Deploy FlowCollector with PacketDrop and Ebpfmanager enabled") + flow := Flowcollector{ + Namespace: namespace, + EBPFeatures: []string{"\"PacketDrop\", \"EbpfManager\""}, + LokiNamespace: lokiStackNS, + Template: flowFixturePath, + } + + defer func() { _ = flow.DeleteFlowcollector(oc) }() + flow.CreateFlowcollector(oc) + + g.By("Wait for 4 mins before logs gets collected and written to loki") + startTime := time.Now() + time.Sleep(240 * time.Second) + + lokilabels := Lokilabels{ + App: "netobserv-flowcollector", + } + + g.By("Verify Packet Drop flows") + lokiParams := []string{"PktDropLatestState=\"TCP_INVALID_STATE\"", "Proto=\"6\""} + flowRecords, err := lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime, lokiParams...) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of TCP Invalid State flows > 0") + for _, r := range flowRecords { + o.Expect(r.Flowlog.PktDropLatestDropCause).NotTo(o.BeEmpty()) + o.Expect(r.Flowlog.PktDropBytes).Should(o.BeNumerically(">", 0)) + o.Expect(r.Flowlog.PktDropPackets).Should(o.BeNumerically(">", 0)) + } + + lokiParams = []string{"PktDropLatestDropCause=\"SKB_DROP_REASON_NO_SOCKET\"", "Proto=\"6\""} + flowRecords, err = lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime, lokiParams...) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of No Socket TCP flows > 0") + for _, r := range flowRecords { + o.Expect(r.Flowlog.PktDropLatestState).NotTo(o.BeEmpty()) + o.Expect(r.Flowlog.PktDropBytes).Should(o.BeNumerically(">", 0)) + o.Expect(r.Flowlog.PktDropPackets).Should(o.BeNumerically(">", 0)) + } + }) + + g.It("Author:memodi-NonPreRelease-High-82637-Verify IPSec feature [Disruptive]", func() { + SkipIfOCPBelow("v4.16") + compat_otp.By("Check if IPSec is enabled in the cluster") + ipsecEnabled, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("networks.operator.openshift.io", "cluster", "-ojsonpath='{.spec.defaultNetwork.ovnKubernetesConfig.ipsecConfig.mode}'").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + ipsecEnabled = strings.Trim(ipsecEnabled, "'") + if ipsecEnabled != "Full" { + g.Skip("IPSec is not enabled in Full mode, skipping test") + } + + g.By("Deploy FlowCollector IPSec enabled") + flow := Flowcollector{ + Namespace: namespace, + EBPFeatures: []string{"\"IPSec\""}, + LokiNamespace: lokiStackNS, + Template: flowFixturePath, + } + + defer func() { _ = flow.DeleteFlowcollector(oc) }() + flow.CreateFlowcollector(oc) + + g.By("Wait for 2 mins before logs gets collected and written to loki") + startTime := time.Now() + time.Sleep(120 * time.Second) + + lokilabels := Lokilabels{ + App: "netobserv-flowcollector", + } + g.By("Verify IPSec flows") + lokiParams := []string{"IPSecStatus=\"success\""} + flowRecords, err := lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime, lokiParams...) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of IPSecStatus==success flows > 0") + metrics, err := getMetric(oc, "sum(netobserv_node_ipsec_flows_total{IPSecStatus=\"success\"})") + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(popMetricValue(metrics)).Should(o.BeNumerically(">", 0)) + o.Expect(err).NotTo(o.HaveOccurred()) + verifyEBPFFeatureMetrics(oc, "additionalmap") // additionalMap for RTT/IPSec map size + }) + + g.It("Author:kapjain-NonPreRelease-Longduration-High-85953-Verify FlowCollector Service deployment model [Serial][Slow]", func() { + SkipIfOCPBelow("v4.14") + g.By("Deploy FlowCollector with Service deployment model") + flow := Flowcollector{ + Namespace: namespace, + DeploymentModel: "Service", + Template: flowFixturePath, + LokiNamespace: lokiStackNS, + } + + defer func() { _ = flow.DeleteFlowcollector(oc) }() + flow.CreateFlowcollector(oc) + + g.By("Wait for pods to fully start and emit startup logs") + time.Sleep(30 * time.Second) + + g.By("Verify FLP logs show 'Starting GRPC server with TLS'") + FLPpods, err := compat_otp.GetAllPodsWithLabel(oc, flow.Namespace, "app=flowlogs-pipeline") + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Get FLP pod logs to check GRPC server startup message") + flpLogs, err := oc.AsAdmin().WithoutNamespace().Run("logs").Args("-n", flow.Namespace, FLPpods[0], "--tail=100").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(flpLogs).To(o.ContainSubstring("Starting GRPC server with TLS"), "FLP logs should contains 'Starting GRPC server with TLS'") + + g.By("Verify agent logs show 'Starting GRPC client with TLS'") + agentPods, err := compat_otp.GetAllPodsWithLabel(oc, flow.Namespace+"-privileged", "app=netobserv-ebpf-agent") + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Get agent pod logs to check GRPC client startup message") + agentLogs, err := oc.AsAdmin().WithoutNamespace().Run("logs").Args("-n", flow.Namespace+"-privileged", agentPods[0], "--tail=100").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(agentLogs).To(o.ContainSubstring("Starting GRPC client with TLS"), "Agent logs should contains 'Starting GRPC client with TLS'") + + g.By("Wait for a min before logs gets collected and written to loki in TLS mode") + startTime := time.Now() + time.Sleep(60 * time.Second) + + g.By("Get flowlogs from loki") + err = verifyLokilogsTime(kubeadminToken, ls.Route, startTime) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Verify default FLP Deployment is created with 3 pods") + result := verifyDeploymentReplicas(oc, "flowlogs-pipeline", flow.Namespace, 3, "") + o.Expect(result).To(o.BeTrue(), "By default the replica count should be 3") + + g.By("Verify Service is created with correct port configuration") + service, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "flowlogs-pipeline", "-n", flow.Namespace, "-o", "jsonpath='{.spec.ports[0].port}'").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + service = strings.Trim(service, "'") + o.Expect(service).To(o.Equal("2055")) + + targetPort, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "flowlogs-pipeline", "-n", flow.Namespace, "-o", "jsonpath='{.spec.ports[0].targetPort}'").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + targetPort = strings.Trim(targetPort, "'") + o.Expect(targetPort).To(o.Equal("2055")) + + // Test replica management with unmanagedReplicas: False by default + g.By("Verify deployment does not upscale when unmanagedReplicas is false or not set") + err = oc.AsAdmin().WithoutNamespace().Run("scale").Args("deployment", "flowlogs-pipeline", "-n", flow.Namespace, "--replicas=4").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + result = verifyDeploymentReplicas(oc, "flowlogs-pipeline", flow.Namespace, 3, "") + o.Expect(result).To(o.BeTrue(), "Deployment should not scale when unmanagedReplicas is false or not set") + + g.By("Verify deployment does not downscale when unmanagedReplicas is false or not set") + err = oc.AsAdmin().WithoutNamespace().Run("scale").Args("deployment", "flowlogs-pipeline", "-n", flow.Namespace, "--replicas=2").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + result = verifyDeploymentReplicas(oc, "flowlogs-pipeline", flow.Namespace, 3, "") + o.Expect(result).To(o.BeTrue(), "Deployment should not scale when unmanagedReplicas is false or not set") + + g.By("Verify deployment scales via consumerReplicas when unmanagedReplicas is false or not set - upscale to 4") + err = oc.AsAdmin().WithoutNamespace().Run("patch").Args("flowcollector", "cluster", "--type=merge", "-p", `{"spec":{"processor":{"consumerReplicas":4}}}`).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + result = verifyDeploymentReplicas(oc, "flowlogs-pipeline", flow.Namespace, 4, "") + o.Expect(result).To(o.BeTrue(), "Deployment should scale via consumerReplicas when unmanagedReplicas is false or not set") + + g.By("Verify deployment scales via consumerReplicas when unmanagedReplicas is false or not set - downscale to 2") + err = oc.AsAdmin().WithoutNamespace().Run("patch").Args("flowcollector", "cluster", "--type=merge", "-p", `{"spec":{"processor":{"consumerReplicas":2}}}`).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + result = verifyDeploymentReplicas(oc, "flowlogs-pipeline", flow.Namespace, 2, "") + o.Expect(result).To(o.BeTrue(), "Deployment should scale via consumerReplicas when unmanagedReplicas is false or not set") + + // Test replica management with unmanagedReplicas: True + g.By("Enable unmanagedReplicas and set consumerReplicas to 3") + err = oc.AsAdmin().WithoutNamespace().Run("patch").Args("flowcollector", "cluster", "--type=merge", "-p", `{"spec":{"processor":{"unmanagedReplicas":true,"consumerReplicas":3}}}`).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Verify deployment upscales when unmanagedReplicas is true") + err = oc.AsAdmin().WithoutNamespace().Run("scale").Args("deployment", "flowlogs-pipeline", "-n", flow.Namespace, "--replicas=4").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + result = verifyDeploymentReplicas(oc, "flowlogs-pipeline", flow.Namespace, 4, "") + o.Expect(result).To(o.BeTrue(), "Deployment should scale when unmanagedReplicas is true") + + g.By("Verify deployment downscales when unmanagedReplicas is true") + err = oc.AsAdmin().WithoutNamespace().Run("scale").Args("deployment", "flowlogs-pipeline", "-n", flow.Namespace, "--replicas=1").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + result = verifyDeploymentReplicas(oc, "flowlogs-pipeline", flow.Namespace, 1, "") + o.Expect(result).To(o.BeTrue(), "Deployment should scale when unmanagedReplicas is true") + + g.By("Verify consumerReplicas change does not scale deployment when unmanagedReplicas is true") + err = oc.AsAdmin().WithoutNamespace().Run("patch").Args("flowcollector", "cluster", "--type=merge", "-p", `{"spec":{"processor":{"consumerReplicas":4}}}`).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + result = verifyDeploymentReplicas(oc, "flowlogs-pipeline", flow.Namespace, 1, "") + o.Expect(result).To(o.BeTrue(), "Deployment should not scale via consumerReplicas when unmanagedReplicas is true") + + g.By("Verify HPA scales the deployment when unmanagedReplicas is true") + hpaYAML := filePath.Join(baseDir, "flowlogs_pipeline_hpa_template.yaml") + hpaFile := compat_otp.ProcessTemplate(oc, "--ignore-unknown-parameters=true", "-f", hpaYAML, "-p", "NAMESPACE="+namespace) + defer func() { + _ = oc.AsAdmin().WithoutNamespace().Run("delete").Args("hpa", "flowlogs-pipeline-hpa", "-n", flow.Namespace).Execute() + }() + err = oc.WithoutNamespace().Run("create").Args("-f", hpaFile).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + result = verifyDeploymentReplicas(oc, "flowlogs-pipeline", flow.Namespace, 4, ">") + o.Expect(result).To(o.BeTrue(), "HPA should scale deployment above 4 replicas when unmanagedReplicas is true") + + g.By("Verify HPA does not scale deployment when unmanagedReplicas is false") + err = oc.AsAdmin().WithoutNamespace().Run("patch").Args("flowcollector", "cluster", "--type=merge", "-p", `{"spec":{"processor":{"unmanagedReplicas":false,"consumerReplicas":2}}}`).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + result = verifyDeploymentReplicas(oc, "flowlogs-pipeline", flow.Namespace, 2, "") + o.Expect(result).To(o.BeTrue(), "Deployment should be reconciled to consumerReplicas=2 when unmanagedReplicas is false") + }) + + g.It("Author:kapjain-Medium-86372-Verify Gateway API three-level owner metadata [Serial]", func() { + SkipIfOCPBelow("v4.19") + startTime := time.Now() + g.By("Deploy flowcollector") + gatewayAPITemplate := filePath.Join(baseDir, "gateway-api-template.yaml") + flow := Flowcollector{ + Namespace: namespace, + Template: flowFixturePath, + LokiNamespace: lokiStackNS, + } + + defer func() { _ = flow.DeleteFlowcollector(oc) }() + flow.CreateFlowcollector(oc) + + g.By("Deploying Gateway API resources from template") + gatewayNS := "netobserv-gateway-test" + gatewayName := "test-gateway-owner" + defer oc.DeleteSpecifiedNamespaceAsAdmin(gatewayNS) + err := applyResourceFromTemplateByAdmin(oc, "-f", gatewayAPITemplate) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Verifying Gateway Deployment exists") + // The Gateway controller creates a Deployment named gateway-name + gatewayclass-name + deploymentName := gatewayName + "-openshift-default" + WaitForDeploymentPodsToBeReady(oc, gatewayNS, deploymentName) + + g.By("Verifying Pods are created by Gateway") + pods, err := compat_otp.GetAllPodsWithLabel(oc, gatewayNS, fmt.Sprintf("gateway.networking.k8s.io/gateway-name=%s", gatewayName)) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(pods)).Should(o.BeNumerically(">", 0), "expected at least one Gateway pod") + + g.By("Waiting for flow data to be collected and written to Loki") + time.Sleep(120 * time.Second) + + g.By("Querying flow data from Loki for Gateway pods") + lokilabels := Lokilabels{ + SrcK8SNamespace: "netobserv-gateway-test", + } + parameters := []string{"SrcK8S_OwnerType=\"Gateway\""} + flowRecords, err := lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime, parameters...) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of Gateway Owner flows > 0") + }) + g.It("Author:kapjain-Medium-88334-Pause Network Observability functions [Serial]", func() { + SkipIfOCPBelow("v4.14") + g.By("Create a FlowCollector") + flow := Flowcollector{ + Namespace: namespace, + Template: flowFixturePath, + LokiNamespace: lokiStackNS, + DeploymentModel: "Service", + SlicesEnable: "true", + } + defer func() { _ = flow.DeleteFlowcollector(oc) }() + flow.CreateFlowcollector(oc) + flow.WaitForFlowcollectorReady(oc) + + g.By("Get all netobserv-managed components before pause (excluding pods with dynamic IDs)") + componentsBeforePause, err := oc.AsAdmin().WithoutNamespace().Run("get").Args( + "service,deployment,daemonset,serviceaccount,networkpolicy,configmap,secret", + "-A", "-l", "netobserv-managed=true", "-o", "name", + ).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("Components before pause (stable names): %s", componentsBeforePause) + + g.By("Pause the FlowCollector") + err = oc.AsAdmin().WithoutNamespace().Run("patch").Args( + "flowcollector", "cluster", + "--type=merge", + "-p", `{"spec":{"execution":{"mode":"OnHold"}}}`, + ).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Wait for FlowCollector status to show 'on hold' message") + err = wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 150*time.Second, false, func(context.Context) (done bool, err error) { + onHoldConditions, err := oc.AsAdmin().WithoutNamespace().Run("get").Args( + "flowcollector", "cluster", + "-o", `jsonpath={.status.conditions[?(@.message=="FlowCollector is on hold")]}`, + ).Output() + if err != nil { + e2e.Logf("Error getting FlowCollector status: %v", err) + return false, nil + } + if onHoldConditions != "" { + e2e.Logf("FlowCollector status shows 'on hold'") + return true, nil + } + e2e.Logf("Waiting for FlowCollector to show 'on hold' status...") + return false, nil + }) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Verify except for netobserv-plugin-static and network policies and persistent configmaps, all components are deleted") + componentsAfterPause, err := oc.AsAdmin().WithoutNamespace().Run("get").Args( + "service,deployment,daemonset,serviceaccount,networkpolicy,configmap,secret", + "-A", "-l", "netobserv-managed=true", "-o", "name", + ).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("Components after pause: %s", componentsAfterPause) + + // Components with stable names that should remain when paused + componentsShouldRemain := []string{ + "deployment.apps/netobserv-plugin-static", + "service/netobserv-plugin-static", + "networkpolicy.networking.k8s.io/netobserv", + "configmap/lokistack-ca-bundle", + "configmap/lokistack-gateway-ca-bundle", + "configmap/grafana-dashboard-netobserv-health", + "configmap/netobserv-main", + "secret/lokistack-query-frontend-http", + } + verifyComponentsExist(componentsAfterPause, componentsShouldRemain) + + // Verify netobserv-plugin-static pod exists and other pods are deleted (using pattern since pod names have dynamic IDs) + podsAfterPause, err := oc.AsAdmin().WithoutNamespace().Run("get").Args( + "pod", "-A", "-l", "netobserv-managed=true", "-o", "name", + ).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(podsAfterPause).Should(o.ContainSubstring("pod/netobserv-plugin-static-"), "netobserv-plugin-static pod should exist after pause") + o.Expect(podsAfterPause).ShouldNot(o.ContainSubstring("pod/flowlogs-pipeline-"), "flowlogs-pipeline pods should be deleted") + o.Expect(podsAfterPause).ShouldNot(o.ContainSubstring("pod/netobserv-ebpf-agent-"), "netobserv-ebpf-agent pods should be deleted") + // Verify regular netobserv-plugin pod is deleted (not the static one) + podLines := strings.Split(podsAfterPause, "\n") + for _, podLine := range podLines { + if strings.Contains(podLine, "pod/netobserv-plugin-") && !strings.Contains(podLine, "pod/netobserv-plugin-static-") { + e2e.Failf("Found non-static netobserv-plugin pod that should be deleted: %s", podLine) + } + } + + // Build list of components that should be deleted = originalComponentsList - componentsShouldRemain + originalComponentsList := strings.Split(strings.TrimSpace(componentsBeforePause), "\n") + var componentsShouldDelete []string + for _, component := range originalComponentsList { + component = strings.TrimSpace(component) + if component == "" { + continue + } + // Check if this component should remain + shouldRemain := false + for _, remainComponent := range componentsShouldRemain { + if component == remainComponent { + shouldRemain = true + break + } + } + // If it shouldn't remain, add to delete list + if !shouldRemain { + componentsShouldDelete = append(componentsShouldDelete, component) + } + } + // Verify all components in the delete list are actually deleted + verifyComponentsDeleted(componentsAfterPause, componentsShouldDelete) + + g.By("Resume the FlowCollector") + resumeTime := time.Now() + err = oc.AsAdmin().WithoutNamespace().Run("patch").Args( + "flowcollector", "cluster", + "--type=merge", + "-p", `{"spec":{"execution":{"mode":"Running"}}}`, + ).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Wait for FlowCollector to be ready") + flow.WaitForFlowcollectorReady(oc) + + g.By("Verify no 'on hold' message in FlowCollector status") + onHoldConditionsAfterResume, err := oc.AsAdmin().WithoutNamespace().Run("get").Args( + "flowcollector", "cluster", + "-o", `jsonpath={.status.conditions[?(@.message=="FlowCollector is on hold")]}`, + ).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(onHoldConditionsAfterResume).Should(o.BeEmpty()) + + g.By("Wait for a min before logs gets collected and written to loki after resume") + time.Sleep(60 * time.Second) + + g.By("Verify flows are being created in Loki after resume") + err = verifyLokilogsTime(kubeadminToken, ls.Route, resumeTime) + o.Expect(err).NotTo(o.HaveOccurred()) + }) + + g.It("Author:aramesha-NonPreRelease-High-88455-Verify TLS Tracking feature [Serial]", func() { + SkipIfOCPBelow("v4.14") + g.By("Deploy TLS test server and client pods") + servertemplate := filePath.Join(baseDir, "test-tls-server_template.yaml") + testServerTemplate := TestServerTemplate{ + ServerNS: "test-server-88455", + Template: servertemplate, + } + defer oc.DeleteSpecifiedNamespaceAsAdmin(testServerTemplate.ServerNS) + err := testServerTemplate.createServer(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + compat_otp.AssertAllPodsToBeReady(oc, testServerTemplate.ServerNS) + + clientTemplate := filePath.Join(baseDir, "test-tls-client_template.yaml") + testClientTemplate := TestClientTemplate{ + ServerNS: testServerTemplate.ServerNS, + ClientNS: "test-client-88455", + Template: clientTemplate, + } + defer oc.DeleteSpecifiedNamespaceAsAdmin(testClientTemplate.ClientNS) + err = testClientTemplate.createClient(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + compat_otp.AssertAllPodsToBeReady(oc, testClientTemplate.ClientNS) + + g.By("Deploy FlowCollector with TLS Tracking feature enabled") + flow := Flowcollector{ + Namespace: namespace, + EBPFeatures: []string{"\"TLSTracking\""}, + LokiNamespace: lokiStackNS, + Template: flowFixturePath, + } + + defer func() { _ = flow.DeleteFlowcollector(oc) }() + flow.CreateFlowcollector(oc) + + g.By("Wait for a min before logs gets collected and written to loki") + startTime := time.Now() + time.Sleep(60 * time.Second) + + lokilabels := Lokilabels{ + App: "netobserv-flowcollector", + SrcK8SNamespace: testClientTemplate.ServerNS, + DstK8SNamespace: testClientTemplate.ClientNS, + DstK8SOwnerName: "tls-client", + SrcK8SOwnerName: "tls-server-service", + } + + g.By("Verify HTTP flows") + lokiParams := []string{"Proto=\"6\"", "SrcPort=\"80\""} + flowRecords, err := lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime, lokiParams...) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of HTTP flows > 0") + // Verify TLS fields are not populated + for _, r := range flowRecords { + o.Expect(r.Flowlog.TLSVersion).Should(o.BeEmpty(), "expected TLS version to be empty for HTTP") + } + + g.By("Verify HTTPS flows with TLSVersion 1.2") + lokiParams = []string{"Proto=\"6\"", "SrcPort=\"443\"", "TLSVersion=\"TLS 1.2\""} + flowRecords, err = lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime, lokiParams...) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of HTTPS flows with TLSv1.2 > 0") + // Verify TLS 1.2 fields + for _, r := range flowRecords { + o.Expect(r.Flowlog.TLSTypes).Should(o.ContainElement("ServerHello"), "expected TLS Types to contain ServerHello") + o.Expect(r.Flowlog.TLSCipherSuite).NotTo(o.BeEmpty()) + // Will be fixed in follow-up + // o.Expect(r.Flowlog.TLSCurve).NotTo(o.BeEmpty()) + } + + g.By("Verify HTTPS flows with TLSVersion 1.3") + lokiParams = []string{"Proto=\"6\"", "SrcPort=\"443\"", "TLSVersion=\"TLS 1.3\""} + flowRecords, err = lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime, lokiParams...) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of HTTPS flows with TLSv1.3 > 0") + // Verify TLS 1.3 fields + for _, r := range flowRecords { + o.Expect(r.Flowlog.TLSTypes).Should(o.ContainElement("ServerHello"), "expected TLS Types to contain ServerHello") + o.Expect(r.Flowlog.TLSCurve).Should(o.ContainSubstring("X25519")) + o.Expect(r.Flowlog.TLSCipherSuite).NotTo(o.BeEmpty()) + } + }) + + g.It("Author:kapjain-Medium-88683-Secure communications between Agent and FLP [Serial]", func() { + SkipIfOCPBelow("v4.14") + var ( + certManagerPackageName = "openshift-cert-manager-operator" + certManagerNS = "cert-manager-operator" + certManagerSource CatalogSourceObjects + certManagerCatalog = "redhat-operators" + certTemplatePath = filePath.Join(baseDir, "cert_manager_certificates_template.yaml") + ) + + certManager := SubscriptionObjects{ + OperatorName: "cert-manager-operator-controller-manager", + Namespace: certManagerNS, + PackageName: certManagerPackageName, + Subscription: filePath.Join(subscriptionDir, "sub-template.yaml"), + OperatorGroup: filePath.Join(subscriptionDir, "allnamespace-og.yaml"), + CatalogSource: &certManagerSource, + } + + g.By("Deploy cert-manager Operator") + // check if cert-manager Operator exists + certManagerExisting, err := CheckOperatorStatus(oc, certManager.Namespace, certManager.PackageName) + o.Expect(err).NotTo(o.HaveOccurred()) + + certManagerChannel, err := getOperatorChannel(oc, certManagerCatalog, certManagerPackageName) + if err != nil || certManagerChannel == "" { + g.Skip("cert-manager channel not found, skipping test") + } + certManagerSource = CatalogSourceObjects{certManagerChannel, certManagerCatalog, "openshift-marketplace"} + + if !certManagerExisting { + // Create namespace for cert-manager operator + certManagerNSObj := OperatorNamespace{ + Name: certManagerNS, + NamespaceTemplate: filePath.Join(subscriptionDir, "namespace.yaml"), + } + certManagerNSObj.DeployOperatorNamespace(oc) + + ensureOperatorDeployed(oc, certManager, certManagerSource, "name=cert-manager-operator") + } + + defer func() { + if !certManagerExisting { + certManager.uninstallOperator(oc) + oc.DeleteSpecifiedNamespaceAsAdmin(certManagerNS) + } + }() + + g.By("Wait for cert-manager CRDs to be available") + err = wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 180*time.Second, false, func(context.Context) (done bool, err error) { + issuerCRD, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("crd", "issuers.cert-manager.io").Output() + certCRD, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("crd", "certificates.cert-manager.io").Output() + if strings.Contains(issuerCRD, "issuers.cert-manager.io") && strings.Contains(certCRD, "certificates.cert-manager.io") { + e2e.Logf("cert-manager CRDs are available") + return true, nil + } + e2e.Logf("Waiting for cert-manager CRDs to be available...") + return false, nil + }) + compat_otp.AssertWaitPollNoErr(err, "cert-manager CRDs did not become available") + + g.By("Create certificates using cert-manager") + certFile := compat_otp.ProcessTemplate(oc, "--ignore-unknown-parameters=true", "-f", certTemplatePath, "-p", "Namespace="+namespace) + err = oc.AsAdmin().WithoutNamespace().Run("apply").Args("-f", certFile, "-n", namespace).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + defer func() { + _ = oc.AsAdmin().WithoutNamespace().Run("delete").Args("-f", certFile, "-n", namespace).Execute() + }() + + g.By("Wait for certificate secrets to be created") + err = wait.PollUntilContextTimeout(context.Background(), 30*time.Second, 300*time.Second, false, func(context.Context) (done bool, err error) { + _, err = oc.AsAdmin().WithoutNamespace().Run("get").Args("secret", "prov-netobserv-ca-secret", "-n", namespace).Output() + if err != nil { + return false, nil + } + _, err = oc.AsAdmin().WithoutNamespace().Run("get").Args("secret", "prov-flowlogs-pipeline-cert", "-n", namespace).Output() + if err != nil { + return false, nil + } + _, err = oc.AsAdmin().WithoutNamespace().Run("get").Args("secret", "prov-ebpf-agent-cert", "-n", namespace).Output() + if err != nil { + return false, nil + } + return true, nil + }) + compat_otp.AssertWaitPollNoErr(err, "certificate secrets did not become available") + + g.By("Deploy FlowCollector in Service mode with Provided TLS certificates") + flow := Flowcollector{ + Namespace: namespace, + Template: flowFixturePath, + DeploymentModel: "Service", + ServiceTLSType: "Provided", + ServiceCASecretName: "prov-netobserv-ca-secret", + ServiceServerCertSecretName: "prov-flowlogs-pipeline-cert", + ServiceClientCertSecretName: "prov-ebpf-agent-cert", + LokiNamespace: lokiStackNS, + } + + defer func() { _ = flow.DeleteFlowcollector(oc) }() + flow.CreateFlowcollector(oc) + + g.By("Verify eBPF agent is using mTLS") + ebpfPods, err := compat_otp.GetAllPods(oc, namespace+"-privileged") + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(ebpfPods)).Should(o.BeNumerically(">", 0), "No eBPF agent pods found") + + ebpfLogs, err := oc.AsAdmin().WithoutNamespace().Run("logs").Args("-n", namespace+"-privileged", ebpfPods[0]).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(ebpfLogs).To(o.ContainSubstring("Starting GRPC client with mTLS"), "eBPF agent logs should show mTLS is enabled") + + g.By("Wait for flow logs to be collected and written to Loki") + startTime := time.Now() + time.Sleep(60 * time.Second) + + g.By("Verify flow logs are being stored in Loki using verifyLokilogsTime") + err = verifyLokilogsTime(kubeadminToken, ls.Route, startTime) + o.Expect(err).NotTo(o.HaveOccurred()) + }) + //Add future NetObserv + Loki test-cases here + + g.Context("with Kafka", func() { + var ( + kafkaDir, kafkaTopicPath, kafkaNodePoolPath string + AMQexisting = false + amq SubscriptionObjects + kafkaMetrics KafkaMetrics + kafka Kafka + kafkaTopic KafkaTopic + kafkaNodePool KafkaNodePool + kafkaUser KafkaUser + kafkaNs = "netobserv-kafka" + kafkaClusterName = "kafka-cluster" + kafkaAddress = fmt.Sprintf("%s-kafka-bootstrap.%s:9093", kafkaClusterName, kafkaNs) + additionalNamespaces = fmt.Sprintf("\"%s\"", kafkaNs) + ) + + g.BeforeEach(func() { + kafkaDir, _ = filePath.Abs("testdata/kafka") + // Kafka NodePool path + kafkaNodePoolPath = filePath.Join(kafkaDir, "kafka-node-pool.yaml") + // Kafka Topic path + kafkaTopicPath = filePath.Join(kafkaDir, "kafka-topic.yaml") + // Kafka TLS Template path + kafkaTLSPath := filePath.Join(kafkaDir, "kafka-tls.yaml") + // Kafka metrics config Template path + kafkaMetricsPath := filePath.Join(kafkaDir, "kafka-metrics-config.yaml") + // Kafka User path + kafkaUserPath := filePath.Join(kafkaDir, "kafka-user.yaml") + + g.By("Subscribe to AMQ operator") + kafkaSource := CatalogSourceObjects{"stable", "redhat-operators", "openshift-marketplace"} + amq = SubscriptionObjects{ + OperatorName: "amq-streams-cluster-operator", + Namespace: "openshift-operators", + PackageName: "amq-streams", + Subscription: filePath.Join(subscriptionDir, "sub-template.yaml"), + CatalogSource: &kafkaSource, + } + + kafkaChannel, err := getOperatorChannel(oc, kafkaSource.SourceName, amq.PackageName) + if err != nil || kafkaChannel == "" { + g.Skip("Kafka channel not found, skip this case") + } + + // check if amq Streams Operator is already present + AMQexisting, err = CheckOperatorStatus(oc, amq.Namespace, amq.PackageName) + o.Expect(err).NotTo(o.HaveOccurred()) + if !AMQexisting { + ensureOperatorDeployed(oc, amq, kafkaSource, "name="+amq.OperatorName) + // before creating kafka, check the existence of crd kafkas.kafka.strimzi.io + checkResource(oc, true, true, "kafka.strimzi.io", []string{"crd", "kafkas.kafka.strimzi.io", "-ojsonpath={.spec.group}"}) + } + + kafkaMetrics = KafkaMetrics{ + Namespace: kafkaNs, + Template: kafkaMetricsPath, + } + + kafka = Kafka{ + Name: kafkaClusterName, + Namespace: kafkaNs, + Template: kafkaTLSPath, + } + + kafkaNodePool = KafkaNodePool{ + NodePoolName: "kafka-pool", + Namespace: kafkaNs, + Name: kafka.Name, + Template: kafkaNodePoolPath, + } + + kafkaTopic = KafkaTopic{ + TopicName: "network-flows", + Name: kafka.Name, + Namespace: kafkaNs, + Template: kafkaTopicPath, + } + + kafkaUser = KafkaUser{ + UserName: "flp-kafka", + Name: kafka.Name, + Namespace: kafkaNs, + Template: kafkaUserPath, + } + + g.By("Deploy Kafka with TLS") + oc.CreateSpecifiedNamespaceAsAdmin(kafkaNs) + kafkaMetrics.deployKafkaMetrics(oc) + kafka.deployKafka(oc) + kafkaNodePool.deployKafkaNodePool(oc) + kafkaTopic.deployKafkaTopic(oc) + kafkaUser.deployKafkaUser(oc) + + g.By("Check if Kafka and Kafka topic are ready") + // wait for KafkaNodePool, Kafka and KafkaTopic to be ready + WaitForPodsReadyWithLabel(oc, kafka.Namespace, "strimzi.io/pool-name=kafka-pool") + waitForKafkaReady(oc, kafka.Name, kafka.Namespace) + waitForKafkaTopicReady(oc, kafkaTopic.TopicName, kafkaTopic.Namespace) + }) + + g.AfterEach(func() { + kafkaUser.deleteKafkaUser(oc) + kafkaTopic.deleteKafkaTopic(oc) + kafkaNodePool.deleteKafkaNodePool(oc) + kafka.deleteKafka(oc) + if !AMQexisting { + amq.uninstallOperator(oc) + } + oc.DeleteSpecifiedNamespaceAsAdmin(kafkaNs) + }) + + g.It("Author:aramesha-NonPreRelease-Longduration-Critical-56362-High-53597-High-56326-High-64880-High-75340-Verify network flows are captured with Kafka with TLS [Serial][Slow]", func() { + SkipIfOCPBelow("v4.14") + + g.By("Deploy FlowCollector with Kafka TLS") + flow := Flowcollector{ + Namespace: namespace, + DeploymentModel: "Kafka", + Template: flowFixturePath, + LokiNamespace: lokiStackNS, + KafkaAddress: kafkaAddress, + KafkaTLSEnable: "true", + KafkaNamespace: kafkaNs, + NetworkPolicyAdditionalNamespaces: []string{additionalNamespaces}, + } + + defer func() { _ = flow.DeleteFlowcollector(oc) }() + flow.CreateFlowcollector(oc) + + g.By("Ensure secrets are synced") + // ensure certs are synced to privileged NS + secrets, err := getSecrets(oc, namespace+"-privileged") + o.Expect(err).ToNot(o.HaveOccurred()) + o.Expect(secrets).To(o.And(o.ContainSubstring(kafkaUser.UserName), o.ContainSubstring(kafka.Name+"-cluster-ca-cert"))) + + g.By("Verify prometheus is able to scrape metrics for FLP-Kafka") + flpPrpmSM := "flowlogs-pipeline-transformer-monitor" + tlsScheme, err := getMetricsScheme(oc, flpPrpmSM, flow.Namespace) + o.Expect(err).NotTo(o.HaveOccurred()) + tlsScheme = strings.Trim(tlsScheme, "'") + o.Expect(tlsScheme).To(o.Equal("https")) + + serverName, err := getMetricsServerName(oc, flpPrpmSM, flow.Namespace) + serverName = strings.Trim(serverName, "'") + o.Expect(err).NotTo(o.HaveOccurred()) + flpPromSA := "flowlogs-pipeline-transformer-prom" + expectedServerName := fmt.Sprintf("%s.%s.svc", flpPromSA, namespace) + o.Expect(serverName).To(o.Equal(expectedServerName)) + + // verify FLP metrics are being populated with Kafka + // Sleep before making any metrics request + g.By("Verify prometheus is able to scrape FLP metrics") + time.Sleep(30 * time.Second) + verifyFLPMetrics(oc) + + // verify logs + g.By("Wait for a min before logs gets collected and written to loki") + startTime := time.Now() + time.Sleep(60 * time.Second) + + g.By("Get flowlogs from loki") + err = verifyLokilogsTime(kubeadminToken, ls.Route, startTime) + o.Expect(err).NotTo(o.HaveOccurred()) + }) + + g.It("Author:aramesha-NonPreRelease-Longduration-High-57397-High-65116-Verify network-flows export with Kafka and netobserv installation without Loki[Serial]", func() { + SkipIfOCPBelow("v4.10") + g.By("Deploy kafka Topic for export") + // deploy kafka topic for export + kafkaTopic2 := KafkaTopic{ + TopicName: "network-flows-export", + Name: kafka.Name, + Namespace: kafkaNs, + Template: kafkaTopicPath, + } + + defer kafkaTopic2.deleteKafkaTopic(oc) + kafkaTopic2.deployKafkaTopic(oc) + waitForKafkaTopicReady(oc, kafkaTopic2.TopicName, kafkaTopic2.Namespace) + + kafkaExporterConfig := map[string]interface{}{ + "kafka": map[string]interface{}{ + "address": kafkaAddress, + "tls": map[string]interface{}{ + "caCert": map[string]interface{}{ + "certFile": "ca.crt", + "name": "kafka-cluster-cluster-ca-cert", + "namespace": kafkaNs, + "type": "secret"}, + "enable": true, + "insecureSkipVerify": false, + "userCert": map[string]interface{}{ + "certFile": "user.crt", + "certKey": "user.key", + "name": kafkaUser.UserName, + "namespace": kafkaNs, + "type": "secret"}, + }, + "topic": kafkaTopic2.TopicName}, + "type": "Kafka", + } + + config, err := json.Marshal(kafkaExporterConfig) + o.Expect(err).ToNot(o.HaveOccurred()) + kafkaConfig := string(config) + + g.By("Deploy FlowCollector with Kafka TLS") + flow := Flowcollector{ + Namespace: namespace, + DeploymentModel: "Kafka", + Template: flowFixturePath, + LokiNamespace: lokiStackNS, + KafkaAddress: kafkaAddress, + KafkaTLSEnable: "true", + KafkaNamespace: kafkaNs, + Exporters: []string{kafkaConfig}, + NetworkPolicyAdditionalNamespaces: []string{additionalNamespaces}, + } + + defer func() { _ = flow.DeleteFlowcollector(oc) }() + flow.CreateFlowcollector(oc) + + // Scenario1: Verify flows are exported with Kafka DeploymentModel and with Loki enabled + g.By("Verify flowcollector is deployed with KAFKA exporter") + exporterType, err := oc.AsAdmin().Run("get").Args("flowcollector", "cluster", "-o", "jsonpath='{.spec.exporters[0].type}'").Output() + o.Expect(err).ToNot(o.HaveOccurred()) + o.Expect(exporterType).To(o.Equal(`'Kafka'`)) + + g.By("Ensure flows are observed, all pods are running and secrets are synced and plugin pod is deployed") + // ensure certs are synced to privileged NS + secrets, err := getSecrets(oc, namespace+"-privileged") + o.Expect(err).ToNot(o.HaveOccurred()) + o.Expect(secrets).To(o.And(o.ContainSubstring(kafkaUser.UserName), o.ContainSubstring(kafka.Name+"-cluster-ca-cert"))) + + // verify logs + g.By("Wait for a min before logs gets collected and written to loki") + startTime := time.Now() + time.Sleep(60 * time.Second) + + g.By("Get flowlogs from loki") + err = verifyLokilogsTime(kubeadminToken, ls.Route, startTime) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Deploy Kafka consumer pod") + // using amq-streams/kafka-34-rhel8:2.5.2 version. Update if imagePull issues are observed + consumerTemplate := filePath.Join(kafkaDir, "topic-consumer-tls.yaml") + consumer := Resource{"job", kafkaTopic2.TopicName + "-consumer", kafkaNs} + defer func() { _ = consumer.clear(oc) }() + err = consumer.applyFromTemplate(oc, "-n", consumer.Namespace, "-f", consumerTemplate, "-p", "NAME="+consumer.Name, "NAMESPACE="+consumer.Namespace, "KAFKA_TOPIC="+kafkaTopic2.TopicName, "CLUSTER_NAME="+kafka.Name, "KAFKA_USER="+kafkaUser.UserName) + o.Expect(err).NotTo(o.HaveOccurred()) + + WaitForPodsReadyWithLabel(oc, consumer.Namespace, "job-name="+consumer.Name) + + consumerPodName, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pods", "-n", consumer.Namespace, "-l", "job-name="+consumer.Name, "-o=jsonpath={.items[0].metadata.name}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Verify Kafka consumer pod logs") + podLogs, err := compat_otp.WaitAndGetSpecificPodLogs(oc, consumer.Namespace, "", consumerPodName, `'{"AgentIP":'`) + compat_otp.AssertWaitPollNoErr(err, "Did not get log for the pod with job-name=network-flows-export-consumer label") + verifyFlowRecordFromLogs(podLogs) + + g.By("Verify NetObserv can be installed without Loki") + _ = flow.DeleteFlowcollector(oc) + // Ensure FLP and eBPF pods are deleted + checkPodDeleted(oc, namespace, "app=flowlogs-pipeline", "flowlogs-pipeline") + checkPodDeleted(oc, namespace+"-privileged", "app=netobserv-ebpf-agent", "netobserv-ebpf-agent") + + flow.DeploymentModel = "Direct" + flow.LokiEnable = "false" + flow.CreateFlowcollector(oc) + + g.By("Verify Kafka consumer pod logs") + podLogs, err = compat_otp.WaitAndGetSpecificPodLogs(oc, consumer.Namespace, "", consumerPodName, `'{"AgentIP":'`) + compat_otp.AssertWaitPollNoErr(err, "Did not get log for the pod with job-name=network-flows-export-consumer label") + verifyFlowRecordFromLogs(podLogs) + + g.By("Verify console plugin pod is not deployed when its disabled in flowcollector") + _ = flow.DeleteFlowcollector(oc) + // Ensure FLP and eBPF pods are deleted + checkPodDeleted(oc, namespace, "app=flowlogs-pipeline", "flowlogs-pipeline") + checkPodDeleted(oc, namespace+"-privileged", "app=netobserv-ebpf-agent", "netobserv-ebpf-agent") + + flow.PluginEnable = "false" + flow.CreateFlowcollector(oc) + + // Scenario3: Verify all pods except plugin pod are present with only Plugin disabled in flowcollector + g.By("Ensure all pods except consolePlugin pod are deployed") + consolePod, err := compat_otp.GetAllPodsWithLabel(oc, namespace, "app=netobserv-plugin") + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(consolePod)).To(o.Equal(0)) + }) + + //Add future NetObserv + Loki + Kafka test-cases here + }) + + g.Context("with VMs", func() { + var ( + // virt operator vars + VOexisting = false + virtOperatorNS = "openshift-cnv" + virtualizationDir, _ = filePath.Abs("testdata/virtualization") + kubevirtHyperconvergedPath = filePath.Join(virtualizationDir, "kubevirt-hyperconverged.yaml") + virtCatsrc = Resource{"catsrc", "redhat-operators", "openshift-marketplace"} + virtPackageName = "kubevirt-hyperconverged" + virtSource = CatalogSourceObjects{"stable", virtCatsrc.Name, virtCatsrc.Namespace} + VO = SubscriptionObjects{ + OperatorName: "kubevirt-hyperconverged", + Namespace: virtOperatorNS, + PackageName: virtPackageName, + Subscription: filePath.Join(subscriptionDir, "sub-template.yaml"), + OperatorGroup: filePath.Join(subscriptionDir, "singlenamespace-og.yaml"), + CatalogSource: &virtSource, + } + ) + + g.BeforeEach(func() { + clusterArch, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("nodes", "-o=jsonpath={.items[0].status.nodeInfo.architecture}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if strings.Contains(clusterArch, "ppc64le") { + g.Skip("Virtualization operator is not supported on ppc64le architecture. Skip this test!") + } + + isMetal, err := isClusterBareMetal(oc) + o.Expect(err).ToNot(o.HaveOccurred()) + if !isMetal && !hasMetalWorkerNodes(oc) { + g.Skip("Cluster does not have baremetal workers. Skip this test!") + } + + g.By("Deploy Openshift Virtualization operator") + VOexisting, err = CheckOperatorStatus(oc, VO.Namespace, VO.PackageName) + o.Expect(err).NotTo(o.HaveOccurred()) + if !VOexisting { + ensureOperatorDeployed(oc, VO, virtSource, "name=virt-operator") + } + + g.By("Deploy OpenShift Virtualization Deployment CR") + _, err = oc.AsAdmin().WithoutNamespace().Run("create").Args("-f", kubevirtHyperconvergedPath).Output() + o.Expect(err).ToNot(o.HaveOccurred()) + waitUntilHyperConvergedReady(oc, "kubevirt-hyperconverged", virtOperatorNS) + WaitForPodsReadyWithLabel(oc, virtOperatorNS, "app.kubernetes.io/managed-by=virt-operator") + }) + + g.AfterEach(func() { + deleteResource(oc, "hyperconverged", "kubevirt-hyperconverged", virtOperatorNS) + if !VOexisting { + VO.uninstallOperator(oc) + } + }) + + g.It("Author:aramesha-NonPreRelease-Longduration-High-76537-Verify flow enrichment for VM's secondary interfaces [Disruptive][Slow]", func() { + SkipIfOCPBelow("v4.13") + var ( + testNS = "test-76537" + // NAD vars + networkName = "l2-network" + layer2NadPath = filePath.Join(virtualizationDir, "layer2-nad.yaml") + // VM vars + testVMStaticIPTemplatePath = filePath.Join(virtualizationDir, "test-vm-static-IP_template.yaml") + ) + + g.By("Deploy Network Attachment Definition in test-76537 namespace") + defer deleteNamespace(oc, testNS) + defer deleteResource(oc, "net-attach-def", networkName, testNS) + _, err := oc.AsAdmin().WithoutNamespace().Run("create").Args("-f", layer2NadPath).Output() + o.Expect(err).ToNot(o.HaveOccurred()) + // Wait a min for NAD to come up + time.Sleep(60 * time.Second) + checkNAD(oc, networkName, testNS) + + g.By("Deploy test VM1") + testVM1 := TestVMStaticIPTemplate{ + Name: "test-vm1", + Namespace: testNS, + NetworkName: networkName, + Mac: "02:00:00:00:00:01", + StaticIP: "10.10.10.15/24", + Template: testVMStaticIPTemplatePath, + } + defer deleteResource(oc, "vm", testVM1.Name, testNS) + err = testVM1.createVMStaticIP(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + waitUntilVMReady(oc, testVM1.Name, testVM1.Namespace) + + g.By("Wait for VM1 to get IP assigned") + vm1Ip, err := waitForVMIPAssignment(oc, testVM1.Name, testVM1.Namespace, 1) + o.Expect(err).ToNot(o.HaveOccurred()) + o.Expect(vm1Ip).To(o.Equal("10.10.10.15")) + + startTime := time.Now() + + g.By("Deploy test VM2") + testVM2 := TestVMStaticIPTemplate{ + Name: "test-vm2", + Namespace: testNS, + NetworkName: networkName, + Mac: "02:00:00:00:00:02", + StaticIP: "10.10.10.14/24", + RunCmd: fmt.Sprintf("[[ping, %s]]", vm1Ip), + Template: testVMStaticIPTemplatePath, + } + defer deleteResource(oc, "vm", testVM2.Name, testNS) + err = testVM2.createVMStaticIP(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + waitUntilVMReady(oc, testVM2.Name, testVM2.Namespace) + + g.By("Wait for VM2 to get IP assigned") + vm2Ip, err := waitForVMIPAssignment(oc, testVM2.Name, testVM2.Namespace, 1) + o.Expect(err).ToNot(o.HaveOccurred()) + o.Expect(vm2Ip).To(o.Equal("10.10.10.14")) + + g.By("Deploy FlowCollector") + flow := Flowcollector{ + Namespace: namespace, + Template: flowFixturePath, + LokiNamespace: lokiStackNS, + EBPFPrivileged: "true", + } + + defer func() { _ = flow.DeleteFlowcollector(oc) }() + flow.CreateFlowcollector(oc) + + g.By("Wait for a min before logs gets collected and written to loki") + time.Sleep(60 * time.Second) + + lokilabels := Lokilabels{ + App: "netobserv-flowcollector", + SrcK8SNamespace: testNS, + SrcK8SOwnerName: testVM2.Name, + DstK8SNamespace: testNS, + DstK8SOwnerName: testVM1.Name, + } + parameters := []string{"DstAddr=\"10.10.10.15\"", "SrcAddr=\"10.10.10.14\""} + + g.By("Verify flows are written to loki") + flowRecords, err := lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime, parameters...) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of flows written to loki > 0") + + g.By("Verify flow logs are enriched") + // Get VM1 pod name and node + vm1PodName, err := compat_otp.GetAllPodsWithLabel(oc, testNS, "vm.kubevirt.io/name="+testVM1.Name) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(vm1PodName).NotTo(o.BeEmpty()) + vm1node, err := compat_otp.GetPodNodeName(oc, testNS, vm1PodName[0]) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(vm1node).NotTo(o.BeEmpty()) + + // Get vm2 pod name and node + vm2PodName, err := compat_otp.GetAllPodsWithLabel(oc, testNS, "vm.kubevirt.io/name="+testVM2.Name) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(vm2PodName).NotTo(o.BeEmpty()) + vm2node, err := compat_otp.GetPodNodeName(oc, testNS, vm2PodName[0]) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(vm2node).NotTo(o.BeEmpty()) + + for _, r := range flowRecords { + o.Expect(r.Flowlog.DstK8SName).Should(o.ContainSubstring(vm1PodName[0])) + o.Expect(r.Flowlog.SrcK8SName).Should(o.ContainSubstring(vm2PodName[0])) + o.Expect(r.Flowlog.DstK8SOwnerType).Should(o.ContainSubstring("VirtualMachineInstance")) + o.Expect(r.Flowlog.SrcK8SOwnerType).Should(o.ContainSubstring("VirtualMachineInstance")) + o.Expect(r.Flowlog.DstK8SNetworkName).Should(o.ContainSubstring("test-76537/l2-network")) + o.Expect(r.Flowlog.SrcK8SNetworkName).Should(o.ContainSubstring("test-76537/l2-network")) + } + }) + + g.It("Author:aramesha-NonPreRelease-Longduration-Medium-85887-Verify UDN with VMs [Disruptive][Slow]", func() { + SkipIfOCPBelow("v4.18") + var ( + // UDN vars + udnNS = "netobserv-udn-85887" + udnName = "udn-network-85887" + // VM vars + testVMUDNTemplatePath = filePath.Join(virtualizationDir, "test-vm-UDN_template.yaml") + ) + + g.By("Deploy UDN in UDN ns") + var cidr, ipv4cidr, ipv6cidr string + defer deleteNamespace(oc, udnNS) + oc.CreateSpecificNamespaceUDN(udnNS) + + if ipStackType == "ipv4single" { + cidr = "10.151.0.0/16" + } else { + if ipStackType == "ipv6single" { + cidr = "2011:100:200::0/48" + } else { + ipv4cidr = "10.151.0.0/16" + ipv6cidr = "2011:100:200::0/48" + } + } + createGeneralUDNCRD(oc, udnNS, udnName, ipv4cidr, ipv6cidr, cidr, "layer2") + + g.By("Deploy test VM3") + testVM3 := TestVMUDNTemplate{ + Name: "test-vm3", + Namespace: udnNS, + NetworkName: udnName, + Template: testVMUDNTemplatePath, + } + defer deleteResource(oc, "vm", testVM3.Name, testVM3.Namespace) + err := testVM3.createVMUDN(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + waitUntilVMReady(oc, testVM3.Name, testVM3.Namespace) + + g.By("Wait for VM3 to get IP assigned") + vm3Ip, err := waitForVMIPAssignment(oc, testVM3.Name, testVM3.Namespace, 0) + o.Expect(err).ToNot(o.HaveOccurred()) + o.Expect(vm3Ip).NotTo(o.BeEmpty()) + + startTime := time.Now() + + g.By("Deploy test VM4") + testVM4 := TestVMUDNTemplate{ + Name: "test-vm4", + Namespace: udnNS, + NetworkName: udnName, + RunCmd: fmt.Sprintf("[[ping, %s]]", vm3Ip), + Template: testVMUDNTemplatePath, + } + defer deleteResource(oc, "vm", testVM4.Name, testVM4.Namespace) + err = testVM4.createVMUDN(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + waitUntilVMReady(oc, testVM4.Name, testVM4.Namespace) + + g.By("Wait for VM4 to get IP assigned") + vm4Ip, err := waitForVMIPAssignment(oc, testVM4.Name, testVM4.Namespace, 0) + o.Expect(err).ToNot(o.HaveOccurred()) + o.Expect(vm4Ip).NotTo(o.BeEmpty()) + + g.By("Deploy FlowCollector with UDNMapping feature enabled with eBPF in privileged mode") + flow := Flowcollector{ + Namespace: namespace, + EBPFPrivileged: "true", + EBPFeatures: []string{"\"UDNMapping\""}, + LokiNamespace: lokiStackNS, + Template: flowFixturePath, + } + + defer func() { _ = flow.DeleteFlowcollector(oc) }() + flow.CreateFlowcollector(oc) + + g.By("Wait for a min before logs gets collected and written to loki") + time.Sleep(60 * time.Second) + + lokilabels := Lokilabels{ + App: "netobserv-flowcollector", + SrcK8SNamespace: udnNS, + SrcK8SOwnerName: testVM4.Name, + DstK8SNamespace: udnNS, + DstK8SOwnerName: testVM3.Name, + } + + g.By("Verify flows are written to loki") + flowRecords, err := lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of flows written to loki > 0") + + g.By("Verify flow logs are enriched") + // Get VM3 launcher pod name + vm3podname, err := compat_otp.GetAllPodsWithLabel(oc, udnNS, "vm.kubevirt.io/name="+testVM3.Name) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(vm3podname).NotTo(o.BeEmpty()) + // Get VM4 launcher pod name + vm4podname, err := compat_otp.GetAllPodsWithLabel(oc, udnNS, "vm.kubevirt.io/name="+testVM4.Name) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(vm4podname).NotTo(o.BeEmpty()) + + for _, r := range flowRecords { + o.Expect(r.Flowlog.DstK8SName).Should(o.ContainSubstring(vm3podname[0])) + o.Expect(r.Flowlog.SrcK8SName).Should(o.ContainSubstring(vm4podname[0])) + o.Expect(r.Flowlog.DstK8SOwnerType).Should(o.ContainSubstring("VirtualMachineInstance")) + o.Expect(r.Flowlog.SrcK8SOwnerType).Should(o.ContainSubstring("VirtualMachineInstance")) + o.Expect(r.Flowlog.Udns).Should(o.ContainElement(fmt.Sprintf("%s/%s", udnNS, udnName))) + o.Expect(r.Flowlog.DstK8SNetworkName).Should(o.ContainSubstring(fmt.Sprintf("%s/%s", udnNS, udnName))) + o.Expect(r.Flowlog.SrcK8SNetworkName).Should(o.ContainSubstring(fmt.Sprintf("%s/%s", udnNS, udnName))) + } + }) + + g.It("Author:aramesha-High-85935-Validate CUDN with Localnet and VM's[Serial]", func() { + SkipIfOCPBelow("v4.19") + var ( + // NMstate operator vars + opNamespace = "openshift-nmstate" + nmStateDir, _ = filePath.Abs("testdata/networking/nmstate") + nmstateCRTemplate = filePath.Join(nmStateDir, "nmstate-cr-template.yaml") + nmstateCR = nmstateCRResource{ + name: "nmstate", + template: nmstateCRTemplate, + } + nodeSelectLabel = "node-role.kubernetes.io/worker" + ovnMappingPolicyTemplate = filePath.Join(nmStateDir, "ovn-mapping-policy-template.yaml") + ovnMappingPolicy = ovnMappingPolicyResource{ + name: "bridge-mapping-85935", + nodelabel: nodeSelectLabel, + labelvalue: "", + localnet1: "mylocalnet", + bridge1: "br-ex", + template: ovnMappingPolicyTemplate, + } + // CUDN vars + matchLabelKey = "test.io" + matchValue = "cudn-network-" + getRandomString() + secondaryCUDNName = "secondary-localnet-85935" + cudnNS = []string{"netobserv-cudn1-85935", "netobserv-cudn2-85935"} + testVMLocalnetTemplatePath = filePath.Join(virtualizationDir, "test-vm-localnet_template.yaml") + ) + + g.By("Check the platform and network plugin type if it is suitable for running the test") + networkType := checkNetworkType(oc) + if !(isPlatformSuitableForNMState(oc)) || !strings.Contains(networkType, "ovn") { + g.Skip("Skipping for unsupported platform or non-OVN network plugin type!") + } + installNMstateOperator(oc) + + workerNode, getNodeErr := compat_otp.GetFirstWorkerNode(oc) + o.Expect(getNodeErr).NotTo(o.HaveOccurred()) + o.Expect(workerNode).NotTo(o.BeEmpty()) + + compat_otp.By("Create NMState CR") + defer deleteNMStateCR(oc, nmstateCR) + result, crErr := createNMStateCR(oc, nmstateCR, opNamespace) + compat_otp.AssertWaitPollNoErr(crErr, "create nmstate cr failed") + o.Expect(result).To(o.BeTrue()) + e2e.Logf("SUCCESS - NMState CR Created") + + compat_otp.By("Configure NNCP for creating OvnMapping NMstate Feature") + defer deleteNNCP(oc, ovnMappingPolicy.name) + defer func() { + ovnmapping, deferErr := compat_otp.DebugNodeWithChroot(oc, workerNode, "ovs-vsctl", "get", "Open_vSwitch", ".", "external_ids:ovn-bridge-mappings") + o.Expect(deferErr).NotTo(o.HaveOccurred()) + if strings.Contains(ovnmapping, ovnMappingPolicy.localnet1) { + // ovs-vsctl can only use "set" to reserve some fields + _, err := compat_otp.DebugNodeWithChroot(oc, workerNode, "ovs-vsctl", "set", "Open_vSwitch", ".", "external_ids:ovn-bridge-mappings=\"physnet:br-ex\"") + o.Expect(err).NotTo(o.HaveOccurred()) + } + }() + configErr3 := ovnMappingPolicy.configNNCP(oc) + o.Expect(configErr3).NotTo(o.HaveOccurred()) + nncpErr3 := checkNNCPStatus(oc, ovnMappingPolicy.name, "Available") + compat_otp.AssertWaitPollNoErr(nncpErr3, fmt.Sprintf("%s policy applied failed", ovnMappingPolicy.name)) + + compat_otp.By("Create two namespaces and label them") + for _, ns := range cudnNS { + defer oc.DeleteSpecifiedNamespaceAsAdmin(ns) + oc.CreateSpecifiedNamespaceAsAdmin(ns) + err := oc.AsAdmin().WithoutNamespace().Run("label").Args("ns", ns, fmt.Sprintf("%s=%s", matchLabelKey, matchValue)).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + } + + compat_otp.By("Create secondary localnet CUDN") + defer removeResource(oc, true, true, "clusteruserdefinednetwork", secondaryCUDNName) + _, err := applyLocalnetCUDNtoMatchLabelNS(oc, matchLabelKey, matchValue, secondaryCUDNName, "mylocalnet", "192.168.200.0/24", "192.168.200.1/32", false) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Deploy test VM5") + testVM5 := TestVMUDNTemplate{ + Name: "test-vm5", + Namespace: cudnNS[0], + NetworkName: secondaryCUDNName, + Template: testVMLocalnetTemplatePath, + } + defer deleteResource(oc, "vm", testVM5.Name, testVM5.Namespace) + err = testVM5.createVMUDN(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + waitUntilVMReady(oc, testVM5.Name, testVM5.Namespace) + + // Even though VM comes up as Ready, the IP assignment takes some time + g.By("Wait for VM5 to get IP assigned") + vm5Ip, err := waitForVMIPAssignment(oc, testVM5.Name, testVM5.Namespace, 1) + o.Expect(err).ToNot(o.HaveOccurred()) + o.Expect(vm5Ip).NotTo(o.BeEmpty()) + + startTime := time.Now() + + g.By("Deploy test VM6") + testVM6 := TestVMUDNTemplate{ + Name: "test-vm6", + Namespace: cudnNS[1], + NetworkName: secondaryCUDNName, + RunCmd: fmt.Sprintf("[[ping, %s]]", vm5Ip), + Template: testVMLocalnetTemplatePath, + } + defer deleteResource(oc, "vm", testVM6.Name, testVM6.Namespace) + err = testVM6.createVMUDN(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + waitUntilVMReady(oc, testVM6.Name, testVM6.Namespace) + + g.By("Wait for VM6 to get IP assigned") + vm6Ip, err := waitForVMIPAssignment(oc, testVM6.Name, testVM6.Namespace, 1) + o.Expect(err).ToNot(o.HaveOccurred()) + o.Expect(vm6Ip).NotTo(o.BeEmpty()) + + g.By("Deploy FlowCollector with UDNMapping feature enabled with eBPF in privileged mode") + flow := Flowcollector{ + Namespace: namespace, + EBPFPrivileged: "true", + EBPFeatures: []string{"\"UDNMapping\""}, + LokiNamespace: lokiStackNS, + Template: flowFixturePath, + } + + defer func() { _ = flow.DeleteFlowcollector(oc) }() + flow.CreateFlowcollector(oc) + + g.By("Wait for a min before logs gets collected and written to loki") + time.Sleep(60 * time.Second) + + g.By("Verify CUDN Localnet flows") + lokilabels := Lokilabels{ + App: "netobserv-flowcollector", + SrcK8SNamespace: cudnNS[1], + SrcK8SOwnerName: testVM6.Name, + DstK8SNamespace: cudnNS[0], + DstK8SOwnerName: testVM5.Name, + } + + flowRecords, err := lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of CUDN Localnet flows > 0") + for _, r := range flowRecords { + o.Expect(r.Flowlog.Udns).Should(o.ContainElement(secondaryCUDNName)) + o.Expect(r.Flowlog.DstK8SNetworkName).Should(o.ContainSubstring(secondaryCUDNName)) + o.Expect(r.Flowlog.SrcK8SNetworkName).Should(o.ContainSubstring(secondaryCUDNName)) + } + }) + //Add future NetObserv + VM test-cases here + }) + }) +}) diff --git a/integration-tests/backend/test_flowcollectorslice.go b/integration-tests/backend/test_flowcollectorslice.go new file mode 100644 index 0000000000..8eccca6895 --- /dev/null +++ b/integration-tests/backend/test_flowcollectorslice.go @@ -0,0 +1,643 @@ +package e2etests + +import ( + "encoding/json" + "os" + "time" + + filePath "path/filepath" + "strings" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" + e2e "k8s.io/kubernetes/test/e2e/framework" + e2eoutput "k8s.io/kubernetes/test/e2e/framework/pod/output" +) + +var _ = g.Describe("[sig-netobserv] Network_Observability", func() { + + defer g.GinkgoRecover() + var ( + oc = compat_otp.NewCLI("netobserv", compat_otp.KubeConfigPath()) + // NetObserv Operator variables + NOcatSrc = Resource{"catsrc", "netobserv-konflux-fbc", netobservNS} + NOSource = CatalogSourceObjects{"stable", NOcatSrc.Name, NOcatSrc.Namespace} + + // Template directories + baseDir, _ = filePath.Abs("testdata") + subscriptionDir = filePath.Join(baseDir, "subscription") + flowFixturePath = filePath.Join(baseDir, "flowcollector_v1beta2_template.yaml") + flowSliceFixturePath = filePath.Join(baseDir, "flowcollectorSlice_v1alpha1_template.yaml") + + // Operator namespace object + OperatorNS = OperatorNamespace{ + Name: netobservNS, + NamespaceTemplate: filePath.Join(subscriptionDir, "namespace.yaml"), + } + NO = SubscriptionObjects{ + OperatorName: "netobserv-operator", + Namespace: netobservNS, + PackageName: NOPackageName, + Subscription: filePath.Join(subscriptionDir, "sub-template.yaml"), + OperatorGroup: filePath.Join(subscriptionDir, "allnamespace-og.yaml"), + CatalogSource: &NOSource, + } + imageDigest = filePath.Join(subscriptionDir, "image-digest-mirror-set.yaml") + catSrcTemplate = filePath.Join(subscriptionDir, "catalog-source.yaml") + catalogSource = os.Getenv("MULTISTAGE_PARAM_OVERRIDE_NETOBSERV_CS_IMAGE") + + kubeadminToken string + kubeAdminPasswd = os.Getenv("QE_KUBEADMIN_PASSWORD") + namespace string + + // Loki Operator variables + lokiDir = filePath.Join(baseDir, "loki") + lokiPackageName = "loki-operator" + lokiSource CatalogSourceObjects + ls *lokiStack + Lokiexisting = false + lokiStackNS = "netobserv-loki" + LO = SubscriptionObjects{ + OperatorName: "loki-operator-controller-manager", + Namespace: loNS, + PackageName: lokiPackageName, + Subscription: filePath.Join(subscriptionDir, "sub-template.yaml"), + OperatorGroup: filePath.Join(subscriptionDir, "allnamespace-og.yaml"), + CatalogSource: &lokiSource, + } + + // LokiStack variables + ipStackType string + lokiStackTemplate = filePath.Join(lokiDir, "lokistack-simple.yaml") + lokiTenant = "openshift-network" + ) + + g.BeforeEach(func() { + if strings.Contains(os.Getenv("E2E_RUN_TAGS"), "disconnected") { + g.Skip("Skipping tests for disconnected profiles") + } + namespace = oc.Namespace() + + g.By("Get kubeadmin token") + if kubeAdminPasswd == "" { + g.Skip("no kubeAdminPasswd is provided in this profile, set QE_KUBEADMIN_PASSWORD env var") + } + serverURL, serverURLErr := oc.AsAdmin().WithoutNamespace().Run("whoami").Args("--show-server").Output() + o.Expect(serverURLErr).NotTo(o.HaveOccurred()) + currentContext, currentContextErr := oc.WithoutNamespace().Run("config").Args("current-context").Output() + o.Expect(currentContextErr).NotTo(o.HaveOccurred()) + defer func() { + rollbackCtxErr := oc.WithoutNamespace().Run("config").Args("set", "current-context", currentContext).Execute() + o.Expect(rollbackCtxErr).NotTo(o.HaveOccurred()) + }() + + kubeadminToken = getKubeAdminToken(oc, kubeAdminPasswd, serverURL, currentContext) + o.Expect(kubeadminToken).NotTo(o.BeEmpty()) + + isHypershift := compat_otp.IsHypershiftHostedCluster(oc) + + // Deploy NetObserv operator + OperatorNS.DeployOperatorNamespace(oc) + deployedUpstreamCatalogSource, catSrcErr := setupCatalogSource(oc, NOcatSrc, catSrcTemplate, imageDigest, catalogSource, isHypershift, &NOSource, &NO) + o.Expect(catSrcErr).NotTo(o.HaveOccurred()) + ensureNetObservOperatorDeployed(oc, NO, NOSource, deployedUpstreamCatalogSource) + + ipStackType = checkIPStackType(oc) + + g.By("Deploy loki operator") + if !validateInfraAndResourcesForLoki(oc, "10Gi", "6") { + g.Skip("Current platform does not have enough resources available for this test!") + } + + // check if Loki Operator exists + var err error + Lokiexisting, err = CheckOperatorStatus(oc, LO.Namespace, LO.PackageName) + o.Expect(err).NotTo(o.HaveOccurred()) + + lokiChannel, err := getOperatorChannel(oc, "redhat-operators", "loki-operator") + if err != nil || lokiChannel == "" { + g.Skip("Loki channel not found, skip this case") + } + lokiSource = CatalogSourceObjects{lokiChannel, "redhat-operators", "openshift-marketplace"} + + // Don't delete if Loki Operator existed already before NetObserv + // unless it is not using the 'stable' operator + // If Loki Operator was installed by NetObserv tests, + // it will install and uninstall after each spec/test. + if !Lokiexisting { + ensureOperatorDeployed(oc, LO, lokiSource, "name="+LO.OperatorName) + } else { + channelName, err := checkOperatorChannel(oc, LO.Namespace, LO.PackageName) + o.Expect(err).NotTo(o.HaveOccurred()) + if channelName != lokiChannel { + e2e.Logf("found %s channel for loki operator, removing and reinstalling with %s channel instead", channelName, lokiSource.Channel) + LO.uninstallOperator(oc) + ensureOperatorDeployed(oc, LO, lokiSource, "name="+LO.OperatorName) + Lokiexisting = false + } + } + + g.By("Deploy lokiStack") + // get storageClass Name + sc, err := getStorageClassName(oc) + if err != nil || len(sc) == 0 { + g.Skip("StorageClass not found in cluster, skip this case") + } + + objectStorageType := getStorageType(oc) + if len(objectStorageType) == 0 && ipStackType != "ipv6single" { + g.Skip("Current cluster doesn't have a proper object storage for this test!") + } + oc.CreateSpecifiedNamespaceAsAdmin(lokiStackNS) + + ls = &lokiStack{ + Name: "lokistack", + Namespace: lokiStackNS, + TSize: "1x.demo", + StorageType: objectStorageType, + StorageSecret: "objectstore-secret", + StorageClass: sc, + BucketName: "netobserv-loki-" + getInfrastructureName(oc), + Tenant: lokiTenant, + Template: lokiStackTemplate, + } + + if ipStackType == "ipv6single" { + e2e.Logf("running IPv6 test") + ls.EnableIPV6 = "true" + } + + err = ls.prepareResourcesForLokiStack(oc) + if err != nil { + g.Skip("Skipping test since LokiStack resources were not deployed") + } + + err = ls.deployLokiStack(oc) + if err != nil { + g.Skip("Skipping test since LokiStack was not deployed") + } + + lokiStackResource := Resource{"lokistack", ls.Name, ls.Namespace} + err = lokiStackResource.WaitForResourceToAppear(oc) + if err != nil { + g.Skip("Skipping test since LokiStack did not become ready") + } + + err = ls.waitForLokiStackToBeReady(oc) + if err != nil { + g.Skip("Skipping test since LokiStack is not ready") + } + ls.Route = "https://" + getRouteAddress(oc, ls.Namespace, ls.Name) + }) + + g.AfterEach(func() { + ls.removeLokiStack(oc) + ls.removeObjectStorage(oc) + if !Lokiexisting { + LO.uninstallOperator(oc) + } + oc.DeleteSpecifiedNamespaceAsAdmin(lokiStackNS) + }) + + g.It("Author:aramesha-Critical-86388-Verify flowCollectorSlice collectionMode: AlwaysCollect [Serial]", func() { + SkipIfOCPBelow("v4.14") + // Test ping pods template variables + pingPodsTemplate := filePath.Join(baseDir, "test-ping-pods_template.yaml") + testPingPodsTemplate := TestPingPodsTemplate{ + ServerNS: "test-ping-server-86388-always", + ClientNS: "test-ping-client-86388-always", + PingTargets: "192.168.1.0 8.8.8.8", + Template: pingPodsTemplate, + } + + subnetLabelsConfig := []map[string]interface{}{ + { + "name": "external-api", + "cidrs": []string{ + "8.8.8.8/32", + "1.1.1.1/32", + }, + }, + { + "name": "internal-service", + "cidrs": []string{ + "192.168.1.0/24", + }, + }, + } + + config, err := json.Marshal(subnetLabelsConfig) + o.Expect(err).ToNot(o.HaveOccurred()) + subnetLabels := string(config) + + g.By("Deploy FlowCollectorSlice") + startTime := time.Now() + defer oc.DeleteSpecifiedNamespaceAsAdmin(testPingPodsTemplate.ClientNS) + oc.CreateSpecifiedNamespaceAsAdmin(testPingPodsTemplate.ClientNS) + flowSlice := FlowcollectorSlice{ + Name: "subnet-label-slice", + Namespace: testPingPodsTemplate.ClientNS, + SubnetLabels: subnetLabels, + Template: flowSliceFixturePath, + } + + defer func() { _ = flowSlice.DeleteFlowcollectorSlice(oc) }() + flowSlice.CreateFlowcollectorSlice(oc) + + g.By("Deploy FlowCollector with SlicesEnabled in AlwaysCollect mode") + flow := Flowcollector{ + Namespace: namespace, + LokiNamespace: lokiStackNS, + CollectionMode: "AlwaysCollect", + SlicesEnable: "true", + Template: flowFixturePath, + } + + defer func() { _ = flow.DeleteFlowcollector(oc) }() + flow.CreateFlowcollector(oc) + flowSlice.WaitForFlowcollectorSliceReady(oc) + + g.By("Deploy test ping server and client pods") + defer oc.DeleteSpecifiedNamespaceAsAdmin(testPingPodsTemplate.ServerNS) + err = testPingPodsTemplate.createPingPods(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + compat_otp.AssertAllPodsToBeReady(oc, testPingPodsTemplate.ServerNS) + compat_otp.AssertAllPodsToBeReady(oc, testPingPodsTemplate.ClientNS) + + g.By("Wait for a min before logs gets collected and written to loki") + time.Sleep(60 * time.Second) + + // Scenario1: Internal IP subnetLabel + g.By("Verify flows with internal-service subnetLabel") + lokilabels := Lokilabels{ + App: "netobserv-flowcollector", + SrcK8SNamespace: testPingPodsTemplate.ClientNS, + } + parameters := []string{"DstAddr=\"192.168.1.0\""} + + flowRecords, err := lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime, parameters...) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of flows from client NS to internal-service > 0") + for _, r := range flowRecords { + o.Expect(r.Flowlog.DstSubnetLabel).Should(o.ContainSubstring("internal-service")) + } + + // Scenario2: External IP subnetLabel + g.By("Verify flows with external-api subnetLabel") + parameters = []string{"DstAddr=\"8.8.8.8\""} + + flowRecords, err = lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime, parameters...) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of flows from client NS to external-api > 0") + for _, r := range flowRecords { + o.Expect(r.Flowlog.DstSubnetLabel).Should(o.ContainSubstring("external-api")) + } + + // Scenario3: Flows are collected from namespaces without Slice deployed too + g.By("Verify flows having no subnet label") + lokilabels = Lokilabels{ + App: "netobserv-flowcollector", + SrcK8SNamespace: testPingPodsTemplate.ServerNS, + } + + flowRecords, err = lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime, parameters...) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of flows from server NS > 0") + for _, r := range flowRecords { + o.Expect(r.Flowlog.DstSubnetLabel).Should(o.ContainSubstring("external-api")) + } + }) + + g.It("Author:aramesha-Critical-86388-Verify flowCollectorSlice collectionMode: AllowList [Serial]", func() { + SkipIfOCPBelow("v4.14") + + // Test ping pods template variables + pingPodsTemplate := filePath.Join(baseDir, "test-ping-pods_template.yaml") + testPingPodsTemplate := TestPingPodsTemplate{ + ServerNS: "test-ping-server-86388-allowlist", + ClientNS: "test-ping-client-86388-allowlist", + PingTargets: "8.8.8.8", + Template: pingPodsTemplate, + } + + g.By("Deploy FlowCollectorSlice") + startTime := time.Now() + defer oc.DeleteSpecifiedNamespaceAsAdmin(testPingPodsTemplate.ClientNS) + oc.CreateSpecifiedNamespaceAsAdmin(testPingPodsTemplate.ClientNS) + flowSlice := FlowcollectorSlice{ + Name: "namespace-slice", + Namespace: testPingPodsTemplate.ClientNS, + Sampling: "3", + Template: flowSliceFixturePath, + } + + defer func() { _ = flowSlice.DeleteFlowcollectorSlice(oc) }() + flowSlice.CreateFlowcollectorSlice(oc) + + g.By("Deploy FlowCollector with Slices enabled in AllowList mode") + flow := Flowcollector{ + Namespace: namespace, + LokiNamespace: lokiStackNS, + CollectionMode: "AllowList", + SlicesEnable: "true", + NamespacesAllow: []string{"\"/openshift-.*/\""}, + Template: flowFixturePath, + } + + defer func() { _ = flow.DeleteFlowcollector(oc) }() + flow.CreateFlowcollector(oc) + flowSlice.WaitForFlowcollectorSliceReady(oc) + + g.By("Deploy test ping server and client pods") + defer oc.DeleteSpecifiedNamespaceAsAdmin(testPingPodsTemplate.ServerNS) + err := testPingPodsTemplate.createPingPods(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + compat_otp.AssertAllPodsToBeReady(oc, testPingPodsTemplate.ServerNS) + compat_otp.AssertAllPodsToBeReady(oc, testPingPodsTemplate.ClientNS) + + g.By("Wait for a min before logs gets collected and written to loki") + time.Sleep(60 * time.Second) + + // Scenario1: Ping from namespace where flowCollectorSlice is deployed + g.By("Verify flows from client NS") + lokilabels := Lokilabels{ + App: "netobserv-flowcollector", + SrcK8SNamespace: testPingPodsTemplate.ClientNS, + } + parameters := []string{"DstAddr=\"8.8.8.8\""} + + flowRecords, err := lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime, parameters...) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of flows from client NS > 0") + for _, r := range flowRecords { + o.Expect(r.Flowlog.Sampling).Should(o.BeNumerically("==", 3)) + } + + // Scenario2: Ping from namespace where flowCollectorSlice is NOT deployed + g.By("Verify NO flows are seen from server NS") + lokilabels = Lokilabels{ + App: "netobserv-flowcollector", + SrcK8SNamespace: testPingPodsTemplate.ServerNS, + } + + flowRecords, err = lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime, parameters...) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically("==", 0), "expected number of flows from server NS = 0") + + // Scenario3: Flows from namespace in allowedNamespaces section of flowcollector + g.By("Verify flows are seen to openshift-dns") + lokilabels = Lokilabels{ + App: "netobserv-flowcollector", + SrcK8SNamespace: "openshift-dns", + } + + flowRecords, err = lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of flows from openshift-dns NS > 0") + + // Scenario4: Flows between namespaces with one in allowedNamespaces section should still be collected + g.By("Verify flows between namespaces") + startTime = time.Now() + // Get server pod IP + serverPodIP, _ := getPodIP(oc, testPingPodsTemplate.ServerNS, "ping-server", ipStackType) + + // Ping server pod from client pod + _, _ = e2eoutput.RunHostCmd(testPingPodsTemplate.ClientNS, "ping-client", "ping -c 100 "+serverPodIP) + time.Sleep(120 * time.Second) + + lokilabels = Lokilabels{ + App: "netobserv-flowcollector", + SrcK8SNamespace: testPingPodsTemplate.ClientNS, + DstK8SNamespace: testPingPodsTemplate.ServerNS, + } + + flowRecords, err = lokilabels.getLokiFlowLogs(kubeadminToken, ls.Route, startTime) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected number of flows between test namespaces > 0") + }) + + g.It("Author:aramesha-NonPreRelease-Longduration-High-87145-Verify FlowCollectorSlices multi-tenancy [Disruptive][Slow]", func() { + SkipIfOCPBelow("v4.14") + g.By("Creating test users") + users, usersHTpassFile, htPassSecret := getNewUser(oc, 1) + defer userCleanup(oc, users, usersHTpassFile, htPassSecret) + + g.By("Deploy FlowCollector with Slices enabled") + flow := Flowcollector{ + Namespace: namespace, + LokiNamespace: lokiStackNS, + CollectionMode: "AllowList", + SlicesEnable: "true", + NamespacesAllow: []string{"\"/openshift-.*/\""}, + Template: flowFixturePath, + } + + defer func() { _ = flow.DeleteFlowcollector(oc) }() + flow.CreateFlowcollector(oc) + + g.By("Verify FlowCollectorSlices ClusterRoles exist") + clusterRoleOutput, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("clusterrole", "-o=jsonpath={.items[*].metadata.name}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(clusterRoleOutput).Should(o.ContainSubstring("flowcollectorslices.flows.netobserv.io-v1alpha1-admin")) + o.Expect(clusterRoleOutput).Should(o.ContainSubstring("flowcollectorslices.flows.netobserv.io-v1alpha1-edit")) + o.Expect(clusterRoleOutput).Should(o.ContainSubstring("flowcollectorslices.flows.netobserv.io-v1alpha1-view")) + + g.By("Deploy test server and client pods for test-a namespace") + serverTemplate := filePath.Join(baseDir, "test-nginx-server_template.yaml") + testServerTemplateA := TestServerTemplate{ + ServerNS: "test-a-server-87145", + Template: serverTemplate, + } + defer oc.DeleteSpecifiedNamespaceAsAdmin(testServerTemplateA.ServerNS) + err = testServerTemplateA.createServer(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + compat_otp.AssertAllPodsToBeReady(oc, testServerTemplateA.ServerNS) + + clientTemplate := filePath.Join(baseDir, "test-nginx-client_template.yaml") + testClientTemplateA := TestClientTemplate{ + ServerNS: testServerTemplateA.ServerNS, + ClientNS: "test-a-client-87145", + Template: clientTemplate, + } + defer oc.DeleteSpecifiedNamespaceAsAdmin(testClientTemplateA.ClientNS) + err = testClientTemplateA.createClient(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + compat_otp.AssertAllPodsToBeReady(oc, testClientTemplateA.ClientNS) + + // Save original context + origContxt, contxtErr := oc.AsAdmin().WithoutNamespace().Run("config").Args("current-context").Output() + o.Expect(contxtErr).NotTo(o.HaveOccurred()) + e2e.Logf("original context is %v", origContxt) + defer func() { _ = oc.AsAdmin().WithoutNamespace().Run("config").Args("use-context", origContxt).Execute() }() + + origUser := oc.Username() + e2e.Logf("original user is %s", origUser) + defer oc.ChangeUser(origUser) + + testUserName := users[0].Username + oc.ChangeUser(testUserName) + e2e.Logf("switched to user: %s", testUserName) + + g.By("Create namespace test-a and grant testuser-0 admin permissions") + testNSA := testClientTemplateA.ClientNS + err = oc.AsAdmin().WithoutNamespace().Run("create").Args("rolebinding", "testuser-0-admin", + "--clusterrole=flowcollectorslices.flows.netobserv.io-v1alpha1-admin", + "--user="+testUserName, "-n", testNSA).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + // Grant testuser admin access to the server namespace as well for flow visibility + err = oc.AsAdmin().WithoutNamespace().Run("create").Args("rolebinding", "testuser-0-admin-server", + "--clusterrole=admin", + "--user="+testUserName, "-n", testServerTemplateA.ServerNS).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + // Grant testuser admin access to client namespace + err = oc.AsAdmin().WithoutNamespace().Run("create").Args("rolebinding", "testuser-0-admin-client", + "--clusterrole=admin", + "--user="+testUserName, "-n", testNSA).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + // Grant loki reader access for multi-tenancy + defer removeUserAsReader(oc, testUserName) + addUserAsReader(oc, testUserName) + + g.By("Verify testuser-0 can create flowcollectorslices in test-a") + canCreate, err := oc.WithoutNamespace().Run("auth").Args("can-i", "create", "flowcollectorslices", "-n", testNSA).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(canCreate).Should(o.ContainSubstring("yes")) + + g.By("Create a FlowCollectorSlice in test-a namespace") + flowSliceA := FlowcollectorSlice{ + Name: "test-a-slice", + Namespace: testNSA, + Sampling: "100", + Template: flowSliceFixturePath, + } + defer func() { _ = flowSliceA.DeleteFlowcollectorSlice(oc) }() + flowSliceA.CreateFlowcollectorSlice(oc) + flowSliceA.WaitForFlowcollectorSliceReady(oc) + + g.By("Verify testuser-0 can view the FlowCollectorSlice in test-a") + sliceOutput, err := oc.WithoutNamespace().Run("get").Args("flowcollectorslice", "-n", testNSA, "-o=jsonpath={.items[*].metadata.name}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(sliceOutput).Should(o.ContainSubstring("test-a-slice")) + + // Verify sampling value + samplingValue, err := oc.WithoutNamespace().Run("get").Args("flowcollectorslice", "test-a-slice", "-n", testNSA, "-o=jsonpath={.spec.sampling}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(samplingValue).Should(o.Equal("100")) + + g.By("Verify testuser-0 can update the FlowCollectorSlice in test-a") + err = oc.WithoutNamespace().Run("patch").Args("flowcollectorslice", "test-a-slice", "-n", testNSA, + "--type=merge", "-p={\"spec\":{\"sampling\":2}}").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + // Verify sampling was updated + samplingValue, err = oc.WithoutNamespace().Run("get").Args("flowcollectorslice", "test-a-slice", "-n", testNSA, "-o=jsonpath={.spec.sampling}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(samplingValue).Should(o.Equal("2")) + + g.By("Get testuser token for loki query") + user0token, err := oc.WithoutNamespace().Run("whoami").Args("-t").Output() + e2e.Logf("testuser token: %s", user0token) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Wait for flows to be collected and written to loki") + startTime := time.Now() + time.Sleep(60 * time.Second) + + g.By("Verify testuser-0 can access flows from test-a namespace") + lokilabels := Lokilabels{ + App: "netobserv-flowcollector", + SrcK8SNamespace: testServerTemplateA.ServerNS, + DstK8SNamespace: testNSA, + SrcK8SOwnerName: "nginx-service", + FlowDirection: "0", + } + flowRecords, err := lokilabels.getLokiFlowLogs(user0token, ls.Route, startTime) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically(">", 0), "expected testuser to see flows from test-a namespace") + + g.By("Create namespace test-b and create a FlowCollectorSlice as kubeadmin") + testServerTemplateB := TestServerTemplate{ + ServerNS: "test-b-server-87145", + Template: serverTemplate, + } + defer oc.DeleteSpecifiedNamespaceAsAdmin(testServerTemplateB.ServerNS) + + testClientTemplateB := TestClientTemplate{ + ServerNS: testServerTemplateB.ServerNS, + ClientNS: "test-b-client-87145", + Template: clientTemplate, + } + defer oc.DeleteSpecifiedNamespaceAsAdmin(testClientTemplateB.ClientNS) + + // Switch to admin to create test-b resources + oc.ChangeUser(origUser) + err = testServerTemplateB.createServer(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + compat_otp.AssertAllPodsToBeReady(oc, testServerTemplateB.ServerNS) + + err = testClientTemplateB.createClient(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + compat_otp.AssertAllPodsToBeReady(oc, testClientTemplateB.ClientNS) + + testNSB := testClientTemplateB.ClientNS + flowSliceB := FlowcollectorSlice{ + Name: "test-b-slice", + Namespace: testNSB, + Sampling: "3", + Template: flowSliceFixturePath, + } + defer func() { _ = flowSliceB.DeleteFlowcollectorSlice(oc) }() + flowSliceB.CreateFlowcollectorSlice(oc) + flowSliceB.WaitForFlowcollectorSliceReady(oc) + + // Switch back to testuser + oc.ChangeUser(testUserName) + + g.By("Verify testuser-0 cannot see test-b slice") + sliceOutputB, err := oc.WithoutNamespace().Run("get").Args("flowcollectorslice", "-n", testNSB).Output() + o.Expect(err).Should(o.HaveOccurred()) + o.Expect(sliceOutputB).Should(o.MatchRegexp(`User ".*" cannot list resource "flowcollectorslices"`)) + + g.By("Verify testuser-0 cannot access flows from test-b namespace") + lokilabels = Lokilabels{ + App: "netobserv-flowcollector", + SrcK8SNamespace: testServerTemplateB.ServerNS, + DstK8SNamespace: testNSB, + SrcK8SOwnerName: "nginx-service", + FlowDirection: "0", + } + flowRecords, err = lokilabels.getLokiFlowLogs(user0token, ls.Route, startTime) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(flowRecords)).Should(o.BeNumerically("==", 0), "expected testuser to NOT see flows from test-b namespace") + + g.By("Add testuser-0 as viewer for test-b namespace") + err = oc.AsAdmin().WithoutNamespace().Run("create").Args("rolebinding", "testuser-0-view", + "--clusterrole=flowcollectorslices.flows.netobserv.io-v1alpha1-view", + "--user="+testUserName, "-n", testNSB).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Verify testuser-0 can view FlowCollectorSlice in test-b") + sliceOutput, err = oc.WithoutNamespace().Run("get").Args("flowcollectorslice", "-n", testNSB, "-o=jsonpath={.items[*].metadata.name}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(sliceOutput).Should(o.ContainSubstring("test-b-slice")) + + g.By("Verify testuser-0 cannot update FlowCollectorSlice in test-b (view-only)") + patchOutput, err := oc.WithoutNamespace().Run("patch").Args("flowcollectorslice", "test-b-slice", "-n", testNSB, + "--type=merge", "-p={\"spec\":{\"sampling\":25}}").Output() + o.Expect(err).Should(o.HaveOccurred()) + o.Expect(patchOutput).Should(o.MatchRegexp(`User ".*" cannot patch resource "flowcollectorslices"`)) + + g.By("Remove testuser-0's view access from test-b") + err = oc.AsAdmin().WithoutNamespace().Run("delete").Args("rolebinding", "testuser-0-view", "-n", testNSB).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Verify access to test-b FlowCollectorSlices is revoked") + sliceOutputRevoked, err := oc.WithoutNamespace().Run("get").Args("flowcollectorslice", "-n", testNSB).Output() + o.Expect(err).Should(o.HaveOccurred()) + o.Expect(sliceOutputRevoked).Should(o.MatchRegexp(`User ".*" cannot list resource "flowcollectorslices"`)) + }) +}) diff --git a/integration-tests/backend/test_flowmetrics.go b/integration-tests/backend/test_flowmetrics.go new file mode 100644 index 0000000000..fb67261a59 --- /dev/null +++ b/integration-tests/backend/test_flowmetrics.go @@ -0,0 +1,115 @@ +package e2etests + +import ( + "os" + filePath "path/filepath" + "regexp" + "strings" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" + e2e "k8s.io/kubernetes/test/e2e/framework" +) + +var _ = g.Describe("[sig-netobserv] Network_Observability", func() { + + defer g.GinkgoRecover() + var ( + oc = compat_otp.NewCLI("netobserv", compat_otp.KubeConfigPath()) + // NetObserv Operator variables + NOcatSrc = Resource{"catsrc", "netobserv-konflux-fbc", netobservNS} + NOSource = CatalogSourceObjects{"stable", NOcatSrc.Name, NOcatSrc.Namespace} + + // Template directories + baseDir, _ = filePath.Abs("testdata") + subscriptionDir = filePath.Join(baseDir, "subscription") + flowFixturePath = filePath.Join(baseDir, "flowcollector_v1beta2_template.yaml") + flowmetricsPath = filePath.Join(baseDir, "flowmetrics_v1alpha1_template.yaml") + + // Operator namespace object + OperatorNS = OperatorNamespace{ + Name: netobservNS, + NamespaceTemplate: filePath.Join(subscriptionDir, "namespace.yaml"), + } + NO = SubscriptionObjects{ + OperatorName: "netobserv-operator", + Namespace: netobservNS, + PackageName: NOPackageName, + Subscription: filePath.Join(subscriptionDir, "sub-template.yaml"), + OperatorGroup: filePath.Join(subscriptionDir, "allnamespace-og.yaml"), + CatalogSource: &NOSource, + } + imageDigest = filePath.Join(subscriptionDir, "image-digest-mirror-set.yaml") + catSrcTemplate = filePath.Join(subscriptionDir, "catalog-source.yaml") + catalogSource = os.Getenv("MULTISTAGE_PARAM_OVERRIDE_NETOBSERV_CS_IMAGE") + + flow Flowcollector + ) + + g.BeforeEach(func() { + if strings.Contains(os.Getenv("E2E_RUN_TAGS"), "disconnected") { + g.Skip("Skipping tests for disconnected profiles") + } + + OperatorNS.DeployOperatorNamespace(oc) + deployedUpstreamCatalogSource, catSrcErr := setupCatalogSource(oc, NOcatSrc, catSrcTemplate, imageDigest, catalogSource, false, &NOSource, &NO) + o.Expect(catSrcErr).NotTo(o.HaveOccurred()) + ensureNetObservOperatorDeployed(oc, NO, NOSource, deployedUpstreamCatalogSource) + + // Create flowcollector in beforeEach + flow = Flowcollector{ + Namespace: oc.Namespace(), + EBPFeatures: []string{"\"FlowRTT\""}, + LokiEnable: "false", + Template: flowFixturePath, + } + flow.CreateFlowcollector(oc) + }) + g.AfterEach(func() { + _ = flow.DeleteFlowcollector(oc) + }) + + g.It("Author:memodi-High-73539-Create custom metrics and charts [Serial]", func() { + SkipIfOCPBelow("v4.12") + namespace := oc.Namespace() + customMetrics := CustomMetrics{ + Namespace: namespace, + Template: flowmetricsPath, + } + + mainDashversion, err := getResourceVersion(oc, "cm", "netobserv-main", "openshift-config-managed") + o.Expect(err).NotTo(o.HaveOccurred()) + curv, err := getResourceVersion(oc, "cm", "flowlogs-pipeline-config-dynamic", namespace) + o.Expect(err).NotTo(o.HaveOccurred()) + + customMetrics.createCustomMetrics(oc) + waitForResourceGenerationUpdate(oc, "cm", "flowlogs-pipeline-config-dynamic", "resourceVersion", curv, namespace) + + customMetricsConfig := customMetrics.getCustomMetricConfigs() + var allUniqueDash = make(map[string]bool) + var uniqueDashboards []string + for _, cmc := range customMetricsConfig { + for _, dashboard := range cmc.DashboardNames { + if _, ok := allUniqueDash[dashboard]; !ok { + allUniqueDash[dashboard] = true + uniqueDashboards = append(uniqueDashboards, dashboard) + } + } + // verify custom metrics queries + for _, query := range cmc.Queries { + metricsQuery := strings.Replace(query, "$METRIC", "netobserv_"+cmc.MetricName, 1) + metricVal := pollMetrics(oc, metricsQuery) + e2e.Logf("metricsQuery %f for query %s", metricVal, metricsQuery) + } + } + // verify dashboard exists + for _, uniqDash := range uniqueDashboards { + dashName := strings.ToLower(regexp.MustCompile(`[^a-zA-Z0-9]+`).ReplaceAllString(uniqDash, "-")) + if dashName == "main" { + waitForResourceGenerationUpdate(oc, "cm", "netobserv-"+dashName, "resourceVersion", mainDashversion, "openshift-config-managed") + } + _, _ = checkResourceExists(oc, "cm", "netobserv-"+dashName, "openshift-config-managed") + } + }) +}) diff --git a/integration-tests/backend/testdata/DNS-pods.yaml b/integration-tests/backend/testdata/DNS-pods.yaml new file mode 100644 index 0000000000..ed0e0382af --- /dev/null +++ b/integration-tests/backend/testdata/DNS-pods.yaml @@ -0,0 +1,73 @@ +--- +kind: Namespace +apiVersion: v1 +metadata: + name: "dns-traffic" + labels: + name: "dns-traffic" + +--- +apiVersion: v1 +kind: Pod +metadata: + name: dnsutils1 + namespace: "dns-traffic" +spec: + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + dnsConfig: + options: + - name: "use-vc" + containers: + - command: + - sh + - -c + - " + \ while : ; do\n + \ dig www.google.com +tcp ; sleep 5 \n + \ done" + image: registry.k8s.io/e2e-test-images/agnhost:2.39 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + privileged: false + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + name: dnsutils1 + +--- +apiVersion: v1 +kind: Pod +metadata: + name: dnsutils2 + namespace: "dns-traffic" +spec: + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + dnsConfig: + options: + - name: "use-vc" + containers: + - command: + - sh + - -c + - " + \ while : ; do\n + \ dig www.google.com ; sleep 5 \n + \ done" + image: registry.k8s.io/e2e-test-images/agnhost:2.39 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + privileged: false + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + name: dnsutils2 diff --git a/integration-tests/backend/testdata/SYN_flood_alert_template.yaml b/integration-tests/backend/testdata/SYN_flood_alert_template.yaml new file mode 100644 index 0000000000..ba3e379369 --- /dev/null +++ b/integration-tests/backend/testdata/SYN_flood_alert_template.yaml @@ -0,0 +1,37 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: netobserv-syn-flood-alert-template +objects: +- apiVersion: monitoring.openshift.io/v1 + kind: AlertingRule + metadata: + name: netobserv-syn-alerts + namespace: ${Namespace} + spec: + groups: + - name: NetObservSYNAlerts + rules: + - alert: NetObserv-SYNFlood-in + annotations: + message: |- + {{ $labels.job }}: incoming SYN-flood attack suspected to Host={{ $labels.DstK8S_HostName}}, Namespace={{ $labels.DstK8S_Namespace }}, Resource={{ $labels.DstK8S_Name }}. This is characterized by a high volume of SYN-only flows with different source IPs and/or ports. + summary: "Incoming SYN-flood" + expr: sum(rate(netobserv_flows_with_flags_per_destination_total{Flags="[SYN]"}[2m])) by (job, DstK8S_HostName, DstK8S_Namespace, DstK8S_Name) > 0.9 + for: 15s + labels: + severity: warning + app: netobserv + - alert: NetObserv-SYNFlood-out + annotations: + message: |- + {{ $labels.job }}: outgoing SYN-flood attack suspected from Host={{ $labels.SrcK8S_HostName}}, Namespace={{ $labels.SrcK8S_Namespace }}, Resource={{ $labels.SrcK8S_Name }}. This is characterized by a high volume of SYN-only flows with different source IPs and/or ports. + summary: "Outgoing SYN-flood" + expr: sum(rate(netobserv_flows_with_flags_per_source_total{Flags="[SYN]"}[2m])) by (job, SrcK8S_HostName, SrcK8S_Namespace, SrcK8S_Name) > 0.9 + for: 15s + labels: + severity: warning + app: netobserv +parameters: +- name: Namespace + value: openshift-monitoring diff --git a/integration-tests/backend/testdata/SYN_flood_metrics_template.yaml b/integration-tests/backend/testdata/SYN_flood_metrics_template.yaml new file mode 100644 index 0000000000..eeccc38cb5 --- /dev/null +++ b/integration-tests/backend/testdata/SYN_flood_metrics_template.yaml @@ -0,0 +1,26 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: netobserv-syn-flood-metrics-template +objects: +- apiVersion: flows.netobserv.io/v1alpha1 + kind: FlowMetric + metadata: + name: flows-with-flags-per-destination + namespace: ${Namespace} + spec: + metricName: flows_with_flags_per_destination_total + type: Counter + labels: [SrcSubnetLabel,DstSubnetLabel,DstK8S_Name,DstK8S_Type,DstK8S_HostName,DstK8S_Namespace,Flags] +- apiVersion: flows.netobserv.io/v1alpha1 + kind: FlowMetric + metadata: + name: flows-with-flags-per-source + namespace: ${Namespace} + spec: + metricName: flows_with_flags_per_source_total + type: Counter + labels: [DstSubnetLabel,SrcSubnetLabel,SrcK8S_Name,SrcK8S_Type,SrcK8S_HostName,SrcK8S_Namespace,Flags] +parameters: +- name: Namespace + value: netobserv diff --git a/integration-tests/backend/testdata/bpfman/catalog-source.yaml b/integration-tests/backend/testdata/bpfman/catalog-source.yaml new file mode 100644 index 0000000000..fc98d2cf38 --- /dev/null +++ b/integration-tests/backend/testdata/bpfman/catalog-source.yaml @@ -0,0 +1,23 @@ +kind: Template +apiVersion: template.openshift.io/v1 +metadata: + name: bpfman-catalog-source-template +objects: +- apiVersion: operators.coreos.com/v1alpha1 + kind: CatalogSource + metadata: + name: "${CATALOG_NAME}" + namespace: "${NAMESPACE}" + spec: + displayName: eBPF Manager Konflux + image: "${IMAGE}" + sourceType: grpc + grpcPodConfig: + securityContextConfig: legacy +parameters: +- name: CATALOG_NAME + value: "bpfman-konflux-fbc" +- name: IMAGE + value: "quay.io/redhat-user-workloads/ocp-bpfman-tenant/catalog-ystream:latest" +- name: NAMESPACE + value: openshift-marketplace diff --git a/integration-tests/backend/testdata/bpfman/image-digest-mirror-set.yaml b/integration-tests/backend/testdata/bpfman/image-digest-mirror-set.yaml new file mode 100644 index 0000000000..50727a4872 --- /dev/null +++ b/integration-tests/backend/testdata/bpfman/image-digest-mirror-set.yaml @@ -0,0 +1,22 @@ +apiVersion: config.openshift.io/v1 +kind: ImageDigestMirrorSet +metadata: + name: bpfman-image-digest-mirror-set +spec: + imageDigestMirrors: + - mirrors: + - quay.io/redhat-user-workloads/ocp-bpfman-tenant/catalog-ystream + - quay.io/redhat-user-workloads/ocp-bpfman-tenant/catalog-zstream + source: registry.redhat.io/bpfman/bpfman-operator-catalog + - mirrors: + - quay.io/redhat-user-workloads/ocp-bpfman-tenant/bpfman-agent-ystream + source: registry.redhat.io/bpfman/bpfman-agent + - mirrors: + - quay.io/redhat-user-workloads/ocp-bpfman-tenant/bpfman-operator-bundle-ystream + source: registry.redhat.io/bpfman/bpfman-operator-bundle + - mirrors: + - quay.io/redhat-user-workloads/ocp-bpfman-tenant/bpfman-operator-ystream + source: registry.redhat.io/bpfman/bpfman-rhel9-operator + - mirrors: + - quay.io/redhat-user-workloads/ocp-bpfman-tenant/bpfman-daemon-ystream + source: registry.redhat.io/bpfman/bpfman diff --git a/integration-tests/backend/testdata/bpfman/namespace.yaml b/integration-tests/backend/testdata/bpfman/namespace.yaml new file mode 100644 index 0000000000..6fe8ec2e34 --- /dev/null +++ b/integration-tests/backend/testdata/bpfman/namespace.yaml @@ -0,0 +1,21 @@ +kind: Template +apiVersion: template.openshift.io/v1 +metadata: + name: bpfman-namespace-template +objects: +- kind: Namespace + apiVersion: v1 + metadata: + name: ${NAMESPACE_NAME} + labels: + kubernetes.io/metadata.name: ${NAMESPACE_NAME} + openshift.io/cluster-monitoring: "true" + pod-security.kubernetes.io/audit: privileged + pod-security.kubernetes.io/audit-version: latest + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/enforce-version: latest + pod-security.kubernetes.io/warn: privileged + pod-security.kubernetes.io/warn-version: latest +parameters: +- name: NAMESPACE_NAME + value: "bpfman" diff --git a/integration-tests/backend/testdata/cert_manager_certificates_template.yaml b/integration-tests/backend/testdata/cert_manager_certificates_template.yaml new file mode 100644 index 0000000000..cadb5b37aa --- /dev/null +++ b/integration-tests/backend/testdata/cert_manager_certificates_template.yaml @@ -0,0 +1,66 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: netobserv-certificates-template +objects: + - apiVersion: cert-manager.io/v1 + kind: Issuer + metadata: + name: netobserv-self-signed + namespace: "${Namespace}" + spec: + selfSigned: {} + - apiVersion: cert-manager.io/v1 + kind: Certificate + metadata: + name: netobserv-ca + namespace: "${Namespace}" + spec: + isCA: true + commonName: netobserv-ca + secretName: prov-netobserv-ca-secret + privateKey: + algorithm: ECDSA + size: 256 + issuerRef: + name: netobserv-self-signed + kind: Issuer + group: cert-manager.io + - apiVersion: cert-manager.io/v1 + kind: Issuer + metadata: + name: netobserv-issuer + namespace: "${Namespace}" + spec: + ca: + secretName: prov-netobserv-ca-secret + - apiVersion: cert-manager.io/v1 + kind: Certificate + metadata: + name: flowlogs-pipeline-cert + namespace: "${Namespace}" + spec: + secretName: prov-flowlogs-pipeline-cert + dnsNames: + - flowlogs-pipeline.${Namespace}.svc + - flowlogs-pipeline.${Namespace}.svc.cluster.local + issuerRef: + name: netobserv-issuer + kind: Issuer + group: cert-manager.io + - apiVersion: cert-manager.io/v1 + kind: Certificate + metadata: + name: ebpf-agent-cert + namespace: "${Namespace}" + spec: + secretName: prov-ebpf-agent-cert + commonName: netobserv-ebpf-agent + issuerRef: + name: netobserv-issuer + kind: Issuer + group: cert-manager.io +parameters: + - name: Namespace + description: "namespace where certificates will be created" + value: "netobserv" diff --git a/integration-tests/backend/testdata/exporters/ipfix-collector.yaml b/integration-tests/backend/testdata/exporters/ipfix-collector.yaml new file mode 100644 index 0000000000..a592bc41c5 --- /dev/null +++ b/integration-tests/backend/testdata/exporters/ipfix-collector.yaml @@ -0,0 +1,52 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ipfix-collector + labels: + app: ipfix-collector + namespace: ipfix +spec: + replicas: 1 + selector: + matchLabels: + app: ipfix-collector + template: + metadata: + labels: + app: ipfix-collector + spec: + containers: + - name: ipfix-collector + image: antrea/ipfix-collector:latest + args: + - "--ipfix.addr=0.0.0.0" + - "--ipfix.port=2055" + - "--ipfix.transport=tcp" + ports: + - containerPort: 2055 + protocol: TCP + name: ipfix + - containerPort: 8080 + protocol: TCP + name: http + imagePullPolicy: IfNotPresent +--- +apiVersion: v1 +kind: Service +metadata: + name: ipfix-collector + labels: + app: ipfix-collector + namespace: ipfix +spec: + ports: + - port: 2055 + targetPort: 2055 + protocol: TCP + name: ipfix + - port: 8080 + targetPort: 8080 + protocol: TCP + name: http + selector: + app: ipfix-collector diff --git a/integration-tests/backend/testdata/exporters/otel-collector-tls.yaml b/integration-tests/backend/testdata/exporters/otel-collector-tls.yaml new file mode 100644 index 0000000000..8934f1f23a --- /dev/null +++ b/integration-tests/backend/testdata/exporters/otel-collector-tls.yaml @@ -0,0 +1,99 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: otel-collector-tls-template +objects: +- apiVersion: v1 + kind: Service + metadata: + name: ${NAME}-collector + annotations: + service.beta.openshift.io/serving-cert-secret-name: ${NAME}-tls + spec: + selector: + app.kubernetes.io/component: opentelemetry-collector + app.kubernetes.io/instance: ${NAME} + ports: + - name: otlp-grpc + port: ${{OTLP_GRPC_ENDPOINT}} + protocol: TCP + targetPort: ${{OTLP_GRPC_ENDPOINT}} + - name: otlp-http + port: 4318 + protocol: TCP + targetPort: 4318 + - name: prometheus + port: ${{OTLP_PROM_PORT}} + protocol: TCP + targetPort: ${{OTLP_PROM_PORT}} + +- apiVersion: v1 + kind: ConfigMap + metadata: + name: service-ca + annotations: + service.beta.openshift.io/inject-cabundle: "true" + data: {} + +- apiVersion: opentelemetry.io/v1beta1 + kind: OpenTelemetryCollector + metadata: + name: ${NAME} + spec: + config: + exporters: + debug: + verbosity: detailed + prometheus: + const_labels: + otel: otel + enable_open_metrics: true + endpoint: 0.0.0.0:${OTLP_PROM_PORT} + processors: + batch: {} + memory_limiter: + check_interval: 1s + limit_percentage: 50 + spike_limit_percentage: 30 + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:${OTLP_GRPC_ENDPOINT} + tls: + cert_file: /var/run/secrets/tls/tls.crt + key_file: /var/run/secrets/tls/tls.key + http: + endpoint: 0.0.0.0:4318 + service: + pipelines: + logs: + exporters: + - debug + processors: + - batch + receivers: + - otlp + metrics: + exporters: + - debug + - prometheus + processors: + - batch + receivers: + - otlp + volumeMounts: + - name: tls-certs + mountPath: /var/run/secrets/tls + volumes: + - name: tls-certs + secret: + secretName: ${NAME}-tls + +parameters: +- name: NAME + value: "otel" +- name: OTLP_GRPC_ENDPOINT + value: "4317" +- name: OTLP_PROM_PORT + value: "8889" diff --git a/integration-tests/backend/testdata/exporters/otel-collector.yaml b/integration-tests/backend/testdata/exporters/otel-collector.yaml new file mode 100644 index 0000000000..8c1c9d2e61 --- /dev/null +++ b/integration-tests/backend/testdata/exporters/otel-collector.yaml @@ -0,0 +1,73 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: otel-collector-template +objects: +- apiVersion: opentelemetry.io/v1beta1 + kind: OpenTelemetryCollector + metadata: + name: ${NAME} + spec: + config: + exporters: + debug: + verbosity: detailed + prometheus: + const_labels: + otel: otel + enable_open_metrics: true + endpoint: 0.0.0.0:${OTLP_PROM_PORT} + processors: + batch: {} + memory_limiter: + check_interval: 1s + limit_percentage: 50 + spike_limit_percentage: 30 + receivers: + jaeger: + protocols: + grpc: {} + thrift_binary: {} + thrift_compact: {} + thrift_http: {} + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:${OTLP_GRPC_ENDPOINT} + http: + endpoint: 0.0.0.0:4318 + zipkin: {} + service: + pipelines: + logs: + exporters: + - debug + processors: + - batch + receivers: + - otlp + metrics: + exporters: + - debug + - prometheus + processors: + - batch + receivers: + - otlp + traces: + exporters: + - debug + processors: + - memory_limiter + - batch + receivers: + - otlp + - jaeger + - zipkin +parameters: +- name: NAME + value: "otel" +- name: OTLP_GRPC_ENDPOINT + value: "4317" +- name: OTLP_PROM_PORT + value: "8889" diff --git a/integration-tests/backend/testdata/flowcollectorSlice_v1alpha1_template.yaml b/integration-tests/backend/testdata/flowcollectorSlice_v1alpha1_template.yaml new file mode 100644 index 0000000000..072cfa2a67 --- /dev/null +++ b/integration-tests/backend/testdata/flowcollectorSlice_v1alpha1_template.yaml @@ -0,0 +1,23 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: flowcollectorslice-template +objects: + - apiVersion: flows.netobserv.io/v1alpha1 + kind: FlowCollectorSlice + metadata: + name: "${Name}" + namespace: "${Namespace}" + spec: + sampling: ${{Sampling}} + subnetLabels: "${{SubnetLabels}}" +parameters: + - name: Name + value: "slice-sample" + - name: Namespace + description: "namespace where you want flowCollectorSlice to be deployed" + value: "netobserv" + - name: Sampling + value: "1" + - name: SubnetLabels + value: '[]' diff --git a/integration-tests/backend/testdata/flowcollector_v1beta2_template.yaml b/integration-tests/backend/testdata/flowcollector_v1beta2_template.yaml new file mode 100644 index 0000000000..f2909fd442 --- /dev/null +++ b/integration-tests/backend/testdata/flowcollector_v1beta2_template.yaml @@ -0,0 +1,231 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: flowcollector-template +objects: + - apiVersion: flows.netobserv.io/v1beta2 + kind: FlowCollector + metadata: + name: cluster + spec: + namespace: "${Namespace}" + deploymentModel: "${DeploymentModel}" + agent: + type: eBPF + ebpf: + imagePullPolicy: IfNotPresent + sampling: ${{Sampling}} + cacheActiveTimeout: ${EBPFCacheActiveTimeout} + cacheMaxFlows: ${{CacheMaxFlows}} + interfaces: [ ] + excludeInterfaces: ["lo"] + features: "${{EBPFeatures}}" + logLevel: info + privileged: "${{EBPFPrivileged}}" + flowFilter: + enable: "${{EBPFFilterEnable}}" + rules: "${{EBPFFilterRules}}" + metrics: + enable: "${{EBPFMetrics}}" + server: + port: 9400 + tls: + type: "${EBPFMetricServerTLSType}" + processor: + multiClusterDeployment: "${{MultiClusterDeployment}}" + addZone: "${{AddZone}}" + service: + providedCertificates: + caFile: + file: ca.crt + name: "${ServiceCASecretName}" + namespace: "${Namespace}" + type: secret + serverCert: + certFile: tls.crt + certKey: tls.key + name: "${ServiceServerCertSecretName}" + namespace: "${Namespace}" + type: secret + clientCert: + certFile: tls.crt + certKey: tls.key + name: "${ServiceClientCertSecretName}" + namespace: "${Namespace}" + type: secret + tlsType: "${ServiceTLSType}" + metrics: + server: + tls: + type: "${FLPMetricServerTLSType}" + disableAlerts: [] + logTypes: "${LogType}" + filters: "${{FLPFilters}}" + slicesConfig: + collectionMode: "${CollectionMode}" + enable: "${{SlicesEnable}}" + namespacesAllowList: "${{NamespacesAllow}}" + advanced: + dropUnusedFields: true + conversationTerminatingTimeout: 5s + conversationHeartbeatInterval: 5s + conversationEndTimeout: 20s + secondaryNetworks: "${{SecondaryNetworks}}" + kafka: + address: "${KafkaAddress}" + topic: "${KafkaTopic}" + tls: + enable: "${{KafkaTLSEnable}}" + caCert: + type: secret + name: "${KafkaClusterName}-cluster-ca-cert" + certFile: ca.crt + namespace: "${KafkaNamespace}" + userCert: + type: secret + name: "${KafkaUser}" + certFile: user.crt + certKey: user.key + namespace: "${KafkaNamespace}" + loki: + mode: "${LokiMode}" + enable: "${{LokiEnable}}" + lokiStack: + name: ${LokistackName} + namespace: ${LokiNamespace} + manual: + authToken: Forward + querierUrl: "${LokiURL}" + ingesterUrl: "${LokiURL}" + statusUrl: "${LokiStatusURL}" + tls: + enable: true + caCert: + type: configmap + name: "${LokiTLSCertName}" + certFile: service-ca.crt + namespace: "${LokiNamespace}" + insecureSkipVerify: false + tenantID: network + statusTls: + enable: "${{LokiStatusTLSEnable}}" + caCert: + certFile: service-ca.crt + name: "${LokiStatusTLSCertName}" + type: configmap + namespace: "${LokiNamespace}" + insecureSkipVerify: false + userCert: + certFile: tls.crt + certKey: tls.key + name: "${LokiStatusTLSUserCertName}" + type: secret + namespace: "${LokiNamespace}" + monolithic: + url: ${MonolithicLokiURL} + networkPolicy: + additionalNamespaces: "${{NetworkPolicyAdditionalNamespaces}}" + enable: "${{NetworkPolicyEnable}}" + consolePlugin: + enable: "${{PluginEnable}}" + portNaming: + enable: true + portNames: + "3100": loki + exporters: "${{Exporters}}" +parameters: + - name: Namespace + description: "namespace where you want flowlogsPipeline and consoleplugin pods to be deployed" + value: "netobserv" + - name: DeploymentModel + value: "Direct" + - name: EBPFCacheActiveTimeout + value: 15s + - name: Sampling + value: "1" + - name: EBPFPrivileged + value: "false" + - name: EBPFMetrics + value: "true" + - name: FLPMetricServerTLSType + value: "Auto" + - name: EBPFMetricServerTLSType + value: "Disabled" + - name: EBPFeatures + value: '[]' + - name: EBPFFilterEnable + value: "true" + - name: EBPFFilterRules + value: '[{"action": "Accept","cidr": "0.0.0.0/0"}]' + - name: CacheMaxFlows + value: "120000" + - name: MultiClusterDeployment + value: "false" + - name: AddZone + value: "false" + - name: LogType + value: "Flows" + - name: FLPFilters + value: '[]' + - name: CollectionMode + value: "AlwaysCollect" + - name: SlicesEnable + value: "false" + - name: NamespacesAllow + value: '["`/netobserv.*/`"]' + - name: LokiMode + value: "LokiStack" + - name: LokiEnable + value: "true" + - name: LokistackName + value: lokistack + - name: LokiURL + value: "https://lokistack-gateway-http.netobserv.svc.cluster.local:8080/api/logs/v1/network/" + - name: LokiTLSCertName + value: "lokistack-gateway-ca-bundle" + - name: LokiStatusURL + value: "" + - name: LokiStatusTLSEnable + value: "false" + - name: LokiStatusTLSCertName + value: "lokistack-ca-bundle" + - name: LokiStatusTLSUserCertName + value: "lokistack-query-frontend-http" + - name: MonolithicLokiURL + value: "http://loki.netobserv.svc:3100/" + - name: LokiNamespace + value: "netobserv" + - name: KafkaAddress + value: "kafka-cluster-kafka-bootstrap.netobserv" + - name: KafkaTLSEnable + value: "false" + - name: KafkaClusterName + value: "kafka-cluster" + - name: KafkaTopic + value: "network-flows" + - name: KafkaUser + value: "flp-kafka" + - name: KafkaNamespace + value: "netobserv" + - name: PluginEnable + value: "true" + - name: NetworkPolicyEnable + value: "true" + - name: NetworkPolicyAdditionalNamespaces + value: '[]' + - name: Exporters + value: '[]' + - name: SecondaryNetworks + value: '[]' + - name: ServiceTLSType + description: "TLS type for Service deployment model (Auto, Provided, Disabled, Auto-mTLS)" + value: "Auto" + - name: ServiceCASecretName + description: "CA certificate secret name for Service mode with Provided certificates" + value: "prov-netobserv-ca-secret" + - name: ServiceServerCertSecretName + description: "Server certificate secret name for Service mode with Provided certificates" + value: "prov-flowlogs-pipeline-cert" + - name: ServiceClientCertSecretName + description: "Client certificate secret name for Service mode with Provided certificates" + value: "prov-ebpf-agent-cert" diff --git a/integration-tests/backend/testdata/flowlogs_pipeline_hpa_template.yaml b/integration-tests/backend/testdata/flowlogs_pipeline_hpa_template.yaml new file mode 100644 index 0000000000..dd37a2f4b3 --- /dev/null +++ b/integration-tests/backend/testdata/flowlogs_pipeline_hpa_template.yaml @@ -0,0 +1,31 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: flowlogs-pipeline-hpa-template + annotations: + description: "Template for Horizontal Pod Autoscaler for flowlogs-pipeline." +objects: + - apiVersion: autoscaling/v2 + kind: HorizontalPodAutoscaler + metadata: + name: flowlogs-pipeline-hpa + namespace: ${NAMESPACE} # Uses the parameter here + spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: flowlogs-pipeline + minReplicas: 2 + maxReplicas: 5 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 1 +parameters: + - name: NAMESPACE + displayName: Namespace + description: The namespace where the flowlogs-pipeline Deployment is located. + value: netobserv diff --git a/integration-tests/backend/testdata/flowmetrics_v1alpha1_template.yaml b/integration-tests/backend/testdata/flowmetrics_v1alpha1_template.yaml new file mode 100644 index 0000000000..ca5ffeb92d --- /dev/null +++ b/integration-tests/backend/testdata/flowmetrics_v1alpha1_template.yaml @@ -0,0 +1,68 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: netobserv-flowmetrics +objects: +- apiVersion: flows.netobserv.io/v1alpha1 + kind: FlowMetric + metadata: + name: port-metrics + namespace: ${Namespace} + spec: + charts: + - dashboardName: Custom + queries: + - legend: '{{DstK8S_Namespace}} / {{DstPort}}' + promQL: 'sum(rate($METRIC[2m])) by (DstK8S_Namespace, DstPort)' + top: 7 + title: Traffic across service ports + type: StackArea + direction: Egress + filters: + - field: DstPort + matchType: Presence + value: "\\d+" + - field: DstK8S_Namespace + matchType: MatchRegex + value: "^openshift-monitoring$" + - field: DstK8S_Namespace + matchType: MatchRegex + value: "^openshift-ingress$" + labels: + - DstPort + - DstK8S_Namespace + metricName: service_ports_total_bytes + type: Counter + valueField: Bytes +- apiVersion: flows.netobserv.io/v1alpha1 + kind: FlowMetric + metadata: + name: flowmetric-cluster-external-ingress-traffic + spec: + metricName: cluster_external_ingress_bytes_total + type: Counter + valueField: Bytes + direction: Ingress + labels: [DstK8S_HostName,DstK8S_Namespace,DstK8S_OwnerName,DstK8S_OwnerType] + filters: + - field: SrcSubnetLabel + matchType: Absence + charts: + - dashboardName: Main + title: External ingress traffic + unit: Bps + type: SingleStat + queries: + - promQL: "sum(rate($METRIC[2m]))" + legend: "" + - dashboardName: Main + sectionName: External + title: Top external ingress traffic per workload + unit: Bps + type: StackArea + queries: + - promQL: "sum(rate($METRIC{DstK8S_Namespace!=\"\"}[2m])) by (DstK8S_Namespace, DstK8S_OwnerName)" + legend: "{{DstK8S_Namespace}} / {{DstK8S_OwnerName}}" +parameters: +- name: Namespace + value: netobserv diff --git a/integration-tests/backend/testdata/gateway-api-template.yaml b/integration-tests/backend/testdata/gateway-api-template.yaml new file mode 100644 index 0000000000..66cd44ae55 --- /dev/null +++ b/integration-tests/backend/testdata/gateway-api-template.yaml @@ -0,0 +1,97 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: gateway-api-resources +objects: +- apiVersion: v1 + kind: Namespace + metadata: + name: ${NAMESPACE} + labels: + app: ${NAMESPACE_LABEL} +- apiVersion: gateway.networking.k8s.io/v1 + kind: GatewayClass + metadata: + name: openshift-default + spec: + controllerName: openshift.io/gateway-controller/v1 +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + name: ${GATEWAY_NAME} + namespace: ${NAMESPACE} + spec: + gatewayClassName: openshift-default + listeners: + - name: demo + hostname: ${HOSTNAME} + port: 80 + protocol: HTTP + allowedRoutes: + namespaces: + from: Selector + selector: + matchLabels: + app: ${NAMESPACE_LABEL} +- apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + name: ${HTTPROUTE_NAME} + namespace: ${NAMESPACE} + spec: + parentRefs: + - name: ${GATEWAY_NAME} + namespace: ${NAMESPACE} + hostnames: ["${HOSTNAME}"] + rules: + - backendRefs: + - name: service-unsecure + port: 27017 +- apiVersion: apps/v1 + kind: Deployment + metadata: + name: traffic-generator + namespace: ${NAMESPACE} + spec: + replicas: 1 + selector: + matchLabels: + app: traffic-generator + template: + metadata: + labels: + app: traffic-generator + spec: + containers: + - name: traffic-gen + image: registry.access.redhat.com/ubi8/ubi-minimal + command: + - /bin/sh + - -c + - | + while true; do + curl -s http://${GATEWAY_NAME}-openshift-default.${NAMESPACE}.svc.cluster.local:8080 || true + wget -q -O- http://${GATEWAY_NAME}-openshift-default.${NAMESPACE}.svc.cluster.local:8080 || true + sleep 5 + done +parameters: + - name: GATEWAY_NAME + description: Name of the Gateway resource + value: test-gateway-owner + required: true + - name: HTTPROUTE_NAME + description: Name of the HTTPRoute resource + value: test-httproute + required: true + - name: NAMESPACE + description: Namespace for Gateway and HTTPRoute resources + value: netobserv-gateway-test + required: true + - name: HOSTNAME + description: Hostname for the Gateway listener and HTTPRoute + value: gwapi.example.com + required: true + - name: NAMESPACE_LABEL + description: Label value for namespace app label + value: gwapi + required: true diff --git a/integration-tests/backend/testdata/kafka/kafka-default.yaml b/integration-tests/backend/testdata/kafka/kafka-default.yaml new file mode 100644 index 0000000000..907e7e2c6f --- /dev/null +++ b/integration-tests/backend/testdata/kafka/kafka-default.yaml @@ -0,0 +1,63 @@ +kind: Template +apiVersion: template.openshift.io/v1 +metadata: + name: kafka-template +objects: +- apiVersion: kafka.strimzi.io/v1beta2 + kind: Kafka + metadata: + name: "${NAME}" + namespace: "${NAMESPACE}" + annotations: + strimzi.io/kraft: enabled + strimzi.io/node-pools: enabled + spec: + kafka: + replicas: 3 + listeners: + - name: plain + port: 9092 + type: internal + tls: false + - name: tls + port: 9093 + type: internal + tls: true + - name: external + port: 9094 + type: nodeport + tls: false + config: + offsets.topic.replication.factor: 1 + transaction.state.log.replication.factor: 1 + transaction.state.log.min.isr: 1 + default.replication.factor: 1 + min.insync.replicas: 1 + auto.create.topics.enable: false + log.cleaner.backoff.ms: 15000 + log.cleaner.dedupe.buffer.size: 134217728 + log.cleaner.enable: true + log.cleaner.io.buffer.load.factor: 0.9 + log.cleaner.threads: 8 + log.cleanup.policy: delete + log.retention.bytes: 107374182400 + log.retention.check.interval.ms: 300000 + log.retention.ms: 1680000 + log.roll.ms: 7200000 + log.segment.bytes: 1073741824 + metadataVersion: 4.0-IV3 + version: 4.0.0 + metricsConfig: + type: jmxPrometheusExporter + valueFrom: + configMapKeyRef: + name: kafka-metrics + key: kafka-metrics-config.yml + entityOperator: + topicOperator: {} + userOperator: {} +parameters: +- name: NAME + value: "kafka-cluster" +- name: NAMESPACE + value: "netobserv" diff --git a/integration-tests/backend/testdata/kafka/kafka-metrics-config.yaml b/integration-tests/backend/testdata/kafka/kafka-metrics-config.yaml new file mode 100644 index 0000000000..2a80349993 --- /dev/null +++ b/integration-tests/backend/testdata/kafka/kafka-metrics-config.yaml @@ -0,0 +1,177 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: kafkametrics-config-template +objects: +- apiVersion: v1 + kind: ConfigMap + metadata: + name: kafka-metrics + labels: + app: strimzi + data: + kafka-metrics-config.yml: | + lowercaseOutputName: true + rules: + - pattern: kafka.server<>Value + name: kafka_server_$1_$2 + type: GAUGE + labels: + clientId: "$3" + topic: "$4" + partition: "$5" + - pattern: kafka.server<>Value + name: kafka_server_$1_$2 + type: GAUGE + labels: + clientId: "$3" + broker: "$4:$5" + - pattern: kafka.server<>connections + name: kafka_server_$1_connections_tls_info + type: GAUGE + labels: + cipher: "$2" + protocol: "$3" + listener: "$4" + networkProcessor: "$5" + - pattern: kafka.server<>connections + name: kafka_server_$1_connections_software + type: GAUGE + labels: + clientSoftwareName: "$2" + clientSoftwareVersion: "$3" + listener: "$4" + networkProcessor: "$5" + - pattern: "kafka.server<>(.+):" + name: kafka_server_$1_$4 + type: GAUGE + labels: + listener: "$2" + networkProcessor: "$3" + - pattern: kafka.server<>(.+) + name: kafka_server_$1_$4 + type: GAUGE + labels: + listener: "$2" + networkProcessor: "$3" + - pattern: kafka.(\w+)<>MeanRate + name: kafka_$1_$2_$3_percent + type: GAUGE + - pattern: kafka.(\w+)<>Value + name: kafka_$1_$2_$3_percent + type: GAUGE + - pattern: kafka.(\w+)<>Value + name: kafka_$1_$2_$3_percent + type: GAUGE + labels: + "$4": "$5" + - pattern: kafka.(\w+)<>Count + name: kafka_$1_$2_$3_total + type: COUNTER + labels: + "$4": "$5" + "$6": "$7" + - pattern: kafka.(\w+)<>Count + name: kafka_$1_$2_$3_total + type: COUNTER + labels: + "$4": "$5" + - pattern: kafka.(\w+)<>Count + name: kafka_$1_$2_$3_total + type: COUNTER + - pattern: kafka.(\w+)<>Value + name: kafka_$1_$2_$3 + type: GAUGE + labels: + "$4": "$5" + "$6": "$7" + - pattern: kafka.(\w+)<>Value + name: kafka_$1_$2_$3 + type: GAUGE + labels: + "$4": "$5" + - pattern: kafka.(\w+)<>Value + name: kafka_$1_$2_$3 + type: GAUGE + - pattern: kafka.(\w+)<>Count + name: kafka_$1_$2_$3_count + type: COUNTER + labels: + "$4": "$5" + "$6": "$7" + - pattern: kafka.(\w+)<>(\d+)thPercentile + name: kafka_$1_$2_$3 + type: GAUGE + labels: + "$4": "$5" + "$6": "$7" + quantile: "0.$8" + - pattern: kafka.(\w+)<>Count + name: kafka_$1_$2_$3_count + type: COUNTER + labels: + "$4": "$5" + - pattern: kafka.(\w+)<>(\d+)thPercentile + name: kafka_$1_$2_$3 + type: GAUGE + labels: + "$4": "$5" + quantile: "0.$6" + - pattern: kafka.(\w+)<>Count + name: kafka_$1_$2_$3_count + type: COUNTER + - pattern: kafka.(\w+)<>(\d+)thPercentile + name: kafka_$1_$2_$3 + type: GAUGE + labels: + quantile: "0.$4" +- apiVersion: monitoring.coreos.com/v1 + kind: PodMonitor + metadata: + name: kafka-resources-metrics + labels: + app: strimzi + spec: + selector: + matchExpressions: + - key: "strimzi.io/kind" + operator: In + values: ["Kafka", "KafkaConnect", "KafkaMirrorMaker", "KafkaMirrorMaker2"] + namespaceSelector: + matchNames: + - ${NAMESPACE} + podMetricsEndpoints: + - path: /metrics + port: tcp-prometheus + relabelings: + - separator: ; + regex: __meta_kubernetes_pod_label_(strimzi_io_.+) + replacement: $1 + action: labelmap + - sourceLabels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + targetLabel: namespace + replacement: $1 + action: replace + - sourceLabels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + targetLabel: kubernetes_pod_name + replacement: $1 + action: replace + - sourceLabels: [__meta_kubernetes_pod_node_name] + separator: ; + regex: (.*) + targetLabel: node_name + replacement: $1 + action: replace + - sourceLabels: [__meta_kubernetes_pod_host_ip] + separator: ; + regex: (.*) + targetLabel: node_ip + replacement: $1 + action: replace +parameters: +- name: NAMESPACE + value: "netobserv" diff --git a/integration-tests/backend/testdata/kafka/kafka-node-pool.yaml b/integration-tests/backend/testdata/kafka/kafka-node-pool.yaml new file mode 100644 index 0000000000..8cd234b7d6 --- /dev/null +++ b/integration-tests/backend/testdata/kafka/kafka-node-pool.yaml @@ -0,0 +1,31 @@ +kind: Template +apiVersion: template.openshift.io/v1 +metadata: + name: kafka-node-pool-template +objects: +- apiVersion: kafka.strimzi.io/v1beta2 + kind: KafkaNodePool + metadata: + name: "${NODEPOOL}" + labels: + strimzi.io/cluster: ${NAME} + namespace: "${NAMESPACE}" + spec: + replicas: 3 + roles: + - broker + - controller + storage: + type: jbod + volumes: + - deleteClaim: false + id: 0 + size: 100Gi + type: persistent-claim +parameters: +- name: NODEPOOL + value: "kafka-pool" +- name: NAME + value: "kafka-cluster" +- name: NAMESPACE + value: "netobserv" diff --git a/integration-tests/backend/testdata/kafka/kafka-tls.yaml b/integration-tests/backend/testdata/kafka/kafka-tls.yaml new file mode 100644 index 0000000000..3d368e2407 --- /dev/null +++ b/integration-tests/backend/testdata/kafka/kafka-tls.yaml @@ -0,0 +1,67 @@ +kind: Template +apiVersion: template.openshift.io/v1 +metadata: + name: kafka-TLS-template +objects: +- apiVersion: kafka.strimzi.io/v1beta2 + kind: Kafka + metadata: + name: "${NAME}" + namespace: "${NAMESPACE}" + annotations: + strimzi.io/kraft: enabled + strimzi.io/node-pools: enabled + spec: + kafka: + replicas: 3 + listeners: + - name: plain + port: 9092 + type: internal + tls: true + - name: tls + port: 9093 + type: internal + tls: true + authentication: + type: tls + - name: external + port: 9094 + type: nodeport + tls: true + authentication: + type: tls + config: + offsets.topic.replication.factor: 1 + transaction.state.log.replication.factor: 1 + transaction.state.log.min.isr: 1 + default.replication.factor: 1 + min.insync.replicas: 1 + auto.create.topics.enable: false + log.cleaner.backoff.ms: 15000 + log.cleaner.dedupe.buffer.size: 134217728 + log.cleaner.enable: true + log.cleaner.io.buffer.load.factor: 0.9 + log.cleaner.threads: 8 + log.cleanup.policy: delete + log.retention.bytes: 107374182400 + log.retention.check.interval.ms: 300000 + log.retention.ms: 1680000 + log.roll.ms: 7200000 + log.segment.bytes: 1073741824 + metadataVersion: 4.0-IV3 + version: 4.0.0 + metricsConfig: + type: jmxPrometheusExporter + valueFrom: + configMapKeyRef: + name: kafka-metrics + key: kafka-metrics-config.yml + entityOperator: + topicOperator: {} + userOperator: {} +parameters: +- name: NAME + value: "kafka-cluster" +- name: NAMESPACE + value: "netobserv" diff --git a/integration-tests/backend/testdata/kafka/kafka-topic.yaml b/integration-tests/backend/testdata/kafka/kafka-topic.yaml new file mode 100644 index 0000000000..5e8e2a51ed --- /dev/null +++ b/integration-tests/backend/testdata/kafka/kafka-topic.yaml @@ -0,0 +1,22 @@ +kind: Template +apiVersion: template.openshift.io/v1 +metadata: + name: kafka-topic-template +objects: +- apiVersion: kafka.strimzi.io/v1beta2 + kind: KafkaTopic + metadata: + name: "${TOPIC}" + labels: + strimzi.io/cluster: "${NAME}" + namespace: "${NAMESPACE}" + spec: + partitions: 6 + replicas: 1 +parameters: +- name: TOPIC + value: "network-flows" +- name: NAME + value: "kafka-cluster" +- name: NAMESPACE + value: "netobserv" diff --git a/integration-tests/backend/testdata/kafka/kafka-user.yaml b/integration-tests/backend/testdata/kafka/kafka-user.yaml new file mode 100644 index 0000000000..ae40f4d778 --- /dev/null +++ b/integration-tests/backend/testdata/kafka/kafka-user.yaml @@ -0,0 +1,22 @@ +kind: Template +apiVersion: template.openshift.io/v1 +metadata: + name: kafka-user-template +objects: +- apiVersion: kafka.strimzi.io/v1beta2 + kind: KafkaUser + metadata: + labels: + strimzi.io/cluster: "${NAME}" + namespace: "${NAMESPACE}" + name: "${USER_NAME}" + spec: + authentication: + type: tls +parameters: +- name: USER_NAME + value: "flp-kafka" +- name: NAME + value: "kafka-cluster" +- name: NAMESPACE + value: "netobserv" diff --git a/integration-tests/backend/testdata/kafka/topic-consumer-tls.yaml b/integration-tests/backend/testdata/kafka/topic-consumer-tls.yaml new file mode 100644 index 0000000000..b7114d7523 --- /dev/null +++ b/integration-tests/backend/testdata/kafka/topic-consumer-tls.yaml @@ -0,0 +1,61 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: kafka-topic-consumer-template +objects: + - apiVersion: batch/v1 + kind: Job + metadata: + name: "${NAME}" + namespace: "${NAMESPACE}" + spec: + ttlSecondsAfterFinished: 60 + template: + spec: + volumes: + - name: cluster-ca + secret: + secretName: "${CLUSTER_NAME}-cluster-ca-cert" + - name: kafkauser + secret: + secretName: "${KAFKA_USER}" + - name: workdir + emptyDir: {} + containers: + - name: kakfa-consumer-tls + image: "${KAFKA_IMAGE}" + volumeMounts: + - mountPath: "/opt/kafka/cluster-ca-certs" + name: cluster-ca + - mountPath: "/opt/kafka/kafkauser" + name: kafkauser + - mountPath: "/opt/kafka/workdir" + name: workdir + command: + - "bash" + - "-c" + - 'keytool -keystore workdir/truststore.p12 -storepass password -noprompt -alias + ca -import -file /opt/kafka/cluster-ca-certs/ca.crt -storetype PKCS12; keytool + -importkeystore -destkeystore workdir/keystore.p12 -srckeystore /opt/kafka/kafkauser/user.p12 + -srcstorepass $(cat /opt/kafka/kafkauser/user.password) -srcstoretype PKCS12 + -deststoretype PKCS12 -destkeypass password -deststorepass password; bin/kafka-console-consumer.sh + --bootstrap-server ${CLUSTER_NAME}-kafka-bootstrap:9093 --consumer-property + security.protocol=SSL --consumer-property ssl.truststore.location=workdir/truststore.p12 + --consumer-property ssl.truststore.password=password --consumer-property ssl.truststore.type=PKCS12 + --consumer-property ssl.keystore.location=workdir/keystore.p12 --consumer-property + ssl.keystore.password=password --consumer-property ssl.keystore.type=PKCS12 + --from-beginning --topic ${KAFKA_TOPIC} --group test-group' + restartPolicy: Never +parameters: +- name: KAFKA_IMAGE + value: "registry.redhat.io/amq-streams/kafka-34-rhel8:2.5.2" +- name: KAFKA_TOPIC + value: "network-flows-export" +- name: KAFKA_USER + value: "flp-kafka" +- name: NAMESPACE + value: "netobserv" +- name: NAME + value: "network-flows-export-consumer-tls" +- name: CLUSTER_NAME + value: "kafka-cluster" diff --git a/integration-tests/backend/testdata/logging/minIO/deploy.yaml b/integration-tests/backend/testdata/logging/minIO/deploy.yaml new file mode 100644 index 0000000000..311f141d6b --- /dev/null +++ b/integration-tests/backend/testdata/logging/minIO/deploy.yaml @@ -0,0 +1,142 @@ +kind: Template +apiVersion: template.openshift.io/v1 +metadata: + name: minio-template + annotations: + description: "A MinIO service" +objects: +- kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: minio-pv-claim + namespace: ${NAMESPACE} + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi +- kind: Deployment + apiVersion: apps/v1 + metadata: + name: ${NAME} + namespace: ${NAMESPACE} + spec: + selector: + matchLabels: + app: ${NAME} + strategy: + type: Recreate + template: + metadata: + labels: + app: ${NAME} + spec: + volumes: + - name: data + persistentVolumeClaim: + claimName: minio-pv-claim + containers: + - name: minio + volumeMounts: + - name: data + mountPath: "/data" + image: ${IMAGE} + args: + - server + - /data + - --console-address + - ":9001" + env: + - name: MINIO_ROOT_USER + valueFrom: + secretKeyRef: + name: ${SECRET_NAME} + key: access_key_id + - name: MINIO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: ${SECRET_NAME} + key: secret_access_key + - name: MINIO_DOMAIN + value: ${MINIO_DOMAIN} + ports: + - containerPort: 9000 + readinessProbe: + httpGet: + path: /minio/health/ready + port: 9000 + initialDelaySeconds: 120 + periodSeconds: 20 + livenessProbe: + httpGet: + path: /minio/health/live + port: 9000 + initialDelaySeconds: 120 + periodSeconds: 20 +- kind: Service + apiVersion: v1 + metadata: + name: ${NAME} + namespace: ${NAMESPACE} + spec: + ports: + - port: 9000 + targetPort: 9000 + protocol: TCP + selector: + app: ${NAME} +- kind: Service + apiVersion: v1 + metadata: + name: minio-service-console + namespace: ${NAMESPACE} + spec: + ports: + - port: 9001 + targetPort: 9001 + protocol: TCP + selector: + app: ${NAME} +- kind: Route + apiVersion: route.openshift.io/v1 + metadata: + labels: + app: ${NAME} + name: ${NAME} + namespace: ${NAMESPACE} + spec: + host: ${MINIO_DOMAIN} + port: + targetPort: 9000 + tls: + termination: edge + to: + kind: Service + name: ${NAME} +- kind: Route + apiVersion: route.openshift.io/v1 + metadata: + labels: + app: ${NAME} + name: minio-console + namespace: ${NAMESPACE} + spec: + port: + targetPort: 9001 + to: + kind: Service + name: minio-service-console +parameters: + - name: IMAGE + displayName: " The MinIO image" + value: "quay.io/openshifttest/minio:latest" + - name: NAMESPACE + displayName: Namespace + value: "minio-aosqe" + - name: NAME + value: "minio" + - name: SECRET_NAME + value: "minio-creds" + - name: MINIO_DOMAIN + value: "" diff --git a/integration-tests/backend/testdata/logging/odf/objectBucketClaim.yaml b/integration-tests/backend/testdata/logging/odf/objectBucketClaim.yaml new file mode 100644 index 0000000000..d6e4eda28d --- /dev/null +++ b/integration-tests/backend/testdata/logging/odf/objectBucketClaim.yaml @@ -0,0 +1,25 @@ +kind: Template +apiVersion: template.openshift.io/v1 +metadata: + name: objectBucketClaim-template +objects: +- apiVersion: objectbucket.io/v1alpha1 + kind: ObjectBucketClaim + metadata: + name: ${NAME} + namespace: ${NAMESPACE} + spec: + additionalConfig: + bucketclass: ${BUCKETCLASS} + generateBucketName: ${NAME} + bucketName: ${NAME} + storageClassName: ${STORAGE_CLASS_NAME} +parameters: +- name: NAME + value: logging-loki +- name: NAMESPACE + value: openshift-storage +- name: BUCKETCLASS + value: noobaa-default-bucket-class +- name: STORAGE_CLASS_NAME + value: openshift-storage.noobaa.io diff --git a/integration-tests/backend/testdata/logging/subscription/namespace.yaml b/integration-tests/backend/testdata/logging/subscription/namespace.yaml new file mode 100644 index 0000000000..cb0f8b5faf --- /dev/null +++ b/integration-tests/backend/testdata/logging/subscription/namespace.yaml @@ -0,0 +1,15 @@ +kind: Template +apiVersion: template.openshift.io/v1 +metadata: + name: namespace-template +objects: +- kind: Namespace + apiVersion: v1 + metadata: + name: ${NAMESPACE_NAME} + annotations: + openshift.io/node-selector: "" + labels: + openshift.io/cluster-monitoring: "true" +parameters: +- name: NAMESPACE_NAME diff --git a/integration-tests/backend/testdata/loki/lokistack-simple.yaml b/integration-tests/backend/testdata/loki/lokistack-simple.yaml new file mode 100644 index 0000000000..23cfe8b2bd --- /dev/null +++ b/integration-tests/backend/testdata/loki/lokistack-simple.yaml @@ -0,0 +1,50 @@ +kind: Template +apiVersion: template.openshift.io/v1 +metadata: + name: lokiStack-template +objects: +- kind: "LokiStack" + apiVersion: "loki.grafana.com/v1" + metadata: + name: ${Name} + namespace: ${Namespace} + spec: + managementState: "Managed" + size: ${TSize} + storage: + secret: + name: ${StorageSecret} + type: ${StorageType} + schemas: + - version: ${StorageSchemaVersion} + effectiveDate: ${SchemaEffectiveDate} + storageClassName: ${StorageClass} + hashRing: + memberlist: + enableIPv6: ${{EnableIPV6}} + tenants: + mode: ${Tenant} + template: + ingester: + replicas: 2 +parameters: +- name: Name + value: "lokistack" +- name: Namespace + value: "netobserv" +- name: TSize + value: "1x.extra-small" +- name: StorageSecret + value: "s3-secret" +- name: StorageType + value: "s3" +- name: StorageClass + value: "gp3-csi" +- name: Tenant + value: openshift-network +- name: EnableIPV6 + value: "false" +- name: StorageSchemaVersion + value: "v13" +- name: SchemaEffectiveDate + value: "2023-10-15" diff --git a/integration-tests/backend/testdata/netobserv-loki-reader-multitenant-crb.yaml b/integration-tests/backend/testdata/netobserv-loki-reader-multitenant-crb.yaml new file mode 100644 index 0000000000..b0cd7c64da --- /dev/null +++ b/integration-tests/backend/testdata/netobserv-loki-reader-multitenant-crb.yaml @@ -0,0 +1,20 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: netobserv-loki-reader-crb +objects: + - apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: netobserv-user-reader + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: netobserv-loki-reader + subjects: + - kind: User + apiGroup: rbac.authorization.k8s.io + name: "${USERNAME}" +parameters: + - name: USERNAME + required: true diff --git a/integration-tests/backend/testdata/networking/adminnetworkPolicy.yaml b/integration-tests/backend/testdata/networking/adminnetworkPolicy.yaml new file mode 100644 index 0000000000..80f0690f20 --- /dev/null +++ b/integration-tests/backend/testdata/networking/adminnetworkPolicy.yaml @@ -0,0 +1,37 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: banp-template +objects: +- apiVersion: policy.networking.k8s.io/v1alpha1 + kind: AdminNetworkPolicy + metadata: + name: ${NAME} + spec: + priority: 10 + subject: + namespaces: + matchLabels: + kubernetes.io/metadata.name: ${SERVER_NS} + ingress: + - name: "allow-ns" + action: "Allow" + from: + - namespaces: + matchLabels: + kubernetes.io/metadata.name: ${ALLOW_NS} + - name: "deny-ns" + action: "Deny" + from: + - namespaces: + matchLabels: + kubernetes.io/metadata.name: ${DENY_NS} +parameters: +- name: NAME + value: server-ns +- name: SERVER_NS + value: test-server +- name: ALLOW_NS + value: test-client1 +- name: DENY_NS + value: test-client2 diff --git a/integration-tests/backend/testdata/networking/baselineadminnetworkPolicy.yaml b/integration-tests/backend/testdata/networking/baselineadminnetworkPolicy.yaml new file mode 100644 index 0000000000..f99c9cce39 --- /dev/null +++ b/integration-tests/backend/testdata/networking/baselineadminnetworkPolicy.yaml @@ -0,0 +1,37 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: banp-template +objects: +- apiVersion: policy.networking.k8s.io/v1alpha1 + kind: BaselineAdminNetworkPolicy + metadata: + name: default + spec: + ingress: + - action: Deny + name: default-deny-ingress1 + from: + - namespaces: + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: ${CLIENT1_NS} + - action: Deny + name: default-deny-ingress2 + from: + - namespaces: + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: ${CLIENT2_NS} + name: default-deny-ns + subject: + namespaces: + matchLabels: + kubernetes.io/metadata.name: ${SERVER_NS} +parameters: +- name: CLIENT1_NS + value: test-client1 +- name: CLIENT2_NS + value: test-client2 +- name: SERVER_NS + value: test-server diff --git a/integration-tests/backend/testdata/networking/egressQoS.yaml b/integration-tests/backend/testdata/networking/egressQoS.yaml new file mode 100644 index 0000000000..83691995ea --- /dev/null +++ b/integration-tests/backend/testdata/networking/egressQoS.yaml @@ -0,0 +1,10 @@ +apiVersion: k8s.ovn.org/v1 +kind: EgressQoS +metadata: + name: default +spec: + egress: + - dscp: 59 + podSelector: + matchLabels: + app: client-dscp diff --git a/integration-tests/backend/testdata/networking/networkPolicy.yaml b/integration-tests/backend/testdata/networking/networkPolicy.yaml new file mode 100644 index 0000000000..587293c18c --- /dev/null +++ b/integration-tests/backend/testdata/networking/networkPolicy.yaml @@ -0,0 +1,27 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: nwpolicy-template +objects: +- apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + name: ${NAME} + namespace: ${SERVER_NS} + spec: + podSelector: {} + policyTypes: + - Ingress + ingress: + - from: + - podSelector: {} + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: ${ALLOW_NS} +parameters: +- name: NAME + value: allow-ingress +- name: SERVER_NS + value: test-server +- name: ALLOW_NS + value: test-client2 diff --git a/integration-tests/backend/testdata/networking/nmstate/catalogsource-template.yaml b/integration-tests/backend/testdata/networking/nmstate/catalogsource-template.yaml new file mode 100644 index 0000000000..1c4a990326 --- /dev/null +++ b/integration-tests/backend/testdata/networking/nmstate/catalogsource-template.yaml @@ -0,0 +1,21 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: kubenetes-nmstate-catalogsource-template +objects: +- apiVersion: operators.coreos.com/v1alpha1 + kind: CatalogSource + metadata: + name: "${CATALOGSOURCENAME}" + namespace: "${CATALOGNAMESPACE}" + spec: + displayName: Kubernetes nmstate FBC Operator Catalog + image: "${IMAGE}" + sourceType: grpc + updateStrategy: + registryPoll: + interval: 10m +parameters: +- name: CATALOGSOURCENAME +- name: CATALOGNAMESPACE +- name: IMAGE diff --git a/integration-tests/backend/testdata/networking/nmstate/image-digest-mirrorset.yaml b/integration-tests/backend/testdata/networking/nmstate/image-digest-mirrorset.yaml new file mode 100644 index 0000000000..51942388e9 --- /dev/null +++ b/integration-tests/backend/testdata/networking/nmstate/image-digest-mirrorset.yaml @@ -0,0 +1,21 @@ +apiVersion: config.openshift.io/v1 +kind: ImageDigestMirrorSet +metadata: + name: kubernetes-nmstate-images-mirror-set +spec: + imageDigestMirrors: + - mirrors: + - quay.io/redhat-user-workloads/ocp-art-tenant/art-images-share + source: registry.redhat.io/openshift4/kubernetes-nmstate-operator-bundle + - mirrors: + - quay.io/redhat-user-workloads/ocp-art-tenant/art-images-share + source: registry.redhat.io/openshift4/kubernetes-nmstate-rhel9-operator + - mirrors: + - quay.io/redhat-user-workloads/ocp-art-tenant/art-images-share + source: registry.redhat.io/openshift4/nmstate-console-plugin-rhel9 + - mirrors: + - quay.io/redhat-user-workloads/ocp-art-tenant/art-images-share + source: registry.redhat.io/openshift4/ose-kube-rbac-proxy-rhel9 + - mirrors: + - quay.io/redhat-user-workloads/ocp-art-tenant/art-images-share + source: registry.redhat.io/openshift4/ose-kubernetes-nmstate-handler-rhel9 diff --git a/integration-tests/backend/testdata/networking/nmstate/namespace-template.yaml b/integration-tests/backend/testdata/networking/nmstate/namespace-template.yaml new file mode 100644 index 0000000000..6b3cc6671a --- /dev/null +++ b/integration-tests/backend/testdata/networking/nmstate/namespace-template.yaml @@ -0,0 +1,17 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: namespace-template +objects: +- kind: Namespace + apiVersion: v1 + metadata: + labels: + kubernetes.io/metadata.name: "${NAME}" + name: "${NAME}" + name: "${NAME}" + spec: + finalizers: + - kubernetes +parameters: +- name: NAME diff --git a/integration-tests/backend/testdata/networking/nmstate/nmstate-cr-template.yaml b/integration-tests/backend/testdata/networking/nmstate/nmstate-cr-template.yaml new file mode 100644 index 0000000000..3182cb1d7f --- /dev/null +++ b/integration-tests/backend/testdata/networking/nmstate/nmstate-cr-template.yaml @@ -0,0 +1,11 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: nmstate-cr-template +objects: +- kind: NMState + apiVersion: nmstate.io/v1 + metadata: + name: "${NAME}" +parameters: +- name: NAME diff --git a/integration-tests/backend/testdata/networking/nmstate/operatorgroup-template.yaml b/integration-tests/backend/testdata/networking/nmstate/operatorgroup-template.yaml new file mode 100644 index 0000000000..5b382219be --- /dev/null +++ b/integration-tests/backend/testdata/networking/nmstate/operatorgroup-template.yaml @@ -0,0 +1,20 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: nmstate-operatorgroup-template +objects: +- kind: OperatorGroup + apiVersion: operators.coreos.com/v1 + metadata: + name: "${NAME}-dzrmx" + namespace: "${NAMESPACE}" + generateName: "${NAME}-" + annotations: + olm.providedAPIs: NMState.v1.nmstate.io + spec: + targetNamespaces: + - "${TARGETNAMESPACES}" +parameters: +- name: NAME +- name: NAMESPACE +- name: TARGETNAMESPACES diff --git a/integration-tests/backend/testdata/networking/nmstate/ovn-mapping-policy-template.yaml b/integration-tests/backend/testdata/networking/nmstate/ovn-mapping-policy-template.yaml new file mode 100644 index 0000000000..53509df305 --- /dev/null +++ b/integration-tests/backend/testdata/networking/nmstate/ovn-mapping-policy-template.yaml @@ -0,0 +1,29 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: ovn-mapping-policy-template +objects: + - apiVersion: nmstate.io/v1 + kind: NodeNetworkConfigurationPolicy + metadata: + name: "${NAME}" + spec: + nodeSelector: + "${NODELABEL}": "${LABELVALUE}" + desiredState: + interfaces: + - name: "${BRIDGE1}" + type: ovs-bridge + state: up + ovn: + bridge-mappings: + - localnet: "${LOCALNET1}" + bridge: "${BRIDGE1}" + +parameters: + - name: NAME + - name: NODELABEL + value: "kubernetes.io/hostname" + - name: LABELVALUE + - name: LOCALNET1 + - name: BRIDGE1 diff --git a/integration-tests/backend/testdata/networking/nmstate/subscription-template.yaml b/integration-tests/backend/testdata/networking/nmstate/subscription-template.yaml new file mode 100644 index 0000000000..834ec75e16 --- /dev/null +++ b/integration-tests/backend/testdata/networking/nmstate/subscription-template.yaml @@ -0,0 +1,26 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: operatorgroup-subscription-template +objects: +- kind: Subscription + apiVersion: operators.coreos.com/v1alpha1 + metadata: + labels: + operators.coreos.com/kubernetes-nmstate-operator.openshift-nmstate: "" + name: "${SUBSCRIPTIONNAME}" + namespace: "${NAMESPACE}" + spec: + name: "${OPERATORNAME}" + channel: "${CHANNEL}" + installPlanApproval: Automatic + source: "${CATALOGSOURCE}" + sourceNamespace: "${CATALOGSOURCENAMESPACE}" +parameters: +- name: OPERATORNAME + value: "kubernetes-nmstate-operator" +- name: SUBSCRIPTIONNAME +- name: NAMESPACE +- name: CHANNEL +- name: CATALOGSOURCE +- name: CATALOGSOURCENAMESPACE diff --git a/integration-tests/backend/testdata/networking/sctpclient.yaml b/integration-tests/backend/testdata/networking/sctpclient.yaml new file mode 100644 index 0000000000..c42222448a --- /dev/null +++ b/integration-tests/backend/testdata/networking/sctpclient.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Pod +metadata: + name: sctpclient + labels: + name: sctpclient +spec: + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - name: sctpclient + image: quay.io/openshifttest/hello-sdn@sha256:c89445416459e7adea9a5a416b3365ed3d74f2491beb904d61dc8d1eb89a72a4 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] diff --git a/integration-tests/backend/testdata/networking/sctpserver.yaml b/integration-tests/backend/testdata/networking/sctpserver.yaml new file mode 100644 index 0000000000..6c7bf3cbef --- /dev/null +++ b/integration-tests/backend/testdata/networking/sctpserver.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Pod +metadata: + name: sctpserver + labels: + name: sctpserver +spec: + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - name: sctpserver + image: quay.io/openshifttest/hello-sdn@sha256:c89445416459e7adea9a5a416b3365ed3d74f2491beb904d61dc8d1eb89a72a4 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + ports: + - containerPort: 30102 + name: sctpserver + protocol: SCTP diff --git a/integration-tests/backend/testdata/networking/test-client-DSCP.yaml b/integration-tests/backend/testdata/networking/test-client-DSCP.yaml new file mode 100644 index 0000000000..d121a55bc9 --- /dev/null +++ b/integration-tests/backend/testdata/networking/test-client-DSCP.yaml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + labels: + app: client-dscp + name: client-dscp + namespace: test-client-68125 +spec: + containers: + - command: + - sh + - -c + - " + \ while : ; do\n + \ curl nginx-service.test-server-68125.svc:80/data/100K 2>&1 > /dev/null ; sleep 5 \n + \ done" + image: quay.io/openshifttest/hello-openshift:1.2.0 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + privileged: false + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + name: client-dscp + resources: {} + restartPolicy: Never +status: {} diff --git a/integration-tests/backend/testdata/networking/udn/cudn_crd_dualstack_template.yaml b/integration-tests/backend/testdata/networking/udn/cudn_crd_dualstack_template.yaml new file mode 100644 index 0000000000..5cdf7e1ac4 --- /dev/null +++ b/integration-tests/backend/testdata/networking/udn/cudn_crd_dualstack_template.yaml @@ -0,0 +1,32 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: cudn-crd-dualstack-template +objects: + - apiVersion: k8s.ovn.org/v1 + kind: ClusterUserDefinedNetwork + metadata: + name: "${CRDNAME}" + spec: + namespaceSelector: + matchLabels: + "${LABELKEY}": "${LABELVALUE}" + network: + topology: Layer3 + layer3: + role: "${ROLE}" + subnets: + - cidr: "${IPv4CIDR}" + hostSubnet: ${{IPv4PREFIX}} + - cidr: "${IPv6CIDR}" + hostSubnet: ${{IPv6PREFIX}} +parameters: +- name: CRDNAME +- name: LABELVALUE +- name: LABELKEY +- name: ROLE +- name: IPv4CIDR +- name: IPv4PREFIX +- name: IPv6CIDR +- name: IPv6PREFIX + diff --git a/integration-tests/backend/testdata/networking/udn/cudn_crd_layer2_dualstack_template.yaml b/integration-tests/backend/testdata/networking/udn/cudn_crd_layer2_dualstack_template.yaml new file mode 100644 index 0000000000..7a95dd3b27 --- /dev/null +++ b/integration-tests/backend/testdata/networking/udn/cudn_crd_layer2_dualstack_template.yaml @@ -0,0 +1,26 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: cudn-crd-layer2-dualstack-template +objects: + - apiVersion: k8s.ovn.org/v1 + kind: ClusterUserDefinedNetwork + metadata: + name: "${CRDNAME}" + spec: + namespaceSelector: + matchLabels: + "${LABELKEY}": "${LABELVALUE}" + network: + topology: Layer2 + layer2: + role: "${ROLE}" + subnets: ["${IPv4CIDR}", "${IPv6CIDR}"] +parameters: +- name: CRDNAME +- name: LABELVALUE +- name: LABELKEY +- name: ROLE +- name: IPv4CIDR +- name: IPv6CIDR + diff --git a/integration-tests/backend/testdata/networking/udn/cudn_crd_layer2_singlestack_template.yaml b/integration-tests/backend/testdata/networking/udn/cudn_crd_layer2_singlestack_template.yaml new file mode 100644 index 0000000000..4ab4cf7a55 --- /dev/null +++ b/integration-tests/backend/testdata/networking/udn/cudn_crd_layer2_singlestack_template.yaml @@ -0,0 +1,25 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: cudn-crd-layer2-singlestack-template +objects: + - apiVersion: k8s.ovn.org/v1 + kind: ClusterUserDefinedNetwork + metadata: + name: "${CRDNAME}" + spec: + namespaceSelector: + matchLabels: + "${LABELKEY}": "${LABELVALUE}" + network: + topology: Layer2 + layer2: + role: "${ROLE}" + subnets: ["${CIDR}"] +parameters: +- name: CRDNAME +- name: LABELVALUE +- name: LABELKEY +- name: CIDR +- name: ROLE + diff --git a/integration-tests/backend/testdata/networking/udn/cudn_crd_localnet_singlestack_template.yaml b/integration-tests/backend/testdata/networking/udn/cudn_crd_localnet_singlestack_template.yaml new file mode 100644 index 0000000000..dcbfc2c9ef --- /dev/null +++ b/integration-tests/backend/testdata/networking/udn/cudn_crd_localnet_singlestack_template.yaml @@ -0,0 +1,30 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: cudn-crd-localnet-singlestack-template +objects: + - apiVersion: k8s.ovn.org/v1 + kind: ClusterUserDefinedNetwork + metadata: + name: "${CRDNAME}" + spec: + namespaceSelector: + matchLabels: + "${LABELKEY}": "${LABELVALUE}" + network: + topology: Localnet + localnet: + role: "${ROLE}" + physicalNetworkName: "${PHYSICALNETWORK}" + subnets: + - ${{SUBNET}} + excludeSubnets: + - ${{EXCLUDESUBNET}} +parameters: +- name: CRDNAME +- name: LABELVALUE +- name: LABELKEY +- name: ROLE +- name: PHYSICALNETWORK +- name: SUBNET +- name: EXCLUDESUBNET diff --git a/integration-tests/backend/testdata/networking/udn/cudn_crd_localnet_singlestack_with_vlan_template.yaml b/integration-tests/backend/testdata/networking/udn/cudn_crd_localnet_singlestack_with_vlan_template.yaml new file mode 100644 index 0000000000..17cbe7369e --- /dev/null +++ b/integration-tests/backend/testdata/networking/udn/cudn_crd_localnet_singlestack_with_vlan_template.yaml @@ -0,0 +1,34 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: cudn-crd-localnet-singlestack-with-vlan-template +objects: + - apiVersion: k8s.ovn.org/v1 + kind: ClusterUserDefinedNetwork + metadata: + name: "${CRDNAME}" + spec: + namespaceSelector: + matchLabels: + "${LABELKEY}": "${LABELVALUE}" + network: + topology: Localnet + localnet: + role: "${ROLE}" + physicalNetworkName: "${PHYSICALNETWORK}" + vlan: + mode: Access + access: + id: 50 + subnets: + - ${{SUBNET}} + excludeSubnets: + - ${{EXCLUDESUBNET}} +parameters: +- name: CRDNAME +- name: LABELVALUE +- name: LABELKEY +- name: ROLE +- name: PHYSICALNETWORK +- name: SUBNET +- name: EXCLUDESUBNET diff --git a/integration-tests/backend/testdata/networking/udn/cudn_crd_singlestack_template.yaml b/integration-tests/backend/testdata/networking/udn/cudn_crd_singlestack_template.yaml new file mode 100644 index 0000000000..d24e452319 --- /dev/null +++ b/integration-tests/backend/testdata/networking/udn/cudn_crd_singlestack_template.yaml @@ -0,0 +1,28 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: cudn-crd-singlestack-template +objects: + - apiVersion: k8s.ovn.org/v1 + kind: ClusterUserDefinedNetwork + metadata: + name: "${CRDNAME}" + spec: + namespaceSelector: + matchLabels: + "${LABELKEY}": "${LABELVALUE}" + network: + topology: Layer3 + layer3: + role: "${ROLE}" + subnets: + - cidr: "${CIDR}" + hostSubnet: ${{PREFIX}} +parameters: +- name: CRDNAME +- name: LABELVALUE +- name: LABELKEY +- name: CIDR +- name: PREFIX +- name: ROLE + diff --git a/integration-tests/backend/testdata/networking/udn/udn_crd_dualstack2_template.yaml b/integration-tests/backend/testdata/networking/udn/udn_crd_dualstack2_template.yaml new file mode 100644 index 0000000000..6ad3c891fc --- /dev/null +++ b/integration-tests/backend/testdata/networking/udn/udn_crd_dualstack2_template.yaml @@ -0,0 +1,27 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: udn-crd-dualstack-template +objects: +- apiVersion: k8s.ovn.org/v1 + kind: UserDefinedNetwork + metadata: + name: "${CRDNAME}" + namespace: "${NAMESPACE}" + spec: + topology: Layer3 + layer3: + role: "${ROLE}" + subnets: + - cidr: "${IPv4CIDR}" + hostSubnet: ${{IPv4PREFIX}} + - cidr: "${IPv6CIDR}" + hostSubnet: ${{IPv6PREFIX}} +parameters: +- name: CRDNAME +- name: NAMESPACE +- name: ROLE +- name: IPv4CIDR +- name: IPv4PREFIX +- name: IPv6CIDR +- name: IPv6PREFIX diff --git a/integration-tests/backend/testdata/networking/udn/udn_crd_layer2_dualstack_template.yaml b/integration-tests/backend/testdata/networking/udn/udn_crd_layer2_dualstack_template.yaml new file mode 100644 index 0000000000..509d665154 --- /dev/null +++ b/integration-tests/backend/testdata/networking/udn/udn_crd_layer2_dualstack_template.yaml @@ -0,0 +1,23 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: udn-crd-layer2-dualstack-template +objects: +- apiVersion: k8s.ovn.org/v1 + kind: UserDefinedNetwork + metadata: + name: "${CRDNAME}" + namespace: "${NAMESPACE}" + spec: + topology: Layer2 + layer2: + role: "${ROLE}" + subnets: ["${IPv4CIDR}","${IPv6CIDR}"] +parameters: +- name: CRDNAME +- name: NAMESPACE +- name: ROLE +- name: IPv4CIDR +- name: IPv6CIDR + + diff --git a/integration-tests/backend/testdata/networking/udn/udn_crd_layer2_singlestack_template.yaml b/integration-tests/backend/testdata/networking/udn/udn_crd_layer2_singlestack_template.yaml new file mode 100644 index 0000000000..011bb03243 --- /dev/null +++ b/integration-tests/backend/testdata/networking/udn/udn_crd_layer2_singlestack_template.yaml @@ -0,0 +1,20 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: udn-crd-layer2-singlestack-template +objects: +- apiVersion: k8s.ovn.org/v1 + kind: UserDefinedNetwork + metadata: + name: "${CRDNAME}" + namespace: "${NAMESPACE}" + spec: + topology: Layer2 + layer2: + role: "${ROLE}" + subnets: ["${CIDR}"] +parameters: +- name: CRDNAME +- name: NAMESPACE +- name: CIDR +- name: ROLE diff --git a/integration-tests/backend/testdata/networking/udn/udn_crd_singlestack_template.yaml b/integration-tests/backend/testdata/networking/udn/udn_crd_singlestack_template.yaml new file mode 100644 index 0000000000..dc2b1aa11b --- /dev/null +++ b/integration-tests/backend/testdata/networking/udn/udn_crd_singlestack_template.yaml @@ -0,0 +1,23 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: udn-crd-singlestack-template +objects: +- apiVersion: k8s.ovn.org/v1 + kind: UserDefinedNetwork + metadata: + name: "${CRDNAME}" + namespace: "${NAMESPACE}" + spec: + topology: Layer3 + layer3: + role: "${ROLE}" + subnets: + - cidr: "${CIDR}" + hostSubnet: ${{PREFIX}} +parameters: +- name: CRDNAME +- name: NAMESPACE +- name: CIDR +- name: PREFIX +- name: ROLE diff --git a/integration-tests/backend/testdata/networking/udn/udn_statefulset_template.yaml b/integration-tests/backend/testdata/networking/udn/udn_statefulset_template.yaml new file mode 100644 index 0000000000..0bf30fdf4d --- /dev/null +++ b/integration-tests/backend/testdata/networking/udn/udn_statefulset_template.yaml @@ -0,0 +1,46 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: udn-statefulset-template +objects: +- apiVersion: apps/v1 + kind: StatefulSet + metadata: + labels: + app: ${LABEL} + name: ${LABEL} + name: ${NAME} + namespace: ${NAMESPACE} + spec: + replicas: 5 + selector: + matchLabels: + app: ${LABEL} + serviceName: hello + template: + metadata: + labels: + app: ${LABEL} + annotations: + k8s.v1.cni.cncf.io/networks: '[{"name": "${NETWORK_NAME}","interface": "${INTERFACE_NAME}"}]' + spec: + containers: + - image: quay.io/openshifttest/hello-sdn@sha256:c89445416459e7adea9a5a416b3365ed3d74f2491beb904d61dc8d1eb89a72a4 + name: hello + ports: + - containerPort: 8080 + name: web + protocol: TCP + resources: + limits: + memory: 340Mi + restartPolicy: Always +parameters: +- name: NAME + value: "hello" +- name: NAMESPACE +- name: LABEL + value: "hello" +- name: NETWORK_NAME +- name: INTERFACE_NAME + value: "ovn-udn1" diff --git a/integration-tests/backend/testdata/networking/udn/udn_test_pod_template.yaml b/integration-tests/backend/testdata/networking/udn/udn_test_pod_template.yaml new file mode 100644 index 0000000000..51516d95e0 --- /dev/null +++ b/integration-tests/backend/testdata/networking/udn/udn_test_pod_template.yaml @@ -0,0 +1,28 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: udn-pod-template +objects: +- kind: Pod + apiVersion: v1 + metadata: + name: "${NAME}" + namespace: "${NAMESPACE}" + labels: + name: "${LABEL}" + spec: + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - image: "quay.io/openshifttest/hello-sdn@sha256:c89445416459e7adea9a5a416b3365ed3d74f2491beb904d61dc8d1eb89a72a4" + name: hello-pod + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] +parameters: +- name: NAME +- name: NAMESPACE +- name: LABEL diff --git a/integration-tests/backend/testdata/subscription/allnamespace-og.yaml b/integration-tests/backend/testdata/subscription/allnamespace-og.yaml new file mode 100644 index 0000000000..2e666deb81 --- /dev/null +++ b/integration-tests/backend/testdata/subscription/allnamespace-og.yaml @@ -0,0 +1,17 @@ +kind: Template +apiVersion: template.openshift.io/v1 +metadata: + name: netobserv-operator-og-template +objects: +- apiVersion: operators.coreos.com/v1 + kind: OperatorGroup + metadata: + name: ${OG_NAME} + namespace: ${NAMESPACE} + spec: + upgradeStrategy: Default +parameters: + - name: OG_NAME + value: "netobserv-operator" + - name: NAMESPACE + value: "openshift-netobserv-operator" diff --git a/integration-tests/backend/testdata/subscription/catalog-source.yaml b/integration-tests/backend/testdata/subscription/catalog-source.yaml new file mode 100644 index 0000000000..6fcf169266 --- /dev/null +++ b/integration-tests/backend/testdata/subscription/catalog-source.yaml @@ -0,0 +1,23 @@ +kind: Template +apiVersion: template.openshift.io/v1 +metadata: + name: netobserv-catalog-source-template +objects: +- apiVersion: operators.coreos.com/v1alpha1 + kind: CatalogSource + metadata: + name: "${CATALOG_NAME}" + namespace: "${NAMESPACE}" + spec: + displayName: NetObserv Konflux + image: "${IMAGE}" + sourceType: grpc + grpcPodConfig: + securityContextConfig: legacy +parameters: +- name: CATALOG_NAME + value: "netobserv-konflux-fbc" +- name: IMAGE + value: "quay.io/redhat-user-workloads/ocp-network-observab-tenant/catalog-ystream:latest" +- name: NAMESPACE + value: openshift-marketplace diff --git a/integration-tests/backend/testdata/subscription/image-digest-mirror-set.yaml b/integration-tests/backend/testdata/subscription/image-digest-mirror-set.yaml new file mode 100644 index 0000000000..3501f52ccc --- /dev/null +++ b/integration-tests/backend/testdata/subscription/image-digest-mirror-set.yaml @@ -0,0 +1,33 @@ +apiVersion: config.openshift.io/v1 +kind: ImageDigestMirrorSet +metadata: + name: netobserv-image-digest-mirror-set +spec: + imageDigestMirrors: + - mirrors: + - quay.io/redhat-user-workloads/ocp-network-observab-tenant/network-observability-operator-ystream + - quay.io/redhat-user-workloads/ocp-network-observab-tenant/network-observability-operator-zstream + source: registry.redhat.io/network-observability/network-observability-rhel9-operator + - mirrors: + - quay.io/redhat-user-workloads/ocp-network-observab-tenant/flowlogs-pipeline-ystream + - quay.io/redhat-user-workloads/ocp-network-observab-tenant/flowlogs-pipeline-zstream + source: registry.redhat.io/network-observability/network-observability-flowlogs-pipeline-rhel9 + - mirrors: + - quay.io/redhat-user-workloads/ocp-network-observab-tenant/netobserv-ebpf-agent-ystream + - quay.io/redhat-user-workloads/ocp-network-observab-tenant/netobserv-ebpf-agent-zstream + source: registry.redhat.io/network-observability/network-observability-ebpf-agent-rhel9 + - mirrors: + - quay.io/redhat-user-workloads/ocp-network-observab-tenant/network-observability-console-plugin-ystream + - quay.io/redhat-user-workloads/ocp-network-observab-tenant/network-observability-console-plugin-zstream + source: registry.redhat.io/network-observability/network-observability-console-plugin-rhel9 + - mirrors: + - quay.io/redhat-user-workloads/ocp-network-observab-tenant/network-observability-cli-ystream + - quay.io/redhat-user-workloads/ocp-network-observab-tenant/network-observability-cli-zstream + source: registry.redhat.io/network-observability/network-observability-cli-rhel9 + - mirrors: + - quay.io/redhat-user-workloads/ocp-network-observab-tenant/network-observability-operator-bundle-ystream + - quay.io/redhat-user-workloads/ocp-network-observab-tenant/network-observability-operator-bundle-zstream + source: registry.redhat.io/network-observability/network-observability-operator-bundle + - mirrors: + - quay.io/redhat-user-workloads/ocp-network-observab-tenant/network-observability-console-plugin-pf4-ystream + source: registry.redhat.io/network-observability/network-observability-console-plugin-compat-rhel9 diff --git a/integration-tests/backend/testdata/subscription/namespace.yaml b/integration-tests/backend/testdata/subscription/namespace.yaml new file mode 100644 index 0000000000..c5e6911286 --- /dev/null +++ b/integration-tests/backend/testdata/subscription/namespace.yaml @@ -0,0 +1,16 @@ +kind: Template +apiVersion: template.openshift.io/v1 +metadata: + name: namespace-template +objects: +- kind: Namespace + apiVersion: v1 + metadata: + name: ${NAMESPACE_NAME} + labels: + openshift.io/cluster-monitoring: "true" + security.openshift.io/scc.podSecurityLabelSync: "false" + pod-security.kubernetes.io/enforce: baseline +parameters: +- name: NAMESPACE_NAME + value: "openshift-netobserv-operator" diff --git a/integration-tests/backend/testdata/subscription/singlenamespace-og.yaml b/integration-tests/backend/testdata/subscription/singlenamespace-og.yaml new file mode 100644 index 0000000000..be683cd281 --- /dev/null +++ b/integration-tests/backend/testdata/subscription/singlenamespace-og.yaml @@ -0,0 +1,18 @@ +kind: Template +apiVersion: template.openshift.io/v1 +metadata: + name: kafka-og-template +objects: +- apiVersion: operators.coreos.com/v1 + kind: OperatorGroup + metadata: + name: ${OG_NAME} + namespace: ${NAMESPACE} + spec: + targetNamespaces: + - ${NAMESPACE} +parameters: + - name: OG_NAME + value: "amq-streams" + - name: NAMESPACE + value: "netobserv" diff --git a/integration-tests/backend/testdata/subscription/sub-template.yaml b/integration-tests/backend/testdata/subscription/sub-template.yaml new file mode 100644 index 0000000000..6bdbfdf15a --- /dev/null +++ b/integration-tests/backend/testdata/subscription/sub-template.yaml @@ -0,0 +1,28 @@ +kind: Template +apiVersion: template.openshift.io/v1 +metadata: + name: subscription-template +objects: +- apiVersion: operators.coreos.com/v1alpha1 + kind: Subscription + metadata: + labels: + operators.coreos.com/${PACKAGE_NAME}.${NAMESPACE}: "" + name: ${PACKAGE_NAME} + namespace: ${NAMESPACE} + spec: + channel: ${CHANNEL} + installPlanApproval: ${INSTALL_PLAN_APPROVAL} + name: ${PACKAGE_NAME} + source: ${SOURCE} + sourceNamespace: ${SOURCE_NAMESPACE} +parameters: + - name: PACKAGE_NAME + - name: NAMESPACE + - name: CHANNEL + - name: SOURCE + value: "qe-app-registry" + - name: SOURCE_NAMESPACE + value: "openshift-marketplace" + - name: INSTALL_PLAN_APPROVAL + value: Automatic diff --git a/integration-tests/backend/testdata/test-SYN-flood-client_template.yaml b/integration-tests/backend/testdata/test-SYN-flood-client_template.yaml new file mode 100644 index 0000000000..a0bc318839 --- /dev/null +++ b/integration-tests/backend/testdata/test-SYN-flood-client_template.yaml @@ -0,0 +1,38 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: netobserv-test-client +objects: +- apiVersion: v1 + kind: Namespace + metadata: + name: ${CLIENT_NS} + labels: + name: ${CLIENT_NS} + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/enforce-version: v1.24 + pod-security.kubernetes.io/audit: privileged +- apiVersion: v1 + kind: Pod + metadata: + creationTimestamp: null + labels: + run: client + name: client + namespace: ${CLIENT_NS} + spec: + containers: + - command: ['/bin/sh', '-c'] + args: ["hping3 -c 5000 -S -p 80 --rand-source 192.168.1.159"] + image: quay.io/openshifttest/hello-sdn:1.2.1 + securityContext: + allowPrivilegeEscalation: true + privileged: true + runAsUser: 0 + seccompProfile: + type: RuntimeDefault + name: client + restartPolicy: Never +parameters: +- name: CLIENT_NS + value: test-client diff --git a/integration-tests/backend/testdata/test-nginx-client_template.yaml b/integration-tests/backend/testdata/test-nginx-client_template.yaml new file mode 100644 index 0000000000..3016583583 --- /dev/null +++ b/integration-tests/backend/testdata/test-nginx-client_template.yaml @@ -0,0 +1,47 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: netobserv-test-client-server +objects: +- apiVersion: v1 + kind: Namespace + metadata: + name: ${CLIENT_NS} + labels: + name: ${CLIENT_NS} +- apiVersion: v1 + kind: Pod + metadata: + creationTimestamp: null + labels: + run: ${POD_NAME} + name: ${POD_NAME} + namespace: ${CLIENT_NS} + spec: + containers: + - command: + - sh + - -c + - " + \ while : ; do\n + \ curl nginx-service.${SERVER_NS}.svc:80/data/${OBJECT_SIZE} 2>&1 > /dev/null ; sleep 5 \n + \ done" + image: quay.io/openshifttest/hello-openshift:1.2.0 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + privileged: false + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + name: client +parameters: +- name: POD_NAME + value: client +- name: SERVER_NS + value: test-server +- name: CLIENT_NS + value: test-client +- name: OBJECT_SIZE + value: 100K diff --git a/integration-tests/backend/testdata/test-nginx-server_template.yaml b/integration-tests/backend/testdata/test-nginx-server_template.yaml new file mode 100644 index 0000000000..4507374f9c --- /dev/null +++ b/integration-tests/backend/testdata/test-nginx-server_template.yaml @@ -0,0 +1,78 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: netobserv-test-client-server +objects: +- apiVersion: v1 + kind: Namespace + metadata: + name: ${SERVER_NS} + labels: + name: ${SERVER_NS} +- apiVersion: apps/v1 + kind: Deployment + metadata: + name: nginx + namespace: ${SERVER_NS} + labels: + app: nginx + spec: + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + replicas: 1 + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + privileged: false + image: quay.io/openshifttest/nginx-alpine:1.2.3 + imagePullPolicy: IfNotPresent + env: + - name: CREATE_LARGE_BLOB_FILES + value: ${LARGE_BLOB} + ports: + - containerPort: 8080 +- apiVersion: v1 + kind: Service + metadata: + namespace: ${SERVER_NS} + name: nginx-service + spec: + selector: + app: nginx + type: ${SERVICE_TYPE} + ports: + - protocol: TCP + port: 80 + targetPort: 8080 +- apiVersion: route.openshift.io/v1 + kind: Route + metadata: + namespace: ${SERVER_NS} + name: nginx-route + spec: + port: + targetPort: 80 + to: + kind: Service + name: nginx-service +parameters: +- name: SERVER_NS + value: test-server +- name: LARGE_BLOB + value: "no" +- name: SERVICE_TYPE + value: "NodePort" + diff --git a/integration-tests/backend/testdata/test-ping-pods_template.yaml b/integration-tests/backend/testdata/test-ping-pods_template.yaml new file mode 100644 index 0000000000..6ccbf02916 --- /dev/null +++ b/integration-tests/backend/testdata/test-ping-pods_template.yaml @@ -0,0 +1,76 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: netobserv-test-ping-pods +objects: +- apiVersion: v1 + kind: Namespace + metadata: + name: ${SERVER_NS} + labels: + name: ${SERVER_NS} +- apiVersion: v1 + kind: Pod + metadata: + creationTimestamp: null + labels: + run: ${SERVER_POD_NAME} + name: ${SERVER_POD_NAME} + namespace: ${SERVER_NS} + spec: + containers: + - command: + - sh + - -c + - | + for target in ${PING_TARGETS}; do + ping $target & + done + wait + image: quay.io/openshifttest/hello-openshift:1.2.0 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + name: ${SERVER_POD_NAME} +- apiVersion: v1 + kind: Pod + metadata: + creationTimestamp: null + labels: + run: ${CLIENT_POD_NAME} + name: ${CLIENT_POD_NAME} + namespace: ${CLIENT_NS} + spec: + containers: + - command: + - sh + - -c + - | + for target in ${PING_TARGETS}; do + ping $target & + done + wait + image: quay.io/openshifttest/hello-openshift:1.2.0 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + name: ${CLIENT_POD_NAME} +parameters: +- name: SERVER_POD_NAME + value: ping-server +- name: CLIENT_POD_NAME + value: ping-client +- name: SERVER_NS + value: test-ping-server +- name: CLIENT_NS + value: test-ping-client +- name: PING_TARGETS + value: "8.8.8.8" diff --git a/integration-tests/backend/testdata/test-tls-client_template.yaml b/integration-tests/backend/testdata/test-tls-client_template.yaml new file mode 100644 index 0000000000..599de92a7c --- /dev/null +++ b/integration-tests/backend/testdata/test-tls-client_template.yaml @@ -0,0 +1,47 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: netobserv-test-tls-client +objects: +- apiVersion: v1 + kind: Namespace + metadata: + name: ${CLIENT_NS} + labels: + name: ${CLIENT_NS} +- apiVersion: v1 + kind: Pod + metadata: + labels: + run: tls-client + name: tls-client + namespace: ${CLIENT_NS} + spec: + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - name: client + image: quay.io/openshifttest/hello-openshift:1.2.0 + command: + - sh + - -c + - | + while : ; do + curl -sk --tlsv1.3 https://tls-server-service.${SERVER_NS}.svc:443/ 2>&1 > /dev/null + sleep 5 + curl -sk --tlsv1.2 --tls-max 1.2 https://tls-server-service.${SERVER_NS}.svc:443/ 2>&1 > /dev/null + sleep 5 + curl -s http://tls-server-service.${SERVER_NS}.svc:80/ 2>&1 > /dev/null + sleep 5 + done + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] +parameters: +- name: SERVER_NS + value: test-tls-server +- name: CLIENT_NS + value: test-tls-client diff --git a/integration-tests/backend/testdata/test-tls-server_template.yaml b/integration-tests/backend/testdata/test-tls-server_template.yaml new file mode 100644 index 0000000000..14cf0504f2 --- /dev/null +++ b/integration-tests/backend/testdata/test-tls-server_template.yaml @@ -0,0 +1,123 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: netobserv-test-tls-server +objects: +- apiVersion: v1 + kind: Namespace + metadata: + name: ${SERVER_NS} + labels: + name: ${SERVER_NS} +- apiVersion: v1 + kind: ConfigMap + metadata: + name: nginx-tls-config + namespace: ${SERVER_NS} + data: + nginx.conf: | + events { + worker_connections 1024; + } + http { + server { + listen 8080; + location / { + return 200 'HTTP OK - Non-TLS\n'; + add_header Content-Type text/plain; + } + } + server { + listen 8443 ssl; + ssl_certificate /etc/nginx/ssl/tls.crt; + ssl_certificate_key /etc/nginx/ssl/tls.key; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ecdh_curve X25519:P-256; + location / { + return 200 'HTTPS OK - TLS Enabled\n'; + add_header Content-Type text/plain; + } + } + } +- apiVersion: apps/v1 + kind: Deployment + metadata: + name: tls-server + namespace: ${SERVER_NS} + labels: + app: tls-server + spec: + replicas: 1 + selector: + matchLabels: + app: tls-server + template: + metadata: + labels: + app: tls-server + spec: + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + initContainers: + - name: cert-generator + image: registry.access.redhat.com/ubi9/ubi:latest + command: + - /bin/bash + - -c + - | + openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ + -keyout /etc/nginx/ssl/tls.key -out /etc/nginx/ssl/tls.crt \ + -subj "/CN=tls-server.${SERVER_NS}.svc.cluster.local" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + volumeMounts: + - name: ssl-certs + mountPath: /etc/nginx/ssl + containers: + - name: nginx + image: quay.io/openshifttest/nginx-alpine:1.2.3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + ports: + - containerPort: 8080 + name: http + - containerPort: 8443 + name: https + volumeMounts: + - name: nginx-config + mountPath: /etc/nginx/nginx.conf + subPath: nginx.conf + - name: ssl-certs + mountPath: /etc/nginx/ssl + volumes: + - name: nginx-config + configMap: + name: nginx-tls-config + - name: ssl-certs + emptyDir: {} +- apiVersion: v1 + kind: Service + metadata: + namespace: ${SERVER_NS} + name: tls-server-service + spec: + selector: + app: tls-server + ports: + - name: http + protocol: TCP + port: 80 + targetPort: 8080 + - name: https + protocol: TCP + port: 443 + targetPort: 8443 +parameters: +- name: SERVER_NS + value: test-tls-server diff --git a/integration-tests/backend/testdata/testuser-client-server_template.yaml b/integration-tests/backend/testdata/testuser-client-server_template.yaml new file mode 100644 index 0000000000..0f6b7a9db4 --- /dev/null +++ b/integration-tests/backend/testdata/testuser-client-server_template.yaml @@ -0,0 +1,103 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: netobserv-test-client-server +objects: +- apiVersion: project.openshift.io/v1 + kind: ProjectRequest + metadata: + name: ${SERVER_NS} + labels: + name: ${SERVER_NS} +- apiVersion: apps/v1 + kind: Deployment + metadata: + name: nginx + namespace: ${SERVER_NS} + labels: + app: nginx + spec: + replicas: 1 + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + privileged: false + image: quay.io/openshifttest/nginx-alpine:1.2.3 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8080 +- apiVersion: v1 + kind: Service + metadata: + namespace: ${SERVER_NS} + name: nginx-service + spec: + selector: + app: nginx + type: NodePort + ports: + - protocol: TCP + port: 8080 + targetPort: 8080 +- apiVersion: route.openshift.io/v1 + kind: Route + metadata: + namespace: ${SERVER_NS} + name: nginx-route + spec: + port: + targetPort: 8080 + to: + kind: Service + name: nginx-service +- apiVersion: project.openshift.io/v1 + kind: ProjectRequest + metadata: + name: ${CLIENT_NS} + labels: + name: ${CLIENT_NS} +- apiVersion: v1 + kind: Pod + metadata: + creationTimestamp: null + labels: + run: client + name: client + namespace: ${CLIENT_NS} + spec: + containers: + - command: + - sh + - -c + - " + \ while : ; do\n + \ curl nginx-service.${SERVER_NS}.svc:8080/data/${OBJECT_SIZE} 2>&1 > /dev/null ; sleep 5 \n + \ done" + image: quay.io/openshifttest/hello-openshift:1.2.0 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + privileged: false + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + name: client +parameters: +- name: SERVER_NS + value: test-server +- name: CLIENT_NS + value: test-client +- name: OBJECT_SIZE + value: 100K diff --git a/integration-tests/backend/testdata/testuser-template-crb.yaml b/integration-tests/backend/testdata/testuser-template-crb.yaml new file mode 100644 index 0000000000..f443df612f --- /dev/null +++ b/integration-tests/backend/testdata/testuser-template-crb.yaml @@ -0,0 +1,34 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: testuser-templating-template +objects: +- apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: testuser-templating-cr + rules: + - apiGroups: + - 'template.openshift.io' + resources: + - templates + - processedtemplates + verbs: + - 'get' + - 'list' + - 'create' +- apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: testuser-templating-crb + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: testuser-templating-cr + subjects: + - kind: User + apiGroup: rbac.authorization.k8s.io + name: "${USERNAME}" +parameters: +- name: USERNAME + required: true diff --git a/integration-tests/backend/testdata/virtualization/kubevirt-hyperconverged.yaml b/integration-tests/backend/testdata/virtualization/kubevirt-hyperconverged.yaml new file mode 100644 index 0000000000..0a5169b95d --- /dev/null +++ b/integration-tests/backend/testdata/virtualization/kubevirt-hyperconverged.yaml @@ -0,0 +1,55 @@ +apiVersion: hco.kubevirt.io/v1beta1 +kind: HyperConverged +metadata: + annotations: + deployOVS: 'false' + name: kubevirt-hyperconverged + namespace: openshift-cnv + labels: + app: kubevirt-hyperconverged +spec: + applicationAwareConfig: + allowApplicationAwareClusterResourceQuota: false + vmiCalcConfigName: DedicatedVirtualResources + certConfig: + ca: + duration: 48h0m0s + renewBefore: 24h0m0s + server: + duration: 24h0m0s + renewBefore: 12h0m0s + deployVmConsoleProxy: false + enableApplicationAwareQuota: false + enableCommonBootImageImport: true + evictionStrategy: LiveMigrate + featureGates: + alignCPUs: false + decentralizedLiveMigration: false + declarativeHotplugVolumes: false + deployKubeSecondaryDNS: false + disableMDevConfiguration: false + downwardMetrics: false + enableMultiArchBootImageImport: false + persistentReservation: false + higherWorkloadDensity: + memoryOvercommitPercentage: 100 + infra: {} + liveMigrationConfig: + allowAutoConverge: false + allowPostCopy: false + completionTimeoutPerGiB: 150 + parallelMigrationsPerCluster: 5 + parallelOutboundMigrationsPerNode: 2 + progressTimeout: 150 + resourceRequirements: + vmiCPUAllocationRatio: 10 + uninstallStrategy: BlockUninstallIfWorkloadsExist + virtualMachineOptions: + disableFreePageReporting: false + disableSerialConsoleLog: false + workloadUpdateStrategy: + batchEvictionInterval: 1m0s + batchEvictionSize: 10 + workloadUpdateMethods: + - LiveMigrate + workloads: {} diff --git a/integration-tests/backend/testdata/virtualization/layer2-nad.yaml b/integration-tests/backend/testdata/virtualization/layer2-nad.yaml new file mode 100644 index 0000000000..db2eb8f37a --- /dev/null +++ b/integration-tests/backend/testdata/virtualization/layer2-nad.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: test-76537 +--- +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + name: l2-network + namespace: test-76537 +spec: + config: '{ + "cniVersion": "0.3.1", + "name": "l2-network", + "type": "ovn-k8s-cni-overlay", + "topology": "layer2", + "mtu": 1300, + "netAttachDefName": "test-76537/l2-network" + }' diff --git a/integration-tests/backend/testdata/virtualization/test-vm-UDN_template.yaml b/integration-tests/backend/testdata/virtualization/test-vm-UDN_template.yaml new file mode 100644 index 0000000000..9ab877f40e --- /dev/null +++ b/integration-tests/backend/testdata/virtualization/test-vm-UDN_template.yaml @@ -0,0 +1,63 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: virtual-machine-udn-template +objects: +- apiVersion: kubevirt.io/v1 + kind: VirtualMachine + metadata: + name: ${NAME} + namespace: ${NAMESPACE} + spec: + instancetype: + kind: virtualmachineclusterinstancetype + name: u1.medium + preference: + kind: virtualmachineclusterpreference + name: rhel.9 + runStrategy: Always + template: + spec: + architecture: amd64 + domain: + devices: + disks: + - disk: + bus: virtio + name: rootdisk + - disk: + bus: virtio + name: cloudinitdisk + interfaces: + - binding: + name: l2bridge + name: ${NETWORK_NAME} + networkInterfaceMultiqueue: true + rng: {} + resources: {} + hostname: ${NAME} + networks: + - name: ${NETWORK_NAME} + pod: {} + terminationGracePeriodSeconds: 180 + volumes: + - containerDisk: + image: quay.io/containerdisks/fedora + name: rootdisk + - cloudInitNoCloud: + userData: |- + #cloud-config + user: cloud-user + password: byje-7cd2-i8et + chpasswd: { expire: False } + runcmd: ${RUN_CMD} + name: cloudinitdisk +parameters: +- name: NAME + value: "test-vm3" +- name: NAMESPACE + value: "netobserv-udn-85887" +- name: NETWORK_NAME + value: "udn-network-85887" +- name: RUN_CMD + value: "[[ping, 8.8.8.8]]" diff --git a/integration-tests/backend/testdata/virtualization/test-vm-localnet_template.yaml b/integration-tests/backend/testdata/virtualization/test-vm-localnet_template.yaml new file mode 100644 index 0000000000..aa3c3b5081 --- /dev/null +++ b/integration-tests/backend/testdata/virtualization/test-vm-localnet_template.yaml @@ -0,0 +1,67 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: virtual-machine-localnet-template +objects: +- apiVersion: kubevirt.io/v1 + kind: VirtualMachine + metadata: + name: ${NAME} + namespace: ${NAMESPACE} + spec: + instancetype: + kind: virtualmachineclusterinstancetype + name: u1.medium + preference: + kind: virtualmachineclusterpreference + name: rhel.9 + runStrategy: Always + template: + spec: + architecture: amd64 + domain: + devices: + disks: + - disk: + bus: virtio + name: rootdisk + - disk: + bus: virtio + name: cloudinitdisk + interfaces: + - name: default + masquerade: {} + - name: secondary + bridge: {} + networkInterfaceMultiqueue: true + rng: {} + resources: {} + hostname: ${NAME} + networks: + - name: default + pod: {} + - multus: + networkName: ${NETWORK_NAME} + name: secondary + terminationGracePeriodSeconds: 180 + volumes: + - containerDisk: + image: quay.io/containerdisks/fedora + name: rootdisk + - cloudInitNoCloud: + userData: |- + #cloud-config + user: cloud-user + password: byje-7cd2-i8et + chpasswd: { expire: False } + runcmd: ${RUN_CMD} + name: cloudinitdisk +parameters: +- name: NAME + value: "test-vm5" +- name: NAMESPACE + value: "netobserv-cudn1-85935" +- name: NETWORK_NAME + value: "secondary-localnet-85935" +- name: RUN_CMD + value: "[[ping, 8.8.8.8]]" diff --git a/integration-tests/backend/testdata/virtualization/test-vm-static-IP_template.yaml b/integration-tests/backend/testdata/virtualization/test-vm-static-IP_template.yaml new file mode 100644 index 0000000000..259ceb589e --- /dev/null +++ b/integration-tests/backend/testdata/virtualization/test-vm-static-IP_template.yaml @@ -0,0 +1,79 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: virtual-machine-static-ip-template +objects: +- apiVersion: kubevirt.io/v1 + kind: VirtualMachine + metadata: + name: ${NAME} + namespace: ${NAMESPACE} + spec: + instancetype: + name: u1.medium + preference: + name: rhel.9 + runStrategy: Always + template: + spec: + architecture: amd64 + domain: + devices: + disks: + - disk: + bus: virtio + name: rootdisk + - disk: + bus: virtio + name: cloudinitdisk + interfaces: + - name: default + masquerade: {} + - name: secondary + bridge: {} + macAddress: ${MAC} + networkInterfaceMultiqueue: true + rng: {} + resources: {} + hostname: ${NAME} + terminationGracePeriodSeconds: 180 + volumes: + - containerDisk: + image: quay.io/containerdisks/fedora + name: rootdisk + - cloudInitNoCloud: + networkData: | + network: + version: 2 + ethernets: + eth1: + match: + macaddress: ${MAC} + addresses: + - ${STATIC_IP} + userData: |- + #cloud-config + user: cloud-user + password: byje-7cd2-i8et + chpasswd: { expire: False } + runcmd: ${RUN_CMD} + name: cloudinitdisk + networks: + - name: default + pod: {} + - multus: + networkName: ${NETWORK_NAME} + name: secondary +parameters: +- name: NAME + value: "test-vm1" +- name: NAMESPACE + value: "test-76537" +- name: NETWORK_NAME + value: "l2-network" +- name: MAC + value: "02:00:00:00:00:01" +- name: STATIC_IP + value: "10.10.10.15/24" +- name: RUN_CMD + value: "[[ping, 8.8.8.8]]" diff --git a/integration-tests/backend/udn.go b/integration-tests/backend/udn.go new file mode 100644 index 0000000000..c84884d420 --- /dev/null +++ b/integration-tests/backend/udn.go @@ -0,0 +1,525 @@ +package e2etests + +import ( + "context" + "fmt" + "net" + filePath "path/filepath" + "strconv" + "strings" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + exutil "github.com/openshift/origin/test/extended/util" + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" + "k8s.io/apimachinery/pkg/util/wait" + e2e "k8s.io/kubernetes/test/e2e/framework" + e2eoutput "k8s.io/kubernetes/test/e2e/framework/pod/output" +) + +type udnCRDResource struct { + crdname string + namespace string + IPv4cidr string + IPv4prefix int32 + IPv6cidr string + IPv6prefix int32 + cidr string + prefix int32 + mtu int32 + role string + template string +} + +type cudnCRDResource struct { + crdname string + labelvalue string + labelkey string + IPv4cidr string + IPv4prefix int32 + IPv6cidr string + IPv6prefix int32 + cidr string + prefix int32 + role string + physicalnetworkname string + subnet string + excludesubnet string + template string +} + +type udnPodResource struct { + name string + namespace string + label string + template string +} + +type nmstateCRResource struct { + name string + template string +} + +type ovnMappingPolicyResource struct { + name string + nodelabel string + labelvalue string + localnet1 string + bridge1 string + template string +} + +func (cudncrd *cudnCRDResource) createCUDNCRDSingleStack(oc *exutil.CLI) { + err := wait.PollUntilContextTimeout(context.TODO(), 2*time.Second, 20*time.Second, false, func(_ context.Context) (bool, error) { + err1 := applyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", cudncrd.template, "-p", "CRDNAME="+cudncrd.crdname, "LABELKEY="+cudncrd.labelkey, "LABELVALUE="+cudncrd.labelvalue, + "CIDR="+cudncrd.cidr, "PREFIX="+strconv.Itoa(int(cudncrd.prefix)), "ROLE="+cudncrd.role) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("fail to create cudn CRD %s due to %v", cudncrd.crdname, err)) +} + +func (cudncrd *cudnCRDResource) createCUDNCRDDualStack(oc *exutil.CLI) { + err := wait.PollUntilContextTimeout(context.TODO(), 2*time.Second, 20*time.Second, false, func(_ context.Context) (bool, error) { + err1 := applyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", cudncrd.template, "-p", "CRDNAME="+cudncrd.crdname, "LABELKEY="+cudncrd.labelkey, "LABELVALUE="+cudncrd.labelvalue, + "IPv4CIDR="+cudncrd.IPv4cidr, "IPv4PREFIX="+strconv.Itoa(int(cudncrd.IPv4prefix)), "IPv6CIDR="+cudncrd.IPv6cidr, "IPv6PREFIX="+strconv.Itoa(int(cudncrd.IPv6prefix)), "ROLE="+cudncrd.role) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("fail to create cudn CRD %s due to %v", cudncrd.crdname, err)) +} + +func (cudncrd *cudnCRDResource) createLayer2SingleStackCUDNCRD(oc *exutil.CLI) { + err := wait.PollUntilContextTimeout(context.TODO(), 5*time.Second, 20*time.Second, false, func(_ context.Context) (bool, error) { + err1 := applyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", cudncrd.template, "-p", "CRDNAME="+cudncrd.crdname, "LABELKEY="+cudncrd.labelkey, "LABELVALUE="+cudncrd.labelvalue, + "CIDR="+cudncrd.cidr, "ROLE="+cudncrd.role) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("fail to create cudn CRD %s due to %v", cudncrd.crdname, err)) +} + +func (cudncrd *cudnCRDResource) createLayer2DualStackCUDNCRD(oc *exutil.CLI) { + err := wait.PollUntilContextTimeout(context.TODO(), 5*time.Second, 20*time.Second, false, func(_ context.Context) (bool, error) { + err1 := applyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", cudncrd.template, "-p", "CRDNAME="+cudncrd.crdname, "LABELKEY="+cudncrd.labelkey, "LABELVALUE="+cudncrd.labelvalue, + "IPv4CIDR="+cudncrd.IPv4cidr, "IPv6CIDR="+cudncrd.IPv6cidr, "ROLE="+cudncrd.role) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("fail to create cudn CRD %s due to %v", cudncrd.crdname, err)) +} + +func (cudncrd *cudnCRDResource) createLayer3LocalnetCUDNCRD(oc *exutil.CLI) { + err := wait.PollUntilContextTimeout(context.TODO(), 2*time.Second, 20*time.Second, false, func(_ context.Context) (bool, error) { + err1 := applyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", cudncrd.template, "-p", "CRDNAME="+cudncrd.crdname, "LABELKEY="+cudncrd.labelkey, "LABELVALUE="+cudncrd.labelvalue, "PHYSICALNETWORK="+cudncrd.physicalnetworkname, "SUBNET="+cudncrd.subnet, "EXCLUDESUBNET="+cudncrd.excludesubnet, "ROLE="+cudncrd.role) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("fail to create cudn CRD %s due to %v", cudncrd.crdname, err)) +} + +func applyCUDNtoMatchLabelNS(oc *exutil.CLI, matchLabelKey, matchValue, crdName, ipv4cidr, ipv6cidr, cidr, topology string) (cudnCRDResource, error) { + + var ( + networkingUDNDir, _ = filePath.Abs("testdata/networking/udn") + cudnCRDSingleStack = filePath.Join(networkingUDNDir, "cudn_crd_singlestack_template.yaml") + cudnCRDdualStack = filePath.Join(networkingUDNDir, "cudn_crd_dualstack_template.yaml") + cudnCRDL2dualStack = filePath.Join(networkingUDNDir, "cudn_crd_layer2_dualstack_template.yaml") + cudnCRDL2SingleStack = filePath.Join(networkingUDNDir, "cudn_crd_layer2_singlestack_template.yaml") + ) + + ipStackType := checkIPStackType(oc) + cudncrd := cudnCRDResource{ + crdname: crdName, + labelkey: matchLabelKey, + labelvalue: matchValue, + role: "Primary", + template: cudnCRDSingleStack, + } + + switch topology { + case "layer3": + switch ipStackType { + case "dualstack": + cudncrd.IPv4cidr = ipv4cidr + cudncrd.IPv4prefix = 24 + cudncrd.IPv6cidr = ipv6cidr + cudncrd.IPv6prefix = 64 + cudncrd.template = cudnCRDdualStack + cudncrd.createCUDNCRDDualStack(oc) + case "ipv6single": + cudncrd.prefix = 64 + cudncrd.cidr = cidr + cudncrd.template = cudnCRDSingleStack + cudncrd.createCUDNCRDSingleStack(oc) + case "ipv4single": + cudncrd.prefix = 24 + cudncrd.cidr = cidr + cudncrd.template = cudnCRDSingleStack + cudncrd.createCUDNCRDSingleStack(oc) + } + case "layer2": + switch ipStackType { + case "dualstack": + cudncrd.IPv4cidr = ipv4cidr + cudncrd.IPv6cidr = ipv6cidr + cudncrd.template = cudnCRDL2dualStack + cudncrd.createLayer2DualStackCUDNCRD(oc) + default: + cudncrd.cidr = cidr + cudncrd.template = cudnCRDL2SingleStack + cudncrd.createLayer2SingleStackCUDNCRD(oc) + } + } + err := waitCUDNCRDApplied(oc, cudncrd.crdname) + if err != nil { + return cudncrd, err + } + return cudncrd, nil +} + +func applyLocalnetCUDNtoMatchLabelNS(oc *exutil.CLI, matchLabelKey, matchValue, crdName, physicalNetworkName, subnet, excludeSubnet string, vlan bool) (cudnCRDResource, error) { + var ( + networkingUDNDir, _ = filePath.Abs("testdata/networking/udn") + cudnCRDLocalnetSingleStack = filePath.Join(networkingUDNDir, "cudn_crd_localnet_singlestack_template.yaml") + cudnCRDLocalnetSingleStackWithVlan = filePath.Join(networkingUDNDir, "cudn_crd_localnet_singlestack_with_vlan_template.yaml") + ) + + cudncrd := cudnCRDResource{ + crdname: crdName, + labelkey: matchLabelKey, + labelvalue: matchValue, + physicalnetworkname: physicalNetworkName, + subnet: subnet, + excludesubnet: excludeSubnet, + role: "Secondary", + } + + if vlan { + cudncrd.template = cudnCRDLocalnetSingleStackWithVlan + } else { + cudncrd.template = cudnCRDLocalnetSingleStack + } + + cudncrd.createLayer3LocalnetCUDNCRD(oc) + err := waitCUDNCRDApplied(oc, cudncrd.crdname) + if err != nil { + return cudncrd, err + } + return cudncrd, nil +} + +func (udncrd *udnCRDResource) createUdnCRDSingleStack(oc *exutil.CLI) { + err := wait.PollUntilContextTimeout(context.TODO(), 5*time.Second, 20*time.Second, false, func(_ context.Context) (bool, error) { + err1 := applyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", udncrd.template, "-p", "CRDNAME="+udncrd.crdname, "NAMESPACE="+udncrd.namespace, "CIDR="+udncrd.cidr, "PREFIX="+strconv.Itoa(int(udncrd.prefix)), "MTU="+strconv.Itoa(int(udncrd.mtu)), "ROLE="+udncrd.role) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("fail to create udn CRD %s due to %v", udncrd.crdname, err)) +} + +func (udncrd *udnCRDResource) createUdnCRDDualStack(oc *exutil.CLI) { + err := wait.PollUntilContextTimeout(context.TODO(), 5*time.Second, 20*time.Second, false, func(_ context.Context) (bool, error) { + err1 := applyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", udncrd.template, "-p", "CRDNAME="+udncrd.crdname, "NAMESPACE="+udncrd.namespace, "IPv4CIDR="+udncrd.IPv4cidr, "IPv4PREFIX="+strconv.Itoa(int(udncrd.IPv4prefix)), "IPv6CIDR="+udncrd.IPv6cidr, "IPv6PREFIX="+strconv.Itoa(int(udncrd.IPv6prefix)), "MTU="+strconv.Itoa(int(udncrd.mtu)), "ROLE="+udncrd.role) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("fail to create udn CRD %s due to %v", udncrd.crdname, err)) +} + +func (udncrd *udnCRDResource) createLayer2DualStackUDNCRD(oc *exutil.CLI) { + err := wait.PollUntilContextTimeout(context.TODO(), 5*time.Second, 20*time.Second, false, func(_ context.Context) (bool, error) { + err1 := applyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", udncrd.template, "-p", "CRDNAME="+udncrd.crdname, "NAMESPACE="+udncrd.namespace, "IPv4CIDR="+udncrd.IPv4cidr, "IPv6CIDR="+udncrd.IPv6cidr, "MTU="+strconv.Itoa(int(udncrd.mtu)), "ROLE="+udncrd.role) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("fail to create udn CRD %s due to %v", udncrd.crdname, err)) +} + +func (udncrd *udnCRDResource) createLayer2SingleStackUDNCRD(oc *exutil.CLI) { + err := wait.PollUntilContextTimeout(context.TODO(), 5*time.Second, 20*time.Second, false, func(_ context.Context) (bool, error) { + err1 := applyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", udncrd.template, "-p", "CRDNAME="+udncrd.crdname, "NAMESPACE="+udncrd.namespace, "CIDR="+udncrd.cidr, "MTU="+strconv.Itoa(int(udncrd.mtu)), "ROLE="+udncrd.role) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("fail to create udn CRD %s due to %v", udncrd.crdname, err)) +} + +func createGeneralUDNCRD(oc *exutil.CLI, namespace, crdName, ipv4cidr, ipv6cidr, cidr, layer string) { + // This is a function for common CRD creation without special requirement for parameters which is can be used for common cases and to reduce code lines in case level. + var ( + networkingUDNDir, _ = filePath.Abs("testdata/networking/udn") + udnCRDdualStack = filePath.Join(networkingUDNDir, "udn_crd_dualstack2_template.yaml") + udnCRDSingleStack = filePath.Join(networkingUDNDir, "udn_crd_singlestack_template.yaml") + udnCRDLayer2dualStack = filePath.Join(networkingUDNDir, "udn_crd_layer2_dualstack_template.yaml") + udnCRDLayer2SingleStack = filePath.Join(networkingUDNDir, "udn_crd_layer2_singlestack_template.yaml") + ) + + ipStackType := checkIPStackType(oc) + var udncrd udnCRDResource + switch layer { + case "layer3": + switch ipStackType { + case "dualstack": + udncrd = udnCRDResource{ + crdname: crdName, + namespace: namespace, + role: "Primary", + IPv4cidr: ipv4cidr, + IPv4prefix: 24, + IPv6cidr: ipv6cidr, + IPv6prefix: 64, + template: udnCRDdualStack, + } + udncrd.createUdnCRDDualStack(oc) + case "ipv6single": + udncrd = udnCRDResource{ + crdname: crdName, + namespace: namespace, + role: "Primary", + cidr: cidr, + prefix: 64, + template: udnCRDSingleStack, + } + udncrd.createUdnCRDSingleStack(oc) + default: + udncrd = udnCRDResource{ + crdname: crdName, + namespace: namespace, + role: "Primary", + cidr: cidr, + prefix: 24, + template: udnCRDSingleStack, + } + udncrd.createUdnCRDSingleStack(oc) + } + err := waitUDNCRDApplied(oc, namespace, udncrd.crdname) + o.Expect(err).NotTo(o.HaveOccurred()) + + case "layer2": + switch ipStackType { + case "dualstack": + udncrd = udnCRDResource{ + crdname: crdName, + namespace: namespace, + role: "Primary", + IPv4cidr: ipv4cidr, + IPv6cidr: ipv6cidr, + template: udnCRDLayer2dualStack, + } + udncrd.createLayer2DualStackUDNCRD(oc) + + default: + udncrd = udnCRDResource{ + crdname: crdName, + namespace: namespace, + role: "Primary", + cidr: cidr, + template: udnCRDLayer2SingleStack, + } + udncrd.createLayer2SingleStackUDNCRD(oc) + err := waitUDNCRDApplied(oc, namespace, udncrd.crdname) + o.Expect(err).NotTo(o.HaveOccurred()) + } + default: + e2e.Logf("Not surpport UDN type for now.") + } +} + +func waitUDNCRDApplied(oc *exutil.CLI, ns, crdName string) error { + checkErr := wait.PollUntilContextTimeout(context.TODO(), 3*time.Second, 60*time.Second, false, func(_ context.Context) (bool, error) { + output, efErr := oc.AsAdmin().WithoutNamespace().Run("wait").Args("UserDefinedNetwork/"+crdName, "-n", ns, "--for", "condition=NetworkAllocationSucceeded=True").Output() + if efErr != nil { + e2e.Logf("Failed to get UDN %v, error: %s. Trying again", crdName, efErr) + return false, nil + } + if !strings.Contains(output, fmt.Sprintf("userdefinednetwork.k8s.ovn.org/%s condition met", crdName)) { + e2e.Logf("UDN CRD was not applied yet, trying again. \n %s", output) + return false, nil + } + return true, nil + }) + return checkErr +} + +func waitCUDNCRDApplied(oc *exutil.CLI, crdName string) error { + checkErr := wait.PollUntilContextTimeout(context.TODO(), 3*time.Second, 30*time.Second, false, func(_ context.Context) (bool, error) { + output, efErr := oc.AsAdmin().WithoutNamespace().Run("wait").Args("ClusterUserDefinedNetwork/"+crdName, "--for", "condition=NetworkCreated=True").Output() + if efErr != nil { + e2e.Logf("Failed to get CUDN %v, error: %s. Trying again", crdName, efErr) + return false, nil + } + if !strings.Contains(output, fmt.Sprintf("clusteruserdefinednetwork.k8s.ovn.org/%s condition met", crdName)) { + e2e.Logf("CUDN CRD was not applied yet, trying again. \n %s", output) + return false, nil + } + return true, nil + }) + return checkErr +} + +func (pod *udnPodResource) createUdnPod(oc *exutil.CLI) { + err := wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 20*time.Second, false, func(_ context.Context) (bool, error) { + err1 := applyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", pod.template, "-p", "NAME="+pod.name, "NAMESPACE="+pod.namespace, "LABEL="+pod.label) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("fail to create pod %v", pod.name)) +} + +// getPodIPUDN returns IPv6 and IPv4 in vars in order on dual stack respectively and main IP in case of single stack (v4 or v6) in 1st var, and nil in 2nd var +func getPodIPUDN(oc *exutil.CLI, namespace, podName, netName string) (string, string) { + ipStack := checkIPStackType(oc) + cmdIPv4 := "ip a sho " + netName + " | awk 'NR==3{print $2}' |grep -Eo '((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])'" + cmdIPv6 := "ip -o -6 addr show dev " + netName + " | awk '$3 == \"inet6\" && $6 == \"global\" {print $4}' | cut -d'/' -f1" + switch ipStack { + case "ipv4single": + podIPv4, err := execCommandInSpecificPod(oc, namespace, podName, cmdIPv4) + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The UDN pod %s IPv4 in namespace %s is %q", podName, namespace, podIPv4) + return podIPv4, "" + case "ipv6single": + podIPv6, err := execCommandInSpecificPod(oc, namespace, podName, cmdIPv6) + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The UDN pod %s IPv6 in namespace %s is %q", podName, namespace, podIPv6) + return podIPv6, "" + default: + podIPv4, err := execCommandInSpecificPod(oc, namespace, podName, cmdIPv4) + o.Expect(err).NotTo(o.HaveOccurred()) + podIPv6, err := execCommandInSpecificPod(oc, namespace, podName, cmdIPv6) + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The UDN pod's %s IPv6 and IPv4 IP in namespace %s is %q %q", podName, namespace, podIPv6, podIPv4) + return podIPv6, podIPv4 + } +} + +// CurlPod2PodFailUDN ensures no connectivity from a udn pod to pod regardless of network addressing type on cluster +func CurlPod2PodFailUDN(oc *exutil.CLI, namespaceSrc, podNameSrc, namespaceDst, podNameDst string) { + // getPodIPUDN will returns IPv6 and IPv4 in vars in order on dual stack respectively and main IP in case of single stack (v4 or v6) in 1st var, and nil in 2nd var + podIP1, podIP2 := getPodIPUDN(oc, namespaceDst, podNameDst, "ovn-udn1") + if podIP2 != "" { + _, err := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl --connect-timeout 5 -s "+net.JoinHostPort(podIP1, "8080")) + o.Expect(err).To(o.HaveOccurred()) + _, err = e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl --connect-timeout 5 -s "+net.JoinHostPort(podIP2, "8080")) + o.Expect(err).To(o.HaveOccurred()) + } else { + _, err := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl --connect-timeout 5 -s "+net.JoinHostPort(podIP1, "8080")) + o.Expect(err).To(o.HaveOccurred()) + } +} + +// CurlPod2PodPass checks connectivity across udn pods regardless of network addressing type on cluster +func CurlPod2PodPassUDN(oc *exutil.CLI, namespaceSrc, podNameSrc, namespaceDst, podNameDst string) { + // getPodIPUDN will returns IPv6 and IPv4 in vars in order on dual stack respectively and main IP in case of single stack (v4 or v6) in 1st var, and nil in 2nd var + podIP1, podIP2 := getPodIPUDN(oc, namespaceDst, podNameDst, "ovn-udn1") + if podIP2 != "" { + _, err := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl --connect-timeout 5 -s "+net.JoinHostPort(podIP1, "8080")) + o.Expect(err).NotTo(o.HaveOccurred()) + _, err = e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl --connect-timeout 5 -s "+net.JoinHostPort(podIP2, "8080")) + o.Expect(err).NotTo(o.HaveOccurred()) + } else { + _, err := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl --connect-timeout 5 -s "+net.JoinHostPort(podIP1, "8080")) + o.Expect(err).NotTo(o.HaveOccurred()) + } +} + +func deleteNMStateCR(oc *exutil.CLI, rs nmstateCRResource) { + e2e.Logf("delete %s CR %s", "nmstate", rs.name) + err := oc.AsAdmin().WithoutNamespace().Run("delete").Args("nmstate", rs.name, "--ignore-not-found=true").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) +} + +func checkNmstateCR(oc *exutil.CLI, namespace string) (bool, error) { + WaitForPodsReadyWithLabel(oc, namespace, "component=kubernetes-nmstate-handler") + WaitForPodsReadyWithLabel(oc, namespace, "component=kubernetes-nmstate-webhook") + /* + Due to bug OCPBUGS-54295 nmstate-console-plugin pod cannot be successfully created, comment it for now + err = waitForPodWithLabelReady(oc, namespace, "app=nmstate-console-plugin") + if err != nil { + e2e.Logf("nmstate-console-plugin pod did not transition to ready state %v", err) + return false, err + }*/ + WaitForPodsReadyWithLabel(oc, namespace, "component=kubernetes-nmstate-metrics") + e2e.Logf("nmstate-handler, nmstate-webhook, nmstate-console-plugin and nmstate-metrics pods created successfully") + return true, nil +} + +func createNMStateCR(oc *exutil.CLI, nmstatecr nmstateCRResource, namespace string) (bool, error) { + g.By("Creating NMState CR from template") + + err := applyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", nmstatecr.template, "-p", "NAME="+nmstatecr.name) + if err != nil { + e2e.Logf("Error creating NMState CR %v", err) + return false, err + } + + result, err := checkNmstateCR(oc, namespace) + return result, err +} + +func deleteNNCP(oc *exutil.CLI, name string) { + e2e.Logf("delete nncp %s", name) + err := oc.AsAdmin().WithoutNamespace().Run("delete").Args("nncp", name, "--ignore-not-found=true").Execute() + if err != nil { + e2e.Logf("Failed to delete nncp %s, error:%s", name, err) + } +} + +func (bvpr *ovnMappingPolicyResource) configNNCP(oc *exutil.CLI) error { + err := applyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", bvpr.template, "-p", "NAME="+bvpr.name, "NODELABEL="+bvpr.nodelabel, "LABELVALUE="+bvpr.labelvalue, + "LOCALNET1="+bvpr.localnet1, "BRIDGE1="+bvpr.bridge1) + if err != nil { + e2e.Logf("Error configure ovnmapping %v", err) + return err + } + return nil +} + +func checkNNCPStatus(oc *exutil.CLI, policyName string, expectedStatus string) error { + err := wait.PollUntilContextTimeout(context.Background(), 10*time.Second, 3*time.Minute, false, func(_ context.Context) (bool, error) { + e2e.Logf("Checking status of nncp %s", policyName) + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("nncp", policyName).Output() + if err != nil { + e2e.Logf("Failed to get nncp status, error:%s. Trying again", err) + return false, nil + } + if !strings.Contains(output, expectedStatus) { + e2e.Logf("nncp status does not meet expectation:%s, error:%s, output:%s. Trying again", expectedStatus, err, output) + return false, nil + } + return true, nil + }) + return err +} diff --git a/integration-tests/backend/util.go b/integration-tests/backend/util.go new file mode 100644 index 0000000000..711608273b --- /dev/null +++ b/integration-tests/backend/util.go @@ -0,0 +1,929 @@ +package e2etests + +import ( + "context" + "crypto/tls" + "fmt" + "io" + "math/rand" + "net/http" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + o "github.com/onsi/gomega" + exutil "github.com/openshift/origin/test/extended/util" + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + e2e "k8s.io/kubernetes/test/e2e/framework" +) + +type TestServerTemplate struct { + ServerNS string + LargeBlob string + ServiceType string + Template string +} + +type TestClientTemplate struct { + ServerNS string + ClientNS string + ObjectSize string + Template string +} + +type TestPingPodsTemplate struct { + ServerNS string + ClientNS string + ServerPodName string + ClientPodName string + PingTargets string + Template string +} + +func getRandomString() string { + chars := "abcdefghijklmnopqrstuvwxyz0123456789" + seed := rand.New(rand.NewSource(time.Now().UnixNano())) + buffer := make([]byte, 8) + for index := range buffer { + buffer[index] = chars[seed.Intn(len(chars))] + } + return string(buffer) +} + +// contain checks if b is an elememt of a +func contain(a []string, b string) bool { + for _, c := range a { + if c == b { + return true + } + } + return false +} + +func getProxyFromEnv() string { + var proxy string + if os.Getenv("http_proxy") != "" { + proxy = os.Getenv("http_proxy") + } else if os.Getenv("http_proxy") != "" { + proxy = os.Getenv("https_proxy") + } + return proxy +} + +func getRouteAddress(oc *exutil.CLI, ns, routeName string) string { + route, err := oc.AdminRouteClient().RouteV1().Routes(ns).Get(context.Background(), routeName, metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + return route.Spec.Host +} + +// return the infrastructureName. For example: anli922-jglp4 +func getInfrastructureName(oc *exutil.CLI) string { + infrastructureName, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("infrastructure/cluster", "-o=jsonpath={.status.infrastructureName}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + return infrastructureName +} + +func (r Resource) applyFromTemplate(oc *exutil.CLI, parameters ...string) error { + parameters = append(parameters, "-n", r.Namespace) + file, err := processTemplate(oc, parameters...) + defer os.Remove(file) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("Can not process %v", parameters)) + output, err := oc.AsAdmin().WithoutNamespace().Run("apply").Args("-f", file, "-n", r.Namespace).Output() + if err != nil { + return fmt.Errorf("%v", output) + } + _ = r.WaitForResourceToAppear(oc) + return nil +} + +func processTemplate(oc *exutil.CLI, parameters ...string) (string, error) { + var configFile string + err := wait.PollUntilContextTimeout(context.Background(), 3*time.Second, 15*time.Second, false, func(context.Context) (bool, error) { + output, err := oc.AsAdmin().Run("process").Args(parameters...).OutputToFile(getRandomString() + ".json") + if err != nil { + e2e.Logf("the err:%v, and try next round", err) + return false, nil + } + configFile = output + return true, nil + }) + return configFile, err +} + +// expect: true means we want the resource contain/compare with the expectedContent, false means the resource is expected not to compare with/contain the expectedContent; +// compare: true means compare the expectedContent with the resource content, false means check if the resource contains the expectedContent; +// args are the arguments used to execute command `oc.AsAdmin.WithoutNamespace().Run("get").Args(args...).Output()`; +func checkResource(oc *exutil.CLI, expect, compare bool, expectedContent string, args []string) { + err := wait.PollUntilContextTimeout(context.Background(), 10*time.Second, 180*time.Second, false, func(context.Context) (done bool, err error) { + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args(args...).Output() + if err != nil { + if strings.Contains(output, "NotFound") { + return false, nil + } + return false, err + } + if compare { + res := strings.Compare(output, expectedContent) + if (res == 0 && expect) || (res != 0 && !expect) { + return true, nil + } + return false, nil + } + res := strings.Contains(output, expectedContent) + if (res && expect) || (!res && !expect) { + return true, nil + } + return false, nil + }) + if expect { + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("The content doesn't match/contain %s", expectedContent)) + } else { + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("The %s still exists in the resource", expectedContent)) + } +} + +func getResourceGeneration(oc *exutil.CLI, resource, name, ns string) (int, error) { + gen, err := oc.AsAdmin().WithoutNamespace().Run("get").Args(resource, name, "-o=jsonpath='{.metadata.generation}'", "-n", ns).Output() + if err != nil { + return -1, err + } + genI, err := strconv.Atoi(strings.Trim(gen, "'")) + if err != nil { + return -1, err + } + return genI, nil + +} + +func getResourceVersion(oc *exutil.CLI, resource, name, ns string) (int, error) { + resV, err := oc.AsAdmin().WithoutNamespace().Run("get").Args(resource, name, "-o=jsonpath='{.metadata.resourceVersion}'", "-n", ns).Output() + if err != nil { + return -1, err + } + vers, err := strconv.Atoi(strings.Trim(resV, "'")) + if err != nil { + return -1, err + } + return vers, nil +} + +func checkResourceExists(oc *exutil.CLI, resource, name, ns string) (bool, error) { + stdout, stderr, err := oc.AsAdmin().WithoutNamespace().Run("get").Args(resource, name, "-n", ns).Outputs() + if err != nil { + return false, err + } + if strings.Contains(stderr, "NotFound") { + return false, nil + } + if strings.Contains(stdout, name) { + return true, nil + } + return false, nil +} + +// Assert the status of a resource +func assertResourceStatus(oc *exutil.CLI, kind, name, namespace, jsonpath, exptdStatus string) { + parameters := []string{kind, name, "-o", "jsonpath=" + jsonpath} + if namespace != "" { + parameters = append(parameters, "-n", namespace) + } + err := wait.PollUntilContextTimeout(context.Background(), 10*time.Second, 180*time.Second, true, func(context.Context) (done bool, err error) { + status, err := oc.AsAdmin().WithoutNamespace().Run("get").Args(parameters...).Output() + if err != nil { + return false, err + } + if strings.Compare(status, exptdStatus) != 0 { + return false, nil + } + return true, nil + }) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("%s/%s value for %s is not %s", kind, name, jsonpath, exptdStatus)) +} + +// For admin user to create resources in the specified namespace from the file (not template) +func ApplyResourceFromFile(oc *exutil.CLI, ns, file string) { + err := oc.AsAdmin().WithoutNamespace().Run("apply").Args("-f", file, "-n", ns).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) +} + +func applyResourceFromTemplateByAdmin(oc *exutil.CLI, parameters ...string) error { + var configFile string + err := wait.PollUntilContextTimeout(context.Background(), 3*time.Second, 15*time.Second, false, func(_ context.Context) (bool, error) { + output, err := oc.AsAdmin().Run("process").Args(parameters...).OutputToFile(getRandomString() + "resource.json") + if err != nil { + e2e.Logf("the err:%v, and try next round", err) + return false, nil + } + configFile = output + return true, nil + }) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("as admin fail to process %v", parameters)) + + e2e.Logf("the file of resource is %s", configFile) + return oc.WithoutNamespace().AsAdmin().Run("apply").Args("-f", configFile).Execute() +} + +// For normal user to create resources in the specified namespace from the file (not template) +func createResourceFromFile(oc *exutil.CLI, ns, file string) { + err := oc.AsAdmin().WithoutNamespace().Run("create").Args("-f", file, "-n", ns).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) +} + +func getSecrets(oc *exutil.CLI, namespace string) (string, error) { + var secrets string + err := wait.PollUntilContextTimeout(context.Background(), 10*time.Second, 360*time.Second, false, func(context.Context) (done bool, err error) { + secrets, err = oc.AsAdmin().WithoutNamespace().Run("get").Args("secrets", "-n", namespace, "-o", "jsonpath='{range .items[*]}{.metadata.name}{\" \"}'").Output() + + if err != nil { + return false, err + } + return true, nil + }) + compat_otp.AssertWaitPollNoErr(err, "Secrets not available") + return secrets, err +} + +// check if pods with label are fully deleted +func checkPodDeleted(oc *exutil.CLI, ns, label, checkValue string) { + podCheck := wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 240*time.Second, false, func(context.Context) (bool, error) { + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", "-n", ns, "-l", label).Output() + if err != nil || strings.Contains(output, checkValue) { + return false, nil + } + return true, nil + }) + compat_otp.AssertWaitPollNoErr(podCheck, fmt.Sprintf("Pod \"%s\" exists or not fully deleted", checkValue)) +} + +func getSAToken(oc *exutil.CLI, name, ns string) string { + token, err := oc.AsAdmin().WithoutNamespace().Run("create").Args("token", name, "-n", ns).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + return token +} + +func doHTTPRequest(header http.Header, address, path, query, method string, quiet bool, attempts int, requestBody io.Reader, expectedStatusCode int) ([]byte, error) { + us, err := buildURL(address, path, query) + if err != nil { + return nil, err + } + if !quiet { + e2e.Logf("%s", us) + } + + req, err := http.NewRequest(strings.ToUpper(method), us, requestBody) + if err != nil { + return nil, err + } + + req.Header = header + + var tr *http.Transport + proxy := getProxyFromEnv() + if len(proxy) > 0 { + proxyURL, err := url.Parse(proxy) + o.Expect(err).NotTo(o.HaveOccurred()) + tr = &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + Proxy: http.ProxyURL(proxyURL), + } + } else { + tr = &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + } + + client := &http.Client{Transport: tr} + + var resp *http.Response + success := false + + for attempts > 0 { + attempts-- + + resp, err = client.Do(req) + if err != nil { + e2e.Logf("error sending request %v", err) + continue + } + if resp.StatusCode != expectedStatusCode { + buf, _ := io.ReadAll(resp.Body) // nolint + e2e.Logf("Error response from server: %s %s (%v), attempts remaining: %d", resp.Status, string(buf), err, attempts) + if err := resp.Body.Close(); err != nil { + e2e.Logf("error closing body %v", err) + } + continue + } + success = true + break + } + if !success { + return nil, fmt.Errorf("run out of attempts while querying the server") + } + + defer func() { + if err := resp.Body.Close(); err != nil { + e2e.Logf("error closing body %v", err) + } + }() + return io.ReadAll(resp.Body) +} + +func (testTemplate *TestServerTemplate) createServer(oc *exutil.CLI) error { + templateParams := []string{"--ignore-unknown-parameters=true", "-f", testTemplate.Template, "-p", "SERVER_NS=" + testTemplate.ServerNS} + + if testTemplate.LargeBlob != "" { + templateParams = append(templateParams, "-p", "LARGE_BLOB="+testTemplate.LargeBlob) + } + if testTemplate.ServiceType != "" { + templateParams = append(templateParams, "-p", "SERVICE_TYPE="+testTemplate.ServiceType) + } + configFile := compat_otp.ProcessTemplate(oc, templateParams...) + + err := oc.AsAdmin().WithoutNamespace().Run("create").Args("-f", configFile).Execute() + if err != nil { + return err + } + return nil +} + +func (testTemplate *TestClientTemplate) createClient(oc *exutil.CLI) error { + templateParams := []string{"--ignore-unknown-parameters=true", "-f", testTemplate.Template, "-p", "SERVER_NS=" + testTemplate.ServerNS, "-p", "CLIENT_NS=" + testTemplate.ClientNS} + + if testTemplate.ObjectSize != "" { + templateParams = append(templateParams, "-p", "OBJECT_SIZE="+testTemplate.ObjectSize) + } + configFile := compat_otp.ProcessTemplate(oc, templateParams...) + + err := oc.AsAdmin().WithoutNamespace().Run("create").Args("-f", configFile).Execute() + if err != nil { + return err + } + return nil +} + +func (testTemplate *TestPingPodsTemplate) createPingPods(oc *exutil.CLI) error { + templateParams := []string{"--ignore-unknown-parameters=true", "-f", testTemplate.Template, "-p", "SERVER_NS=" + testTemplate.ServerNS, "-p", "CLIENT_NS=" + testTemplate.ClientNS} + + if testTemplate.ServerPodName != "" { + templateParams = append(templateParams, "-p", "SERVER_POD_NAME="+testTemplate.ServerPodName) + } + if testTemplate.ClientPodName != "" { + templateParams = append(templateParams, "-p", "CLIENT_POD_NAME="+testTemplate.ClientPodName) + } + if testTemplate.PingTargets != "" { + templateParams = append(templateParams, "-p", "PING_TARGETS="+testTemplate.PingTargets) + } + configFile := compat_otp.ProcessTemplate(oc, templateParams...) + + err := oc.AsAdmin().WithoutNamespace().Run("create").Args("-f", configFile).Execute() + if err != nil { + return err + } + return nil +} + +func waitForResourceGenerationUpdate(oc *exutil.CLI, resource, name, field string, prev int, ns string) { + err := wait.PollUntilContextTimeout(context.Background(), 10*time.Second, 300*time.Second, false, func(context.Context) (done bool, err error) { + var cur int + switch field { + case "generation": + cur, err = getResourceGeneration(oc, resource, name, ns) + case "resourceVersion": + cur, err = getResourceVersion(oc, resource, name, ns) + } + if err != nil { + return false, err + } + if cur != prev { + return true, nil + } + return false, nil + }) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("%s/%s generation did not update", resource, name)) +} + +func (r Resource) WaitForResourceToAppear(oc *exutil.CLI) error { + err := wait.PollUntilContextTimeout(context.Background(), 3*time.Second, 180*time.Second, true, func(context.Context) (done bool, err error) { + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("-n", r.Namespace, r.Kind, r.Name).Output() + if err != nil { + msg := fmt.Sprintf("%v", output) + if strings.Contains(msg, "NotFound") { + return false, nil + } + return false, err + } + e2e.Logf("Find %s %s", r.Kind, r.Name) + return true, nil + }) + return err +} + +func (r Resource) WaitUntilResourceIsGone(oc *exutil.CLI) error { + err := wait.PollUntilContextTimeout(context.Background(), 3*time.Second, 180*time.Second, true, func(context.Context) (done bool, err error) { + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("-n", r.Namespace, r.Kind, r.Name).Output() + if err != nil { + errstring := fmt.Sprintf("%v", output) + if strings.Contains(errstring, "NotFound") || strings.Contains(errstring, "the server doesn't have a resource type") { + return true, nil + } + return true, err + } + return false, nil + }) + if err != nil { + return fmt.Errorf("can't remove %s/%s in %s project", r.Kind, r.Name, r.Namespace) + } + return nil +} + +func WaitForPodsReadyWithLabel(oc *exutil.CLI, ns, label string) { + err := wait.PollUntilContextTimeout(context.Background(), 10*time.Second, 360*time.Second, false, func(context.Context) (done bool, err error) { + pods, err := oc.AdminKubeClient().CoreV1().Pods(ns).List(context.Background(), metav1.ListOptions{LabelSelector: label}) + if err != nil { + return false, err + } + if len(pods.Items) == 0 { + e2e.Logf("Waiting for pod with label %s to appear\n", label) + return false, nil + } + ready := true + for _, pod := range pods.Items { + for _, containerStatus := range pod.Status.ContainerStatuses { + if !containerStatus.Ready { + ready = false + break + } + } + } + if !ready { + e2e.Logf("Waiting for pod with label %s to be ready...\n", label) + } + return ready, nil + }) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("The pod with label %s is not availabile", label)) +} + +// waitForConfigMapDataInjection waits for a configmap to have its data field populated +// This is useful for waiting on service-ca configmap injection or other dynamic configmap updates +func waitForConfigMapDataInjection(oc *exutil.CLI, namespace, configMapName, dataKey string) { + err := wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 180*time.Second, false, func(context.Context) (bool, error) { + // Check if .data field is populated (returns {} when empty, populated JSON when injected) + dataValue, getErr := oc.AsAdmin().WithoutNamespace().Run("get").Args("configmap", configMapName, "-n", namespace, "-o=jsonpath={.data}").Output() + if getErr != nil { + e2e.Logf("ConfigMap %s/%s not found yet, will retry: %v", namespace, configMapName, getErr) + return false, nil + } + // Check if data is populated (more than just empty braces "{}") + if len(dataValue) > 2 { + e2e.Logf("ConfigMap %s/%s has been populated with data", namespace, configMapName) + return true, nil + } + e2e.Logf("ConfigMap %s/%s exists but data not populated yet, will retry", namespace, configMapName) + return false, nil + }) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("ConfigMap %s/%s data was not populated within timeout", namespace, configMapName)) +} + +// WaitForDeploymentPodsToBeReady waits for the specific deployment to be ready +func waitForDeploymentPodsToBeReady(oc *exutil.CLI, namespace, name string) error { + err := wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 180*time.Second, false, func(context.Context) (done bool, err error) { + deployment, err := oc.AdminKubeClient().AppsV1().Deployments(namespace).Get(context.Background(), name, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + e2e.Logf("Waiting for availability of deployment/%s\n", name) + return false, nil + } + return false, err + } + if deployment.Status.AvailableReplicas == *deployment.Spec.Replicas && deployment.Status.UpdatedReplicas == *deployment.Spec.Replicas { + e2e.Logf("Deployment %s available (%d/%d)\n", name, deployment.Status.AvailableReplicas, *deployment.Spec.Replicas) + return true, nil + } + e2e.Logf("Waiting for full availability of %s deployment (%d/%d)\n", name, deployment.Status.AvailableReplicas, *deployment.Spec.Replicas) + return false, nil + }) + return err +} + +func waitForStatefulsetReady(oc *exutil.CLI, namespace, name string) error { + err := wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 180*time.Second, false, func(context.Context) (done bool, err error) { + ss, err := oc.AdminKubeClient().AppsV1().StatefulSets(namespace).Get(context.Background(), name, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + e2e.Logf("Waiting for availability of %s statefulset\n", name) + return false, nil + } + return false, err + } + if ss.Status.ReadyReplicas == *ss.Spec.Replicas && ss.Status.UpdatedReplicas == *ss.Spec.Replicas { + e2e.Logf("statefulset %s available (%d/%d)\n", name, ss.Status.ReadyReplicas, *ss.Spec.Replicas) + return true, nil + } + e2e.Logf("Waiting for full availability of %s statefulset (%d/%d)\n", name, ss.Status.ReadyReplicas, *ss.Spec.Replicas) + return false, nil + }) + return err +} + +// wait until DaemonSet is Ready +func waitUntilDaemonSetReady(oc *exutil.CLI, daemonset, namespace string) { + err := wait.PollUntilContextTimeout(context.Background(), 10*time.Second, 600*time.Second, false, func(context.Context) (done bool, err error) { + desiredNumber, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("daemonset", daemonset, "-n", namespace, "-o", "jsonpath='{.status.desiredNumberScheduled}'").Output() + + if err != nil { + // loop until daemonset is found or until timeout + if strings.Contains(err.Error(), "not found") { + return false, nil + } + return false, err + } + numberReady, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("daemonset", daemonset, "-n", namespace, "-o", "jsonpath='{.status.numberReady}'").Output() + if err != nil { + return false, err + } + numberReadyi, err := strconv.Atoi(strings.Trim(numberReady, "'")) + if err != nil { + return false, err + } + + desiredNumberi, err := strconv.Atoi(strings.Trim(desiredNumber, "'")) + if err != nil { + return false, err + } + if numberReadyi != desiredNumberi { + return false, nil + } + updatedNumber, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("daemonset", daemonset, "-n", namespace, "-o", "jsonpath='{.status.updatedNumberScheduled}'").Output() + if err != nil { + return false, err + } + updatedNumberi, err := strconv.Atoi(strings.Trim(updatedNumber, "'")) + if err != nil { + return false, err + } + if updatedNumberi != desiredNumberi { + return false, nil + } + + return true, nil + }) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("Daemonset %s did not become Ready", daemonset)) +} + +// WaitForDeploymentPodsToBeReady waits for the specific deployment to be ready +func WaitForDeploymentPodsToBeReady(oc *exutil.CLI, namespace string, name string) { + var selectors map[string]string + err := wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 180*time.Second, true, func(context.Context) (done bool, err error) { + deployment, err := oc.AdminKubeClient().AppsV1().Deployments(namespace).Get(context.Background(), name, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + e2e.Logf("Waiting for deployment/%s to appear\n", name) + return false, nil + } + return false, err + } + selectors = deployment.Spec.Selector.MatchLabels + if deployment.Status.AvailableReplicas == *deployment.Spec.Replicas && deployment.Status.UpdatedReplicas == *deployment.Spec.Replicas { + e2e.Logf("Deployment %s available (%d/%d)\n", name, deployment.Status.AvailableReplicas, *deployment.Spec.Replicas) + return true, nil + } + e2e.Logf("Waiting for full availability of %s deployment (%d/%d)\n", name, deployment.Status.AvailableReplicas, *deployment.Spec.Replicas) + return false, nil + }) + if err != nil && len(selectors) > 0 { + var labels []string + for k, v := range selectors { + labels = append(labels, k+"="+v) + } + label := strings.Join(labels, ",") + _ = oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", "-n", namespace, "-l", label).Execute() + podStatus, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", "-n", namespace, "-l", label, "-ojsonpath={.items[*].status.conditions}").Output() + containerStatus, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", "-n", namespace, "-l", label, "-ojsonpath={.items[*].status.containerStatuses}").Output() + e2e.Failf("deployment %s is not ready:\nconditions: %s\ncontainer status: %s", name, podStatus, containerStatus) + } + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("deployment %s is not available", name)) +} + +// wait until Deployment is Ready +func waitUntilDeploymentReady(oc *exutil.CLI, deployment, ns string) { + err := wait.PollUntilContextTimeout(context.Background(), 10*time.Second, 600*time.Second, false, func(context.Context) (done bool, err error) { + status, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("deployment", deployment, "-n", ns, "-o", "jsonpath='{.status.conditions[0].type}'").Output() + + if err != nil { + // loop until deployment is found or until timeout + if strings.Contains(err.Error(), "not found") { + return false, nil + } + return false, err + } + + if strings.Trim(status, "'") != "Available" { + return false, nil + } + return true, nil + }) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("Deployment %s did not become Available", deployment)) +} + +// verifyDeploymentReplicas waits for and verifies the deployment replica count +// Returns true if verification passes within timeout, false otherwise +// For exact match: verifyDeploymentReplicas(oc, "deploy", "ns", 3, "") +// For numeric comparison: verifyDeploymentReplicas(oc, "deploy", "ns", 5, ">") +// Supported operators: ">", "<", ">=", "<=", "==", "~" +// Calling code should check the result and provide custom message to o.Expect() +func verifyDeploymentReplicas(oc *exutil.CLI, deployment, namespace string, expectedValue int, operator string) bool { + err := wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 180*time.Second, false, func(context.Context) (done bool, err error) { + dep, err := oc.AdminKubeClient().AppsV1().Deployments(namespace).Get(context.Background(), deployment, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + e2e.Logf("Waiting for deployment/%s to be created\n", deployment) + return false, nil + } + e2e.Logf("Error getting deployment %s in namespace %s: %v", deployment, namespace, err) + return false, nil + } + + currentReplicas := int(dep.Status.Replicas) + availableReplicas := int(dep.Status.AvailableReplicas) + updatedReplicas := int(dep.Status.UpdatedReplicas) + + // Check if replicas match the condition + var conditionMet bool + if operator == "" || operator == "==" { + // Exact match - check all three replica counts + conditionMet = (currentReplicas == expectedValue && availableReplicas == expectedValue && updatedReplicas == expectedValue) + } else { + // Numeric comparison - use available replicas + switch operator { + case ">": + conditionMet = availableReplicas > expectedValue && availableReplicas == currentReplicas + case "<": + conditionMet = availableReplicas < expectedValue && availableReplicas == currentReplicas + case ">=": + conditionMet = availableReplicas >= expectedValue && availableReplicas == currentReplicas + case "<=": + conditionMet = availableReplicas <= expectedValue && availableReplicas == currentReplicas + case "~": + conditionMet = currentReplicas == expectedValue && availableReplicas == currentReplicas + default: + e2e.Logf("Unknown operator: %s", operator) + return false, nil + } + } + + if conditionMet { + e2e.Logf("Deployment %s replica condition met - spec=%d, available=%d, updated=%d (expected: %s %d)\n", + deployment, currentReplicas, availableReplicas, updatedReplicas, operator, expectedValue) + return true, nil + } + + e2e.Logf("Waiting for deployment %s replica condition (expected: %s %d) - current: spec=%d, available=%d, updated=%d\n", + deployment, operator, expectedValue, currentReplicas, availableReplicas, updatedReplicas) + return false, nil + }) + + return err == nil +} + +// get pod logs absolute path +func getPodLogs(oc *exutil.CLI, namespace, podname string) (string, error) { + cargs := []string{"-n", namespace, podname} + var podLogs string + var err error + + // add polling as logs could be rotated + err = wait.PollUntilContextTimeout(context.Background(), 10*time.Second, 600*time.Second, false, func(_ context.Context) (bool, error) { + podLogs, err = oc.AsAdmin().WithoutNamespace().Run("logs").Args(cargs...).OutputToFile("podLogs.txt") + + if err != nil { + e2e.Logf("unable to get the pod (%s) logs", podname) + return false, err + } + podLogsf, err := os.Stat(podLogs) + + if err != nil { + return false, err + } + return podLogsf.Size() > 0, nil + }) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("%s pod logs were not collected", podname)) + + e2e.Logf("pod logs file is %s", podLogs) + return filepath.Abs(podLogs) +} + +// wait until NetworkAttachDefinition is Ready +func checkNAD(oc *exutil.CLI, nad, ns string) { + err := wait.PollUntilContextTimeout(context.Background(), 10*time.Second, 600*time.Second, false, func(context.Context) (done bool, err error) { + nadOutput, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("net-attach-def", nad, "-n", ns).Output() + if err != nil { + // loop until NAD is found or until timeout + if strings.Contains(err.Error(), "not found") { + return false, nil + } + return false, err + } + if !strings.Contains(nadOutput, nad) { + return false, nil + } + return true, nil + }) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("Network Attach Definition %s did not become Available", nad)) +} + +// wait until catalogSource is Ready +func (r Resource) WaitUntilCatSrcReady(oc *exutil.CLI) { + err := wait.PollUntilContextTimeout(context.Background(), 10*time.Second, 600*time.Second, false, func(context.Context) (done bool, err error) { + state, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("catalogsource", r.Name, "-n", r.Namespace, "-o", "jsonpath='{.status.connectionState.lastObservedState}'").Output() + if err != nil { + // loop until catalogSource is found or until timeout + if strings.Contains(err.Error(), "not found") { + return false, nil + } + return false, err + } + + if strings.Trim(state, "'") != "READY" { + return false, nil + } + return true, nil + }) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("Catalog Source %s did not become Ready", r.Name)) +} + +// check resource is fully deleted +func checkResourceDeleted(oc *exutil.CLI, resourceType, resourceName, namespace string) { + resourceCheck := wait.PollUntilContextTimeout(context.Background(), 30*time.Second, 600*time.Second, false, func(context.Context) (bool, error) { + output, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args(resourceType, resourceName, "-n", namespace).Output() + if !strings.Contains(output, "NotFound") { + return false, nil + } + return true, nil + }) + compat_otp.AssertWaitPollNoErr(resourceCheck, fmt.Sprintf("found %s \"%s\" exist or not fully deleted", resourceType, resourceName)) +} + +// delete a resource +func deleteResource(oc *exutil.CLI, resourceType, resourceName, namespace string, optionalParameters ...string) { + cmdArgs := []string{resourceType, resourceName, "-n", namespace} + cmdArgs = append(cmdArgs, optionalParameters...) + err := oc.AsAdmin().WithoutNamespace().Run("delete").Args(cmdArgs...).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + checkResourceDeleted(oc, resourceType, resourceName, namespace) +} + +// get kubeadmin token of the cluster +func getKubeAdminToken(oc *exutil.CLI, kubeAdminPasswd, serverURL, currentContext string) string { + longinErr := oc.WithoutNamespace().Run("login").Args("-u", "kubeadmin", "-p", kubeAdminPasswd, serverURL).NotShowInfo().Execute() + o.Expect(longinErr).NotTo(o.HaveOccurred()) + kubeadminToken, kubeadminTokenErr := oc.WithoutNamespace().Run("whoami").Args("-t").Output() + o.Expect(kubeadminTokenErr).NotTo(o.HaveOccurred()) + + rollbackCtxErr := oc.WithoutNamespace().Run("config").Args("set", "current-context", currentContext).Execute() + o.Expect(rollbackCtxErr).NotTo(o.HaveOccurred()) + return kubeadminToken +} + +// get nginx pod name, IP and client IP +func getClientServerInfo(oc *exutil.CLI, serverNS, clientNS, ipStackType string) (map[string]map[string]string, error) { + nginxPodName, err := compat_otp.GetAllPodsWithLabel(oc, serverNS, "app=nginx") + nginxPodIP, _ := getPodIP(oc, serverNS, nginxPodName[0], ipStackType) + + clientPodIP, _ := getPodIP(oc, clientNS, "client", ipStackType) + + serviceIP := getServiceIPv4(oc, serverNS, "nginx-service") + + clientServerMap := map[string]map[string]string{ + "client": { + "ip": clientPodIP, + "name": "client", + }, + "server": { + "ip": nginxPodIP, + "name": nginxPodName[0], + }, + "service": { + "ip": serviceIP, + "name": "nginx-service", + }, + } + return clientServerMap, err +} + +func doAction(oc *exutil.CLI, action string, asAdmin bool, withoutNamespace bool, parameters ...string) (string, error) { + if asAdmin && withoutNamespace { + return oc.AsAdmin().WithoutNamespace().Run(action).Args(parameters...).Output() + } + if asAdmin && !withoutNamespace { + return oc.AsAdmin().Run(action).Args(parameters...).Output() + } + if !asAdmin && withoutNamespace { + return oc.WithoutNamespace().Run(action).Args(parameters...).Output() + } + if !asAdmin && !withoutNamespace { + return oc.Run(action).Args(parameters...).Output() + } + return "", nil +} + +func removeResource(oc *exutil.CLI, asAdmin bool, withoutNamespace bool, parameters ...string) { + output, err := doAction(oc, "delete", asAdmin, withoutNamespace, parameters...) + if err != nil && (strings.Contains(output, "NotFound") || strings.Contains(output, "No resources found")) { + e2e.Logf("the resource is deleted already") + return + } + o.Expect(err).NotTo(o.HaveOccurred()) + + err = wait.PollUntilContextTimeout(context.Background(), 3*time.Second, 120*time.Second, false, func(_ context.Context) (bool, error) { + output, err := doAction(oc, "get", asAdmin, withoutNamespace, parameters...) + if err != nil && (strings.Contains(output, "NotFound") || strings.Contains(output, "No resources found")) { + e2e.Logf("the resource is delete successfully") + return true, nil + } + return false, nil + }) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("fail to delete resource %v", parameters)) +} + +func execCommandInSpecificPod(oc *exutil.CLI, namespace string, podName string, command string) (string, error) { + e2e.Logf("The command is: %v", command) + command1 := []string{"-n", namespace, podName, "--", "bash", "-c", command} + msg, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args(command1...).Output() + if err != nil { + e2e.Logf("Execute command failed with err:%v and output is %v.", err, msg) + return msg, err + } + o.Expect(err).NotTo(o.HaveOccurred()) + return msg, nil +} + +func checkNetworkType(oc *exutil.CLI) string { + output, _ := oc.WithoutNamespace().AsAdmin().Run("get").Args("network.operator", "cluster", "-o=jsonpath={.spec.defaultNetwork.type}").Output() + return strings.ToLower(output) +} + +func checkPlatform(oc *exutil.CLI) string { + output, _ := oc.WithoutNamespace().AsAdmin().Run("get").Args("infrastructure", "cluster", "-o=jsonpath={.status.platformStatus.type}").Output() + return strings.ToLower(output) +} + +func isPlatformSuitableForNMState(oc *exutil.CLI) bool { + platform := checkPlatform(oc) + if !strings.Contains(platform, "baremetal") && !strings.Contains(platform, "none") && !strings.Contains(platform, "vsphere") && !strings.Contains(platform, "openstack") { + e2e.Logf("Skipping for unsupported platform, not baremetal/vsphere/openstack!") + return false + } + return true +} + +// Check if BaselineCapabilities have been set +func isBaselineCapsSet(oc *exutil.CLI) bool { + baselineCapabilitySet, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("clusterversion", "version", "-o=jsonpath={.spec.capabilities.baselineCapabilitySet}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("baselineCapabilitySet parameters: %v\n", baselineCapabilitySet) + return len(baselineCapabilitySet) != 0 +} + +// Check if component is listed in clusterversion.status.capabilities.enabledCapabilities +func isEnabledCapability(oc *exutil.CLI, component string) bool { + enabledCapabilities, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("clusterversion", "-o=jsonpath={.items[*].status.capabilities.enabledCapabilities}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("Cluster enabled capability parameters: %v\n", enabledCapabilities) + return strings.Contains(enabledCapabilities, component) +} + +// verifyComponentsDeleted verifies that specified components are NOT present in the output (exact line match) +func verifyComponentsDeleted(componentsOutput string, componentsList []string) { + outputLines := strings.Split(strings.TrimSpace(componentsOutput), "\n") + for _, component := range componentsList { + componentFound := false + for _, line := range outputLines { + if strings.TrimSpace(line) == component { + componentFound = true + break + } + } + o.Expect(componentFound).Should(o.BeFalse(), fmt.Sprintf("%s should be deleted but was found", component)) + } +} + +// verifyComponentsExist verifies that specified components ARE present in the output (exact line match) +func verifyComponentsExist(componentsOutput string, componentsList []string) { + outputLines := strings.Split(strings.TrimSpace(componentsOutput), "\n") + for _, component := range componentsList { + componentFound := false + for _, line := range outputLines { + if strings.TrimSpace(line) == component { + componentFound = true + break + } + } + o.Expect(componentFound).Should(o.BeTrue(), fmt.Sprintf("%s should exist but was not found", component)) + } +} diff --git a/integration-tests/backend/version_checker.go b/integration-tests/backend/version_checker.go new file mode 100644 index 0000000000..ff93465fbb --- /dev/null +++ b/integration-tests/backend/version_checker.go @@ -0,0 +1,43 @@ +package e2etests + +import ( + "fmt" + + "github.com/onsi/ginkgo/v2" + exutil "github.com/openshift/origin/test/extended/util" + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" + + "golang.org/x/mod/semver" +) + +var clusterVersion string + +func GetOCPVersion(oc *exutil.CLI) (string, error) { + if clusterVersion != "" { + return clusterVersion, nil + } + + var err error + _, clusterVersion, err = compat_otp.GetClusterVersion(oc) + clusterVersion = semver.Canonical("v" + clusterVersion) + clusterVersion = semver.MajorMinor(clusterVersion) + fmt.Printf("Detected OCP version: %s\n", clusterVersion) + return clusterVersion, err +} + +// SkipIfOCPBelow skips test if cluster version is below requirement +// expects "v4.19" format +func SkipIfOCPBelow(requiredVersion string) { + if clusterVersion == "" { + ginkgo.Fail("Cluster version not initialized") + } + + requiredVersion = semver.Canonical(requiredVersion) + if !semver.IsValid(requiredVersion) { + ginkgo.Fail("Requested cluster version is invalid") + } + + if semver.Compare(clusterVersion, requiredVersion) == -1 { + ginkgo.Skip(fmt.Sprintf("Requires at least OCP %s+, cluster is %s", requiredVersion, clusterVersion)) + } +} diff --git a/integration-tests/backend/virtualization.go b/integration-tests/backend/virtualization.go new file mode 100644 index 0000000000..9dd08c617d --- /dev/null +++ b/integration-tests/backend/virtualization.go @@ -0,0 +1,152 @@ +package e2etests + +import ( + "context" + "fmt" + "strings" + "time" + + o "github.com/onsi/gomega" + exutil "github.com/openshift/origin/test/extended/util" + compat_otp "github.com/openshift/origin/test/extended/util/compat_otp" + "k8s.io/apimachinery/pkg/util/wait" + e2e "k8s.io/kubernetes/test/e2e/framework" +) + +type TestVMStaticIPTemplate struct { + Name string + Namespace string + Mac string + StaticIP string + NetworkName string + RunCmd string + Template string +} + +type TestVMUDNTemplate struct { + Name string + Namespace string + NetworkName string + RunCmd string + Template string +} + +// check if cluster has baremetal workers +func hasMetalWorkerNodes(oc *exutil.CLI) bool { + workers, err := compat_otp.GetClusterNodesBy(oc, "worker") + o.Expect(err).NotTo(o.HaveOccurred()) + for _, w := range workers { + Output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("node", w, "-o", "jsonpath='{.metadata.labels.node\\.kubernetes\\.io/instance-type}'").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if !strings.Contains(Output, "metal") { + e2e.Logf("Cluster does not have metal worker nodes") + return false + } + } + return true +} + +func isClusterBareMetal(oc *exutil.CLI) (bool, error) { + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("infrastructure", "cluster", "-o=jsonpath={.status.platformStatus.type}").Output() + if err != nil { + return false, err + } + if !strings.Contains(output, "BareMetal") && !strings.Contains(output, "None") { + return false, nil + } + return true, nil +} + +// wait until hyperconverged is ready +func waitUntilHyperConvergedReady(oc *exutil.CLI, hc, ns string) { + err := wait.PollUntilContextTimeout(context.Background(), 10*time.Second, 600*time.Second, false, func(context.Context) (done bool, err error) { + status, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("hyperconverged", hc, "-n", ns, "-o", "jsonpath='{.status.conditions[0].status}'").Output() + + if err != nil { + // loop until hyperconverged is found or until timeout + if strings.Contains(err.Error(), "not found") { + return false, nil + } + return false, err + } + + if strings.Trim(status, "'") != "True" { + return false, nil + } + return true, nil + }) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("HyperConverged %s did not become Available", hc)) +} + +func (testTemplate *TestVMStaticIPTemplate) createVMStaticIP(oc *exutil.CLI) error { + templateParams := []string{"--ignore-unknown-parameters=true", "-f", testTemplate.Template, "-p", "NAME=" + testTemplate.Name, "-p", "NAMESPACE=" + testTemplate.Namespace, "-p", "NETWORK_NAME=" + testTemplate.NetworkName, "-p", "MAC=" + testTemplate.Mac, "-p", "STATIC_IP=" + testTemplate.StaticIP} + + if testTemplate.RunCmd != "" { + templateParams = append(templateParams, "-p", "RUN_CMD="+testTemplate.RunCmd) + } + configFile := compat_otp.ProcessTemplate(oc, templateParams...) + + err := oc.AsAdmin().WithoutNamespace().Run("create").Args("-f", configFile).Execute() + if err != nil { + return err + } + return nil +} + +func (testTemplate *TestVMUDNTemplate) createVMUDN(oc *exutil.CLI) error { + templateParams := []string{"--ignore-unknown-parameters=true", "-f", testTemplate.Template, "-p", "NAME=" + testTemplate.Name, "-p", "NAMESPACE=" + testTemplate.Namespace, "-p", "NETWORK_NAME=" + testTemplate.NetworkName} + + if testTemplate.RunCmd != "" { + templateParams = append(templateParams, "-p", "RUN_CMD="+testTemplate.RunCmd) + } + configFile := compat_otp.ProcessTemplate(oc, templateParams...) + + err := oc.AsAdmin().WithoutNamespace().Run("create").Args("-f", configFile).Execute() + if err != nil { + return err + } + return nil +} + +// wait until virtual machine is Ready +func waitUntilVMReady(oc *exutil.CLI, vm, ns string) { + err := wait.PollUntilContextTimeout(context.Background(), 30*time.Second, 1200*time.Second, false, func(context.Context) (done bool, err error) { + status, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("virtualmachine", vm, "-n", ns, "-o", "jsonpath='{.status.conditions[0].status}'").Output() + + if err != nil { + // loop until virtual machine is found or until timeout + if strings.Contains(err.Error(), "not found") { + return false, nil + } + return false, err + } + + if strings.Trim(status, "'") != "True" { + return false, nil + } + return true, nil + }) + compat_otp.AssertWaitPollNoErr(err, fmt.Sprintf("Virtual machine %s did not become Available", vm)) +} + +// waitForVMIPAssignment waits until the VM has an IP assigned to the specified interface index +func waitForVMIPAssignment(oc *exutil.CLI, vmName, namespace string, interfaceIndex int) (string, error) { + var vmIP string + err := wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 300*time.Second, false, func(context.Context) (done bool, err error) { + ip, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("vmi", vmName, "-n", namespace, fmt.Sprintf("-ojsonpath={.status.interfaces[%d].ipAddress}", interfaceIndex)).Output() + if err != nil { + // If VMI not found yet, keep polling + if strings.Contains(err.Error(), "not found") { + return false, nil + } + return false, err + } + // Check if IP is actually assigned (not empty) + if ip != "" { + vmIP = ip + return true, nil + } + return false, nil + }) + return vmIP, err +}