diff --git a/Makefile b/Makefile index a22f9a03a..a736fd15d 100644 --- a/Makefile +++ b/Makefile @@ -258,7 +258,23 @@ E2E_TIMEOUT ?= 20m GODOG_ARGS ?= .PHONY: e2e e2e: #EXHELP Run the e2e tests. - go test -count=1 -v ./test/e2e/features_test.go -timeout=$(E2E_TIMEOUT) $(if $(GODOG_ARGS),-args $(GODOG_ARGS)) +ifeq ($(strip $(GODOG_ARGS)),) + set +e; \ + timeout -s SIGTERM $(E2E_TIMEOUT) bash -c 'go test -count=1 -v ./test/e2e/features_test.go -args --godog.tags="~@Serial" --godog.concurrency=100; \ + parallelExitCode=$$?; \ + go test -count=1 -v ./test/e2e/features_test.go -args --godog.tags="@Serial" --godog.concurrency=1; \ + serialExitCode=$$?'; \ + timeoutExitCode=$$?; \ + if [[ $$timeoutExitCode -ne 0 ]]; then \ + echo "e2e tests failed: tests timed out"; \ + exit 1; \ + elif [[ $$parallelExitCode -ne 0 ]] || [[ $$serialExitCode -ne 0 ]]; then \ + echo "e2e tests failed: parallel=$$parallelExitCode serial=$$serialExitCode"; \ + exit 1; \ + fi +else + go test -count=1 -v ./test/e2e/features_test.go -timeout=$(E2E_TIMEOUT) -args $(GODOG_ARGS) +endif export CLUSTER_REGISTRY_HOST := docker-registry.operator-controller-e2e.svc:5000 .PHONY: extension-developer-e2e diff --git a/test/e2e/README.md b/test/e2e/README.md index 98b99b9ca..817a5c383 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -207,6 +207,10 @@ Use these variables in YAML templates: ### 5. Feature Tags +Tags can be used for different purposes in the test suite: + +#### Feature Gate Tags + Use tags to conditionally run scenarios based on feature gates: ```gherkin @@ -216,6 +220,28 @@ Scenario: Install operator having webhooks Scenarios are skipped if the feature gate is not enabled on the deployed controller. +#### Serial Execution Tag + +By default, scenarios run concurrently (up to 100 parallel scenarios). However, some tests must run serially, typically because they: +- Modify shared cluster resources (e.g., cluster-wide TLS configuration) +- Have resource constraints that prevent parallel execution +- Require exclusive access to a resource + +To mark a test for serial execution, add the `@Serial` tag: + +```gherkin +@Serial +Feature: TLS profile enforcement on metrics endpoints + + Scenario: Test TLS configuration + Given the "catalogd" deployment is configured with custom TLS settings + ... +``` + +The `Makefile` automatically separates scenarios when run without additional `GODOG_ARGS`: +- Scenarios **without** `@Serial` run concurrently in the first test phase +- Scenarios **with** `@Serial` run sequentially in a separate serial test phase + ## Running Tests ### Run All Tests @@ -230,6 +256,15 @@ or make test-experimental-e2e ``` +Custom godog arguments can be modified by setting the following: +```bash +GODOG_ARGS=--godog.tags=@WebhookProviderCertManager make test-experimental-e2e +``` + +Note that when this is done the `make` target will no longer automatically split the test run into parallel and serial runs, and test execution time may increase. If you wish to add concurrency back into the arguments, it is recommended to also disable the `@Serial` tests: +```bash +GODOG_ARGS="--godog.tags=~@Serial --godog.concurrency=100" make test-experimental-e2e +``` ### Run Specific Feature diff --git a/test/e2e/features/ha.feature b/test/e2e/features/ha.feature index 6b889c99b..bb20cfafa 100644 --- a/test/e2e/features/ha.feature +++ b/test/e2e/features/ha.feature @@ -1,3 +1,4 @@ +@Serial Feature: HA failover for catalogd When catalogd is deployed with multiple replicas, the remaining pods must diff --git a/test/e2e/features/proxy.feature b/test/e2e/features/proxy.feature index c4ddcc091..9f30a47f0 100644 --- a/test/e2e/features/proxy.feature +++ b/test/e2e/features/proxy.feature @@ -1,3 +1,4 @@ +@Serial Feature: HTTPS proxy support for outbound catalog requests OLM's operator-controller fetches catalog data from catalogd over HTTPS. diff --git a/test/e2e/features/revision.feature b/test/e2e/features/revision.feature index 866e8195f..0ad542f4d 100644 --- a/test/e2e/features/revision.feature +++ b/test/e2e/features/revision.feature @@ -672,6 +672,7 @@ Feature: Install ClusterObjectSet And resource "deployment/test-deployment" is eventually not found @ProgressDeadline + @Serial Scenario: COS recovers from ProgressDeadlineExceeded to Succeeded when probes pass Given min value for ClusterObjectSet .spec.progressDeadlineMinutes is set to 1 And ServiceAccount "olm-sa" with needed permissions is available in test namespace @@ -721,6 +722,7 @@ Feature: Install ClusterObjectSet containers: - name: delayed-ready image: busybox:1.36 + imagePullPolicy: IfNotPresent command: ["sleep", "1000"] readinessProbe: exec: diff --git a/test/e2e/features/tls.feature b/test/e2e/features/tls.feature index 77293c12f..84c8c5086 100644 --- a/test/e2e/features/tls.feature +++ b/test/e2e/features/tls.feature @@ -1,3 +1,4 @@ +@Serial Feature: TLS profile enforcement on metrics endpoints Background: @@ -6,11 +7,15 @@ Feature: TLS profile enforcement on metrics endpoints # Each scenario patches the deployment with the TLS settings under test and # restores the original configuration during cleanup, so scenarios are independent. + # This feature file is run serially to avoid potential issues modifying the catalogd + # deployment during functional testing. + # All three scenarios test catalogd only: the enforcement logic lives in the shared # tlsprofiles package, so one component is sufficient. TLS 1.2 is used for cipher # and curve enforcement because Go's crypto/tls does not allow the server to restrict # TLS 1.3 cipher suites — CipherSuites config only applies to TLS 1.2. The e2e cert # uses ECDSA, so ECDHE_ECDSA cipher families are required. + @TLSProfile Scenario: catalogd metrics endpoint enforces configured minimum TLS version Given the "catalogd" deployment is configured with custom TLS minimum version "TLSv1.3"