From f1f2ceac1a4bc1bf73f4ccb521a7a5a206a8096b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20=22decko=22=20de=20Brito?= Date: Fri, 8 May 2026 15:56:01 -0300 Subject: [PATCH] feat: parallelize deploy-and-test pipeline for faster CI The integration test pipeline ran ~1 hour with all tasks in a strict sequential chain. This change splits the monolithic functional test task into 3 parallel groups and decouples the API schema upload, reducing wall-clock time by an estimated 15-25 minutes. Changes: - Split pulp-functional-tests into 3 parallel Tekton tasks: - functional-tests-rpm (RPM parallel + serial) - functional-tests-core-service (pulpcore + pulp_service) - functional-tests-npm-maven (Maven + NPM) - Decouple push-api-json-files-to-pulp from test results (real dependency is install-bindings, not test output) - Move test dependency installation into install-bindings task to avoid duplicating setup across parallel tasks - Fix JUnit XML overwriting (unique filenames per test group) - Use unique pytest cache_dir per task to prevent corruption - Reduce wait-for-api-schema retry interval from 15s to 5s Assisted-by: Claude Code Co-Authored-By: Claude Opus 4.6 (1M context) --- .tekton/pulp-deploy-and-test.yaml | 245 ++++++++++++++++++++++++------ 1 file changed, 199 insertions(+), 46 deletions(-) diff --git a/.tekton/pulp-deploy-and-test.yaml b/.tekton/pulp-deploy-and-test.yaml index b3e5dc5c..42ce8375 100644 --- a/.tekton/pulp-deploy-and-test.yaml +++ b/.tekton/pulp-deploy-and-test.yaml @@ -409,7 +409,7 @@ spec: command: ["/bin/sh", "-c"] args: - > - curl --retry-all-errors --fail --retry-delay 15 --retry 30 --retry-max-time 900 https://${PULP_API_HOST}/api/pulp/api/v3/docs/api.json + curl --retry-all-errors --fail --retry-delay 5 --retry 90 --retry-max-time 900 https://${PULP_API_HOST}/api/pulp/api/v3/docs/api.json - name: get-pulp-api-schema args: - -k @@ -697,7 +697,52 @@ spec: cat __tmp_init__.py | cmd_stdin_prefix bash -c "cat > /tmp/home/.local/lib/python3.11/site-packages/pulpcore/__init__.py" cat __tmp_init__.py | cmd_stdin_prefix bash -c "cat > /tmp/home/.local/lib/python3.11/site-packages/pulpcore/client/__init__.py" - - name: pulp-functional-tests + - name: install-test-deps + image: "$(params.BONFIRE_IMAGE)" + env: + - name: OC_LOGIN_TOKEN + valueFrom: + secretKeyRef: + name: $(params.EPHEMERAL_ENV_PROVIDER_SECRET) + key: token + - name: OC_LOGIN_SERVER + valueFrom: + secretKeyRef: + name: $(params.EPHEMERAL_ENV_PROVIDER_SECRET) + key: url + - name: NS + value: $(params.NS) + script: | + set -ex + + login.sh + + if [ -n "$NS" ]; then + oc_wrapper project $NS + else + export NS=$(oc_wrapper project | grep -oE 'ephemeral-......') + fi + echo "Namespace is $NS" + + cmd_prefix() { + oc_wrapper exec -c pulp-api deployment/pulp-api -- "$@" + } + + cmd_stdin_prefix() { + oc_wrapper exec -c pulp-api -i deployment/pulp-api -- "$@" + } + + cmd_prefix bash -c "HOME=/tmp/home pip3 install pytest\<8 gnupg" + + curl -o functest_requirements.txt https://raw.githubusercontent.com/pulp/pulp_rpm/main/functest_requirements.txt + curl -o unittest_requirements.txt https://raw.githubusercontent.com/pulp/pulp_rpm/main/unittest_requirements.txt + + cat unittest_requirements.txt | cmd_stdin_prefix bash -c "cat > /tmp/unittest_requirements.txt" + cat functest_requirements.txt | cmd_stdin_prefix bash -c "cat > /tmp/functest_requirements.txt" + cmd_prefix bash -c "HOME=/tmp/home pip3 install -r /tmp/unittest_requirements.txt -r /tmp/functest_requirements.txt" + cmd_prefix bash -c "ln -s /usr/local/lib/pulp/bin/pulpcore-manager /tmp/home/.local/bin/pulpcore-manager || /bin/true" + + - name: functional-tests-rpm when: - input: '$(tasks.test-metadata.results.test-event-type)' operator: in @@ -724,7 +769,7 @@ spec: type: string description: Namespace name to deploy the application to steps: - - name: functional-tests + - name: run-rpm-tests image: "$(params.BONFIRE_IMAGE)" env: - name: OC_LOGIN_TOKEN @@ -753,26 +798,88 @@ spec: PASSWORD=$(oc_wrapper extract secret/pulp-admin-password --to=-) - POD=$(oc_wrapper get pod | grep -oE "pulp-api\S*") - echo $POD - oc_wrapper get pod $POD -o yaml | grep memory: + cmd_prefix() { + oc_wrapper exec -c pulp-api deployment/pulp-api -- "$@" + } - oc_wrapper get clowdenvironment env-$(oc_wrapper project | grep -oE 'ephemeral-......') -o yaml + debug_and_fail() { + oc_wrapper logs $(oc_wrapper get pod | grep -oE "pulp-content\S*") + oc_wrapper logs $(oc_wrapper get pod | grep -oE "pulp-api\S*") + for pod in $(oc_wrapper get pod | grep -oE "pulp-worker\S*"); do + oc_wrapper logs "$pod" + done + echo "CURL OUTPUT" + curl https://env-${NS}.apps.crc-eph.r9lp.p1.openshiftapps.com/api/pulp-content/default/ + echo "ROUTES" + oc_wrapper get route + exit 1 + } + + set +e + cmd_prefix bash -c "HOME=/tmp/home PYTHONPATH=/tmp/home/.local/lib/python3.11/site-packages/ XDG_CONFIG_HOME=/tmp/home/.config API_PROTOCOL=http API_HOST=pulp-api API_PORT=8000 ADMIN_USERNAME=admin ADMIN_PASSWORD=$PASSWORD /tmp/home/.local/bin/pytest -o cache_dir=/tmp/home/.cache/pytest_cache_rpm -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_rpm.tests.functional -m parallel -n 8 -k 'test_download_content' --junitxml=/tmp/home/junit-rpm-parallel.xml" || debug_and_fail - oc_wrapper get clowdapp pulp -o yaml + cmd_prefix bash -c "HOME=/tmp/home PYTHONPATH=/tmp/home/.local/lib/python3.11/site-packages/ XDG_CONFIG_HOME=/tmp/home/.config API_PROTOCOL=http API_HOST=pulp-api API_PORT=8000 ADMIN_USERNAME=admin ADMIN_PASSWORD=$PASSWORD /tmp/home/.local/bin/pytest -o cache_dir=/tmp/home/.cache/pytest_cache_rpm -v -r sx --color=yes --pyargs pulp_rpm.tests.functional -m 'not parallel' -k 'test_download_policies' --junitxml=/tmp/home/junit-rpm-serial.xml" || debug_and_fail + + - name: functional-tests-core-service + when: + - input: '$(tasks.test-metadata.results.test-event-type)' + operator: in + values: ["pull_request"] + runAfter: + - install-bindings + params: + - name: BONFIRE_IMAGE + value: "$(params.BONFIRE_IMAGE)" + - name: NS + value: "$(tasks.reserve-namespace.results.NS)" + - name: EPHEMERAL_ENV_PROVIDER_SECRET + value: "$(params.EPHEMERAL_ENV_PROVIDER_SECRET)" + taskSpec: + params: + - name: BONFIRE_IMAGE + type: string + description: The container Bonfire image to use for the tekton tasks + default: quay.io/redhat-user-workloads/hcc-devprod-tenant/hcc-cicd-tools/cicd-tools:834176766e3f911ffa24bfacff59dd15126e4b3a + - name: EPHEMERAL_ENV_PROVIDER_SECRET + type: string + default: ephemeral-env-provider + - name: NS + type: string + description: Namespace name to deploy the application to + steps: + - name: run-core-service-tests + image: "$(params.BONFIRE_IMAGE)" + env: + - name: OC_LOGIN_TOKEN + valueFrom: + secretKeyRef: + name: $(params.EPHEMERAL_ENV_PROVIDER_SECRET) + key: token + - name: OC_LOGIN_SERVER + valueFrom: + secretKeyRef: + name: $(params.EPHEMERAL_ENV_PROVIDER_SECRET) + key: url + - name: NS + value: $(params.NS) + script: | + set -ex + + login.sh + + if [ -n "$NS" ]; then + oc_wrapper project $NS + else + export NS=$(oc_wrapper project | grep -oE 'ephemeral-......') + fi + echo "Namespace is $NS" + + PASSWORD=$(oc_wrapper extract secret/pulp-admin-password --to=-) - ### Adapted from ./.github/workflows/scripts/utils.sh - # Run a command cmd_prefix() { oc_wrapper exec -c pulp-api deployment/pulp-api -- "$@" } - # Run a command, and pass STDIN - cmd_stdin_prefix() { - oc_wrapper exec -c pulp-api -i deployment/pulp-api -- "$@" - } - ### END Adapted from ./.github/workflows/scripts/utils.sh - debug_and_fail() { oc_wrapper logs $(oc_wrapper get pod | grep -oE "pulp-content\S*") oc_wrapper logs $(oc_wrapper get pod | grep -oE "pulp-api\S*") @@ -786,42 +893,88 @@ spec: exit 1 } - cmd_prefix bash -c "HOME=/tmp/home pip3 install pytest\<8 gnupg" + set +e + cmd_prefix bash -c "HOME=/tmp/home PYTHONPATH=/tmp/home/.local/lib/python3.11/site-packages/ XDG_CONFIG_HOME=/tmp/home/.config API_PROTOCOL=http API_HOST=pulp-api API_PORT=8000 ADMIN_USERNAME=admin ADMIN_PASSWORD=$PASSWORD /tmp/home/.local/bin/pytest -o cache_dir=/tmp/home/.cache/pytest_cache_core -v -r sx --color=yes --pyargs pulpcore.tests.functional -m 'parallel' -n 8 -k 'test_jq_header_remote_auth' --junitxml=/tmp/home/junit-pulpcore-parallel.xml" || debug_and_fail - curl -o functest_requirements.txt https://raw.githubusercontent.com/pulp/pulp_rpm/main/functest_requirements.txt - curl -o unittest_requirements.txt https://raw.githubusercontent.com/pulp/pulp_rpm/main/unittest_requirements.txt + cmd_prefix bash -c "HOME=/tmp/home PYTHONPATH=/tmp/home/.local/lib/python3.11/site-packages/ XDG_CONFIG_HOME=/tmp/home/.config API_PROTOCOL=http API_HOST=pulp-api API_PORT=8000 ADMIN_USERNAME=admin ADMIN_PASSWORD=$PASSWORD /tmp/home/.local/bin/pytest -o cache_dir=/tmp/home/.cache/pytest_cache_core -v -r sx --color=yes --pyargs pulp_service.tests.functional -m 'not parallel' --junitxml=/tmp/home/junit-service.xml" || debug_and_fail - ### Adapted from ./.github/workflows/scripts/script.sh - cat unittest_requirements.txt | cmd_stdin_prefix bash -c "cat > /tmp/unittest_requirements.txt" - cat functest_requirements.txt | cmd_stdin_prefix bash -c "cat > /tmp/functest_requirements.txt" - cmd_prefix bash -c "HOME=/tmp/home pip3 install -r /tmp/unittest_requirements.txt -r /tmp/functest_requirements.txt" - # Because we pass the path to pytest -o cache_dir=/tmp/home/.cache/pytest_cache, pulpcore-manager must be in the same dir - cmd_prefix bash -c "ln -s /usr/local/lib/pulp/bin/pulpcore-manager /tmp/home/.local/bin/pulpcore-manager || /bin/true" - echo "CURL OUTPUT" - curl https://env-${NS}.apps.crc-eph.r9lp.p1.openshiftapps.com/api/pulp-content/default/ - echo "ROUTES" - oc_wrapper get route - set +e - # Only testing test_download_content because it is a very thorough test that tests that all the components of pulp can work - cmd_prefix bash -c "HOME=/tmp/home PYTHONPATH=/tmp/home/.local/lib/python3.11/site-packages/ XDG_CONFIG_HOME=/tmp/home/.config API_PROTOCOL=http API_HOST=pulp-api API_PORT=8000 ADMIN_USERNAME=admin ADMIN_PASSWORD=$PASSWORD /tmp/home/.local/bin/pytest -o cache_dir=/tmp/home/.cache/pytest_cache -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_rpm.tests.functional -m parallel -n 8 -k 'test_download_content' --junitxml=/tmp/home/junit-pulp-parallel.xml" || debug_and_fail - # Never test test_package_manager_consume because they require sudo - # Do not test test_domain_create because it requires more than 2GB of RAM - # Only testing test_download_policies because they are very thorough tests that test that all the components of pulp can work - cmd_prefix bash -c "HOME=/tmp/home PYTHONPATH=/tmp/home/.local/lib/python3.11/site-packages/ XDG_CONFIG_HOME=/tmp/home/.config API_PROTOCOL=http API_HOST=pulp-api API_PORT=8000 ADMIN_USERNAME=admin ADMIN_PASSWORD=$PASSWORD /tmp/home/.local/bin/pytest -o cache_dir=/tmp/home/.cache/pytest_cache -v -r sx --color=yes --pyargs pulp_rpm.tests.functional -m 'not parallel' -k 'test_download_policies' --junitxml=/tmp/home/junit-pulp-serial.xml" || debug_and_fail + - name: functional-tests-npm-maven + when: + - input: '$(tasks.test-metadata.results.test-event-type)' + operator: in + values: ["pull_request"] + runAfter: + - install-bindings + params: + - name: BONFIRE_IMAGE + value: "$(params.BONFIRE_IMAGE)" + - name: NS + value: "$(tasks.reserve-namespace.results.NS)" + - name: EPHEMERAL_ENV_PROVIDER_SECRET + value: "$(params.EPHEMERAL_ENV_PROVIDER_SECRET)" + taskSpec: + params: + - name: BONFIRE_IMAGE + type: string + description: The container Bonfire image to use for the tekton tasks + default: quay.io/redhat-user-workloads/hcc-devprod-tenant/hcc-cicd-tools/cicd-tools:834176766e3f911ffa24bfacff59dd15126e4b3a + - name: EPHEMERAL_ENV_PROVIDER_SECRET + type: string + default: ephemeral-env-provider + - name: NS + type: string + description: Namespace name to deploy the application to + steps: + - name: run-npm-maven-tests + image: "$(params.BONFIRE_IMAGE)" + env: + - name: OC_LOGIN_TOKEN + valueFrom: + secretKeyRef: + name: $(params.EPHEMERAL_ENV_PROVIDER_SECRET) + key: token + - name: OC_LOGIN_SERVER + valueFrom: + secretKeyRef: + name: $(params.EPHEMERAL_ENV_PROVIDER_SECRET) + key: url + - name: NS + value: $(params.NS) + script: | + set -ex - # Run the jq header auth test - cmd_prefix bash -c "HOME=/tmp/home PYTHONPATH=/tmp/home/.local/lib/python3.11/site-packages/ XDG_CONFIG_HOME=/tmp/home/.config API_PROTOCOL=http API_HOST=pulp-api API_PORT=8000 ADMIN_USERNAME=admin ADMIN_PASSWORD=$PASSWORD /tmp/home/.local/bin/pytest -o cache_dir=/tmp/home/.cache/pytest_cache -v -r sx --color=yes --pyargs pulpcore.tests.functional -m 'parallel' -n 8 -k 'test_jq_header_remote_auth' --junitxml=/tmp/home/junit-pulp-serial.xml" || debug_and_fail + login.sh - ### END Adapted from ./.github/workflows/scripts/script.sh + if [ -n "$NS" ]; then + oc_wrapper project $NS + else + export NS=$(oc_wrapper project | grep -oE 'ephemeral-......') + fi + echo "Namespace is $NS" + + PASSWORD=$(oc_wrapper extract secret/pulp-admin-password --to=-) - # Run pulp_maven functional tests - cmd_prefix bash -c "HOME=/tmp/home PYTHONPATH=/tmp/home/.local/lib/python3.11/site-packages/ XDG_CONFIG_HOME=/tmp/home/.config API_PROTOCOL=http API_HOST=pulp-api API_PORT=8000 ADMIN_USERNAME=admin ADMIN_PASSWORD=$PASSWORD /tmp/home/.local/bin/pytest -o cache_dir=/tmp/home/.cache/pytest_cache -v -r sx --color=yes --pyargs pulp_maven.tests.functional.api.test_download_content --junitxml=/tmp/home/junit-pulp-serial.xml" || debug_and_fail + cmd_prefix() { + oc_wrapper exec -c pulp-api deployment/pulp-api -- "$@" + } + + debug_and_fail() { + oc_wrapper logs $(oc_wrapper get pod | grep -oE "pulp-content\S*") + oc_wrapper logs $(oc_wrapper get pod | grep -oE "pulp-api\S*") + for pod in $(oc_wrapper get pod | grep -oE "pulp-worker\S*"); do + oc_wrapper logs "$pod" + done + echo "CURL OUTPUT" + curl https://env-${NS}.apps.crc-eph.r9lp.p1.openshiftapps.com/api/pulp-content/default/ + echo "ROUTES" + oc_wrapper get route + exit 1 + } - # Run pulp_npm functional tests - cmd_prefix bash -c "HOME=/tmp/home PYTHONPATH=/tmp/home/.local/lib/python3.11/site-packages/ XDG_CONFIG_HOME=/tmp/home/.config API_PROTOCOL=http API_HOST=pulp-api API_PORT=8000 ADMIN_USERNAME=admin ADMIN_PASSWORD=$PASSWORD /tmp/home/.local/bin/pytest -o cache_dir=/tmp/home/.cache/pytest_cache -v -r sx --color=yes --pyargs pulp_npm.tests.functional -k 'test_pull_through_install' --junitxml=/tmp/home/junit-pulp-serial.xml" || debug_and_fail + set +e + cmd_prefix bash -c "HOME=/tmp/home PYTHONPATH=/tmp/home/.local/lib/python3.11/site-packages/ XDG_CONFIG_HOME=/tmp/home/.config API_PROTOCOL=http API_HOST=pulp-api API_PORT=8000 ADMIN_USERNAME=admin ADMIN_PASSWORD=$PASSWORD /tmp/home/.local/bin/pytest -o cache_dir=/tmp/home/.cache/pytest_cache_npm -v -r sx --color=yes --pyargs pulp_maven.tests.functional.api.test_download_content --junitxml=/tmp/home/junit-maven.xml" || debug_and_fail - # Run pulp_service functional tests - cmd_prefix bash -c "HOME=/tmp/home PYTHONPATH=/tmp/home/.local/lib/python3.11/site-packages/ XDG_CONFIG_HOME=/tmp/home/.config API_PROTOCOL=http API_HOST=pulp-api API_PORT=8000 ADMIN_USERNAME=admin ADMIN_PASSWORD=$PASSWORD /tmp/home/.local/bin/pytest -o cache_dir=/tmp/home/.cache/pytest_cache -v -r sx --color=yes --pyargs pulp_service.tests.functional -m 'not parallel' --junitxml=/tmp/home/junit-pulp-serial.xml" || debug_and_fail + cmd_prefix bash -c "HOME=/tmp/home PYTHONPATH=/tmp/home/.local/lib/python3.11/site-packages/ XDG_CONFIG_HOME=/tmp/home/.config API_PROTOCOL=http API_HOST=pulp-api API_PORT=8000 ADMIN_USERNAME=admin ADMIN_PASSWORD=$PASSWORD /tmp/home/.local/bin/pytest -o cache_dir=/tmp/home/.cache/pytest_cache_npm -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_npm.tests.functional -k 'test_pull_through_install' --junitxml=/tmp/home/junit-npm.xml" || debug_and_fail - name: push-api-json-files-to-pulp when: @@ -838,7 +991,7 @@ spec: - name: SNAPSHOT value: "$(params.SNAPSHOT)" runAfter: - - pulp-functional-tests + - install-bindings taskSpec: params: - name: BONFIRE_IMAGE