Skip to content

feat: parallelize deploy-and-test pipeline #1191

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
decko wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat: parallelize deploy-and-test pipeline #1191

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
245 changes: 199 additions & 46 deletions .tekton/pulp-deploy-and-test.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -409,7 +409,7 @@ spec:
command: ["/bin/sh", "-c"]
args:
- >
curl --retry-all-errors --fail --retry-delay 15 --retry 30 --retry-max-time 900 https://${PULP_API_HOST}/api/pulp/api/v3/docs/api.json
curl --retry-all-errors --fail --retry-delay 5 --retry 90 --retry-max-time 900 https://${PULP_API_HOST}/api/pulp/api/v3/docs/api.json
- name: get-pulp-api-schema
args:
- -k
Expand Down Expand Up @@ -697,7 +697,52 @@ spec:
cat __tmp_init__.py | cmd_stdin_prefix bash -c "cat > /tmp/home/.local/lib/python3.11/site-packages/pulpcore/__init__.py"
cat __tmp_init__.py | cmd_stdin_prefix bash -c "cat > /tmp/home/.local/lib/python3.11/site-packages/pulpcore/client/__init__.py"

- name: pulp-functional-tests
- name: install-test-deps
image: "$(params.BONFIRE_IMAGE)"
env:
- name: OC_LOGIN_TOKEN
valueFrom:
secretKeyRef:
name: $(params.EPHEMERAL_ENV_PROVIDER_SECRET)
key: token
- name: OC_LOGIN_SERVER
valueFrom:
secretKeyRef:
name: $(params.EPHEMERAL_ENV_PROVIDER_SECRET)
key: url
- name: NS
value: $(params.NS)
script: |
set -ex

login.sh

if [ -n "$NS" ]; then
oc_wrapper project $NS
else
export NS=$(oc_wrapper project | grep -oE 'ephemeral-......')
fi
echo "Namespace is $NS"

cmd_prefix() {
oc_wrapper exec -c pulp-api deployment/pulp-api -- "$@"
}

cmd_stdin_prefix() {
oc_wrapper exec -c pulp-api -i deployment/pulp-api -- "$@"
}

cmd_prefix bash -c "HOME=/tmp/home pip3 install pytest\<8 gnupg"

curl -o functest_requirements.txt https://raw.githubusercontent.com/pulp/pulp_rpm/main/functest_requirements.txt
curl -o unittest_requirements.txt https://raw.githubusercontent.com/pulp/pulp_rpm/main/unittest_requirements.txt

cat unittest_requirements.txt | cmd_stdin_prefix bash -c "cat > /tmp/unittest_requirements.txt"
cat functest_requirements.txt | cmd_stdin_prefix bash -c "cat > /tmp/functest_requirements.txt"
cmd_prefix bash -c "HOME=/tmp/home pip3 install -r /tmp/unittest_requirements.txt -r /tmp/functest_requirements.txt"
cmd_prefix bash -c "ln -s /usr/local/lib/pulp/bin/pulpcore-manager /tmp/home/.local/bin/pulpcore-manager || /bin/true"

- name: functional-tests-rpm
when:
- input: '$(tasks.test-metadata.results.test-event-type)'
operator: in
Expand All @@ -724,7 +769,7 @@ spec:
type: string
description: Namespace name to deploy the application to
steps:
- name: functional-tests
- name: run-rpm-tests
image: "$(params.BONFIRE_IMAGE)"
env:
- name: OC_LOGIN_TOKEN
Expand Down Expand Up @@ -753,26 +798,88 @@ spec:

PASSWORD=$(oc_wrapper extract secret/pulp-admin-password --to=-)

POD=$(oc_wrapper get pod | grep -oE "pulp-api\S*")
echo $POD
oc_wrapper get pod $POD -o yaml | grep memory:
cmd_prefix() {
oc_wrapper exec -c pulp-api deployment/pulp-api -- "$@"
}

oc_wrapper get clowdenvironment env-$(oc_wrapper project | grep -oE 'ephemeral-......') -o yaml
debug_and_fail() {
oc_wrapper logs $(oc_wrapper get pod | grep -oE "pulp-content\S*")
oc_wrapper logs $(oc_wrapper get pod | grep -oE "pulp-api\S*")
for pod in $(oc_wrapper get pod | grep -oE "pulp-worker\S*"); do
oc_wrapper logs "$pod"
done
echo "CURL OUTPUT"
curl https://env-${NS}.apps.crc-eph.r9lp.p1.openshiftapps.com/api/pulp-content/default/
echo "ROUTES"
oc_wrapper get route
exit 1
}

set +e
cmd_prefix bash -c "HOME=/tmp/home PYTHONPATH=/tmp/home/.local/lib/python3.11/site-packages/ XDG_CONFIG_HOME=/tmp/home/.config API_PROTOCOL=http API_HOST=pulp-api API_PORT=8000 ADMIN_USERNAME=admin ADMIN_PASSWORD=$PASSWORD /tmp/home/.local/bin/pytest -o cache_dir=/tmp/home/.cache/pytest_cache_rpm -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_rpm.tests.functional -m parallel -n 8 -k 'test_download_content' --junitxml=/tmp/home/junit-rpm-parallel.xml" || debug_and_fail

oc_wrapper get clowdapp pulp -o yaml
cmd_prefix bash -c "HOME=/tmp/home PYTHONPATH=/tmp/home/.local/lib/python3.11/site-packages/ XDG_CONFIG_HOME=/tmp/home/.config API_PROTOCOL=http API_HOST=pulp-api API_PORT=8000 ADMIN_USERNAME=admin ADMIN_PASSWORD=$PASSWORD /tmp/home/.local/bin/pytest -o cache_dir=/tmp/home/.cache/pytest_cache_rpm -v -r sx --color=yes --pyargs pulp_rpm.tests.functional -m 'not parallel' -k 'test_download_policies' --junitxml=/tmp/home/junit-rpm-serial.xml" || debug_and_fail

- name: functional-tests-core-service
when:
- input: '$(tasks.test-metadata.results.test-event-type)'
operator: in
values: ["pull_request"]
runAfter:
- install-bindings
params:
- name: BONFIRE_IMAGE
value: "$(params.BONFIRE_IMAGE)"
- name: NS
value: "$(tasks.reserve-namespace.results.NS)"
- name: EPHEMERAL_ENV_PROVIDER_SECRET
value: "$(params.EPHEMERAL_ENV_PROVIDER_SECRET)"
taskSpec:
params:
- name: BONFIRE_IMAGE
type: string
description: The container Bonfire image to use for the tekton tasks
default: quay.io/redhat-user-workloads/hcc-devprod-tenant/hcc-cicd-tools/cicd-tools:834176766e3f911ffa24bfacff59dd15126e4b3a
- name: EPHEMERAL_ENV_PROVIDER_SECRET
type: string
default: ephemeral-env-provider
- name: NS
type: string
description: Namespace name to deploy the application to
steps:
- name: run-core-service-tests
image: "$(params.BONFIRE_IMAGE)"
env:
- name: OC_LOGIN_TOKEN
valueFrom:
secretKeyRef:
name: $(params.EPHEMERAL_ENV_PROVIDER_SECRET)
key: token
- name: OC_LOGIN_SERVER
valueFrom:
secretKeyRef:
name: $(params.EPHEMERAL_ENV_PROVIDER_SECRET)
key: url
- name: NS
value: $(params.NS)
script: |
set -ex

login.sh

if [ -n "$NS" ]; then
oc_wrapper project $NS
else
export NS=$(oc_wrapper project | grep -oE 'ephemeral-......')
fi
echo "Namespace is $NS"

PASSWORD=$(oc_wrapper extract secret/pulp-admin-password --to=-)

### Adapted from ./.github/workflows/scripts/utils.sh
# Run a command
cmd_prefix() {
oc_wrapper exec -c pulp-api deployment/pulp-api -- "$@"
}

# Run a command, and pass STDIN
cmd_stdin_prefix() {
oc_wrapper exec -c pulp-api -i deployment/pulp-api -- "$@"
}
### END Adapted from ./.github/workflows/scripts/utils.sh

debug_and_fail() {
oc_wrapper logs $(oc_wrapper get pod | grep -oE "pulp-content\S*")
oc_wrapper logs $(oc_wrapper get pod | grep -oE "pulp-api\S*")
Expand All @@ -786,42 +893,88 @@ spec:
exit 1
}

cmd_prefix bash -c "HOME=/tmp/home pip3 install pytest\<8 gnupg"
set +e
cmd_prefix bash -c "HOME=/tmp/home PYTHONPATH=/tmp/home/.local/lib/python3.11/site-packages/ XDG_CONFIG_HOME=/tmp/home/.config API_PROTOCOL=http API_HOST=pulp-api API_PORT=8000 ADMIN_USERNAME=admin ADMIN_PASSWORD=$PASSWORD /tmp/home/.local/bin/pytest -o cache_dir=/tmp/home/.cache/pytest_cache_core -v -r sx --color=yes --pyargs pulpcore.tests.functional -m 'parallel' -n 8 -k 'test_jq_header_remote_auth' --junitxml=/tmp/home/junit-pulpcore-parallel.xml" || debug_and_fail

curl -o functest_requirements.txt https://raw.githubusercontent.com/pulp/pulp_rpm/main/functest_requirements.txt
curl -o unittest_requirements.txt https://raw.githubusercontent.com/pulp/pulp_rpm/main/unittest_requirements.txt
cmd_prefix bash -c "HOME=/tmp/home PYTHONPATH=/tmp/home/.local/lib/python3.11/site-packages/ XDG_CONFIG_HOME=/tmp/home/.config API_PROTOCOL=http API_HOST=pulp-api API_PORT=8000 ADMIN_USERNAME=admin ADMIN_PASSWORD=$PASSWORD /tmp/home/.local/bin/pytest -o cache_dir=/tmp/home/.cache/pytest_cache_core -v -r sx --color=yes --pyargs pulp_service.tests.functional -m 'not parallel' --junitxml=/tmp/home/junit-service.xml" || debug_and_fail

### Adapted from ./.github/workflows/scripts/script.sh
cat unittest_requirements.txt | cmd_stdin_prefix bash -c "cat > /tmp/unittest_requirements.txt"
cat functest_requirements.txt | cmd_stdin_prefix bash -c "cat > /tmp/functest_requirements.txt"
cmd_prefix bash -c "HOME=/tmp/home pip3 install -r /tmp/unittest_requirements.txt -r /tmp/functest_requirements.txt"
# Because we pass the path to pytest -o cache_dir=/tmp/home/.cache/pytest_cache, pulpcore-manager must be in the same dir
cmd_prefix bash -c "ln -s /usr/local/lib/pulp/bin/pulpcore-manager /tmp/home/.local/bin/pulpcore-manager || /bin/true"
echo "CURL OUTPUT"
curl https://env-${NS}.apps.crc-eph.r9lp.p1.openshiftapps.com/api/pulp-content/default/
echo "ROUTES"
oc_wrapper get route
set +e
# Only testing test_download_content because it is a very thorough test that tests that all the components of pulp can work
cmd_prefix bash -c "HOME=/tmp/home PYTHONPATH=/tmp/home/.local/lib/python3.11/site-packages/ XDG_CONFIG_HOME=/tmp/home/.config API_PROTOCOL=http API_HOST=pulp-api API_PORT=8000 ADMIN_USERNAME=admin ADMIN_PASSWORD=$PASSWORD /tmp/home/.local/bin/pytest -o cache_dir=/tmp/home/.cache/pytest_cache -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_rpm.tests.functional -m parallel -n 8 -k 'test_download_content' --junitxml=/tmp/home/junit-pulp-parallel.xml" || debug_and_fail
# Never test test_package_manager_consume because they require sudo
# Do not test test_domain_create because it requires more than 2GB of RAM
# Only testing test_download_policies because they are very thorough tests that test that all the components of pulp can work
cmd_prefix bash -c "HOME=/tmp/home PYTHONPATH=/tmp/home/.local/lib/python3.11/site-packages/ XDG_CONFIG_HOME=/tmp/home/.config API_PROTOCOL=http API_HOST=pulp-api API_PORT=8000 ADMIN_USERNAME=admin ADMIN_PASSWORD=$PASSWORD /tmp/home/.local/bin/pytest -o cache_dir=/tmp/home/.cache/pytest_cache -v -r sx --color=yes --pyargs pulp_rpm.tests.functional -m 'not parallel' -k 'test_download_policies' --junitxml=/tmp/home/junit-pulp-serial.xml" || debug_and_fail
- name: functional-tests-npm-maven
when:
- input: '$(tasks.test-metadata.results.test-event-type)'
operator: in
values: ["pull_request"]
runAfter:
- install-bindings
params:
- name: BONFIRE_IMAGE
value: "$(params.BONFIRE_IMAGE)"
- name: NS
value: "$(tasks.reserve-namespace.results.NS)"
- name: EPHEMERAL_ENV_PROVIDER_SECRET
value: "$(params.EPHEMERAL_ENV_PROVIDER_SECRET)"
taskSpec:
params:
- name: BONFIRE_IMAGE
type: string
description: The container Bonfire image to use for the tekton tasks
default: quay.io/redhat-user-workloads/hcc-devprod-tenant/hcc-cicd-tools/cicd-tools:834176766e3f911ffa24bfacff59dd15126e4b3a
- name: EPHEMERAL_ENV_PROVIDER_SECRET
type: string
default: ephemeral-env-provider
- name: NS
type: string
description: Namespace name to deploy the application to
steps:
- name: run-npm-maven-tests
image: "$(params.BONFIRE_IMAGE)"
env:
- name: OC_LOGIN_TOKEN
valueFrom:
secretKeyRef:
name: $(params.EPHEMERAL_ENV_PROVIDER_SECRET)
key: token
- name: OC_LOGIN_SERVER
valueFrom:
secretKeyRef:
name: $(params.EPHEMERAL_ENV_PROVIDER_SECRET)
key: url
- name: NS
value: $(params.NS)
script: |
set -ex

# Run the jq header auth test
cmd_prefix bash -c "HOME=/tmp/home PYTHONPATH=/tmp/home/.local/lib/python3.11/site-packages/ XDG_CONFIG_HOME=/tmp/home/.config API_PROTOCOL=http API_HOST=pulp-api API_PORT=8000 ADMIN_USERNAME=admin ADMIN_PASSWORD=$PASSWORD /tmp/home/.local/bin/pytest -o cache_dir=/tmp/home/.cache/pytest_cache -v -r sx --color=yes --pyargs pulpcore.tests.functional -m 'parallel' -n 8 -k 'test_jq_header_remote_auth' --junitxml=/tmp/home/junit-pulp-serial.xml" || debug_and_fail
login.sh

### END Adapted from ./.github/workflows/scripts/script.sh
if [ -n "$NS" ]; then
oc_wrapper project $NS
else
export NS=$(oc_wrapper project | grep -oE 'ephemeral-......')
fi
echo "Namespace is $NS"

PASSWORD=$(oc_wrapper extract secret/pulp-admin-password --to=-)

# Run pulp_maven functional tests
cmd_prefix bash -c "HOME=/tmp/home PYTHONPATH=/tmp/home/.local/lib/python3.11/site-packages/ XDG_CONFIG_HOME=/tmp/home/.config API_PROTOCOL=http API_HOST=pulp-api API_PORT=8000 ADMIN_USERNAME=admin ADMIN_PASSWORD=$PASSWORD /tmp/home/.local/bin/pytest -o cache_dir=/tmp/home/.cache/pytest_cache -v -r sx --color=yes --pyargs pulp_maven.tests.functional.api.test_download_content --junitxml=/tmp/home/junit-pulp-serial.xml" || debug_and_fail
cmd_prefix() {
oc_wrapper exec -c pulp-api deployment/pulp-api -- "$@"
}

debug_and_fail() {
oc_wrapper logs $(oc_wrapper get pod | grep -oE "pulp-content\S*")
oc_wrapper logs $(oc_wrapper get pod | grep -oE "pulp-api\S*")
for pod in $(oc_wrapper get pod | grep -oE "pulp-worker\S*"); do
oc_wrapper logs "$pod"
done
echo "CURL OUTPUT"
curl https://env-${NS}.apps.crc-eph.r9lp.p1.openshiftapps.com/api/pulp-content/default/
echo "ROUTES"
oc_wrapper get route
exit 1
}

# Run pulp_npm functional tests
cmd_prefix bash -c "HOME=/tmp/home PYTHONPATH=/tmp/home/.local/lib/python3.11/site-packages/ XDG_CONFIG_HOME=/tmp/home/.config API_PROTOCOL=http API_HOST=pulp-api API_PORT=8000 ADMIN_USERNAME=admin ADMIN_PASSWORD=$PASSWORD /tmp/home/.local/bin/pytest -o cache_dir=/tmp/home/.cache/pytest_cache -v -r sx --color=yes --pyargs pulp_npm.tests.functional -k 'test_pull_through_install' --junitxml=/tmp/home/junit-pulp-serial.xml" || debug_and_fail
set +e
cmd_prefix bash -c "HOME=/tmp/home PYTHONPATH=/tmp/home/.local/lib/python3.11/site-packages/ XDG_CONFIG_HOME=/tmp/home/.config API_PROTOCOL=http API_HOST=pulp-api API_PORT=8000 ADMIN_USERNAME=admin ADMIN_PASSWORD=$PASSWORD /tmp/home/.local/bin/pytest -o cache_dir=/tmp/home/.cache/pytest_cache_npm -v -r sx --color=yes --pyargs pulp_maven.tests.functional.api.test_download_content --junitxml=/tmp/home/junit-maven.xml" || debug_and_fail

# Run pulp_service functional tests
cmd_prefix bash -c "HOME=/tmp/home PYTHONPATH=/tmp/home/.local/lib/python3.11/site-packages/ XDG_CONFIG_HOME=/tmp/home/.config API_PROTOCOL=http API_HOST=pulp-api API_PORT=8000 ADMIN_USERNAME=admin ADMIN_PASSWORD=$PASSWORD /tmp/home/.local/bin/pytest -o cache_dir=/tmp/home/.cache/pytest_cache -v -r sx --color=yes --pyargs pulp_service.tests.functional -m 'not parallel' --junitxml=/tmp/home/junit-pulp-serial.xml" || debug_and_fail
cmd_prefix bash -c "HOME=/tmp/home PYTHONPATH=/tmp/home/.local/lib/python3.11/site-packages/ XDG_CONFIG_HOME=/tmp/home/.config API_PROTOCOL=http API_HOST=pulp-api API_PORT=8000 ADMIN_USERNAME=admin ADMIN_PASSWORD=$PASSWORD /tmp/home/.local/bin/pytest -o cache_dir=/tmp/home/.cache/pytest_cache_npm -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_npm.tests.functional -k 'test_pull_through_install' --junitxml=/tmp/home/junit-npm.xml" || debug_and_fail

- name: push-api-json-files-to-pulp
when:
Expand All @@ -838,7 +991,7 @@ spec:
- name: SNAPSHOT
value: "$(params.SNAPSHOT)"
runAfter:
- pulp-functional-tests
- install-bindings
taskSpec:
params:
- name: BONFIRE_IMAGE
Expand Down