From 00b0d3996e190d386205d06ffd1f57818f37cf04 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Thu, 17 Apr 2025 09:41:17 +0100 Subject: [PATCH] [ci] Try to combine the FVT and tempest (#256) * [ci] Try to combine the FVT and tempest This approach runs the FVTs before tempest and then collects the results after running test-operator This means that the tempest tests will trigger a failure if they fail, and the FVTs will trigger a failure only after the tempest tests are run, which means that the results are reported as expected. This will work for github-check pipeline and the component pipeline. However, this approach may not be usable with the unified jobs. This change adds a tempest config to the logging, graphing, metrics-verification tests. --- .zuul.yaml | 20 +++++++++++++++----- ci/vars-functional-test.yml | 10 ++++++++-- ci/vars-graphing-test.yml | 9 +++++++-- ci/vars-logging-test.yml | 8 +++++++- ci/vars-metric-verification-test.yml | 5 ++++- 5 files changed, 41 insertions(+), 11 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index 4ee18ce06..5dd7bf146 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -3,12 +3,14 @@ name: functional-autoscaling-tests-osp18 parent: telemetry-operator-multinode-autoscaling description: | - Run the autoscaling functional test on osp18+patched version of aodh and heat. + Run the autoscaling functional tests and tempest tests on osp18+patched + versions of aodh and heat. vars: patch_openstackversions: true cifmw_extras: - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/ci-framework'].src_dir }}/scenarios/centos-9/multinode-ci.yml" - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-autoscaling.yml" + - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-autoscaling-tempest.yml" - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-functional-test.yml" - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-use-master-containers.yml" roles: @@ -58,11 +60,14 @@ name: functional-logging-tests-osp18 parent: telemetry-operator-multinode-logging description: | - Run the logging functional tests on osp18 + Run the logging functional tests and tempest smoketests on osp18 vars: cifmw_extras: - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/ci-framework'].src_dir }}/scenarios/centos-9/multinode-ci.yml" - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-logging.yml" + # Use the tempest config we have for autoscaling tests + - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-autoscaling-tempest.yml" + # There's an override in here to modify the enabled tests to include only the smoke tests - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-logging-test.yml" roles: - zuul: github.com/openstack-k8s-operators/ci-framework @@ -73,12 +78,15 @@ name: functional-metric-verification-tests-osp18 parent: telemetry-operator-multinode-autoscaling description: | - Run the metric verification functional test on osp18. + Run the metric verification functional test and tempest smoketests on osp18. vars: patch_observabilityclient: true cifmw_extras: - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/ci-framework'].src_dir }}/scenarios/centos-9/multinode-ci.yml" - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-autoscaling.yml" + # Use the tempest config we have for autoscaling tests + - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-autoscaling-tempest.yml" + # tempest enabled tests list is overwritten in the metrics-verification-tests, so needs to be added after the autoscaling-tempest, so that we only run the smoke-tests - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-metric-verification-test.yml" - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-power-monitoring.yml" - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-use-master-containers.yml" @@ -100,13 +108,15 @@ name: functional-graphing-tests-osp18 parent: telemetry-operator-multinode-autoscaling description: | - Run the UI Graphing test on osp18. + Run the UI Graphing test and tempest smoketests on osp18. vars: - cifmw_run_tests: false cifmw_extras: - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/ci-framework'].src_dir }}/scenarios/centos-9/multinode-ci.yml" - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-autoscaling.yml" - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-power-monitoring.yml" + # Use the tempest config we have for autoscaling tests + - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-autoscaling-tempest.yml" + # Currently, this runs tempest smoketests after the dashboards have been disabled again - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-graphing-test.yml" roles: - zuul: github.com/openstack-k8s-operators/ci-framework diff --git a/ci/vars-functional-test.yml b/ci/vars-functional-test.yml index 081d86d5c..b45f00746 100644 --- a/ci/vars-functional-test.yml +++ b/ci/vars-functional-test.yml @@ -5,10 +5,16 @@ # tests. # skip os-net-setup cifmw_os_net_setup_config: [] -post_deploy_00_run_functional_test: +pre_tests_00_run_functional_test: source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/run_autoscaling_osp18.yml" type: playbook config_file: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/ansible.cfg" -post_deploy_99_collect_results: + +cifmw_run_tests: true +# redefine this to clear the skipped test +# this should be done in the parent job IF the openstack versions are updated +cifmw_test_operator_tempest_exclude_list: "" + +post_tests_99_collect_results: source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/report_result.yml" type: playbook diff --git a/ci/vars-graphing-test.yml b/ci/vars-graphing-test.yml index c852624a8..5b7055029 100644 --- a/ci/vars-graphing-test.yml +++ b/ci/vars-graphing-test.yml @@ -2,7 +2,12 @@ post_deploy_00_run_graphing_test: source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/run_graphing_test.yml" config_file: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/ansible.cfg" - type: playbook -post_deploy_99_collect_results: + type: playbook + +cifmw_run_tests: true +# run only smoke tests +cifmw_test_operator_tempest_include_list: | + ^tempest.*\[.*\bsmoke\b.*\] +post_tests_99_collect_results: source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/report_result.yml" type: playbook diff --git a/ci/vars-logging-test.yml b/ci/vars-logging-test.yml index 89b17d92f..11b866fb8 100644 --- a/ci/vars-logging-test.yml +++ b/ci/vars-logging-test.yml @@ -4,6 +4,12 @@ post_deploy_00_fvt_logging: source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/logging_tests_all.yml" config_file: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/ansible.cfg" type: playbook -post_deploy_99_collect_results: + +cifmw_run_tests: true +# enable only the smoketests +cifmw_test_operator_tempest_include_list: | + ^tempest.*\[.*\bsmoke\b.*\] + +post_tests_99_collect_results: source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/report_result.yml" type: playbook diff --git a/ci/vars-metric-verification-test.yml b/ci/vars-metric-verification-test.yml index cca5ef0ba..7799a3a10 100644 --- a/ci/vars-metric-verification-test.yml +++ b/ci/vars-metric-verification-test.yml @@ -3,6 +3,9 @@ post_deploy_00_fvt_verify_metrics: source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/run_verify_metrics_osp18.yml" type: playbook config_file: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/ansible.cfg" -post_deploy_99_collect_results: +cifmw_run_tests: true +cifmw_test_operator_tempest_include_list: | + ^tempest.*\[.*\bsmoke\b.*\] +post_tests_99_collect_results: source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/report_result.yml" type: playbook