From 609496add2a7819cb4f1982ad8edded42f2218ff Mon Sep 17 00:00:00 2001 From: Connor Ward Date: Fri, 20 Mar 2026 12:55:12 +0000 Subject: [PATCH 1/4] Debug performance regression --- .github/workflows/core.yml | 260 ++++++++++++++++++------------------- 1 file changed, 130 insertions(+), 130 deletions(-) diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index e738561b9c..62dae75e07 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -95,7 +95,7 @@ jobs: PYOP2_CFLAGS: -fno-fast-math # NOTE: One should occasionally update test_durations.json by running # 'make test_durations' inside a 'firedrake:latest' Docker image. - EXTRA_PYTEST_ARGS: --splitting-algorithm least_duration --timeout=600 --timeout-method=thread -o faulthandler_timeout=660 --durations-path=./firedrake-repo/tests/test_durations.json --durations=50 + EXTRA_PYTEST_ARGS: --splitting-algorithm least_duration --timeout=100 --timeout-method=thread -o faulthandler_timeout=660 --durations-path=./firedrake-repo/tests/test_durations.json --durations=50 PYTEST_MPI_MAX_NPROCS: 8 steps: - name: Fix HOME @@ -230,37 +230,37 @@ jobs: firedrake-check timeout-minutes: 5 - - name: Run TSFC tests - # Run even if earlier tests failed - if: inputs.run_tests && (success() || steps.install.conclusion == 'success') - run: | - . venv/bin/activate - : # Use pytest-xdist here so we can have a single collated output (not possible - : # for parallel tests) - firedrake-run-split-tests 1 1 -n 8 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/tsfc - timeout-minutes: 10 - - - name: Run PyOP2 tests - if: inputs.run_tests && (success() || steps.install.conclusion == 'success') - run: | - . venv/bin/activate - : # Use pytest-xdist here so we can have a single collated output (not possible - : # for parallel tests) - firedrake-run-split-tests 1 1 -n 8 "$EXTRA_PYTEST_ARGS" --timeout 30 firedrake-repo/tests/pyop2 - firedrake-run-split-tests 2 4 "$EXTRA_PYTEST_ARGS" --timeout 30 firedrake-repo/tests/pyop2 - firedrake-run-split-tests 3 2 "$EXTRA_PYTEST_ARGS" --timeout 30 firedrake-repo/tests/pyop2 - firedrake-run-split-tests 4 2 "$EXTRA_PYTEST_ARGS" --timeout 30 firedrake-repo/tests/pyop2 - timeout-minutes: 10 - - - - name: Run Firedrake tests (nprocs = 1) - if: inputs.run_tests && (success() || steps.install.conclusion == 'success') - run: | - . venv/bin/activate - : # Use pytest-xdist here so we can have a single collated output (not possible - : # for parallel tests) - firedrake-run-split-tests 1 1 -n 8 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake - timeout-minutes: 90 + # - name: Run TSFC tests + # # Run even if earlier tests failed + # if: inputs.run_tests && (success() || steps.install.conclusion == 'success') + # run: | + # . venv/bin/activate + # : # Use pytest-xdist here so we can have a single collated output (not possible + # : # for parallel tests) + # firedrake-run-split-tests 1 1 -n 8 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/tsfc + # timeout-minutes: 10 + # + # - name: Run PyOP2 tests + # if: inputs.run_tests && (success() || steps.install.conclusion == 'success') + # run: | + # . venv/bin/activate + # : # Use pytest-xdist here so we can have a single collated output (not possible + # : # for parallel tests) + # firedrake-run-split-tests 1 1 -n 8 "$EXTRA_PYTEST_ARGS" --timeout 30 firedrake-repo/tests/pyop2 + # firedrake-run-split-tests 2 4 "$EXTRA_PYTEST_ARGS" --timeout 30 firedrake-repo/tests/pyop2 + # firedrake-run-split-tests 3 2 "$EXTRA_PYTEST_ARGS" --timeout 30 firedrake-repo/tests/pyop2 + # firedrake-run-split-tests 4 2 "$EXTRA_PYTEST_ARGS" --timeout 30 firedrake-repo/tests/pyop2 + # timeout-minutes: 10 + + + # - name: Run Firedrake tests (nprocs = 1) + # if: inputs.run_tests && (success() || steps.install.conclusion == 'success') + # run: | + # . venv/bin/activate + # : # Use pytest-xdist here so we can have a single collated output (not possible + # : # for parallel tests) + # firedrake-run-split-tests 1 1 -n 8 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake + # timeout-minutes: 90 - name: Run tests (nprocs = 2) if: inputs.run_tests && (success() || steps.install.conclusion == 'success') @@ -269,104 +269,104 @@ jobs: firedrake-run-split-tests 2 4 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake timeout-minutes: 60 - - name: Run tests (nprocs = 3) - if: inputs.run_tests && (success() || steps.install.conclusion == 'success') - run: | - . venv/bin/activate - firedrake-run-split-tests 3 2 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake - timeout-minutes: 60 - - - name: Run tests (nprocs = 4) - if: inputs.run_tests && (success() || steps.install.conclusion == 'success') - run: | - . venv/bin/activate - firedrake-run-split-tests 4 2 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake - timeout-minutes: 15 - - - name: Run tests (nprocs = 5) - if: inputs.run_tests && (success() || steps.install.conclusion == 'success') - run: | - . venv/bin/activate - firedrake-run-split-tests 5 1 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake - timeout-minutes: 15 - - - name: Run tests (nprocs = 6) - if: inputs.run_tests && (success() || steps.install.conclusion == 'success') - run: | - . venv/bin/activate - firedrake-run-split-tests 6 1 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake - timeout-minutes: 15 - - - name: Run tests (nprocs = 7) - if: inputs.run_tests && (success() || steps.install.conclusion == 'success') - run: | - . venv/bin/activate - firedrake-run-split-tests 7 1 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake - timeout-minutes: 15 - - - name: Run tests (nprocs = 8) - if: inputs.run_tests && (success() || steps.install.conclusion == 'success') - run: | - . venv/bin/activate - firedrake-run-split-tests 8 1 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake - timeout-minutes: 15 - - - name: Run Gusto smoke tests - # Only test Gusto in real mode - if: | - inputs.run_tests && - (success() || steps.install.conclusion == 'success') && - matrix.arch == 'default' - run: | - . venv/bin/activate - if [ ${{ inputs.target_branch }} = 'release' ]; then - GUSTO_BRANCH='main' - else - GUSTO_BRANCH='future' - fi - git clone --depth 1 https://github.com/firedrakeproject/gusto.git gusto-repo --branch $GUSTO_BRANCH - pip install --verbose ./gusto-repo - python -m pytest -n 8 --verbose \ - gusto-repo/integration-tests/balance/test_saturated_balance.py \ - gusto-repo/integration-tests/equations/test_thermal_sw.py \ - gusto-repo/integration-tests/transport/test_embedded_dg_advection.py - timeout-minutes: 10 - - - name: Run Thetis smoke tests - if: | - inputs.run_tests && - (success() || steps.install.conclusion == 'success') && - matrix.arch == 'default' - run: | - . venv/bin/activate - git clone --depth 1 https://github.com/thetisproject/thetis.git thetis-repo --branch ${{ inputs.target_branch }} - pip install --verbose ./thetis-repo - python -m pytest -n 8 --verbose thetis-repo/test_adjoint/test_swe_adjoint.py - timeout-minutes: 10 - - - name: Run spyro smoke tests - if: | - inputs.run_tests && - (success() || steps.install.conclusion == 'success') && - matrix.arch == 'default' - run: | - . venv/bin/activate - git clone --depth 1 https://github.com/NDF-Poli-USP/spyro.git spyro-repo - pip install --verbose ./spyro-repo - mpiexec -n 6 python -m pytest spyro-repo/test_integration/ -m parallel[6] - timeout-minutes: 5 - - - name: Run G-ADOPT smoke tests - if: | - inputs.run_tests && - (success() || steps.install.conclusion == 'success') && - matrix.arch == 'default' - run: | - . venv/bin/activate - git clone --depth 1 https://github.com/g-adopt/g-adopt.git g-adopt-repo --branch ${{ inputs.target_branch }} - pip install --verbose ./g-adopt-repo - make -C g-adopt-repo/demos/mantle_convection/base_case check - timeout-minutes: 5 + # - name: Run tests (nprocs = 3) + # if: inputs.run_tests && (success() || steps.install.conclusion == 'success') + # run: | + # . venv/bin/activate + # firedrake-run-split-tests 3 2 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake + # timeout-minutes: 60 + # + # - name: Run tests (nprocs = 4) + # if: inputs.run_tests && (success() || steps.install.conclusion == 'success') + # run: | + # . venv/bin/activate + # firedrake-run-split-tests 4 2 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake + # timeout-minutes: 15 + # + # - name: Run tests (nprocs = 5) + # if: inputs.run_tests && (success() || steps.install.conclusion == 'success') + # run: | + # . venv/bin/activate + # firedrake-run-split-tests 5 1 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake + # timeout-minutes: 15 + # + # - name: Run tests (nprocs = 6) + # if: inputs.run_tests && (success() || steps.install.conclusion == 'success') + # run: | + # . venv/bin/activate + # firedrake-run-split-tests 6 1 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake + # timeout-minutes: 15 + # + # - name: Run tests (nprocs = 7) + # if: inputs.run_tests && (success() || steps.install.conclusion == 'success') + # run: | + # . venv/bin/activate + # firedrake-run-split-tests 7 1 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake + # timeout-minutes: 15 + # + # - name: Run tests (nprocs = 8) + # if: inputs.run_tests && (success() || steps.install.conclusion == 'success') + # run: | + # . venv/bin/activate + # firedrake-run-split-tests 8 1 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake + # timeout-minutes: 15 + # + # - name: Run Gusto smoke tests + # # Only test Gusto in real mode + # if: | + # inputs.run_tests && + # (success() || steps.install.conclusion == 'success') && + # matrix.arch == 'default' + # run: | + # . venv/bin/activate + # if [ ${{ inputs.target_branch }} = 'release' ]; then + # GUSTO_BRANCH='main' + # else + # GUSTO_BRANCH='future' + # fi + # git clone --depth 1 https://github.com/firedrakeproject/gusto.git gusto-repo --branch $GUSTO_BRANCH + # pip install --verbose ./gusto-repo + # python -m pytest -n 8 --verbose \ + # gusto-repo/integration-tests/balance/test_saturated_balance.py \ + # gusto-repo/integration-tests/equations/test_thermal_sw.py \ + # gusto-repo/integration-tests/transport/test_embedded_dg_advection.py + # timeout-minutes: 10 + # + # - name: Run Thetis smoke tests + # if: | + # inputs.run_tests && + # (success() || steps.install.conclusion == 'success') && + # matrix.arch == 'default' + # run: | + # . venv/bin/activate + # git clone --depth 1 https://github.com/thetisproject/thetis.git thetis-repo --branch ${{ inputs.target_branch }} + # pip install --verbose ./thetis-repo + # python -m pytest -n 8 --verbose thetis-repo/test_adjoint/test_swe_adjoint.py + # timeout-minutes: 10 + # + # - name: Run spyro smoke tests + # if: | + # inputs.run_tests && + # (success() || steps.install.conclusion == 'success') && + # matrix.arch == 'default' + # run: | + # . venv/bin/activate + # git clone --depth 1 https://github.com/NDF-Poli-USP/spyro.git spyro-repo + # pip install --verbose ./spyro-repo + # mpiexec -n 6 python -m pytest spyro-repo/test_integration/ -m parallel[6] + # timeout-minutes: 5 + # + # - name: Run G-ADOPT smoke tests + # if: | + # inputs.run_tests && + # (success() || steps.install.conclusion == 'success') && + # matrix.arch == 'default' + # run: | + # . venv/bin/activate + # git clone --depth 1 https://github.com/g-adopt/g-adopt.git g-adopt-repo --branch ${{ inputs.target_branch }} + # pip install --verbose ./g-adopt-repo + # make -C g-adopt-repo/demos/mantle_convection/base_case check + # timeout-minutes: 5 - name: Upload log files uses: actions/upload-artifact@v4 From fa7f280d44847af7373ce7b5e6cb2d9e2d201df6 Mon Sep 17 00:00:00 2001 From: Connor Ward Date: Fri, 20 Mar 2026 13:53:11 +0000 Subject: [PATCH 2/4] fixup --- .github/workflows/core.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index 62dae75e07..67d50d1046 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -95,7 +95,7 @@ jobs: PYOP2_CFLAGS: -fno-fast-math # NOTE: One should occasionally update test_durations.json by running # 'make test_durations' inside a 'firedrake:latest' Docker image. - EXTRA_PYTEST_ARGS: --splitting-algorithm least_duration --timeout=100 --timeout-method=thread -o faulthandler_timeout=660 --durations-path=./firedrake-repo/tests/test_durations.json --durations=50 + EXTRA_PYTEST_ARGS: --splitting-algorithm least_duration --timeout=600 --timeout-method=thread -o faulthandler_timeout=660 --durations-path=./firedrake-repo/tests/test_durations.json --durations=50 PYTEST_MPI_MAX_NPROCS: 8 steps: - name: Fix HOME @@ -267,7 +267,7 @@ jobs: run: | . venv/bin/activate firedrake-run-split-tests 2 4 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake - timeout-minutes: 60 + timeout-minutes: 240 # - name: Run tests (nprocs = 3) # if: inputs.run_tests && (success() || steps.install.conclusion == 'success') From a8130ae41d2bc1231bae895860cb54259eae42c4 Mon Sep 17 00:00:00 2001 From: Connor Ward Date: Fri, 20 Mar 2026 14:40:47 +0000 Subject: [PATCH 3/4] gc every test --- tests/firedrake/regression/test_covariance_operator.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/firedrake/regression/test_covariance_operator.py b/tests/firedrake/regression/test_covariance_operator.py index 4a5ab20257..38ede96759 100644 --- a/tests/firedrake/regression/test_covariance_operator.py +++ b/tests/firedrake/regression/test_covariance_operator.py @@ -32,6 +32,12 @@ def rng(): return RandomGenerator(PCG64(seed=13)) +@pytest.fixture(autouse=True) +def garbage_cleanup(): + yield + PETSc.garbage_cleanup(COMM_WORLD) + + @pytest.mark.skipcomplex @pytest.mark.parallel([1, 2]) @pytest.mark.parametrize("degree", (1, 2), ids=["degree1", "degree2"]) From 5e891e4051b8e1eaa0646d914864eff0447c1157 Mon Sep 17 00:00:00 2001 From: Connor Ward Date: Mon, 23 Mar 2026 10:29:48 +0000 Subject: [PATCH 4/4] empty