From e568ed7d824c03e7c764d36d3819063ab4d1db0f Mon Sep 17 00:00:00 2001 From: Hans Johnson Date: Wed, 22 Apr 2026 00:01:48 +0000 Subject: [PATCH 1/2] ENH: Add performance benchmark CI builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds .github/workflows/perf-benchmark.yml — a GitHub Actions workflow that builds ITK twice (merge-base and PR HEAD) with the PerformanceBenchmarking remote module enabled, then drives the ASV harness from InsightSoftwareConsortium/ITKPerformanceBenchmarking#111 against each build to produce an asv compare report. Bumps the PerformanceBenchmarking remote module pin from 7950c1d76095033edbf7601a925cdacc2fe717f9 to 41bf1b9cfbaa1b146e1b3e5d3ad3571b2203ce1e (master HEAD, post-#111). The new pin carries: * asv.conf.json, benchmarks/{core,filtering,segmentation}.py, python/itk_perf_shim/ (the ASV harness). * Hardened examples/Core/itkCopyIterationBenchmark.cxx that no longer crashes on VectorImage under ITK >= 1d87efa52 (the VLV null-data regression; see InsightSoftwareConsortium/ITK#6087 for the parallel ITK-side defensive null-guard). Supersedes the 2019 Azure Pipelines macos-11 approach; macos-11 is retired and the older evaluate-itk-performance.py driver has been replaced by the ASV-native harness. For the very first PR (this one) the merge-base lacks the new pin, so its benchmark run is best-effort and the asv compare step is wrapped with continue-on-error. The workflow becomes fully useful for regression detection after this PR lands. --- .github/workflows/perf-benchmark.yml | 177 ++++++++++++++++++ .../PerformanceBenchmarking.remote.cmake | 2 +- 2 files changed, 178 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/perf-benchmark.yml diff --git a/.github/workflows/perf-benchmark.yml b/.github/workflows/perf-benchmark.yml new file mode 100644 index 000000000000..6ecd1e745d6e --- /dev/null +++ b/.github/workflows/perf-benchmark.yml @@ -0,0 +1,177 @@ +name: ITK.Performance + +on: + pull_request: + paths-ignore: + - '*.md' + - LICENSE + - NOTICE + - 'Documentation/**' + - 'Utilities/Debugger/**' + - 'Utilities/ITKv5Preparation/**' + - 'Utilities/Maintenance/**' + workflow_dispatch: + +concurrency: + group: '${{ github.workflow }}@${{ github.head_ref || github.ref }}' + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + # ASV-wired targets — see Modules/Remote/PerformanceBenchmarking's python/itk_perf_shim/registry.py. + BenchmarkTargets: >- + MedianBenchmark + BinaryAddBenchmark + UnaryAddBenchmark + GradientMagnitudeBenchmark + MinMaxCurvatureFlowBenchmark + CopyIterationBenchmark + VectorIterationBenchmark + RegionGrowingBenchmark + WatershedBenchmark + MorphologicalWatershedBenchmark + LevelSetBenchmark + ITKBenchmarksData + +jobs: + asv-compare: + name: asv base vs head (Ubuntu 24.04) + runs-on: ubuntu-24.04 + timeout-minutes: 180 + + steps: + - name: Checkout ITK PR HEAD (full history) + uses: actions/checkout@v6 + with: + fetch-depth: 0 + path: itk-head + + - name: Resolve merge-base and HEAD SHAs + id: sha + working-directory: itk-head + env: + # github.base_ref is empty for workflow_dispatch; fall back to main + # so manual reruns still work. For pull_request events this is the + # target branch name (e.g. "main"). + BASE_REF: ${{ github.base_ref || 'main' }} + run: | + git fetch origin "${BASE_REF}" + MERGE_BASE=$(git merge-base HEAD "origin/${BASE_REF}") + HEAD_SHA=$(git rev-parse HEAD) + echo "merge_base=${MERGE_BASE}" >> "$GITHUB_OUTPUT" + echo "head=${HEAD_SHA}" >> "$GITHUB_OUTPUT" + echo "merge_base=${MERGE_BASE} head=${HEAD_SHA}" + + - name: Checkout ITK at merge-base (shallow) + uses: actions/checkout@v6 + with: + ref: ${{ steps.sha.outputs.merge_base }} + path: itk-base + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install Ninja + ASV + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends ninja-build + python -m pip install "asv>=0.6,<0.7" + + - name: Configure + build merge-base benchmark targets + env: + ITK_SRC: ${{ github.workspace }}/itk-base + ITK_BUILD: ${{ runner.temp }}/ITK-build-base + run: | + set -e + cmake -S "${ITK_SRC}" -B "${ITK_BUILD}" -GNinja \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=OFF \ + -DBUILD_TESTING=ON \ + -DBUILD_EXAMPLES=ON \ + -DITK_WRAP_PYTHON=OFF \ + -DModule_PerformanceBenchmarking=ON + cmake --build "${ITK_BUILD}" --target ${BenchmarkTargets} || \ + echo "NOTE: merge-base may lack some benchmark targets if the pin predates them; asv will surface per-benchmark failures as '--'." + + - name: Configure + build PR HEAD benchmark targets + env: + ITK_SRC: ${{ github.workspace }}/itk-head + ITK_BUILD: ${{ runner.temp }}/ITK-build-head + run: | + set -e + cmake -S "${ITK_SRC}" -B "${ITK_BUILD}" -GNinja \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=OFF \ + -DBUILD_TESTING=ON \ + -DBUILD_EXAMPLES=ON \ + -DITK_WRAP_PYTHON=OFF \ + -DModule_PerformanceBenchmarking=ON + cmake --build "${ITK_BUILD}" --target ${BenchmarkTargets} + + - name: Stage ASV harness (PR HEAD pin) with itk-repo symlink + id: harness + run: | + set -e + HARNESS="${GITHUB_WORKSPACE}/itk-head/Modules/Remote/PerformanceBenchmarking" + if [ ! -f "${HARNESS}/asv.conf.json" ]; then + echo "ERROR: ASV harness not found at ${HARNESS}/asv.conf.json." >&2 + echo " Modules/Remote/PerformanceBenchmarking.remote.cmake must pin" >&2 + echo " a commit with the ASV harness (introduced by" >&2 + echo " InsightSoftwareConsortium/ITKPerformanceBenchmarking#111)." >&2 + exit 1 + fi + ln -sfn "${GITHUB_WORKSPACE}/itk-head" "${HARNESS}/itk-repo" + python -m pip install -e "${HARNESS}/python" + echo "harness=${HARNESS}" >> "$GITHUB_OUTPUT" + + - name: Register ASV machine + working-directory: ${{ steps.harness.outputs.harness }} + run: asv machine --yes --machine gha-ubuntu-24.04 + + - name: Run ASV at merge-base + working-directory: ${{ steps.harness.outputs.harness }} + continue-on-error: true + env: + ITK_BENCHMARK_BIN: ${{ runner.temp }}/ITK-build-base/bin + ITK_BENCHMARK_DATA: ${{ runner.temp }}/ITK-build-base/ExternalData/Modules/Remote/PerformanceBenchmarking/examples/Data/Input + ITK_BENCHMARK_SCRATCH: ${{ runner.temp }}/asv-scratch-base + run: | + mkdir -p "${ITK_BENCHMARK_SCRATCH}" + # asv 0.6.x does not accept --min-runs on `asv run` (only on + # `asv continuous`); rely on per-benchmark repeat/number defaults + # to collect multiple samples. See greptile review on PR #612. + asv run --machine gha-ubuntu-24.04 \ + --set-commit-hash "${{ steps.sha.outputs.merge_base }}" + + - name: Run ASV at PR HEAD + working-directory: ${{ steps.harness.outputs.harness }} + env: + ITK_BENCHMARK_BIN: ${{ runner.temp }}/ITK-build-head/bin + ITK_BENCHMARK_DATA: ${{ runner.temp }}/ITK-build-head/ExternalData/Modules/Remote/PerformanceBenchmarking/examples/Data/Input + ITK_BENCHMARK_SCRATCH: ${{ runner.temp }}/asv-scratch-head + run: | + mkdir -p "${ITK_BENCHMARK_SCRATCH}" + asv run --machine gha-ubuntu-24.04 \ + --set-commit-hash "${{ steps.sha.outputs.head }}" + + - name: Compare base vs HEAD + working-directory: ${{ steps.harness.outputs.harness }} + continue-on-error: true + run: | + asv compare \ + "${{ steps.sha.outputs.merge_base }}" \ + "${{ steps.sha.outputs.head }}" \ + --machine gha-ubuntu-24.04 \ + | tee asv-compare.txt + + - name: Upload ASV results + if: always() + uses: actions/upload-artifact@v7 + with: + name: asv-ubuntu-24.04-${{ github.run_id }} + path: | + ${{ steps.harness.outputs.harness }}/.asv/results/ + ${{ steps.harness.outputs.harness }}/asv-compare.txt + retention-days: 14 + if-no-files-found: warn diff --git a/Modules/Remote/PerformanceBenchmarking.remote.cmake b/Modules/Remote/PerformanceBenchmarking.remote.cmake index 834df502299a..dd8ce5f1c3ff 100644 --- a/Modules/Remote/PerformanceBenchmarking.remote.cmake +++ b/Modules/Remote/PerformanceBenchmarking.remote.cmake @@ -59,5 +59,5 @@ For more information, see:: " MODULE_COMPLIANCE_LEVEL 2 GIT_REPOSITORY https://github.com/InsightSoftwareConsortium/ITKPerformanceBenchmarking.git - GIT_TAG 7950c1d76095033edbf7601a925cdacc2fe717f9 + GIT_TAG 41bf1b9cfbaa1b146e1b3e5d3ad3571b2203ce1e ) From 93b0f7dacbb0003f7ef9dd671210cf6b3d07a0ea Mon Sep 17 00:00:00 2001 From: Hans Johnson Date: Thu, 23 Apr 2026 02:28:44 +0000 Subject: [PATCH 2/2] COMP: Bump PerformanceBenchmarking remote pin to current master Advance the GIT_TAG in Modules/Remote/PerformanceBenchmarking.remote.cmake to 63fec3013cccfe7e8b1b88a1d2d889f7e64e4469, the current tip of InsightSoftwareConsortium/ITKPerformanceBenchmarking master, which includes the COMP: Drop unused InputPixelType typedefs cleanup that silences the -Wunused-local-typedef warnings the perf-benchmark CI job emitted on the prior pin. --- Modules/Remote/PerformanceBenchmarking.remote.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/Remote/PerformanceBenchmarking.remote.cmake b/Modules/Remote/PerformanceBenchmarking.remote.cmake index dd8ce5f1c3ff..c1f1b4db90f6 100644 --- a/Modules/Remote/PerformanceBenchmarking.remote.cmake +++ b/Modules/Remote/PerformanceBenchmarking.remote.cmake @@ -59,5 +59,5 @@ For more information, see:: " MODULE_COMPLIANCE_LEVEL 2 GIT_REPOSITORY https://github.com/InsightSoftwareConsortium/ITKPerformanceBenchmarking.git - GIT_TAG 41bf1b9cfbaa1b146e1b3e5d3ad3571b2203ce1e + GIT_TAG 63fec3013cccfe7e8b1b88a1d2d889f7e64e4469 )