ci(deps): parallelize the image build (per-CPython matrix + assemble)

.github/workflows/build-ci-image.yml

@@ -1,16 +1,13 @@
 name: Build CI deps image 🐳
 
-# Builds the manylinux + prebuilt Boost/Boost.Python + eigenpy image consumed by the
-# wheel matrix in build_and_test.yml, and pushes it to
-# ghcr.io/bertiniteam/b2-manylinux-deps.  The image is tagged by its TOOLCHAIN KEY
-# (boost/eigen/eigenpy + manylinux), so a version bump makes a new tag, never a silent
-# ABI swap.  This runs rarely -- only when the Dockerfile or the pinned versions change,
-# or on demand -- so the ~hour of Boost builds happens here, once, instead of in every
-# wheel run.
+# Builds the manylinux + prebuilt Boost/Boost.Python + eigenpy image consumed by the wheel matrix.
+# The expensive per-CPython Boost builds run in PARALLEL (one matrix runner each, inside the
+# manylinux container via docker run); a final job assembles them into the image and pushes it to
+# ghcr.io/bertiniteam/b2-manylinux-deps, tagged by TOOLCHAIN KEY.  Rare: only on docker/ changes or
+# dispatch.
 #
-# OWNER: the package lands under whichever org runs this workflow.  It MUST run in
-# bertiniteam/b2 (its GITHUB_TOKEN can only push to that org's GHCR); a run on a fork
-# would (correctly) fail to push to bertiniteam.
+# OWNER: the package lands under whichever org runs this; it MUST run in bertiniteam/b2 (its token
+# can only push to that org's GHCR).
 
 on:
   workflow_dispatch:
@@ -25,29 +22,59 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  # Keep in lockstep with BOOST_VERSION / EIGENPY_VERSION in build_and_test.yml.
+  # Keep in lockstep with build_and_test.yml.
   BOOST_VERSION: "1.90.0"
   EIGEN_VERSION: "3.4.0"
   EIGENPY_VERSION: "3.13.0"
   MANYLINUX_TAG: "mlx2_34"
-  IMAGE: ghcr.io/bertiniteam/b2-manylinux-deps
+  MANYLINUX_IMAGE: "quay.io/pypa/manylinux_2_34_x86_64"
+  IMAGE: "ghcr.io/bertiniteam/b2-manylinux-deps"
 
 jobs:
-  build-image:
-    name: Build & push manylinux deps image
+  build-deps:
+    name: Boost+eigenpy ${{ matrix.pytag }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        pytag: [cp310-cp310, cp311-cp311, cp312-cp312, cp313-cp313, cp314-cp314]
+    steps:
+      - uses: actions/checkout@v5
+      - name: Build Boost+eigenpy for ${{ matrix.pytag }} (manylinux)
+        run: |
+          docker run --rm -v "$PWD:/work" -w /work \
+            -e BOOST_VERSION -e EIGEN_VERSION -e EIGENPY_VERSION \
+            "$MANYLINUX_IMAGE" \
+            bash docker/manylinux-deps/build-python-deps.sh "${{ matrix.pytag }}"
+      - uses: actions/upload-artifact@v5
+        with:
+          name: deps-${{ matrix.pytag }}
+          path: deps-${{ matrix.pytag }}.tar.gz
+          if-no-files-found: error
+
+  assemble-and-push:
+    name: Assemble & push image
+    needs: build-deps
     runs-on: ubuntu-latest
     permissions:
       contents: read
       packages: write
     steps:
       - uses: actions/checkout@v5
 
+      - name: Collect prebuilt deps into the build context
+        uses: actions/download-artifact@v5
+        with:
+          pattern: deps-*
+          merge-multiple: true
+          path: docker/manylinux-deps/deps
+
       - name: Compute toolchain tag
         id: tag
         run: |
-          KEY="boost${BOOST_VERSION}-eigen${EIGEN_VERSION}-eigenpy${EIGENPY_VERSION}-${MANYLINUX_TAG}"
-          echo "key=$KEY" >> "$GITHUB_OUTPUT"
-          echo "Image: $IMAGE:$KEY"
+          echo "key=boost${BOOST_VERSION}-eigen${EIGEN_VERSION}-eigenpy${EIGENPY_VERSION}-${MANYLINUX_TAG}" >> "$GITHUB_OUTPUT"
+          echo "Image: $IMAGE:boost${BOOST_VERSION}-eigen${EIGEN_VERSION}-eigenpy${EIGENPY_VERSION}-${MANYLINUX_TAG}"
+          ls -lh docker/manylinux-deps/deps
 
       - name: Log in to GHCR
         uses: docker/login-action@v3
@@ -56,23 +83,15 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Set up Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Build and push
+      - name: Build & push (assembly only -- fast)
         uses: docker/build-push-action@v6
         with:
           context: docker/manylinux-deps
           push: true
-          # Tag with the exact toolchain key (what main CI pins to) plus a moving
-          # :latest for humans.  Never pin main CI to :latest -- pin the key.
           tags: |
             ${{ env.IMAGE }}:${{ steps.tag.outputs.key }}
             ${{ env.IMAGE }}:latest
           build-args: |
             BOOST_VERSION=${{ env.BOOST_VERSION }}
             EIGEN_VERSION=${{ env.EIGEN_VERSION }}
             EIGENPY_VERSION=${{ env.EIGENPY_VERSION }}
-          # GHCR layer cache keeps rebuilds cheap when only one thing changes.
-          cache-from: type=registry,ref=${{ env.IMAGE }}:buildcache
-          cache-to: type=registry,ref=${{ env.IMAGE }}:buildcache,mode=max

docker/manylinux-deps/.gitignore

@@ -0,0 +1 @@
+deps/

docker/manylinux-deps/Dockerfile

@@ -1,106 +1,56 @@
 # syntax=docker/dockerfile:1
 #
-# Prebuilt dependencies for bertini2 Linux wheels: Boost (incl. a per-CPython
-# Boost.Python) + eigenpy, on top of the manylinux image cibuildwheel already uses.
+# Assembly image: the manylinux system layer + prebuilt per-CPython Boost/eigenpy bundles COPYed in.
 #
-# WHY: the wheel matrix recompiles Boost.Python + eigenpy in every run (the ~hour
-# sink), because libboost_python3X is ABI-locked per CPython.  Baking them into a
-# GHCR image moves that cost out of every run and off the 10 GB Actions-cache ceiling
-# (image storage is a separate, effectively-unbounded quota; the runner layer-caches
-# the pull).  The build recipe below MIRRORS build_and_test.yml's
-# CIBW_BEFORE_ALL/BEFORE_BUILD_LINUX exactly -- just relocated here.
+# The slow per-CPython Boost+eigenpy builds run in PARALLEL matrix jobs (build-ci-image.yml, via
+# build-python-deps.sh) and are handed here as tarballs.  This Dockerfile therefore only installs
+# the CPython-independent system layer and unpacks those tarballs, so it builds in ~a couple of
+# minutes instead of the ~hour a serial per-Python loop took.
 #
-# ABI CONTRACT (load-bearing): the image *tag* encodes the toolchain
-# (boost/eigen/eigenpy + manylinux), and main CI ASSERTS a match before using it.  A
-# version bump = a new tag = a new image; a stale image can never be silently consumed.
-# This repo has a SIGABRT history from ABI drift (ADR-0006), so this is not optional.
+# NB: not standalone-buildable -- it expects deps/*.tar.gz in the build context, which the CI
+# assemble job populates from the matrix artifacts.
 #
-# Per-CPython prefixes land at /opt/deps/<tag> (e.g. /opt/deps/cp310-cp310).  Main CI
-# points CMAKE_PREFIX_PATH / LD_LIBRARY_PATH at the active Python's prefix.
-#
-# SIZE NOTE: Boost headers (~200 MB) are duplicated across the per-Python prefixes for
-# correctness/simplicity (this mirrors the known-good recipe).  A follow-up can share
-# the CPython-independent headers + non-python libs once and keep only libboost_python
-# + eigenpy per Python -- see README.md.
+# ABI CONTRACT: the image *tag* encodes the toolchain (boost/eigen/eigenpy + manylinux), stamped
+# also as labels; main CI pins the exact tag and asserts the labels match.  See README.md, ADR-0006.
 
 ARG MANYLINUX_IMAGE=quay.io/pypa/manylinux_2_34_x86_64
 FROM ${MANYLINUX_IMAGE}
 
-# Pins -- MUST stay in lockstep with BOOST_VERSION / EIGENPY_VERSION in build_and_test.yml.
 ARG BOOST_VERSION=1.90.0
 ARG EIGEN_VERSION=3.4.0
 ARG EIGENPY_VERSION=3.13.0
-# CPython tags to build for (manylinux ships these under /opt/python/<tag>).
-ARG PYTHON_TAGS="cp310-cp310 cp311-cp311 cp312-cp312 cp313-cp313 cp314-cp314"
 
 LABEL org.opencontainers.image.source="https://github.com/bertiniteam/b2"
 LABEL org.opencontainers.image.description="manylinux_2_34 + prebuilt Boost/Boost.Python + eigenpy for bertini2 wheels"
-# Toolchain stamp -- main CI reads these labels to assert an ABI match.
 LABEL org.bertini.boost_version="${BOOST_VERSION}"
 LABEL org.bertini.eigen_version="${EIGEN_VERSION}"
 LABEL org.bertini.eigenpy_version="${EIGENPY_VERSION}"
 
-# System layer (CPython-independent): numeric -devel libs, ccache, and OpenMPI so
-# `pip install mpi4py` can build in the wheel *test* venv -- which un-skips the MPI tests
-# (test_mpi_zerodim.py, test_doc_example_scripts.py).  This is the "complete build+test
-# env" idea: everything the Linux wheel run needs, factored here and assembled once.
+# System layer (CPython-independent): numeric -devel libs, ccache, OpenMPI (so `pip install mpi4py`
+# builds in the wheel test venv -> un-skips the MPI tests), and a pinned patchelf (the stock one
+# corrupts the bundled bertini2 CLI's ELF and segfaults it at load).
 RUN yum install -y epel-release \
  && yum install -y wget gmp-devel mpfr-devel libmpc-devel libtool ccache openmpi openmpi-devel \
  && yum clean all \
  && echo /usr/lib64/openmpi/lib > /etc/ld.so.conf.d/openmpi.conf && ldconfig
-
-# OpenMPI's mpicc/mpirun are not on the default PATH on AlmaLinux; expose them so mpi4py
-# builds and (for the dedicated MPI job / serial runs) mpirun is available.
 ENV PATH=/usr/lib64/openmpi/bin:${PATH}
-
-# Pin patchelf 0.17.2.1 (mirrors CIBW_BEFORE_ALL_LINUX): the image's stock patchelf
-# mis-rewrites the bundled bertini2 CLI's ELF and segfaults it at load.
 RUN python3 -m venv /tmp/pe \
  && /tmp/pe/bin/pip install -q "patchelf==0.17.2.1" \
  && install -m755 /tmp/pe/bin/patchelf /usr/local/bin/patchelf \
  && rm -rf /tmp/pe \
  && patchelf --version
 
-# Eigen (header-only, CPython-independent) -> /usr/local, same as CIBW_BEFORE_ALL_LINUX.
+# Eigen headers (CPython-independent) -> /usr/local, for the bertini compile in main CI.
 RUN cd /tmp \
  && wget -q "https://gitlab.com/libeigen/eigen/-/archive/${EIGEN_VERSION}/eigen-${EIGEN_VERSION}.tar.gz" \
  && tar xzf "eigen-${EIGEN_VERSION}.tar.gz" \
  && cmake -S "eigen-${EIGEN_VERSION}" -B /tmp/eigen-bld -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_BUILD_TYPE=Release \
  && cmake --install /tmp/eigen-bld \
  && rm -rf /tmp/eigen*
 
-# Fetch Boost + eigenpy sources once.
-RUN cd /tmp \
- && BU="boost_$(echo "${BOOST_VERSION}" | tr . _)" \
- && wget -q "https://archives.boost.io/release/${BOOST_VERSION}/source/${BU}.tar.bz2" \
- && wget -q "https://github.com/stack-of-tasks/eigenpy/releases/download/v${EIGENPY_VERSION}/eigenpy-${EIGENPY_VERSION}.tar.gz"
-
-# Build Boost.Python + eigenpy per CPython into /opt/deps/<tag>.  A fresh Boost tree
-# per Python avoids cross-variant b2 contamination.  Mirrors CIBW_BEFORE_BUILD_LINUX.
-RUN set -eux; cd /tmp; \
-    BU="boost_$(echo "${BOOST_VERSION}" | tr . _)"; \
-    for tag in ${PYTHON_TAGS}; do \
-      PY="/opt/python/${tag}/bin/python"; \
-      [ -x "$PY" ] || { echo "missing interpreter: $PY"; exit 1; }; \
-      PREFIX="/opt/deps/${tag}"; \
-      "$PY" -m pip install -q numpy scipy; \
-      PY_VER="$("$PY" -c 'import sys;print(f"{sys.version_info.major}.{sys.version_info.minor}")')"; \
-      PY_INC="$("$PY" -c 'import sysconfig;print(sysconfig.get_path("include"))')"; \
-      PY_LIB="$("$PY" -c 'import sysconfig;print(sysconfig.get_config_var("LIBDIR") or "")')"; \
-      rm -rf "/tmp/${BU}"; tar xjf "/tmp/${BU}.tar.bz2" -C /tmp; \
-      cd "/tmp/${BU}"; \
-      ./bootstrap.sh --with-python="$PY" --prefix="$PREFIX"; \
-      printf 'using python : %s : %s : %s : %s ;\n' "$PY_VER" "$PY" "$PY_INC" "$PY_LIB" > user-config.jam; \
-      ./b2 install -j"$(nproc)" --user-config=user-config.jam python="$PY_VER" --without-mpi; \
-      cd /tmp; \
-      rm -rf "eigenpy-${EIGENPY_VERSION}"; tar xzf "eigenpy-${EIGENPY_VERSION}.tar.gz"; \
-      cmake -S "eigenpy-${EIGENPY_VERSION}" -B "/tmp/eigenpy-bld-${tag}" \
-        -DCMAKE_PREFIX_PATH="$PREFIX" -DCMAKE_INSTALL_PREFIX="$PREFIX" -DCMAKE_BUILD_TYPE=Release \
-        -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=ON -DPython3_EXECUTABLE="$PY" \
-        -DPython3_NumPy_INCLUDE_DIR="$("$PY" -c 'import numpy;print(numpy.get_include())')" \
-        -DBUILD_TESTING=OFF -DCMAKE_INSTALL_DO_STRIP=ON; \
-      cmake --build "/tmp/eigenpy-bld-${tag}" -j2 --target install; \
-      find "$PREFIX" -name '*.so*' -type f -exec strip --strip-unneeded {} + 2>/dev/null || true; \
-      rm -rf "/tmp/eigenpy-bld-${tag}"; \
-    done; \
-    rm -rf "/tmp/${BU}" "/tmp/${BU}.tar.bz2" /tmp/eigenpy-${EIGENPY_VERSION}*
+# Prebuilt per-CPython Boost + eigenpy bundles from the parallel build-deps matrix jobs.  Each
+# tarball is rooted at opt/deps/<tag>, so `tar -C /` recreates /opt/deps/<tag>.
+COPY deps/ /tmp/deps-tars/
+RUN set -eux; for f in /tmp/deps-tars/*.tar.gz; do echo "unpacking $f"; tar xzf "$f" -C /; done \
+ && rm -rf /tmp/deps-tars \
+ && ls -1 /opt/deps

docker/manylinux-deps/README.md

@@ -49,6 +49,13 @@ Built and pushed by `.github/workflows/build-ci-image.yml` → `ghcr.io/bertinit
 Triggers: a push to `develop` touching `docker/manylinux-deps/**`, or manual dispatch.
 It **must run in `bertiniteam/b2`** (its token can only push to that org's GHCR).
 
+**Parallel build.** The expensive per-CPython Boost+eigenpy builds run as a **matrix** (one
+runner each, inside the manylinux container via `docker run`, using `build-python-deps.sh`),
+each uploading a `deps-<tag>.tar.gz` artifact.  A final `assemble-and-push` job downloads them
+into `deps/` and builds the (thin, ~2-minute) assembly `Dockerfile` that just installs the
+system layer and unpacks the tarballs.  So wall-clock is one Python's build, not five in
+series.  `deps/` is gitignored (populated only in CI).
+
 **One-time setup:** after the first push, make the package **public** in the org's
 Packages settings so any CI (incl. fork PRs) can pull it without auth. (Owner action —
 GHCR package visibility is a package setting, not something CI can flip.)

docker/manylinux-deps/build-python-deps.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+#
+# Build Boost (incl. Boost.Python) + eigenpy for ONE CPython tag into /opt/deps/<tag> and tar it.
+#
+# Run inside the manylinux container, ONE tag per invocation, so build-ci-image.yml can fan the
+# per-CPython builds across a matrix (parallel) instead of a serial Dockerfile loop.  The assembly
+# Dockerfile then just COPYs the resulting tarballs in.  Recipe mirrors the wheel job's historical
+# CIBW_BEFORE_BUILD_LINUX exactly.
+#
+# Usage (from the repo root, cwd mounted at /work):
+#   docker run --rm -v "$PWD:/work" -w /work -e BOOST_VERSION -e EIGEN_VERSION -e EIGENPY_VERSION \
+#     quay.io/pypa/manylinux_2_34_x86_64 bash docker/manylinux-deps/build-python-deps.sh cp312-cp312
+#
+# Emits ./deps-<tag>.tar.gz (paths rooted at opt/deps/<tag>, so the image extracts it with -C /).
+
+set -euxo pipefail
+TAG="${1:?usage: build-python-deps.sh <cpXY-cpXY>}"
+: "${BOOST_VERSION:?}"; : "${EIGEN_VERSION:?}"; : "${EIGENPY_VERSION:?}"
+OUT="$PWD"   # mounted workspace, before we cd away
+
+yum install -y wget gmp-devel mpfr-devel libmpc-devel libtool >/dev/null
+yum clean all
+
+# Eigen -> /usr/local, needed only to BUILD eigenpy here (the assembly image ships its own copy for
+# the bertini compile; the tarball carries only /opt/deps/<tag>).
+cd /tmp
+wget -q "https://gitlab.com/libeigen/eigen/-/archive/${EIGEN_VERSION}/eigen-${EIGEN_VERSION}.tar.gz"
+tar xzf "eigen-${EIGEN_VERSION}.tar.gz"
+cmake -S "eigen-${EIGEN_VERSION}" -B /tmp/eigen-bld -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_BUILD_TYPE=Release
+cmake --install /tmp/eigen-bld
+
+PY="/opt/python/${TAG}/bin/python"
+[ -x "$PY" ] || { echo "missing interpreter: $PY"; exit 1; }
+"$PY" -m pip install -q numpy scipy
+PREFIX="/opt/deps/${TAG}"
+PY_VER="$("$PY" -c 'import sys;print(f"{sys.version_info.major}.{sys.version_info.minor}")')"
+PY_INC="$("$PY" -c 'import sysconfig;print(sysconfig.get_path("include"))')"
+PY_LIB="$("$PY" -c 'import sysconfig;print(sysconfig.get_config_var("LIBDIR") or "")')"
+
+cd /tmp
+BU="boost_$(echo "${BOOST_VERSION}" | tr . _)"
+wget -q "https://archives.boost.io/release/${BOOST_VERSION}/source/${BU}.tar.bz2"
+tar xjf "${BU}.tar.bz2"
+cd "${BU}"
+./bootstrap.sh --with-python="$PY" --prefix="$PREFIX"
+printf 'using python : %s : %s : %s : %s ;\n' "$PY_VER" "$PY" "$PY_INC" "$PY_LIB" > user-config.jam
+./b2 install -j"$(nproc)" --user-config=user-config.jam python="$PY_VER" --without-mpi
+
+cd /tmp
+wget -q "https://github.com/stack-of-tasks/eigenpy/releases/download/v${EIGENPY_VERSION}/eigenpy-${EIGENPY_VERSION}.tar.gz"
+tar xzf "eigenpy-${EIGENPY_VERSION}.tar.gz"
+cmake -S "eigenpy-${EIGENPY_VERSION}" -B /tmp/eigenpy-bld \
+  -DCMAKE_PREFIX_PATH="$PREFIX" -DCMAKE_INSTALL_PREFIX="$PREFIX" -DCMAKE_BUILD_TYPE=Release \
+  -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=ON -DPython3_EXECUTABLE="$PY" \
+  -DPython3_NumPy_INCLUDE_DIR="$("$PY" -c 'import numpy;print(numpy.get_include())')" \
+  -DBUILD_TESTING=OFF -DCMAKE_INSTALL_DO_STRIP=ON
+cmake --build /tmp/eigenpy-bld -j2 --target install
+
+find "$PREFIX" -name '*.so*' -type f -exec strip --strip-unneeded {} + 2>/dev/null || true
+
+tar czf "${OUT}/deps-${TAG}.tar.gz" -C / "opt/deps/${TAG}"
+chmod a+r "${OUT}/deps-${TAG}.tar.gz"
+ls -lh "${OUT}/deps-${TAG}.tar.gz"