From 2c29a6b1c3fa523934c265d76070df0032c8e742 Mon Sep 17 00:00:00 2001 From: silviana amethyst Date: Tue, 7 Jul 2026 14:21:28 +0000 Subject: [PATCH] ci(deps): parallelize the image build (per-CPython matrix + assemble) The single-Dockerfile producer built Boost+eigenpy for all 5 CPythons serially -- ~an hour, and the run we cancelled was still on cp310 after 85 min. Split it: - build-python-deps.sh: builds one CPython's Boost+eigenpy into /opt/deps/ and tars it. - build-ci-image.yml: a `build-deps` MATRIX runs that script per tag in parallel (inside the manylinux container via `docker run`, dodging the node-in-container issue), uploading a deps-.tar.gz artifact each; `assemble-and-push` downloads them and builds a thin assembly image that just installs the system layer (openmpi/ccache/patchelf/eigen) and unpacks the tarballs -> ~2 min. Dropped the registry buildcache (assembly is cheap). - Dockerfile: reworked from the inline per-Python loop to the COPY-and-unpack assembly. Wall-clock is now one Python's build, not five in series. `deps/` is gitignored. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/build-ci-image.yml | 71 ++++++++++------- docker/manylinux-deps/.gitignore | 1 + docker/manylinux-deps/Dockerfile | 88 +++++----------------- docker/manylinux-deps/README.md | 7 ++ docker/manylinux-deps/build-python-deps.sh | 63 ++++++++++++++++ 5 files changed, 135 insertions(+), 95 deletions(-) create mode 100644 docker/manylinux-deps/.gitignore create mode 100755 docker/manylinux-deps/build-python-deps.sh diff --git a/.github/workflows/build-ci-image.yml b/.github/workflows/build-ci-image.yml index ac79bde50..1bd53bc48 100644 --- a/.github/workflows/build-ci-image.yml +++ b/.github/workflows/build-ci-image.yml @@ -1,16 +1,13 @@ name: Build CI deps image 🐳 -# Builds the manylinux + prebuilt Boost/Boost.Python + eigenpy image consumed by the -# wheel matrix in build_and_test.yml, and pushes it to -# ghcr.io/bertiniteam/b2-manylinux-deps. The image is tagged by its TOOLCHAIN KEY -# (boost/eigen/eigenpy + manylinux), so a version bump makes a new tag, never a silent -# ABI swap. This runs rarely -- only when the Dockerfile or the pinned versions change, -# or on demand -- so the ~hour of Boost builds happens here, once, instead of in every -# wheel run. +# Builds the manylinux + prebuilt Boost/Boost.Python + eigenpy image consumed by the wheel matrix. +# The expensive per-CPython Boost builds run in PARALLEL (one matrix runner each, inside the +# manylinux container via docker run); a final job assembles them into the image and pushes it to +# ghcr.io/bertiniteam/b2-manylinux-deps, tagged by TOOLCHAIN KEY. Rare: only on docker/ changes or +# dispatch. # -# OWNER: the package lands under whichever org runs this workflow. It MUST run in -# bertiniteam/b2 (its GITHUB_TOKEN can only push to that org's GHCR); a run on a fork -# would (correctly) fail to push to bertiniteam. +# OWNER: the package lands under whichever org runs this; it MUST run in bertiniteam/b2 (its token +# can only push to that org's GHCR). on: workflow_dispatch: @@ -25,16 +22,39 @@ concurrency: cancel-in-progress: true env: - # Keep in lockstep with BOOST_VERSION / EIGENPY_VERSION in build_and_test.yml. + # Keep in lockstep with build_and_test.yml. BOOST_VERSION: "1.90.0" EIGEN_VERSION: "3.4.0" EIGENPY_VERSION: "3.13.0" MANYLINUX_TAG: "mlx2_34" - IMAGE: ghcr.io/bertiniteam/b2-manylinux-deps + MANYLINUX_IMAGE: "quay.io/pypa/manylinux_2_34_x86_64" + IMAGE: "ghcr.io/bertiniteam/b2-manylinux-deps" jobs: - build-image: - name: Build & push manylinux deps image + build-deps: + name: Boost+eigenpy ${{ matrix.pytag }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + pytag: [cp310-cp310, cp311-cp311, cp312-cp312, cp313-cp313, cp314-cp314] + steps: + - uses: actions/checkout@v5 + - name: Build Boost+eigenpy for ${{ matrix.pytag }} (manylinux) + run: | + docker run --rm -v "$PWD:/work" -w /work \ + -e BOOST_VERSION -e EIGEN_VERSION -e EIGENPY_VERSION \ + "$MANYLINUX_IMAGE" \ + bash docker/manylinux-deps/build-python-deps.sh "${{ matrix.pytag }}" + - uses: actions/upload-artifact@v5 + with: + name: deps-${{ matrix.pytag }} + path: deps-${{ matrix.pytag }}.tar.gz + if-no-files-found: error + + assemble-and-push: + name: Assemble & push image + needs: build-deps runs-on: ubuntu-latest permissions: contents: read @@ -42,12 +62,19 @@ jobs: steps: - uses: actions/checkout@v5 + - name: Collect prebuilt deps into the build context + uses: actions/download-artifact@v5 + with: + pattern: deps-* + merge-multiple: true + path: docker/manylinux-deps/deps + - name: Compute toolchain tag id: tag run: | - KEY="boost${BOOST_VERSION}-eigen${EIGEN_VERSION}-eigenpy${EIGENPY_VERSION}-${MANYLINUX_TAG}" - echo "key=$KEY" >> "$GITHUB_OUTPUT" - echo "Image: $IMAGE:$KEY" + echo "key=boost${BOOST_VERSION}-eigen${EIGEN_VERSION}-eigenpy${EIGENPY_VERSION}-${MANYLINUX_TAG}" >> "$GITHUB_OUTPUT" + echo "Image: $IMAGE:boost${BOOST_VERSION}-eigen${EIGEN_VERSION}-eigenpy${EIGENPY_VERSION}-${MANYLINUX_TAG}" + ls -lh docker/manylinux-deps/deps - name: Log in to GHCR uses: docker/login-action@v3 @@ -56,16 +83,11 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Set up Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build and push + - name: Build & push (assembly only -- fast) uses: docker/build-push-action@v6 with: context: docker/manylinux-deps push: true - # Tag with the exact toolchain key (what main CI pins to) plus a moving - # :latest for humans. Never pin main CI to :latest -- pin the key. tags: | ${{ env.IMAGE }}:${{ steps.tag.outputs.key }} ${{ env.IMAGE }}:latest @@ -73,6 +95,3 @@ jobs: BOOST_VERSION=${{ env.BOOST_VERSION }} EIGEN_VERSION=${{ env.EIGEN_VERSION }} EIGENPY_VERSION=${{ env.EIGENPY_VERSION }} - # GHCR layer cache keeps rebuilds cheap when only one thing changes. - cache-from: type=registry,ref=${{ env.IMAGE }}:buildcache - cache-to: type=registry,ref=${{ env.IMAGE }}:buildcache,mode=max diff --git a/docker/manylinux-deps/.gitignore b/docker/manylinux-deps/.gitignore new file mode 100644 index 000000000..6912fefe1 --- /dev/null +++ b/docker/manylinux-deps/.gitignore @@ -0,0 +1 @@ +deps/ diff --git a/docker/manylinux-deps/Dockerfile b/docker/manylinux-deps/Dockerfile index 486178eae..7812e6115 100644 --- a/docker/manylinux-deps/Dockerfile +++ b/docker/manylinux-deps/Dockerfile @@ -1,67 +1,46 @@ # syntax=docker/dockerfile:1 # -# Prebuilt dependencies for bertini2 Linux wheels: Boost (incl. a per-CPython -# Boost.Python) + eigenpy, on top of the manylinux image cibuildwheel already uses. +# Assembly image: the manylinux system layer + prebuilt per-CPython Boost/eigenpy bundles COPYed in. # -# WHY: the wheel matrix recompiles Boost.Python + eigenpy in every run (the ~hour -# sink), because libboost_python3X is ABI-locked per CPython. Baking them into a -# GHCR image moves that cost out of every run and off the 10 GB Actions-cache ceiling -# (image storage is a separate, effectively-unbounded quota; the runner layer-caches -# the pull). The build recipe below MIRRORS build_and_test.yml's -# CIBW_BEFORE_ALL/BEFORE_BUILD_LINUX exactly -- just relocated here. +# The slow per-CPython Boost+eigenpy builds run in PARALLEL matrix jobs (build-ci-image.yml, via +# build-python-deps.sh) and are handed here as tarballs. This Dockerfile therefore only installs +# the CPython-independent system layer and unpacks those tarballs, so it builds in ~a couple of +# minutes instead of the ~hour a serial per-Python loop took. # -# ABI CONTRACT (load-bearing): the image *tag* encodes the toolchain -# (boost/eigen/eigenpy + manylinux), and main CI ASSERTS a match before using it. A -# version bump = a new tag = a new image; a stale image can never be silently consumed. -# This repo has a SIGABRT history from ABI drift (ADR-0006), so this is not optional. +# NB: not standalone-buildable -- it expects deps/*.tar.gz in the build context, which the CI +# assemble job populates from the matrix artifacts. # -# Per-CPython prefixes land at /opt/deps/ (e.g. /opt/deps/cp310-cp310). Main CI -# points CMAKE_PREFIX_PATH / LD_LIBRARY_PATH at the active Python's prefix. -# -# SIZE NOTE: Boost headers (~200 MB) are duplicated across the per-Python prefixes for -# correctness/simplicity (this mirrors the known-good recipe). A follow-up can share -# the CPython-independent headers + non-python libs once and keep only libboost_python -# + eigenpy per Python -- see README.md. +# ABI CONTRACT: the image *tag* encodes the toolchain (boost/eigen/eigenpy + manylinux), stamped +# also as labels; main CI pins the exact tag and asserts the labels match. See README.md, ADR-0006. ARG MANYLINUX_IMAGE=quay.io/pypa/manylinux_2_34_x86_64 FROM ${MANYLINUX_IMAGE} -# Pins -- MUST stay in lockstep with BOOST_VERSION / EIGENPY_VERSION in build_and_test.yml. ARG BOOST_VERSION=1.90.0 ARG EIGEN_VERSION=3.4.0 ARG EIGENPY_VERSION=3.13.0 -# CPython tags to build for (manylinux ships these under /opt/python/). -ARG PYTHON_TAGS="cp310-cp310 cp311-cp311 cp312-cp312 cp313-cp313 cp314-cp314" LABEL org.opencontainers.image.source="https://github.com/bertiniteam/b2" LABEL org.opencontainers.image.description="manylinux_2_34 + prebuilt Boost/Boost.Python + eigenpy for bertini2 wheels" -# Toolchain stamp -- main CI reads these labels to assert an ABI match. LABEL org.bertini.boost_version="${BOOST_VERSION}" LABEL org.bertini.eigen_version="${EIGEN_VERSION}" LABEL org.bertini.eigenpy_version="${EIGENPY_VERSION}" -# System layer (CPython-independent): numeric -devel libs, ccache, and OpenMPI so -# `pip install mpi4py` can build in the wheel *test* venv -- which un-skips the MPI tests -# (test_mpi_zerodim.py, test_doc_example_scripts.py). This is the "complete build+test -# env" idea: everything the Linux wheel run needs, factored here and assembled once. +# System layer (CPython-independent): numeric -devel libs, ccache, OpenMPI (so `pip install mpi4py` +# builds in the wheel test venv -> un-skips the MPI tests), and a pinned patchelf (the stock one +# corrupts the bundled bertini2 CLI's ELF and segfaults it at load). RUN yum install -y epel-release \ && yum install -y wget gmp-devel mpfr-devel libmpc-devel libtool ccache openmpi openmpi-devel \ && yum clean all \ && echo /usr/lib64/openmpi/lib > /etc/ld.so.conf.d/openmpi.conf && ldconfig - -# OpenMPI's mpicc/mpirun are not on the default PATH on AlmaLinux; expose them so mpi4py -# builds and (for the dedicated MPI job / serial runs) mpirun is available. ENV PATH=/usr/lib64/openmpi/bin:${PATH} - -# Pin patchelf 0.17.2.1 (mirrors CIBW_BEFORE_ALL_LINUX): the image's stock patchelf -# mis-rewrites the bundled bertini2 CLI's ELF and segfaults it at load. RUN python3 -m venv /tmp/pe \ && /tmp/pe/bin/pip install -q "patchelf==0.17.2.1" \ && install -m755 /tmp/pe/bin/patchelf /usr/local/bin/patchelf \ && rm -rf /tmp/pe \ && patchelf --version -# Eigen (header-only, CPython-independent) -> /usr/local, same as CIBW_BEFORE_ALL_LINUX. +# Eigen headers (CPython-independent) -> /usr/local, for the bertini compile in main CI. RUN cd /tmp \ && wget -q "https://gitlab.com/libeigen/eigen/-/archive/${EIGEN_VERSION}/eigen-${EIGEN_VERSION}.tar.gz" \ && tar xzf "eigen-${EIGEN_VERSION}.tar.gz" \ @@ -69,38 +48,9 @@ RUN cd /tmp \ && cmake --install /tmp/eigen-bld \ && rm -rf /tmp/eigen* -# Fetch Boost + eigenpy sources once. -RUN cd /tmp \ - && BU="boost_$(echo "${BOOST_VERSION}" | tr . _)" \ - && wget -q "https://archives.boost.io/release/${BOOST_VERSION}/source/${BU}.tar.bz2" \ - && wget -q "https://github.com/stack-of-tasks/eigenpy/releases/download/v${EIGENPY_VERSION}/eigenpy-${EIGENPY_VERSION}.tar.gz" - -# Build Boost.Python + eigenpy per CPython into /opt/deps/. A fresh Boost tree -# per Python avoids cross-variant b2 contamination. Mirrors CIBW_BEFORE_BUILD_LINUX. -RUN set -eux; cd /tmp; \ - BU="boost_$(echo "${BOOST_VERSION}" | tr . _)"; \ - for tag in ${PYTHON_TAGS}; do \ - PY="/opt/python/${tag}/bin/python"; \ - [ -x "$PY" ] || { echo "missing interpreter: $PY"; exit 1; }; \ - PREFIX="/opt/deps/${tag}"; \ - "$PY" -m pip install -q numpy scipy; \ - PY_VER="$("$PY" -c 'import sys;print(f"{sys.version_info.major}.{sys.version_info.minor}")')"; \ - PY_INC="$("$PY" -c 'import sysconfig;print(sysconfig.get_path("include"))')"; \ - PY_LIB="$("$PY" -c 'import sysconfig;print(sysconfig.get_config_var("LIBDIR") or "")')"; \ - rm -rf "/tmp/${BU}"; tar xjf "/tmp/${BU}.tar.bz2" -C /tmp; \ - cd "/tmp/${BU}"; \ - ./bootstrap.sh --with-python="$PY" --prefix="$PREFIX"; \ - printf 'using python : %s : %s : %s : %s ;\n' "$PY_VER" "$PY" "$PY_INC" "$PY_LIB" > user-config.jam; \ - ./b2 install -j"$(nproc)" --user-config=user-config.jam python="$PY_VER" --without-mpi; \ - cd /tmp; \ - rm -rf "eigenpy-${EIGENPY_VERSION}"; tar xzf "eigenpy-${EIGENPY_VERSION}.tar.gz"; \ - cmake -S "eigenpy-${EIGENPY_VERSION}" -B "/tmp/eigenpy-bld-${tag}" \ - -DCMAKE_PREFIX_PATH="$PREFIX" -DCMAKE_INSTALL_PREFIX="$PREFIX" -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=ON -DPython3_EXECUTABLE="$PY" \ - -DPython3_NumPy_INCLUDE_DIR="$("$PY" -c 'import numpy;print(numpy.get_include())')" \ - -DBUILD_TESTING=OFF -DCMAKE_INSTALL_DO_STRIP=ON; \ - cmake --build "/tmp/eigenpy-bld-${tag}" -j2 --target install; \ - find "$PREFIX" -name '*.so*' -type f -exec strip --strip-unneeded {} + 2>/dev/null || true; \ - rm -rf "/tmp/eigenpy-bld-${tag}"; \ - done; \ - rm -rf "/tmp/${BU}" "/tmp/${BU}.tar.bz2" /tmp/eigenpy-${EIGENPY_VERSION}* +# Prebuilt per-CPython Boost + eigenpy bundles from the parallel build-deps matrix jobs. Each +# tarball is rooted at opt/deps/, so `tar -C /` recreates /opt/deps/. +COPY deps/ /tmp/deps-tars/ +RUN set -eux; for f in /tmp/deps-tars/*.tar.gz; do echo "unpacking $f"; tar xzf "$f" -C /; done \ + && rm -rf /tmp/deps-tars \ + && ls -1 /opt/deps diff --git a/docker/manylinux-deps/README.md b/docker/manylinux-deps/README.md index 6a6ef9a40..4cdc0de35 100644 --- a/docker/manylinux-deps/README.md +++ b/docker/manylinux-deps/README.md @@ -49,6 +49,13 @@ Built and pushed by `.github/workflows/build-ci-image.yml` → `ghcr.io/bertinit Triggers: a push to `develop` touching `docker/manylinux-deps/**`, or manual dispatch. It **must run in `bertiniteam/b2`** (its token can only push to that org's GHCR). +**Parallel build.** The expensive per-CPython Boost+eigenpy builds run as a **matrix** (one +runner each, inside the manylinux container via `docker run`, using `build-python-deps.sh`), +each uploading a `deps-.tar.gz` artifact. A final `assemble-and-push` job downloads them +into `deps/` and builds the (thin, ~2-minute) assembly `Dockerfile` that just installs the +system layer and unpacks the tarballs. So wall-clock is one Python's build, not five in +series. `deps/` is gitignored (populated only in CI). + **One-time setup:** after the first push, make the package **public** in the org's Packages settings so any CI (incl. fork PRs) can pull it without auth. (Owner action — GHCR package visibility is a package setting, not something CI can flip.) diff --git a/docker/manylinux-deps/build-python-deps.sh b/docker/manylinux-deps/build-python-deps.sh new file mode 100755 index 000000000..6bef9a2ed --- /dev/null +++ b/docker/manylinux-deps/build-python-deps.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# +# Build Boost (incl. Boost.Python) + eigenpy for ONE CPython tag into /opt/deps/ and tar it. +# +# Run inside the manylinux container, ONE tag per invocation, so build-ci-image.yml can fan the +# per-CPython builds across a matrix (parallel) instead of a serial Dockerfile loop. The assembly +# Dockerfile then just COPYs the resulting tarballs in. Recipe mirrors the wheel job's historical +# CIBW_BEFORE_BUILD_LINUX exactly. +# +# Usage (from the repo root, cwd mounted at /work): +# docker run --rm -v "$PWD:/work" -w /work -e BOOST_VERSION -e EIGEN_VERSION -e EIGENPY_VERSION \ +# quay.io/pypa/manylinux_2_34_x86_64 bash docker/manylinux-deps/build-python-deps.sh cp312-cp312 +# +# Emits ./deps-.tar.gz (paths rooted at opt/deps/, so the image extracts it with -C /). + +set -euxo pipefail +TAG="${1:?usage: build-python-deps.sh }" +: "${BOOST_VERSION:?}"; : "${EIGEN_VERSION:?}"; : "${EIGENPY_VERSION:?}" +OUT="$PWD" # mounted workspace, before we cd away + +yum install -y wget gmp-devel mpfr-devel libmpc-devel libtool >/dev/null +yum clean all + +# Eigen -> /usr/local, needed only to BUILD eigenpy here (the assembly image ships its own copy for +# the bertini compile; the tarball carries only /opt/deps/). +cd /tmp +wget -q "https://gitlab.com/libeigen/eigen/-/archive/${EIGEN_VERSION}/eigen-${EIGEN_VERSION}.tar.gz" +tar xzf "eigen-${EIGEN_VERSION}.tar.gz" +cmake -S "eigen-${EIGEN_VERSION}" -B /tmp/eigen-bld -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_BUILD_TYPE=Release +cmake --install /tmp/eigen-bld + +PY="/opt/python/${TAG}/bin/python" +[ -x "$PY" ] || { echo "missing interpreter: $PY"; exit 1; } +"$PY" -m pip install -q numpy scipy +PREFIX="/opt/deps/${TAG}" +PY_VER="$("$PY" -c 'import sys;print(f"{sys.version_info.major}.{sys.version_info.minor}")')" +PY_INC="$("$PY" -c 'import sysconfig;print(sysconfig.get_path("include"))')" +PY_LIB="$("$PY" -c 'import sysconfig;print(sysconfig.get_config_var("LIBDIR") or "")')" + +cd /tmp +BU="boost_$(echo "${BOOST_VERSION}" | tr . _)" +wget -q "https://archives.boost.io/release/${BOOST_VERSION}/source/${BU}.tar.bz2" +tar xjf "${BU}.tar.bz2" +cd "${BU}" +./bootstrap.sh --with-python="$PY" --prefix="$PREFIX" +printf 'using python : %s : %s : %s : %s ;\n' "$PY_VER" "$PY" "$PY_INC" "$PY_LIB" > user-config.jam +./b2 install -j"$(nproc)" --user-config=user-config.jam python="$PY_VER" --without-mpi + +cd /tmp +wget -q "https://github.com/stack-of-tasks/eigenpy/releases/download/v${EIGENPY_VERSION}/eigenpy-${EIGENPY_VERSION}.tar.gz" +tar xzf "eigenpy-${EIGENPY_VERSION}.tar.gz" +cmake -S "eigenpy-${EIGENPY_VERSION}" -B /tmp/eigenpy-bld \ + -DCMAKE_PREFIX_PATH="$PREFIX" -DCMAKE_INSTALL_PREFIX="$PREFIX" -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=ON -DPython3_EXECUTABLE="$PY" \ + -DPython3_NumPy_INCLUDE_DIR="$("$PY" -c 'import numpy;print(numpy.get_include())')" \ + -DBUILD_TESTING=OFF -DCMAKE_INSTALL_DO_STRIP=ON +cmake --build /tmp/eigenpy-bld -j2 --target install + +find "$PREFIX" -name '*.so*' -type f -exec strip --strip-unneeded {} + 2>/dev/null || true + +tar czf "${OUT}/deps-${TAG}.tar.gz" -C / "opt/deps/${TAG}" +chmod a+r "${OUT}/deps-${TAG}.tar.gz" +ls -lh "${OUT}/deps-${TAG}.tar.gz"