From ca73c1bf4f59903e792f3c2c349917f575c5cb5e Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Mon, 20 Oct 2025 21:54:26 +0200 Subject: [PATCH 1/8] pin cibuildwheel images and manage them with dependabot --- .github/dependabot.yml | 13 ++++ .../python-dockerimages-cibuildwheel.yml | 16 +---- docker/manylinux-builder-aarch64/Dockerfile | 15 +++++ docker/manylinux-builder-x86_64/Dockerfile | 15 +++++ docker/manylinux-builder/Dockerfile | 60 ------------------- docker/manylinux-builder/build_and_push.sh | 28 --------- docker/musllinux-builder-aarch64/Dockerfile | 13 ++++ docker/musllinux-builder-x86_64/Dockerfile | 13 ++++ docker/scripts/install_boost.sh | 42 +++++++++++++ docker/scripts/install_ccache.sh | 55 +++++++++++++++++ docker/scripts/install_cmake.sh | 43 +++++++++++++ docker/scripts/install_zlib.sh | 43 +++++++++++++ 12 files changed, 255 insertions(+), 101 deletions(-) create mode 100644 docker/manylinux-builder-aarch64/Dockerfile create mode 100644 docker/manylinux-builder-x86_64/Dockerfile delete mode 100644 docker/manylinux-builder/Dockerfile delete mode 100755 docker/manylinux-builder/build_and_push.sh create mode 100644 docker/musllinux-builder-aarch64/Dockerfile create mode 100644 docker/musllinux-builder-x86_64/Dockerfile create mode 100755 docker/scripts/install_boost.sh create mode 100755 docker/scripts/install_ccache.sh create mode 100755 docker/scripts/install_cmake.sh create mode 100755 docker/scripts/install_zlib.sh diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 6fddca0d6..be7ab4ba0 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,3 +5,16 @@ updates: directory: "/" schedule: interval: "weekly" + # Docker images for cibuildwheel + - package-ecosystem: "docker" + directories: + - "/docker/manylinux-builder-aarch64" + - "/docker/manylinux-builder-x86_64" + - "/docker/musllinux-builder-aarch64" + - "/docker/musllinux-builder-x86_64" + schedule: + interval: "weekly" + groups: + docker: + patterns: + - "*" diff --git a/.github/workflows/python-dockerimages-cibuildwheel.yml b/.github/workflows/python-dockerimages-cibuildwheel.yml index d3ac05c7b..72929baff 100644 --- a/.github/workflows/python-dockerimages-cibuildwheel.yml +++ b/.github/workflows/python-dockerimages-cibuildwheel.yml @@ -4,12 +4,12 @@ on: push: branches: [ master, release-* ] paths: - - 'docker/*/Dockerfile' + - 'docker/**' - '.github/workflows/python-dockerimages-cibuildwheel.yml' pull_request: branches: [ master ] paths: - - 'docker/*/Dockerfile' + - 'docker/**' - '.github/workflows/python-dockerimages-cibuildwheel.yml' workflow_dispatch: @@ -39,22 +39,12 @@ jobs: echo "PLATFORM=linux/amd64" >> $GITHUB_ENV echo "ARCH=x86_64" >> $GITHUB_ENV if: matrix.os == 'ubuntu-24.04' - - name: Sets env for manylinux - run: | - echo "BASE_IMAGE=quay.io/pypa/manylinux_2_28_${{ env.ARCH }}" >> $GITHUB_ENV - if: matrix.base_image == 'many' - - name: Sets env for musllinux - run: | - echo "BASE_IMAGE=quay.io/pypa/musllinux_1_2_${{ env.ARCH }}" >> $GITHUB_ENV - if: matrix.base_image == 'musl' - name: Build and push ${{ matrix.base_image }}-${{ env.ARCH }} docker image uses: docker/build-push-action@v6 with: - context: docker/manylinux-builder + context: docker/${{ matrix.base_image }}linux-builder-${{ env.ARCH }} cache-from: type=gha cache-to: type=gha,mode=max platforms: ${{ env.PLATFORM }} push: ${{ github.event_name == 'push' }} - build-args: | - base_image=${{ env.BASE_IMAGE }} tags: keyvidev/${{ matrix.base_image }}linux-builder-${{ env.ARCH }} diff --git a/docker/manylinux-builder-aarch64/Dockerfile b/docker/manylinux-builder-aarch64/Dockerfile new file mode 100644 index 000000000..d1e322e26 --- /dev/null +++ b/docker/manylinux-builder-aarch64/Dockerfile @@ -0,0 +1,15 @@ +FROM quay.io/pypa/manylinux_2_28_aarch64@sha256:202a1466ef81720b304986cdb98a057def76135259642ff011218a4eef4ae55e + +RUN \ + yum -y update && \ + yum clean all && \ + yum -y install bzip2-devel snappy-devel xxhash-devel libzstd-devel python3-devel openssl-devel && \ + yum clean all; + +COPY ../scripts/ /scripts/ +WORKDIR /scripts + +RUN ./install_zlib.sh +RUN ./install_boost.sh +RUN ./install_cmake.sh +RUN ./install_ccache.sh diff --git a/docker/manylinux-builder-x86_64/Dockerfile b/docker/manylinux-builder-x86_64/Dockerfile new file mode 100644 index 000000000..9ef2eb201 --- /dev/null +++ b/docker/manylinux-builder-x86_64/Dockerfile @@ -0,0 +1,15 @@ +FROM quay.io/pypa/manylinux_2_28_x86_64@sha256:ef34ac42712ea34065f9036ec72b09b465a6da9e80f94d819a67399957229296 + +RUN \ + yum -y update && \ + yum clean all && \ + yum -y install bzip2-devel snappy-devel xxhash-devel libzstd-devel python3-devel openssl-devel && \ + yum clean all; + +COPY ../scripts/ /scripts/ +WORKDIR /scripts + +RUN ./install_zlib.sh +RUN ./install_boost.sh +RUN ./install_cmake.sh +RUN ./install_ccache.sh diff --git a/docker/manylinux-builder/Dockerfile b/docker/manylinux-builder/Dockerfile deleted file mode 100644 index 1fdb70668..000000000 --- a/docker/manylinux-builder/Dockerfile +++ /dev/null @@ -1,60 +0,0 @@ -ARG base_image -FROM $base_image - -RUN \ - if which yum; then \ - yum -y update && \ - yum clean all && \ - yum -y install bzip2-devel snappy-devel xxhash-devel libzstd-devel python3-devel openssl-devel && \ - yum clean all; \ - elif which apk; then \ - apk upgrade --no-cache && \ - apk add --no-cache procps bzip2-dev snappy-dev python3-dev zstd-dev xxhash-dev openssl-dev; \ - else \ - echo "unsupported os"; \ - exit 1; \ - fi - -ENV ZLIB_MAJOR=1 ZLIB_MINOR=3 ZLIB_PATCH=1 - -RUN curl https://zlib.net/zlib-${ZLIB_MAJOR}.${ZLIB_MINOR}.${ZLIB_PATCH}.tar.gz --output zlib-${ZLIB_MAJOR}.${ZLIB_MINOR}.${ZLIB_PATCH}.tar.gz && \ - tar -xvf zlib-${ZLIB_MAJOR}.${ZLIB_MINOR}.${ZLIB_PATCH}.tar.gz && \ - cd zlib-${ZLIB_MAJOR}.${ZLIB_MINOR}.${ZLIB_PATCH} && ./configure && make -j`nproc` && make install && \ - cd .. && rm -rf zlib-${ZLIB_MAJOR}.${ZLIB_MINOR}.${ZLIB_PATCH} zlib-${ZLIB_MAJOR}.${ZLIB_MINOR}.${ZLIB_PATCH}.tar.gz - -ENV BOOST_MAJOR=1 BOOST_MINOR=88 BOOST_PATCH=0 -ENV BOOST_URL="https://boostorg.jfrog.io/artifactory/main/release/${BOOST_MAJOR}.${BOOST_MINOR}.${BOOST_PATCH}/source/boost_${BOOST_MAJOR}_${BOOST_MINOR}_${BOOST_PATCH}.tar.gz" -ENV BOOST_FALLBACK_URL="https://sourceforge.net/projects/boost/files/boost/${BOOST_MAJOR}.${BOOST_MINOR}.${BOOST_PATCH}/boost_${BOOST_MAJOR}_${BOOST_MINOR}_${BOOST_PATCH}.tar.gz" - -RUN echo "downhload boost ${BOOST_URL}, fallback: ${BOOST_FALLBACK_URL}" -RUN curl -s -SL ${BOOST_URL} -o boost.tar.gz -RUN tar xzf boost.tar.gz 2> /dev/null || (echo "switching to fallback" && curl -s -SL ${BOOST_FALLBACK_URL} -o boost.tar.gz && tar xzf boost.tar.gz) -RUN cd boost_${BOOST_MAJOR}_${BOOST_MINOR}_${BOOST_PATCH} && \ - ./bootstrap.sh --without-libraries=graph_parallel,python, && \ - ./b2 -j`nproc` -d0 --prefix=/usr/local/ install && \ - cd .. && \ -rm -rf boost_* - -ENV CMAKE_MAJOR=3 CMAKE_MINOR=31 CMAKE_PATCH=8 - -RUN curl -sSL https://cmake.org/files/v${CMAKE_MAJOR}.${CMAKE_MINOR}/cmake-${CMAKE_MAJOR}.${CMAKE_MINOR}.${CMAKE_PATCH}.tar.gz | tar -xz && \ - cd cmake-${CMAKE_MAJOR}.${CMAKE_MINOR}.${CMAKE_PATCH} && ./bootstrap --parallel=`nproc` -- && make -j`nproc` && make install && \ - cd .. && rm -rf cmake-${CMAKE_MAJOR}.${CMAKE_MINOR}.${CMAKE_PATCH} - -ENV CCACHE_MAJOR=4 CCACHE_MINOR=11 CCACHE_PATCH=3 - -RUN \ - if [ $AUDITWHEEL_ARCH == "aarch64" ] && [ $AUDITWHEEL_POLICY == "manylinux2014" ]; then \ - echo "skiping ccache, not available for this platform"; \ - else \ - curl -sSL https://github.com/ccache/ccache/archive/refs/tags/v${CCACHE_MAJOR}.${CCACHE_MINOR}.${CCACHE_PATCH}.tar.gz | tar -xz && \ - cd ccache-${CCACHE_MAJOR}.${CCACHE_MINOR}.${CCACHE_PATCH} && mkdir build && cd build && cmake -D ENABLE_TESTING=OFF -D REDIS_STORAGE_BACKEND=OFF -D CMAKE_BUILD_TYPE=Release .. && make -j`nproc` && make install && \ - cd ../.. && rm -rf ccache-${CCACHE_MAJOR}.${CCACHE_MINOR}.${CCACHE_PATCH}; \ - fi - -RUN ln -s /usr/local/bin/ccache /usr/local/bin/gcc && \ - ln -s /usr/local/bin/ccache /usr/local/bin/g++ && \ - ln -s /usr/local/bin/ccache /usr/local/bin/cc && \ - ln -s /usr/local/bin/ccache /usr/local/bin/c++ && \ - ln -s /usr/local/bin/ccache /usr/local/bin/clang && \ - ln -s /usr/local/bin/ccache /usr/local/bin/clang++ diff --git a/docker/manylinux-builder/build_and_push.sh b/docker/manylinux-builder/build_and_push.sh deleted file mode 100755 index 2e94256f0..000000000 --- a/docker/manylinux-builder/build_and_push.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash - -set -euxo pipefail - -docker build . \ - -f Dockerfile \ - --build-arg base_image="quay.io/pypa/manylinux_2_28_x86_64" \ - -t keyvidev/manylinux-builder-x86_64 - -docker build . \ - -f Dockerfile \ - --build-arg base_image="quay.io/pypa/manylinux_2_28_aarch64" \ - -t keyvidev/manylinux-builder-aarch64 - -docker build . \ - -f Dockerfile \ - --build-arg base_image="quay.io/pypa/musllinux_1_1_x86_64" \ - -t keyvidev/musllinux-builder-x86_64 - -docker build . \ - -f Dockerfile \ - --build-arg base_image="quay.io/pypa/musllinux_1_1_aarch64" \ - -t keyvidev/musllinux-builder-aarch64 - -docker push keyvidev/manylinux-builder-x86_64 -docker push keyvidev/manylinux-builder-aarch64 -docker push keyvidev/musllinux-builder-x86_64 -docker push keyvidev/musllinux-builder-aarch64 diff --git a/docker/musllinux-builder-aarch64/Dockerfile b/docker/musllinux-builder-aarch64/Dockerfile new file mode 100644 index 000000000..fc24e08a3 --- /dev/null +++ b/docker/musllinux-builder-aarch64/Dockerfile @@ -0,0 +1,13 @@ +FROM quay.io/pypa/musllinux_1_2_aarch64@sha256:5ebf27586ed7e77f54b41a3c4cecd8e271e62b2d30917a951f2e3b80201774e8 + +RUN \ + apk upgrade --no-cache && \ + apk add --no-cache procps bzip2-dev snappy-dev python3-dev zstd-dev xxhash-dev openssl-dev; + +COPY ../scripts/ /scripts/ +WORKDIR /scripts + +RUN ./install_zlib.sh +RUN ./install_boost.sh +RUN ./install_cmake.sh +RUN ./install_ccache.sh diff --git a/docker/musllinux-builder-x86_64/Dockerfile b/docker/musllinux-builder-x86_64/Dockerfile new file mode 100644 index 000000000..49fb30f1d --- /dev/null +++ b/docker/musllinux-builder-x86_64/Dockerfile @@ -0,0 +1,13 @@ +FROM quay.io/pypa/musllinux_1_2_x86_64@sha256:c5071dd790d06f34e8ef0a36310988d6a1ad77d3b743ba1495e09529145833d0 + +RUN \ + apk upgrade --no-cache && \ + apk add --no-cache procps bzip2-dev snappy-dev python3-dev zstd-dev xxhash-dev openssl-dev; + +COPY ../scripts/ /scripts/ +WORKDIR /scripts + +RUN ./install_zlib.sh +RUN ./install_boost.sh +RUN ./install_cmake.sh +RUN ./install_ccache.sh diff --git a/docker/scripts/install_boost.sh b/docker/scripts/install_boost.sh new file mode 100755 index 000000000..14134133e --- /dev/null +++ b/docker/scripts/install_boost.sh @@ -0,0 +1,42 @@ +#!/bin/sh +set -eu + +BOOST_MAJOR=1 +BOOST_MINOR=88 +BOOST_PATCH=0 + +BOOST_URL="https://archives.boost.io/release/${BOOST_MAJOR}.${BOOST_MINOR}.${BOOST_PATCH}/source/boost_${BOOST_MAJOR}_${BOOST_MINOR}_${BOOST_PATCH}.tar.gz" +BOOST_FALLBACK_URL="https://sourceforge.net/projects/boost/files/boost/${BOOST_MAJOR}.${BOOST_MINOR}.${BOOST_PATCH}/boost_${BOOST_MAJOR}_${BOOST_MINOR}_${BOOST_PATCH}.tar.gz" +BOOST_DIR="boost_${BOOST_MAJOR}_${BOOST_MINOR}_${BOOST_PATCH}" + +echo "Downloading Boost from ${BOOST_URL} (fallback: ${BOOST_FALLBACK_URL})" + +if curl -fsSL "${BOOST_URL}" -o boost.tar.gz; then + echo "Downloaded Boost successfully." +else + echo "Primary download failed. Switching to fallback..." + curl -fsSL "${BOOST_FALLBACK_URL}" -o boost.tar.gz +fi + +echo "Extracting boost.tar.gz ..." +tar xzf boost.tar.gz + +cd "${BOOST_DIR}" + +echo "Bootstrapping Boost..." +./bootstrap.sh --without-libraries=graph_parallel,python + +# Use nproc if available, otherwise default to 1 +if command -v nproc >/dev/null 2>&1; then + JOBS=$(nproc) +else + JOBS=1 +fi + +echo "Building and installing Boost with ${JOBS} jobs..." +./b2 -j"${JOBS}" -d0 --prefix=/usr/local/ install + +cd .. +rm -rf "boost_*" boost.tar.gz + +echo "Boost ${BOOST_MAJOR}.${BOOST_MINOR}.${BOOST_PATCH} installed successfully." diff --git a/docker/scripts/install_ccache.sh b/docker/scripts/install_ccache.sh new file mode 100755 index 000000000..7a6720c0e --- /dev/null +++ b/docker/scripts/install_ccache.sh @@ -0,0 +1,55 @@ +#!/bin/sh +set -eu + +CCACHE_MAJOR=4 +CCACHE_MINOR=11 +CCACHE_PATCH=3 + +CCACHE_VERSION="${CCACHE_MAJOR}.${CCACHE_MINOR}.${CCACHE_PATCH}" +CCACHE_TAR="ccache-${CCACHE_VERSION}.tar.gz" +CCACHE_DIR="ccache-${CCACHE_VERSION}" +CCACHE_URL="https://github.com/ccache/ccache/archive/refs/tags/v${CCACHE_VERSION}.tar.gz" + +# --- Helpers --- +if command -v nproc >/dev/null 2>&1; then + JOBS=$(nproc) +else + JOBS=1 +fi + +# --- Download & Extract --- +echo "Downloading ccache ${CCACHE_VERSION} from ${CCACHE_URL}" +curl -fsSL "${CCACHE_URL}" -o "${CCACHE_TAR}" + +echo "Extracting ${CCACHE_TAR}..." +tar -xzf "${CCACHE_TAR}" + +# --- Build & Install --- +cd "${CCACHE_DIR}" +mkdir build +cd build + +echo "Configuring ccache..." +cmake -D ENABLE_TESTING=OFF \ + -D REDIS_STORAGE_BACKEND=OFF \ + -D CMAKE_BUILD_TYPE=Release .. + +echo "Building with ${JOBS} jobs..." +make -j"${JOBS}" + +echo "Installing ccache..." +make install + +# --- Cleanup --- +cd ../.. +rm -rf "${CCACHE_DIR}" "${CCACHE_TAR}" + +# --- Symlinks --- +echo "Creating compiler symlinks for ccache..." +for bin in gcc g++ cc c++ clang clang++; do + if [ ! -e "/usr/local/bin/$bin" ]; then + ln -s /usr/local/bin/ccache "/usr/local/bin/$bin" || true + fi +done + +echo "ccache ${CCACHE_VERSION} installed successfully." diff --git a/docker/scripts/install_cmake.sh b/docker/scripts/install_cmake.sh new file mode 100755 index 000000000..527a72c05 --- /dev/null +++ b/docker/scripts/install_cmake.sh @@ -0,0 +1,43 @@ +#!/bin/sh +set -eu + +CMAKE_MAJOR=4 +CMAKE_MINOR=1 +CMAKE_PATCH=2 + +CMAKE_VERSION="${CMAKE_MAJOR}.${CMAKE_MINOR}.${CMAKE_PATCH}" +CMAKE_TAR="cmake-${CMAKE_VERSION}.tar.gz" +CMAKE_DIR="cmake-${CMAKE_VERSION}" +CMAKE_URL="https://cmake.org/files/v${CMAKE_MAJOR}.${CMAKE_MINOR}/${CMAKE_TAR}" + +# --- Helpers --- +if command -v nproc >/dev/null 2>&1; then + JOBS=$(nproc) +else + JOBS=1 +fi + +# --- Download & Extract --- +echo "Downloading CMake ${CMAKE_VERSION} from ${CMAKE_URL}" +curl -fsSL "${CMAKE_URL}" -o "${CMAKE_TAR}" + +echo "Extracting ${CMAKE_TAR}..." +tar -xzf "${CMAKE_TAR}" + +# --- Build & Install --- +cd "${CMAKE_DIR}" + +echo "Bootstrapping CMake..." +./bootstrap --parallel="${JOBS}" -- + +echo "Building CMake with ${JOBS} jobs..." +make -j"${JOBS}" + +echo "Installing CMake..." +make install + +# --- Cleanup --- +cd .. +rm -rf "${CMAKE_DIR}" "${CMAKE_TAR}" + +echo "CMake ${CMAKE_VERSION} installed successfully." diff --git a/docker/scripts/install_zlib.sh b/docker/scripts/install_zlib.sh new file mode 100755 index 000000000..48b223e31 --- /dev/null +++ b/docker/scripts/install_zlib.sh @@ -0,0 +1,43 @@ +#!/bin/sh +set -eu + +ZLIB_MAJOR=1 +ZLIB_MINOR=3 +ZLIB_PATCH=1 + +ZLIB_VERSION="${ZLIB_MAJOR}.${ZLIB_MINOR}.${ZLIB_PATCH}" +ZLIB_TAR="zlib-${ZLIB_VERSION}.tar.gz" +ZLIB_DIR="zlib-${ZLIB_VERSION}" +ZLIB_URL="https://zlib.net/${ZLIB_TAR}" + +# --- Helpers --- +if command -v nproc >/dev/null 2>&1; then + JOBS=$(nproc) +else + JOBS=1 +fi + +# --- Download & Extract --- +echo "Downloading zlib ${ZLIB_VERSION} from ${ZLIB_URL}" +curl -fsSL "${ZLIB_URL}" -o "${ZLIB_TAR}" + +echo "Extracting ${ZLIB_TAR}..." +tar -xzf "${ZLIB_TAR}" + +# --- Build & Install --- +cd "${ZLIB_DIR}" + +echo "Configuring zlib..." +./configure + +echo "Building zlib with ${JOBS} jobs..." +make -j"${JOBS}" + +echo "Installing zlib..." +make install + +# --- Cleanup --- +cd .. +rm -rf "${ZLIB_DIR}" "${ZLIB_TAR}" + +echo "zlib ${ZLIB_VERSION} installed successfully." From f5d247a1445444f3e33ed23353b62a6c198f6c91 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Mon, 20 Oct 2025 21:58:20 +0200 Subject: [PATCH 2/8] fix directory --- docker/manylinux-builder-aarch64/Dockerfile | 2 +- docker/manylinux-builder-x86_64/Dockerfile | 2 +- docker/musllinux-builder-aarch64/Dockerfile | 2 +- docker/musllinux-builder-x86_64/Dockerfile | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/manylinux-builder-aarch64/Dockerfile b/docker/manylinux-builder-aarch64/Dockerfile index d1e322e26..fb39da1c9 100644 --- a/docker/manylinux-builder-aarch64/Dockerfile +++ b/docker/manylinux-builder-aarch64/Dockerfile @@ -6,7 +6,7 @@ RUN \ yum -y install bzip2-devel snappy-devel xxhash-devel libzstd-devel python3-devel openssl-devel && \ yum clean all; -COPY ../scripts/ /scripts/ +COPY docker/scripts/ /scripts/ WORKDIR /scripts RUN ./install_zlib.sh diff --git a/docker/manylinux-builder-x86_64/Dockerfile b/docker/manylinux-builder-x86_64/Dockerfile index 9ef2eb201..1d3239013 100644 --- a/docker/manylinux-builder-x86_64/Dockerfile +++ b/docker/manylinux-builder-x86_64/Dockerfile @@ -6,7 +6,7 @@ RUN \ yum -y install bzip2-devel snappy-devel xxhash-devel libzstd-devel python3-devel openssl-devel && \ yum clean all; -COPY ../scripts/ /scripts/ +COPY docker/scripts/ /scripts/ WORKDIR /scripts RUN ./install_zlib.sh diff --git a/docker/musllinux-builder-aarch64/Dockerfile b/docker/musllinux-builder-aarch64/Dockerfile index fc24e08a3..f0f52109a 100644 --- a/docker/musllinux-builder-aarch64/Dockerfile +++ b/docker/musllinux-builder-aarch64/Dockerfile @@ -4,7 +4,7 @@ RUN \ apk upgrade --no-cache && \ apk add --no-cache procps bzip2-dev snappy-dev python3-dev zstd-dev xxhash-dev openssl-dev; -COPY ../scripts/ /scripts/ +COPY docker/scripts/ /scripts/ WORKDIR /scripts RUN ./install_zlib.sh diff --git a/docker/musllinux-builder-x86_64/Dockerfile b/docker/musllinux-builder-x86_64/Dockerfile index 49fb30f1d..965fab55e 100644 --- a/docker/musllinux-builder-x86_64/Dockerfile +++ b/docker/musllinux-builder-x86_64/Dockerfile @@ -4,7 +4,7 @@ RUN \ apk upgrade --no-cache && \ apk add --no-cache procps bzip2-dev snappy-dev python3-dev zstd-dev xxhash-dev openssl-dev; -COPY ../scripts/ /scripts/ +COPY docker/scripts/ /scripts/ WORKDIR /scripts RUN ./install_zlib.sh From 7104f63b180bf59508c885b646bd40b205e79f43 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Mon, 20 Oct 2025 22:13:31 +0200 Subject: [PATCH 3/8] make work with buildx --- docker/manylinux-builder-aarch64/Dockerfile | 2 +- docker/manylinux-builder-x86_64/Dockerfile | 2 +- docker/musllinux-builder-aarch64/Dockerfile | 2 +- docker/musllinux-builder-x86_64/Dockerfile | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/manylinux-builder-aarch64/Dockerfile b/docker/manylinux-builder-aarch64/Dockerfile index fb39da1c9..f84edf75d 100644 --- a/docker/manylinux-builder-aarch64/Dockerfile +++ b/docker/manylinux-builder-aarch64/Dockerfile @@ -6,7 +6,7 @@ RUN \ yum -y install bzip2-devel snappy-devel xxhash-devel libzstd-devel python3-devel openssl-devel && \ yum clean all; -COPY docker/scripts/ /scripts/ +COPY --from=scripts . /scripts/ WORKDIR /scripts RUN ./install_zlib.sh diff --git a/docker/manylinux-builder-x86_64/Dockerfile b/docker/manylinux-builder-x86_64/Dockerfile index 1d3239013..aa0e0b2cb 100644 --- a/docker/manylinux-builder-x86_64/Dockerfile +++ b/docker/manylinux-builder-x86_64/Dockerfile @@ -6,7 +6,7 @@ RUN \ yum -y install bzip2-devel snappy-devel xxhash-devel libzstd-devel python3-devel openssl-devel && \ yum clean all; -COPY docker/scripts/ /scripts/ +COPY --from=scripts . /scripts/ WORKDIR /scripts RUN ./install_zlib.sh diff --git a/docker/musllinux-builder-aarch64/Dockerfile b/docker/musllinux-builder-aarch64/Dockerfile index f0f52109a..ac7d94fbc 100644 --- a/docker/musllinux-builder-aarch64/Dockerfile +++ b/docker/musllinux-builder-aarch64/Dockerfile @@ -4,7 +4,7 @@ RUN \ apk upgrade --no-cache && \ apk add --no-cache procps bzip2-dev snappy-dev python3-dev zstd-dev xxhash-dev openssl-dev; -COPY docker/scripts/ /scripts/ +COPY --from=scripts . /scripts/ WORKDIR /scripts RUN ./install_zlib.sh diff --git a/docker/musllinux-builder-x86_64/Dockerfile b/docker/musllinux-builder-x86_64/Dockerfile index 965fab55e..5811e4aeb 100644 --- a/docker/musllinux-builder-x86_64/Dockerfile +++ b/docker/musllinux-builder-x86_64/Dockerfile @@ -4,7 +4,7 @@ RUN \ apk upgrade --no-cache && \ apk add --no-cache procps bzip2-dev snappy-dev python3-dev zstd-dev xxhash-dev openssl-dev; -COPY docker/scripts/ /scripts/ +COPY --from=scripts . /scripts/ WORKDIR /scripts RUN ./install_zlib.sh From d7536a67f45a82a4c48254984a060525d93ea536 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Mon, 20 Oct 2025 22:28:31 +0200 Subject: [PATCH 4/8] add build context --- .github/dependabot.yml | 2 +- .github/workflows/python-dockerimages-cibuildwheel.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index be7ab4ba0..f11b10049 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -7,7 +7,7 @@ updates: interval: "weekly" # Docker images for cibuildwheel - package-ecosystem: "docker" - directories: + directories: - "/docker/manylinux-builder-aarch64" - "/docker/manylinux-builder-x86_64" - "/docker/musllinux-builder-aarch64" diff --git a/.github/workflows/python-dockerimages-cibuildwheel.yml b/.github/workflows/python-dockerimages-cibuildwheel.yml index 72929baff..215c401d9 100644 --- a/.github/workflows/python-dockerimages-cibuildwheel.yml +++ b/.github/workflows/python-dockerimages-cibuildwheel.yml @@ -43,6 +43,7 @@ jobs: uses: docker/build-push-action@v6 with: context: docker/${{ matrix.base_image }}linux-builder-${{ env.ARCH }} + build-contexts: scripts=docker/scripts cache-from: type=gha cache-to: type=gha,mode=max platforms: ${{ env.PLATFORM }} From 1a3c85c75efe6b67959c2fc174b17c346581e66c Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Mon, 20 Oct 2025 22:35:08 +0200 Subject: [PATCH 5/8] only build(and upload) from push on master --- .github/workflows/python-dockerimages-cibuildwheel.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-dockerimages-cibuildwheel.yml b/.github/workflows/python-dockerimages-cibuildwheel.yml index 215c401d9..d8e517730 100644 --- a/.github/workflows/python-dockerimages-cibuildwheel.yml +++ b/.github/workflows/python-dockerimages-cibuildwheel.yml @@ -2,7 +2,7 @@ name: Python Linux docker cibuildwheel on: push: - branches: [ master, release-* ] + branches: [ master ] paths: - 'docker/**' - '.github/workflows/python-dockerimages-cibuildwheel.yml' From 9217329b983428a3412874a1711f283faea0e5c6 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Tue, 21 Oct 2025 04:59:11 +0200 Subject: [PATCH 6/8] update path filters --- .github/workflows/docs.yml | 11 +++++++++++ .github/workflows/keyvi.yml | 19 ++++++++++++++++++- .github/workflows/python-cibuildwheel.yml | 12 ++++++++++++ .github/workflows/rust.yml | 14 ++++++++++++++ 4 files changed, 55 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 5afbeff4e..c7586e1af 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -6,8 +6,19 @@ on: # Triggers the workflow on push or pull request events but only for the master branch push: branches: [ master ] + paths-ignore: + - '.github/workflows/keyvi.yml' + - '.github/workflows/python**.yml' + - '.github/workflows/rust**.yml' + - 'docker/**' + pull_request: branches: [ master ] + paths-ignore: + - '.github/workflows/keyvi.yml' + - '.github/workflows/python**.yml' + - '.github/workflows/rust**.yml' + - 'docker/**' # Allows you to run this workflow manually from the Actions tab workflow_dispatch: diff --git a/.github/workflows/keyvi.yml b/.github/workflows/keyvi.yml index 16f03c995..e5ebf2f65 100644 --- a/.github/workflows/keyvi.yml +++ b/.github/workflows/keyvi.yml @@ -1,13 +1,30 @@ # Build on Linux -name: Build keyvi +name: Build keyvi core # Controls when the action will run. on: # Triggers the workflow on push or pull request events but only for the master branch push: + paths-ignore: + - '.github/workflows/docs.yml' + - '.github/workflows/python**.yml' + - '.github/workflows/rust**.yml' + - 'docker/**' + - 'doc/**' + - 'python/**' + - 'rust/**' + pull_request: branches: [ master ] + paths-ignore: + - '.github/workflows/docs.yml' + - '.github/workflows/python**.yml' + - '.github/workflows/rust**.yml' + - 'docker/**' + - 'doc/**' + - 'python/**' + - 'rust/**' # Allows you to run this workflow manually from the Actions tab workflow_dispatch: diff --git a/.github/workflows/python-cibuildwheel.yml b/.github/workflows/python-cibuildwheel.yml index d34f462da..9a4a062cc 100644 --- a/.github/workflows/python-cibuildwheel.yml +++ b/.github/workflows/python-cibuildwheel.yml @@ -3,8 +3,20 @@ name: Python cibuildwheel on: push: branches: [ master, release-* ] + paths-ignore: + - '.github/workflows/python-dockerimages-**.yml' + - '.github/workflows/rust**.yml' + - 'docker/**' + - 'doc/**' + - 'rust/**' pull_request: branches: [ master ] + paths-ignore: + - '.github/workflows/python-dockerimages-**.yml' + - '.github/workflows/rust**.yml' + - 'docker/**' + - 'doc/**' + - 'rust/**' release: types: [published] workflow_dispatch: diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 94ba13e94..5f96b5a92 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -3,8 +3,22 @@ name: Rust on: push: branches: [ master, release-* ] + paths-ignore: + - '.github/workflows/docs.yml' + - '.github/workflows/keyvi.yml' + - '.github/workflows/python**.yml' + - 'docker/**' + - 'doc/**' + - 'python/**' pull_request: branches: [master] + paths-ignore: + - '.github/workflows/docs.yml' + - '.github/workflows/keyvi.yml' + - '.github/workflows/python**.yml' + - 'docker/**' + - 'doc/**' + - 'python/**' release: types: [published] workflow_dispatch: From 206116501290fdf3636e07c8ddd89f7eea80b756 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Tue, 21 Oct 2025 05:03:01 +0200 Subject: [PATCH 7/8] pre-commit --- .github/workflows/docs.yml | 6 +++--- .github/workflows/keyvi.yml | 2 +- .github/workflows/python-cibuildwheel.yml | 10 +++++----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index c7586e1af..9d580da84 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,7 +1,7 @@ # Build on Linux name: Build keyvi documentation -# Controls when the action will run. +# Controls when the action will run. on: # Triggers the workflow on push or pull request events but only for the master branch push: @@ -58,14 +58,14 @@ jobs: export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" python -m pip install -r python/requirements.txt python -m pip install sphinx breathe sphinx-wagtail-theme - + - name: build python extension run: | export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" cd python python setup.py build --mode ${BUILD_TYPE} python setup.py install --user - + - name: build with cmake uses: lukka/run-cmake@v3 with: diff --git a/.github/workflows/keyvi.yml b/.github/workflows/keyvi.yml index e5ebf2f65..a7710e474 100644 --- a/.github/workflows/keyvi.yml +++ b/.github/workflows/keyvi.yml @@ -2,7 +2,7 @@ name: Build keyvi core -# Controls when the action will run. +# Controls when the action will run. on: # Triggers the workflow on push or pull request events but only for the master branch push: diff --git a/.github/workflows/python-cibuildwheel.yml b/.github/workflows/python-cibuildwheel.yml index 9a4a062cc..7fd688f07 100644 --- a/.github/workflows/python-cibuildwheel.yml +++ b/.github/workflows/python-cibuildwheel.yml @@ -102,14 +102,14 @@ jobs: CIBW_MANYLINUX_AARCH64_IMAGE: 'keyvidev/manylinux-builder-aarch64' CIBW_MUSLLINUX_X86_64_IMAGE: 'keyvidev/musllinux-builder-x86_64' CIBW_MUSLLINUX_AARCH64_IMAGE: 'keyvidev/musllinux-builder-aarch64' - + # ccache using path CIBW_ENVIRONMENT_MACOS: PATH=/usr/local/opt/ccache/libexec:$PATH CIBW_ENVIRONMENT_LINUX: PATH=/usr/local/bin:/usr/lib/ccache:$PATH CCACHE_DIR=/host${{ github.workspace }}/.ccache CCACHE_CONFIGPATH=/host/home/runner/.config/ccache/ccache.conf - + # python dependencies CIBW_BEFORE_BUILD: pip install -r python/requirements.txt - + # testing CIBW_TEST_REQUIRES: pytest python-snappy zstd CIBW_TEST_COMMAND: > @@ -120,7 +120,7 @@ jobs: # CIBW_BUILD_VERBOSITY: 2 with: package-dir: python - + - uses: actions/upload-artifact@v4 with: name: artifact-${{ matrix.os }}-${{ matrix.flavor }}-${{ matrix.target }} @@ -172,5 +172,5 @@ jobs: pattern: artifact-* merge-multiple: true path: dist - + - uses: pypa/gh-action-pypi-publish@release/v1 From f06743a321b457cc19e1fded09829fdc0678854d Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Tue, 21 Oct 2025 05:07:36 +0200 Subject: [PATCH 8/8] filter --- .github/workflows/python-cibuildwheel.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/python-cibuildwheel.yml b/.github/workflows/python-cibuildwheel.yml index 7fd688f07..66b5cfe96 100644 --- a/.github/workflows/python-cibuildwheel.yml +++ b/.github/workflows/python-cibuildwheel.yml @@ -4,6 +4,8 @@ on: push: branches: [ master, release-* ] paths-ignore: + - '.github/workflows/docs.yml' + - '.github/workflows/keyvi.yml' - '.github/workflows/python-dockerimages-**.yml' - '.github/workflows/rust**.yml' - 'docker/**' @@ -12,6 +14,8 @@ on: pull_request: branches: [ master ] paths-ignore: + - '.github/workflows/docs.yml' + - '.github/workflows/keyvi.yml' - '.github/workflows/python-dockerimages-**.yml' - '.github/workflows/rust**.yml' - 'docker/**'