From 790acf76d15b3c37f75a07a7551dd6a482d95198 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Sat, 29 Mar 2025 19:32:13 +0100 Subject: [PATCH 1/5] drop QEMU builds and use native arm runners for Linux and Mac --- .github/workflows/python-cibuildwheel.yml | 68 ++++------------------- 1 file changed, 10 insertions(+), 58 deletions(-) diff --git a/.github/workflows/python-cibuildwheel.yml b/.github/workflows/python-cibuildwheel.yml index 783ee2abe..7b7155493 100644 --- a/.github/workflows/python-cibuildwheel.yml +++ b/.github/workflows/python-cibuildwheel.yml @@ -11,14 +11,13 @@ on: jobs: build_wheels: - name: cibuildwheel ${{ matrix.os }}/${{ matrix.arch }}/${{ matrix.flavor }}/${{ matrix.target }} + name: cibuildwheel ${{ matrix.os }}/${{ matrix.flavor }}/${{ matrix.target }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - os: [ubuntu-22.04, macos-13] - # separate archs, so they use individual caches - arch: [ 'x86_64', 'arm64' ] + # macos-13: x86, macos-14: Arm64 + os: [ubuntu-22.04, ubuntu-24.04-arm, macos-13, macos-14] # skip pypy, https://github.com/pypa/distutils/issues/283 flavor: ['cpython'] # separate musl and many on linux, for mac we just skip one of those @@ -31,31 +30,13 @@ jobs: # target: musl # flavor: pypy #- os: macos-13 - # arch: arm64 # flavor: pypy steps: - uses: actions/checkout@v4 - - name: Set up QEMU - if: ${{ (runner.os == 'Linux') && (matrix.arch == 'arm64') }} - uses: docker/setup-qemu-action@v3 - with: - platforms: all - name: ccache uses: hendrikmuhs/ccache-action@v1.2.11 with: - key: ${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.target }}-${{ matrix.flavor }}-python - - - name: Sets env for x86_64 - run: | - echo "CIBW_ARCHS_LINUX=auto64" >> $GITHUB_ENV - echo "CIBW_ARCHS_MACOS=x86_64" >> $GITHUB_ENV - if: matrix.arch == 'x86_64' - - - name: Sets env for arm64 - run: | - echo "CIBW_ARCHS_LINUX=aarch64" >> $GITHUB_ENV - echo "CIBW_ARCHS_MACOS=arm64" >> $GITHUB_ENV - if: matrix.arch == 'arm64' + key: ${{ matrix.os }}-${{ matrix.target }}-${{ matrix.flavor }}-python - name: Skip manylinux for musllinux target if: ${{ (runner.os == 'Linux') && (matrix.target == 'musl') }} @@ -83,45 +64,16 @@ jobs: run: | brew update && \ brew list -1 | grep python | while read formula; do brew unlink $formula; brew link --overwrite $formula; done && \ - brew install ccache - - - name: install mac dependencies X86_64 - if: ${{ (runner.os == 'macOS') && (matrix.arch == 'x86_64') }} - run: | - echo "MACOSX_DEPLOYMENT_TARGET=13.0" >> $GITHUB_ENV && \ - brew update && \ - brew install zlib snappy boost - - - name: install mac dependencies arm64 - if: ${{ (runner.os == 'macOS') && (matrix.arch == 'arm64') }} - # explicitly install icu4c@76 and boost@1.87, not icu4c/boost to avoid unsupported combinations - # Note: when brew reinstall finds unsolved deps, it downloads and installs using the wrong arch(x86_64 instead of arm64) - run: | - set -e - echo "MACOSX_DEPLOYMENT_TARGET=13.0" >> $GITHUB_ENV - echo "_CMAKE_PREFIX_PATH=${{ github.workspace }}/arm64-homebrew" >> $GITHUB_ENV - echo "CIBW_REPAIR_WHEEL_COMMAND_MACOS=DYLD_LIBRARY_PATH=${{ github.workspace }}/arm64-homebrew delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}" >> $GITHUB_ENV - mkdir arm64-homebrew && curl -L https://github.com/Homebrew/brew/tarball/master | tar xz --strip 1 -C arm64-homebrew - PACKAGES=( icu4c@76 xz lz4 zstd zlib snappy boost@1.87 ) - for PACKAGE in "${PACKAGES[@]}" - do - arm64-homebrew/bin/brew info $PACKAGE - response=$(arm64-homebrew/bin/brew fetch --force --bottle-tag=arm64_ventura $PACKAGE | grep Downloaded ) - download_path=$(echo $response | xargs -n 1 | tail -1) - arm64-homebrew/bin/brew reinstall -vd $download_path - done - arm64-homebrew/bin/brew config - ls /Users/runner/work/keyvi/keyvi/arm64-homebrew + brew install ccache zlib snappy boost - - name: Build python wheels for ${{ matrix.os }} on ${{ matrix.arch }} - uses: pypa/cibuildwheel@v2.22.0 + - name: Build python wheels for ${{ matrix.os }} + uses: pypa/cibuildwheel@v2.23.1 env: # Skip CPython 3.6 and CPython 3.7 CIBW_SKIP: ${{ env.CIBW_SKIP }} cp36-* cp37-* - # skip testing all python versions on linux arm, only test 3.12 # skip tests on pypy, currently fails for indexer tests - CIBW_TEST_SKIP: "*p{38,39,310,311}-m*linux_aarch64 pp*" + CIBW_TEST_SKIP: "pp*" # (many)linux custom docker images CIBW_MANYLINUX_X86_64_IMAGE: 'keyvidev/manylinux-builder-x86_64' @@ -149,7 +101,7 @@ jobs: - uses: actions/upload-artifact@v4 with: - name: artifact-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.flavor }}-${{ matrix.target }} + name: artifact-${{ matrix.os }}-${{ matrix.flavor }}-${{ matrix.target }} path: ./wheelhouse/*.whl build_sdist: @@ -164,7 +116,7 @@ jobs: - name: ccache uses: hendrikmuhs/ccache-action@v1.2.11 with: - key: ${{ matrix.os }}-sdist-python + key: ubuntu-sdist-python - name: Build SDist run: | From 880ade2a717dcda27643f677f3556a5d0789fcdf Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Sat, 29 Mar 2025 19:51:31 +0100 Subject: [PATCH 2/5] set MACOSX_DEPLOYMENT_TARGET and drop python 3.8 --- .github/workflows/python-cibuildwheel.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-cibuildwheel.yml b/.github/workflows/python-cibuildwheel.yml index 7b7155493..1279bf241 100644 --- a/.github/workflows/python-cibuildwheel.yml +++ b/.github/workflows/python-cibuildwheel.yml @@ -66,11 +66,21 @@ jobs: brew list -1 | grep python | while read formula; do brew unlink $formula; brew link --overwrite $formula; done && \ brew install ccache zlib snappy boost + - name: set mac deployment target X86_64 + if: runner.os == 'macOS' && runner.arch == 'x86_64' + run: | + echo "MACOSX_DEPLOYMENT_TARGET=13.0" + + - name: set mac deployment target arm64 + if: runner.os == 'macOS' && runner.arch == 'ARM64' + run: | + echo "MACOSX_DEPLOYMENT_TARGET=14.0" + - name: Build python wheels for ${{ matrix.os }} uses: pypa/cibuildwheel@v2.23.1 env: - # Skip CPython 3.6 and CPython 3.7 - CIBW_SKIP: ${{ env.CIBW_SKIP }} cp36-* cp37-* + # Skip CPython 3.{6, 7, 8} + CIBW_SKIP: ${{ env.CIBW_SKIP }} cp36-* cp37-* cp38-* # skip tests on pypy, currently fails for indexer tests CIBW_TEST_SKIP: "pp*" From e36e02035709b3de677cf5a297aed762fcdeab0c Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Sat, 29 Mar 2025 19:53:16 +0100 Subject: [PATCH 3/5] drop python 3.8 --- python/setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/setup.py b/python/setup.py index 6f3279d68..df785c0d2 100644 --- a/python/setup.py +++ b/python/setup.py @@ -413,7 +413,6 @@ class build_ext(custom_opts, build_cxx): "Programming Language :: C++", "Programming Language :: Cython", "Programming Language :: Python", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", From 17110e3bcfc209a360c5b1d5bf8f3596da094fc0 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Sat, 29 Mar 2025 20:00:31 +0100 Subject: [PATCH 4/5] fix runner variable --- .github/workflows/python-cibuildwheel.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-cibuildwheel.yml b/.github/workflows/python-cibuildwheel.yml index 1279bf241..471bf9d60 100644 --- a/.github/workflows/python-cibuildwheel.yml +++ b/.github/workflows/python-cibuildwheel.yml @@ -66,8 +66,8 @@ jobs: brew list -1 | grep python | while read formula; do brew unlink $formula; brew link --overwrite $formula; done && \ brew install ccache zlib snappy boost - - name: set mac deployment target X86_64 - if: runner.os == 'macOS' && runner.arch == 'x86_64' + - name: set mac deployment target X64 + if: runner.os == 'macOS' && runner.arch == 'X64' run: | echo "MACOSX_DEPLOYMENT_TARGET=13.0" From de5c8a6de851393a746b896d92b84b42bf7d0bcc Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Sat, 29 Mar 2025 21:11:08 +0100 Subject: [PATCH 5/5] only build native packages --- .github/workflows/python-cibuildwheel.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-cibuildwheel.yml b/.github/workflows/python-cibuildwheel.yml index 471bf9d60..68d1b9f6d 100644 --- a/.github/workflows/python-cibuildwheel.yml +++ b/.github/workflows/python-cibuildwheel.yml @@ -69,12 +69,12 @@ jobs: - name: set mac deployment target X64 if: runner.os == 'macOS' && runner.arch == 'X64' run: | - echo "MACOSX_DEPLOYMENT_TARGET=13.0" + echo "MACOSX_DEPLOYMENT_TARGET=13.0" >> $GITHUB_ENV - name: set mac deployment target arm64 if: runner.os == 'macOS' && runner.arch == 'ARM64' run: | - echo "MACOSX_DEPLOYMENT_TARGET=14.0" + echo "MACOSX_DEPLOYMENT_TARGET=14.0" >> $GITHUB_ENV - name: Build python wheels for ${{ matrix.os }} uses: pypa/cibuildwheel@v2.23.1 @@ -82,6 +82,9 @@ jobs: # Skip CPython 3.{6, 7, 8} CIBW_SKIP: ${{ env.CIBW_SKIP }} cp36-* cp37-* cp38-* + # only build native packages + CIBW_ARCHS: native + # skip tests on pypy, currently fails for indexer tests CIBW_TEST_SKIP: "pp*"