diff --git a/dev/tasks/java-jars/github.yml b/dev/tasks/java-jars/github.yml index 9dd69a41f0b8..a99e3ee78ea2 100644 --- a/dev/tasks/java-jars/github.yml +++ b/dev/tasks/java-jars/github.yml @@ -30,16 +30,17 @@ jobs: ARCH: {{ '${{ matrix.platform.archery_arch }}' }} ARCH_ALIAS: {{ '${{ matrix.platform.archery_arch_alias }}' }} ARCH_SHORT: {{ '${{ matrix.platform.archery_arch_short }}' }} + ARCHERY_USE_DOCKER_CLI: {{ "${{matrix.platform.archery_use_docker_cli || '1'}}" }} strategy: fail-fast: false matrix: platform: - - runs_on: ["ubuntu-latest"] + - runs_on: ubuntu-latest arch: "x86_64" archery_arch: "amd64" archery_arch_alias: "x86_64" archery_arch_short: "amd64" - - runs_on: ["buildjet-8vcpu-ubuntu-2204-arm"] + - runs_on: ubuntu-24.04-arm arch: "aarch_64" archery_arch: "arm64v8" archery_arch_alias: "aarch64" @@ -72,7 +73,7 @@ jobs: - name: Compress into single artifact to keep directory structure run: tar -cvzf arrow-shared-libs-linux-{{ arch }}.tar.gz arrow/java-dist/ - name: Upload artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 with: name: ubuntu-shared-lib-{{ arch }} path: arrow-shared-libs-linux-{{ arch }}.tar.gz @@ -91,33 +92,55 @@ jobs: fail-fast: false matrix: platform: - - { runs_on: ["macos-15-intel"], arch: "x86_64"} + - { runs_on: macos-15, arch: "aarch_64" } env: - MACOSX_DEPLOYMENT_TARGET: "12.0" + MACOSX_DEPLOYMENT_TARGET: "15.0" steps: {{ macros.github_checkout_arrow()|indent }} - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 with: cache: 'pip' python-version: 3.12 - name: Install Archery shell: bash run: pip install -e arrow/dev/archery[all] + - name: Checkout vcpkg + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: Microsoft/vcpkg + path: arrow/vcpkg + fetch-depth: 0 + - name: Install vcpkg + run: | + cd arrow/vcpkg + ./bootstrap-vcpkg.sh + echo "VCPKG_ROOT_LOCAL=${PWD}" >> ${GITHUB_ENV} + echo "${PWD}" >> ${GITHUB_PATH} + - name: Clean up disk space + run: | + echo "=== Free disk space before cleanup ===" + df -h / + + echo "" + echo "=== Removing Xcode simulators ===" + sudo rm -rf /Library/Developer/CoreSimulator/Caches || : + echo "Removed /Library/Developer/CoreSimulator/Caches" + + echo "" + echo "=== Removing user simulator data ===" + rm -rf ~/Library/Developer/CoreSimulator || : + echo "Removed ~/Library/Developer/CoreSimulator" + + echo "" + echo "=== Free disk space after cleanup ===" + df -h / - name: Install dependencies run: | - # We want to use llvm@14 to avoid shared z3 - # dependency. llvm@14 doesn't depend on z3 and llvm depends - # on z3. And Homebrew's z3 provides only shared library. It - # doesn't provides static z3 because z3's CMake doesn't accept - # building both shared and static libraries at once. - # See also: Z3_BUILD_LIBZ3_SHARED in - # https://github.com/Z3Prover/z3/blob/master/README-CMake.md - # - # If llvm is installed, Apache Arrow C++ uses llvm rather than - # llvm@14 because llvm is newer than llvm@14. - brew uninstall llvm || : + echo "=== Free disk space at start of dependency installation ===" + df -h / + echo "" # Ensure updating python@XXX with the "--overwrite" option. # If python@XXX is updated without "--overwrite", it causes # a conflict error. Because Python 3 installed not by @@ -131,58 +154,123 @@ jobs: done brew install --overwrite python + if [ "$(uname -m)" = "arm64" ]; then + # pkg-config formula is deprecated but it's still installed + # in GitHub Actions runner now. We can remove this once + # pkg-config formula is removed from GitHub Actions runner. + brew uninstall pkg-config || : + brew uninstall pkg-config@0.29.2 || : + fi + + # Install basic build tools via brew (vcpkg needs these) + brew install cmake ninja pkg-config brew bundle --file=arrow/cpp/Brewfile + + # Clean up any existing LLVM installations in favor of vcpkg. + # Need to uninstall all versioned LLVM packages (llvm@18, llvm@17, etc.) + for llvm_pkg in $(brew list | grep -E '^llvm(@[0-9]+)?$'); do + brew uninstall "${llvm_pkg}" || : + done + # We want to link aws-sdk-cpp statically but Homebrew's # aws-sdk-cpp provides only shared library. If we have # Homebrew's aws-sdk-cpp, our build mix Homebrew's # aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's # aws-sdk-cpp to ensure using only bundled aws-sdk-cpp. - brew uninstall aws-sdk-cpp + brew uninstall aws-sdk-cpp || : # We want to use bundled RE2 for static linking. If # Homebrew's RE2 is installed, its header file may be used. # We uninstall Homebrew's RE2 to ensure using bundled RE2. brew uninstall grpc || : # gRPC depends on RE2 - brew uninstall re2 + brew uninstall grpc@1.54 || : # gRPC 1.54 may be installed too + brew uninstall re2 || : # We want to use bundled Protobuf for static linking. If # Homebrew's Protobuf is installed, its library file may be # used on test We uninstall Homebrew's Protobuf to ensure using # bundled Protobuf. - brew uninstall protobuf - # fix cmake and boost versionsAdd commentMore actions - brew uninstall -f boost || true - brew uninstall -f cmake || true - mkdir -p homebrew-custom/Formula - curl -o homebrew-custom/Formula/cmake.rb https://raw.githubusercontent.com/Homebrew/homebrew-core/f68532bfe5cb87474093df8a839c3818c6aa44dd/Formula/c/cmake.rb - curl -o homebrew-custom/Formula/boost.rb https://raw.githubusercontent.com/Homebrew/homebrew-core/23f9c56c5075dd56b4471e2c93f89f6400b49ddd/Formula/b/boost.rb - brew tap-new local/homebrew-custom - cp ./homebrew-custom/Formula/*.rb "$(brew --repo local/homebrew-custom)/Formula/" - brew install -v local/homebrew-custom/cmake - brew install -v local/homebrew-custom/boost - brew pin cmake - brew pin boost - # + brew uninstall protobuf || : + + echo "" + echo "=== Free disk space before LLVM build ===" + df -h / + + echo "" + # Use vcpkg to install LLVM. + vcpkg install \ + --clean-after-build \ + --vcpkg-root=${VCPKG_ROOT_LOCAL} \ + --x-install-root=${VCPKG_ROOT_LOCAL}/installed \ + --x-manifest-root=arrow/ci/vcpkg \ + --overlay-ports=arrow/ci/vcpkg/overlay/llvm/ \ + --x-feature=gandiva-llvm + echo "" + echo "=== Free disk space after LLVM build ===" + df -h / + echo "" brew bundle --file=arrow/java/Brewfile - name: Build C++ libraries env: ARROW_USE_CCACHE: "ON" run: | set -e + echo "=== Free disk space at start of build ===" + df -h / + + echo "" # make brew Java available to CMake export JAVA_HOME=$(brew --prefix openjdk@11)/libexec/openjdk.jdk/Contents/Home arrow/ci/scripts/java_jni_macos_build.sh \ $GITHUB_WORKSPACE/arrow \ $GITHUB_WORKSPACE/arrow/cpp-build \ $GITHUB_WORKSPACE/arrow/java-dist + + echo "" + echo "=== Free disk space at end of build ===" + df -h / - name: Compress into single artifact to keep directory structure run: tar -cvzf arrow-shared-libs-macos-{{ arch }}.tar.gz arrow/java-dist/ - name: Upload artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 with: name: macos-shared-lib-{{ arch }} path: arrow-shared-libs-macos-{{ arch }}.tar.gz + build-cpp-windows: + name: Build C++ libraries Windows + runs-on: windows-2019 + steps: + {{ macros.github_checkout_arrow()|indent }} + - name: Set up Java + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'temurin' + - name: Download Timezone Database + shell: bash + run: arrow/ci/scripts/download_tz_database.sh + - name: Install sccache + shell: bash + run: arrow/ci/scripts/install_sccache.sh pc-windows-msvc $(pwd)/sccache + - name: Build C++ libraries + shell: cmd + env: + {{ macros.github_set_sccache_envvars()|indent(8) }} + run: | + call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 + REM For ORC + set TZDIR=/c/msys64/usr/share/zoneinfo + bash -c "arrow/ci/scripts/java_jni_windows_build.sh $(pwd)/arrow $(pwd)/arrow/cpp-build $(pwd)/arrow/java-dist" + - name: Compress into single artifact to keep directory structure + shell: bash + run: tar -cvzf arrow-shared-libs-windows.tar.gz arrow/java-dist/ + - name: Upload artifacts + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + with: + name: windows-shared-lib + path: arrow-shared-libs-windows.tar.gz + package-jars: name: Build jar files runs-on: {{ '${{ matrix.platform.runs_on }}' }} @@ -190,14 +278,15 @@ jobs: fail-fast: false matrix: platform: - - { runs_on: ["macos-15-intel"], arch: "x86_64"} + - { runs_on: macos-13, arch: "x86_64"} needs: - build-cpp-ubuntu - build-cpp-macos + - build-cpp-windows steps: {{ macros.github_checkout_arrow(fetch_depth=0)|indent }} - name: Download Libraries - uses: actions/download-artifact@v4 + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: path: artifacts - name: Decompress artifacts @@ -206,6 +295,8 @@ jobs: tar -xvzf arrow-shared-libs-linux-x86_64.tar.gz tar -xvzf arrow-shared-libs-linux-aarch_64.tar.gz tar -xvzf arrow-shared-libs-macos-x86_64.tar.gz + tar -xvzf arrow-shared-libs-macos-aarch_64.tar.gz + tar -xvzf arrow-shared-libs-windows.tar.gz - name: Test that shared libraries exist run: | set -x @@ -224,6 +315,15 @@ jobs: test -f arrow/java-dist/arrow_dataset_jni/x86_64/libarrow_dataset_jni.dylib test -f arrow/java-dist/arrow_orc_jni/x86_64/libarrow_orc_jni.dylib test -f arrow/java-dist/gandiva_jni/x86_64/libgandiva_jni.dylib + + test -f arrow/java-dist/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.dylib + test -f arrow/java-dist/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.dylib + test -f arrow/java-dist/arrow_orc_jni/aarch_64/libarrow_orc_jni.dylib + test -f arrow/java-dist/gandiva_jni/aarch_64/libgandiva_jni.dylib + + test -f arrow/java-dist/arrow_cdata_jni/x86_64/arrow_cdata_jni.dll + test -f arrow/java-dist/arrow_dataset_jni/x86_64/arrow_dataset_jni.dll + test -f arrow/java-dist/arrow_orc_jni/x86_64/arrow_orc_jni.dll - name: Build bundled jar env: MAVEN_ARGS: >- @@ -233,6 +333,7 @@ jobs: pushd arrow/java mvn versions:set -DnewVersion={{ arrow.no_rc_snapshot_version }} mvn versions:set -DnewVersion={{ arrow.no_rc_snapshot_version }} -f bom + mvn versions:set -DnewVersion={{ arrow.no_rc_snapshot_version }} -f maven popd arrow/ci/scripts/java_full_build.sh \ $GITHUB_WORKSPACE/arrow \ @@ -242,3 +343,4 @@ jobs: "arrow/java-dist/*.pom", "arrow/java-dist/*.xml", "arrow/java-dist/*.zip"])|indent }} +