Upgrade Java JAR build workflow with modern improvements

timhurskidremio · timhurskidremio · commit 26818682dcd3 · 2026-02-25T23:11:00.000-06:00
- Update GitHub Actions to v4+ with SHA pinning
     - Add ARM64 support for Ubuntu and macOS
     - Use vcpkg for LLVM to remove z3 runtime dependency
     - Add disk space cleanup for macOS runners
     - Add disk space monitoring throughout build
     - Enhance testing for ARM64 libraries

     Ref: DX-103340
diff --git a/dev/tasks/java-jars/github.yml b/dev/tasks/java-jars/github.yml
@@ -30,16 +30,17 @@ jobs:
       ARCH: {{ '${{ matrix.platform.archery_arch }}' }}
       ARCH_ALIAS: {{ '${{ matrix.platform.archery_arch_alias }}' }}
       ARCH_SHORT: {{ '${{ matrix.platform.archery_arch_short }}' }}
+      ARCHERY_USE_DOCKER_CLI: {{ "${{matrix.platform.archery_use_docker_cli || '1'}}" }}
     strategy:
       fail-fast: false
       matrix:
         platform:
-          - runs_on: ["ubuntu-latest"]
+          - runs_on: ubuntu-latest
             arch: "x86_64"
             archery_arch: "amd64"
             archery_arch_alias: "x86_64"
             archery_arch_short: "amd64"
-          - runs_on: ["buildjet-8vcpu-ubuntu-2204-arm"]
+          - runs_on: ubuntu-24.04-arm
             arch: "aarch_64"
             archery_arch: "arm64v8"
             archery_arch_alias: "aarch64"
@@ -72,7 +73,7 @@ jobs:
       - name: Compress into single artifact to keep directory structure
         run: tar -cvzf arrow-shared-libs-linux-{{ arch }}.tar.gz arrow/java-dist/
       - name: Upload artifacts
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
         with:
           name: ubuntu-shared-lib-{{ arch }}
           path: arrow-shared-libs-linux-{{ arch }}.tar.gz
@@ -91,113 +92,201 @@ jobs:
       fail-fast: false
       matrix:
         platform:
-          - { runs_on: ["macos-15-intel"], arch: "x86_64"}
+          - { runs_on: macos-15, arch: "aarch_64" }
     env:
-      MACOSX_DEPLOYMENT_TARGET: "12.0"
+      MACOSX_DEPLOYMENT_TARGET: "15.0"
     steps:
       {{ macros.github_checkout_arrow()|indent }}
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
         with:
           cache: 'pip'
           python-version: 3.12
       - name: Install Archery
         shell: bash
         run: pip install -e arrow/dev/archery[all]
+      - name: Checkout vcpkg
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          repository: Microsoft/vcpkg
+          path: arrow/vcpkg
+          fetch-depth: 0
+      - name: Install vcpkg
+        run: |
+          cd arrow/vcpkg
+          ./bootstrap-vcpkg.sh
+          echo "VCPKG_ROOT_LOCAL=${PWD}" >> ${GITHUB_ENV}
+          echo "${PWD}" >> ${GITHUB_PATH}
+      - name: Clean up disk space
+        run: |
+          echo "=== Free disk space before cleanup ==="
+          df -h /
+
+          echo ""
+          echo "=== Removing Xcode simulators ==="
+          sudo rm -rf /Library/Developer/CoreSimulator/Caches || :
+          echo "Removed /Library/Developer/CoreSimulator/Caches"
+
+          echo ""
+          echo "=== Removing user simulator data ==="
+          rm -rf ~/Library/Developer/CoreSimulator || :
+          echo "Removed ~/Library/Developer/CoreSimulator"
+
+          echo ""
+          echo "=== Free disk space after cleanup ==="
+          df -h /
       - name: Install dependencies
         run: |
-          # We want to use llvm@14 to avoid shared z3
-          # dependency. llvm@14 doesn't depend on z3 and llvm depends
-          # on z3. And Homebrew's z3 provides only shared library. It
-          # doesn't provides static z3 because z3's CMake doesn't accept
-          # building both shared and static libraries at once.
-          # See also: Z3_BUILD_LIBZ3_SHARED in
-          # https://github.com/Z3Prover/z3/blob/master/README-CMake.md
-          #
-          # If llvm is installed, Apache Arrow C++ uses llvm rather than
-          # llvm@14 because llvm is newer than llvm@14.
-          brew uninstall llvm || :
+          echo "=== Free disk space at start of dependency installation ==="
+          df -h /
 
+          echo ""
           # Ensure updating python@XXX with the "--overwrite" option.
           # If python@XXX is updated without "--overwrite", it causes
           # a conflict error. Because Python 3 installed not by
           # Homebrew exists in /usr/local on GitHub Actions. If
           # Homebrew's python@XXX is updated without "--overwrite", it
           # tries to replace /usr/local/bin/2to3 and so on and causes
           # a conflict error.
-          # brew update
+          brew update
           for python_package in $(brew list | grep python@); do
             brew install --overwrite ${python_package}
           done
           brew install --overwrite python
 
+          if [ "$(uname -m)" = "arm64" ]; then
+            # pkg-config formula is deprecated but it's still installed
+            # in GitHub Actions runner now. We can remove this once
+            # pkg-config formula is removed from GitHub Actions runner.
+            brew uninstall pkg-config || :
+          fi
+
+          # Install basic build tools via brew (vcpkg needs these)
+          brew install cmake ninja pkg-config
           brew bundle --file=arrow/cpp/Brewfile
+
+          # Clean up any existing LLVM installations in favor of vcpkg.
+          # Need to uninstall all versioned LLVM packages (llvm@18, llvm@17, etc.)
+          for llvm_pkg in $(brew list | grep -E '^llvm(@[0-9]+)?$'); do
+            brew uninstall "${llvm_pkg}" || :
+          done
+
           # We want to link aws-sdk-cpp statically but Homebrew's
           # aws-sdk-cpp provides only shared library. If we have
           # Homebrew's aws-sdk-cpp, our build mix Homebrew's
           # aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's
           # aws-sdk-cpp to ensure using only bundled aws-sdk-cpp.
-          brew uninstall aws-sdk-cpp
+          brew uninstall aws-sdk-cpp || :
           # We want to use bundled RE2 for static linking. If
           # Homebrew's RE2 is installed, its header file may be used.
           # We uninstall Homebrew's RE2 to ensure using bundled RE2.
           brew uninstall grpc || : # gRPC depends on RE2
-          brew uninstall re2
+          brew uninstall re2 || :
           # We want to use bundled Protobuf for static linking. If
           # Homebrew's Protobuf is installed, its library file may be
           # used on test  We uninstall Homebrew's Protobuf to ensure using
           # bundled Protobuf.
-          brew uninstall protobuf
-          # fix cmake and boost versionsAdd commentMore actions
-          brew uninstall -f boost || true
-          brew uninstall -f cmake || true
-          mkdir -p homebrew-custom/Formula
-          curl -o homebrew-custom/Formula/cmake.rb https://raw.githubusercontent.com/Homebrew/homebrew-core/f68532bfe5cb87474093df8a839c3818c6aa44dd/Formula/c/cmake.rb
-          curl -o homebrew-custom/Formula/boost.rb https://raw.githubusercontent.com/Homebrew/homebrew-core/23f9c56c5075dd56b4471e2c93f89f6400b49ddd/Formula/b/boost.rb
-          brew tap-new local/homebrew-custom
-          cp ./homebrew-custom/Formula/*.rb "$(brew --repo local/homebrew-custom)/Formula/"
-          brew install -v local/homebrew-custom/cmake
-          brew install -v local/homebrew-custom/boost
-          brew pin cmake
-          brew pin boost
-          #
+          brew uninstall protobuf || :
+
+          echo ""
+          echo "=== Free disk space before LLVM build ==="
+          df -h /
+
+          echo ""
+          # Use vcpkg to install LLVM.
+          # Create overlay directory if it doesn't exist
+          mkdir -p arrow/ci/vcpkg/overlay/llvm
+          vcpkg install \
+            --clean-after-build \
+            --vcpkg-root=${VCPKG_ROOT_LOCAL} \
+            --x-install-root=${VCPKG_ROOT_LOCAL}/installed \
+            --x-manifest-root=arrow/ci/vcpkg \
+            --overlay-ports=arrow/ci/vcpkg/overlay/llvm/ \
+            --x-feature=gandiva
 
+          echo ""
+          echo "=== Free disk space after LLVM build ==="
+          df -h /
 
+          echo ""
           brew bundle --file=arrow/java/Brewfile
       - name: Build C++ libraries
         env:
           ARROW_USE_CCACHE: "ON"
         run: |
           set -e
+          echo "=== Free disk space at start of build ==="
+          df -h /
+
+          echo ""
           # make brew Java available to CMake
           export JAVA_HOME=$(brew --prefix openjdk@11)/libexec/openjdk.jdk/Contents/Home
           arrow/ci/scripts/java_jni_macos_build.sh \
             $GITHUB_WORKSPACE/arrow \
             $GITHUB_WORKSPACE/arrow/cpp-build \
             $GITHUB_WORKSPACE/arrow/java-dist
+
+          echo ""
+          echo "=== Free disk space at end of build ==="
+          df -h /
       - name: Compress into single artifact to keep directory structure
         run: tar -cvzf arrow-shared-libs-macos-{{ arch }}.tar.gz arrow/java-dist/
       - name: Upload artifacts
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
         with:
           name: macos-shared-lib-{{ arch }}
           path: arrow-shared-libs-macos-{{ arch }}.tar.gz
 
+  build-cpp-windows:
+    name: Build C++ libraries Windows
+    runs-on: windows-2019
+    steps:
+      {{ macros.github_checkout_arrow()|indent }}
+      - name: Set up Java
+        uses: actions/setup-java@v4
+        with:
+          java-version: '11'
+          distribution: 'temurin'
+      - name: Download Timezone Database
+        shell: bash
+        run: arrow/ci/scripts/download_tz_database.sh
+      - name: Install sccache
+        shell: bash
+        run: arrow/ci/scripts/install_sccache.sh pc-windows-msvc $(pwd)/sccache
+      - name: Build C++ libraries
+        shell: cmd
+        env:
+        {{ macros.github_set_sccache_envvars()|indent(8) }}
+        run: |
+          call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
+          REM For ORC
+          set TZDIR=/c/msys64/usr/share/zoneinfo
+          bash -c "arrow/ci/scripts/java_jni_windows_build.sh $(pwd)/arrow $(pwd)/arrow/cpp-build $(pwd)/arrow/java-dist"
+      - name: Compress into single artifact to keep directory structure
+        shell: bash
+        run: tar -cvzf arrow-shared-libs-windows.tar.gz arrow/java-dist/
+      - name: Upload artifacts
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
+        with:
+          name: windows-shared-lib
+          path: arrow-shared-libs-windows.tar.gz
+
   package-jars:
     name: Build jar files
     runs-on: {{ '${{ matrix.platform.runs_on }}' }}
     strategy:
       fail-fast: false
       matrix:
         platform:
-          - { runs_on: ["macos-15-intel"], arch: "x86_64"}
+          - { runs_on: macos-13, arch: "x86_64"}
     needs:
       - build-cpp-ubuntu
       - build-cpp-macos
+      - build-cpp-windows
     steps:
       {{ macros.github_checkout_arrow(fetch_depth=0)|indent }}
       - name: Download Libraries
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
         with:
           path: artifacts
       - name: Decompress artifacts
@@ -206,6 +295,8 @@ jobs:
           tar -xvzf arrow-shared-libs-linux-x86_64.tar.gz
           tar -xvzf arrow-shared-libs-linux-aarch_64.tar.gz
           tar -xvzf arrow-shared-libs-macos-x86_64.tar.gz
+          tar -xvzf arrow-shared-libs-macos-aarch_64.tar.gz
+          tar -xvzf arrow-shared-libs-windows.tar.gz
       - name: Test that shared libraries exist
         run: |
           set -x
@@ -224,6 +315,15 @@ jobs:
           test -f arrow/java-dist/arrow_dataset_jni/x86_64/libarrow_dataset_jni.dylib
           test -f arrow/java-dist/arrow_orc_jni/x86_64/libarrow_orc_jni.dylib
           test -f arrow/java-dist/gandiva_jni/x86_64/libgandiva_jni.dylib
+
+          test -f arrow/java-dist/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.dylib
+          test -f arrow/java-dist/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.dylib
+          test -f arrow/java-dist/arrow_orc_jni/aarch_64/libarrow_orc_jni.dylib
+          test -f arrow/java-dist/gandiva_jni/aarch_64/libgandiva_jni.dylib
+
+          test -f arrow/java-dist/arrow_cdata_jni/x86_64/arrow_cdata_jni.dll
+          test -f arrow/java-dist/arrow_dataset_jni/x86_64/arrow_dataset_jni.dll
+          test -f arrow/java-dist/arrow_orc_jni/x86_64/arrow_orc_jni.dll
       - name: Build bundled jar
         env:
           MAVEN_ARGS: >-
@@ -233,6 +333,7 @@ jobs:
           pushd arrow/java
           mvn versions:set -DnewVersion={{ arrow.no_rc_snapshot_version }}
           mvn versions:set -DnewVersion={{ arrow.no_rc_snapshot_version }} -f bom
+          mvn versions:set -DnewVersion={{ arrow.no_rc_snapshot_version }} -f maven
           popd
           arrow/ci/scripts/java_full_build.sh \
             $GITHUB_WORKSPACE/arrow \
@@ -242,3 +343,4 @@ jobs:
                                         "arrow/java-dist/*.pom",
                                         "arrow/java-dist/*.xml",
                                         "arrow/java-dist/*.zip"])|indent }}
+
