From 08323e7626dd3cd1bc9b0d72d7f368e60662331f Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Tue, 23 Jun 2026 21:19:06 +0200 Subject: [PATCH 1/6] Resolve verify-rc --- ci/conda_env_gandiva.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/conda_env_gandiva.txt b/ci/conda_env_gandiva.txt index 217936e2c947..5a19aead8e41 100644 --- a/ci/conda_env_gandiva.txt +++ b/ci/conda_env_gandiva.txt @@ -15,5 +15,7 @@ # specific language governing permissions and limitations # under the License. +# Exclude LLVM 22.1.8: its conda-forge macOS llvm-tools package cannot load +# @rpath/libLLVM.22.1.dylib, breaking the C++ build (mimalloc, Gandiva IR) for verify-rc. clang>=11 -llvmdev>=11 +llvmdev>=11,<22 From 50ff3f517f701f8c7d72257b5a770f9d529a6fee Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Wed, 24 Jun 2026 13:43:32 +0200 Subject: [PATCH 2/6] Debug flaky conda verify-rc --- ci/conda_env_gandiva.txt | 4 +--- dev/release/verify-release-candidate.sh | 9 +++++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/ci/conda_env_gandiva.txt b/ci/conda_env_gandiva.txt index 5a19aead8e41..217936e2c947 100644 --- a/ci/conda_env_gandiva.txt +++ b/ci/conda_env_gandiva.txt @@ -15,7 +15,5 @@ # specific language governing permissions and limitations # under the License. -# Exclude LLVM 22.1.8: its conda-forge macOS llvm-tools package cannot load -# @rpath/libLLVM.22.1.dylib, breaking the C++ build (mimalloc, Gandiva IR) for verify-rc. clang>=11 -llvmdev>=11,<22 +llvmdev>=11 diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index f4d2800eaf0e..726fe7e71437 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -456,6 +456,11 @@ test_and_install_cpp() { if [ "${USE_CONDA}" -gt 0 ]; then DEFAULT_DEPENDENCY_SOURCE="CONDA" CMAKE_PREFIX_PATH="${CONDA_BACKUP_CMAKE_PREFIX_PATH}:${CMAKE_PREFIX_PATH}" + # Ensure conda's LLVM tools (llvm-link, llvm-ranlib) find libLLVM.*.dylib. + # Otherwise @rpath resolution is flaky and the C++ build aborts on macOS. + if [ "$(uname)" = "Darwin" ] && [ -n "${CONDA_PREFIX:-}" ]; then + export DYLD_LIBRARY_PATH="${CONDA_PREFIX}/lib${DYLD_LIBRARY_PATH:+:${DYLD_LIBRARY_PATH}}" + fi else DEFAULT_DEPENDENCY_SOURCE="AUTO" fi @@ -779,10 +784,10 @@ test_source_distribution() { if [ "$(uname)" == "Darwin" ]; then NPROC=$(sysctl -n hw.ncpu) - export DYLD_LIBRARY_PATH=$ARROW_HOME/lib:${DYLD_LIBRARY_PATH:-} + export DYLD_LIBRARY_PATH=$ARROW_HOME/lib${DYLD_LIBRARY_PATH:+:${DYLD_LIBRARY_PATH}} else NPROC=$(nproc) - export LD_LIBRARY_PATH=$ARROW_HOME/lib:${LD_LIBRARY_PATH:-} + export LD_LIBRARY_PATH=$ARROW_HOME/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} fi pushd $ARROW_SOURCE_DIR From 0bc831e5d46d3763131ad981110545833d6f7f37 Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Wed, 24 Jun 2026 15:39:36 +0200 Subject: [PATCH 3/6] Do not merge - VERIFY_RC_DEBUG --- dev/release/verify-release-candidate.sh | 39 +++++++++++++++++++++++++ dev/tasks/tasks.yml | 3 ++ 2 files changed, 42 insertions(+) diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index 726fe7e71437..61e88bfffa32 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -465,6 +465,45 @@ test_and_install_cpp() { DEFAULT_DEPENDENCY_SOURCE="AUTO" fi + # TEMP - remove before merge: VERIFY_RC_DEBUG=1 probes the libLLVM @rpath cause. + # set +e: this is expected to abort (exit 134) and must not kill the script + if [ "${VERIFY_RC_DEBUG:-0}" -gt 0 ] && [ "$(uname)" = "Darwin" ] && [ -n "${CONDA_PREFIX:-}" ]; then + set +e + echo "===== VERIFY_RC_DEBUG: libLLVM @rpath probe =====" + echo "CONDA_PREFIX=${CONDA_PREFIX}" + echo "ARROW_HOME=${ARROW_HOME:-}" + + envlib=$(ls -d "${CONDA_PREFIX}"/lib/libLLVM.*.dylib 2>/dev/null | head -1) + echo "env libLLVM : ${envlib:-MISSING}" + if [ -n "${envlib}" ]; then + otool -L "${envlib}" >/dev/null 2>&1 && echo "env libLLVM loadable : yes" + fi + + # env binary resolving into the env = hardlinked (safe) vs. into pkgs = vulnerable + for tool in llvm-link llvm-ranlib; do + envln="${CONDA_PREFIX}/bin/${tool}" + [ -e "${envln}" ] || continue + real=$(python3 -c 'import os,sys;print(os.path.realpath(sys.argv[1]))' "${envln}" 2>/dev/null) + echo "env ${tool} realpath: ${real}" + case "${real}" in + "${CONDA_PREFIX}"/*) echo " -> INTO env (safe)" ;; + *) echo " -> pkgs cache (VULNERABLE)" ;; + esac + done + + # controlled comparison on the pkgs binary (its @loader_path is always the pkgs cache) + pkgbin=$(ls -d "${CONDA_PREFIX}"/../../pkgs/llvm-tools-*/bin/llvm-link-* 2>/dev/null | head -1) + echo "pkgs llvm-link : ${pkgbin:-MISSING}" + if [ -n "${pkgbin}" ]; then + DYLD_LIBRARY_PATH="${ARROW_HOME}/lib" "${pkgbin}" --version >/dev/null 2>&1 + echo "A) DYLD=ARROW_HOME/lib only : exit=$? (expect 134 = bug)" + DYLD_LIBRARY_PATH="${CONDA_PREFIX}/lib:${ARROW_HOME}/lib" "${pkgbin}" --version >/dev/null 2>&1 + echo "B) DYLD=CONDA_PREFIX/lib + above : exit=$? (expect 0 = fixed)" + fi + echo "===== end VERIFY_RC_DEBUG =====" + set -e + fi + mkdir -p $ARROW_TMPDIR/cpp-build pushd $ARROW_TMPDIR/cpp-build diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 56a9ce1472ac..8e7b403e8d02 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -298,6 +298,9 @@ tasks: ci: github template: verify-rc/github.macos.yml params: + # TEMP - remove before merge: enable the libLLVM @rpath probe + env: + VERIFY_RC_DEBUG: 1 target: {{ target }} use_conda: True github_runner: "macos-15-intel" From 6a8426d5e07df60eb47dc69b65164d7a58746b9c Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Wed, 24 Jun 2026 17:51:53 +0200 Subject: [PATCH 4/6] Expose conda lib for build LLVM tools --- dev/release/verify-release-candidate.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index 61e88bfffa32..e9aaf5548d1f 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -456,11 +456,6 @@ test_and_install_cpp() { if [ "${USE_CONDA}" -gt 0 ]; then DEFAULT_DEPENDENCY_SOURCE="CONDA" CMAKE_PREFIX_PATH="${CONDA_BACKUP_CMAKE_PREFIX_PATH}:${CMAKE_PREFIX_PATH}" - # Ensure conda's LLVM tools (llvm-link, llvm-ranlib) find libLLVM.*.dylib. - # Otherwise @rpath resolution is flaky and the C++ build aborts on macOS. - if [ "$(uname)" = "Darwin" ] && [ -n "${CONDA_PREFIX:-}" ]; then - export DYLD_LIBRARY_PATH="${CONDA_PREFIX}/lib${DYLD_LIBRARY_PATH:+:${DYLD_LIBRARY_PATH}}" - fi else DEFAULT_DEPENDENCY_SOURCE="AUTO" fi @@ -563,7 +558,13 @@ test_and_install_cpp() { ${ARROW_CMAKE_OPTIONS:-} \ ${ARROW_SOURCE_DIR}/cpp export CMAKE_BUILD_PARALLEL_LEVEL=${CMAKE_BUILD_PARALLEL_LEVEL:-${NPROC}} - cmake --build . --target install + # Expose conda's lib so LLVM tools (llvm-link, llvm-ranlib) find libLLVM.*.dylib via + # @rpath else build flakily fails on macOS (Scoped here so libiconv can't leak into tests) + if [ "$(uname)" = "Darwin" ] && [ "${USE_CONDA}" -gt 0 ] && [ -n "${CONDA_PREFIX:-}" ]; then + DYLD_LIBRARY_PATH="${CONDA_PREFIX}/lib:${DYLD_LIBRARY_PATH:-}" cmake --build . --target install + else + cmake --build . --target install + fi if [ ${TEST_CPP} -gt 0 ]; then LD_LIBRARY_PATH=$PWD/release:$LD_LIBRARY_PATH ctest \ From 8f573bb03089dfd1a79dda8ebabd9430802ea0a3 Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Wed, 24 Jun 2026 19:11:38 +0200 Subject: [PATCH 5/6] Use DYLD_FALLBACK --- dev/release/verify-release-candidate.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index e9aaf5548d1f..9b18a251490f 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -491,9 +491,11 @@ test_and_install_cpp() { echo "pkgs llvm-link : ${pkgbin:-MISSING}" if [ -n "${pkgbin}" ]; then DYLD_LIBRARY_PATH="${ARROW_HOME}/lib" "${pkgbin}" --version >/dev/null 2>&1 - echo "A) DYLD=ARROW_HOME/lib only : exit=$? (expect 134 = bug)" + echo "A) no conda lib : exit=$? (134 or 0, pkgs binary varies by run)" DYLD_LIBRARY_PATH="${CONDA_PREFIX}/lib:${ARROW_HOME}/lib" "${pkgbin}" --version >/dev/null 2>&1 - echo "B) DYLD=CONDA_PREFIX/lib + above : exit=$? (expect 0 = fixed)" + echo "B) conda lib via DYLD_LIBRARY_PATH : exit=$? (expect 0)" + DYLD_LIBRARY_PATH="${ARROW_HOME}/lib" DYLD_FALLBACK_LIBRARY_PATH="${CONDA_PREFIX}/lib" "${pkgbin}" --version >/dev/null 2>&1 + echo "C) conda lib via DYLD_FALLBACK (fix) : exit=$? (expect 0)" fi echo "===== end VERIFY_RC_DEBUG =====" set -e @@ -558,10 +560,11 @@ test_and_install_cpp() { ${ARROW_CMAKE_OPTIONS:-} \ ${ARROW_SOURCE_DIR}/cpp export CMAKE_BUILD_PARALLEL_LEVEL=${CMAKE_BUILD_PARALLEL_LEVEL:-${NPROC}} - # Expose conda's lib so LLVM tools (llvm-link, llvm-ranlib) find libLLVM.*.dylib via - # @rpath else build flakily fails on macOS (Scoped here so libiconv can't leak into tests) + # Expose conda's lib on the *fallback* path so the LLVM tools (llvm-link, + # llvm-ranlib) find libLLVM.*.dylib on flaky @rpath miss on macOS. (Fallback is searched last, + # so it doesn't shadow system libs like libiconv during the build or leak into tests.) if [ "$(uname)" = "Darwin" ] && [ "${USE_CONDA}" -gt 0 ] && [ -n "${CONDA_PREFIX:-}" ]; then - DYLD_LIBRARY_PATH="${CONDA_PREFIX}/lib:${DYLD_LIBRARY_PATH:-}" cmake --build . --target install + DYLD_FALLBACK_LIBRARY_PATH="${CONDA_PREFIX}/lib:${DYLD_FALLBACK_LIBRARY_PATH:-/usr/local/lib:/usr/lib}" cmake --build . --target install else cmake --build . --target install fi From 8f1149d535b9e8f5b5fbb00c8d36c5c6956f4f14 Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Wed, 24 Jun 2026 20:46:28 +0200 Subject: [PATCH 6/6] Final cleanup of temp debug --- dev/release/verify-release-candidate.sh | 41 ------------------------- dev/tasks/tasks.yml | 3 -- 2 files changed, 44 deletions(-) diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index 9b18a251490f..f1b11e3ec7f4 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -460,47 +460,6 @@ test_and_install_cpp() { DEFAULT_DEPENDENCY_SOURCE="AUTO" fi - # TEMP - remove before merge: VERIFY_RC_DEBUG=1 probes the libLLVM @rpath cause. - # set +e: this is expected to abort (exit 134) and must not kill the script - if [ "${VERIFY_RC_DEBUG:-0}" -gt 0 ] && [ "$(uname)" = "Darwin" ] && [ -n "${CONDA_PREFIX:-}" ]; then - set +e - echo "===== VERIFY_RC_DEBUG: libLLVM @rpath probe =====" - echo "CONDA_PREFIX=${CONDA_PREFIX}" - echo "ARROW_HOME=${ARROW_HOME:-}" - - envlib=$(ls -d "${CONDA_PREFIX}"/lib/libLLVM.*.dylib 2>/dev/null | head -1) - echo "env libLLVM : ${envlib:-MISSING}" - if [ -n "${envlib}" ]; then - otool -L "${envlib}" >/dev/null 2>&1 && echo "env libLLVM loadable : yes" - fi - - # env binary resolving into the env = hardlinked (safe) vs. into pkgs = vulnerable - for tool in llvm-link llvm-ranlib; do - envln="${CONDA_PREFIX}/bin/${tool}" - [ -e "${envln}" ] || continue - real=$(python3 -c 'import os,sys;print(os.path.realpath(sys.argv[1]))' "${envln}" 2>/dev/null) - echo "env ${tool} realpath: ${real}" - case "${real}" in - "${CONDA_PREFIX}"/*) echo " -> INTO env (safe)" ;; - *) echo " -> pkgs cache (VULNERABLE)" ;; - esac - done - - # controlled comparison on the pkgs binary (its @loader_path is always the pkgs cache) - pkgbin=$(ls -d "${CONDA_PREFIX}"/../../pkgs/llvm-tools-*/bin/llvm-link-* 2>/dev/null | head -1) - echo "pkgs llvm-link : ${pkgbin:-MISSING}" - if [ -n "${pkgbin}" ]; then - DYLD_LIBRARY_PATH="${ARROW_HOME}/lib" "${pkgbin}" --version >/dev/null 2>&1 - echo "A) no conda lib : exit=$? (134 or 0, pkgs binary varies by run)" - DYLD_LIBRARY_PATH="${CONDA_PREFIX}/lib:${ARROW_HOME}/lib" "${pkgbin}" --version >/dev/null 2>&1 - echo "B) conda lib via DYLD_LIBRARY_PATH : exit=$? (expect 0)" - DYLD_LIBRARY_PATH="${ARROW_HOME}/lib" DYLD_FALLBACK_LIBRARY_PATH="${CONDA_PREFIX}/lib" "${pkgbin}" --version >/dev/null 2>&1 - echo "C) conda lib via DYLD_FALLBACK (fix) : exit=$? (expect 0)" - fi - echo "===== end VERIFY_RC_DEBUG =====" - set -e - fi - mkdir -p $ARROW_TMPDIR/cpp-build pushd $ARROW_TMPDIR/cpp-build diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 8e7b403e8d02..56a9ce1472ac 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -298,9 +298,6 @@ tasks: ci: github template: verify-rc/github.macos.yml params: - # TEMP - remove before merge: enable the libLLVM @rpath probe - env: - VERIFY_RC_DEBUG: 1 target: {{ target }} use_conda: True github_runner: "macos-15-intel"