From 04d1867fbd658ec39db4ba18607995a47ef5ecde Mon Sep 17 00:00:00 2001 From: James A Sutherland Date: Mon, 9 Mar 2026 10:12:49 -0500 Subject: [PATCH 1/4] fix: add SHA256 checksum verification for native dependency downloads Fixes CI failure where autoconf download via ftpmirror.gnu.org failed because --proto =https blocked HTTP mirror redirects. Downloads now go to files with SHA256 verification instead of piping to tar, making HTTP redirects safe. Also adds checksums for all 8 library downloads in build-config.sh and auto-updates them in check-native-updates.sh. --- .github/workflows/build.yml | 11 +++++---- native/build-config.sh | 39 +++++++++++++++++++++++++------- scripts/check-native-updates.sh | 40 ++++++++++++++++++++++++++++++--- 3 files changed, 75 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 21e3463..e799404 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -148,17 +148,20 @@ jobs: - name: Install autoconf and automake run: | # xz 5.8.2 was built with automake 1.18.1 and autoconf 2.72 - # Use ftpmirror.gnu.org to auto-select nearest mirror (more reliable than ftp.gnu.org) - # --proto =https ensures redirects stay on HTTPS (security requirement) + # Download to files and verify checksums (allows HTTP mirror redirects) # Install autoconf 2.72 - curl -sSL --proto =https --retry 3 --retry-delay 5 https://ftpmirror.gnu.org/gnu/autoconf/autoconf-2.72.tar.xz | tar xJ + curl -sSL --retry 3 --retry-delay 5 https://ftpmirror.gnu.org/gnu/autoconf/autoconf-2.72.tar.xz -o autoconf-2.72.tar.xz + echo "ba885c1319578d6c94d46e9b0dceb4014caafe2490e437a0dbca3f270a223f5a autoconf-2.72.tar.xz" | shasum -a 256 --check + tar xJf autoconf-2.72.tar.xz cd autoconf-2.72 ./configure --prefix=$HOME/.local make -j$(nproc) make install cd .. # Install automake 1.18.1 (exact version used to build xz 5.8.2) - curl -sSL --proto =https --retry 3 --retry-delay 5 https://ftpmirror.gnu.org/gnu/automake/automake-1.18.1.tar.xz | tar xJ + curl -sSL --retry 3 --retry-delay 5 https://ftpmirror.gnu.org/gnu/automake/automake-1.18.1.tar.xz -o automake-1.18.1.tar.xz + echo "168aa363278351b89af56684448f525a5bce5079d0b6842bd910fdd3f1646887 automake-1.18.1.tar.xz" | shasum -a 256 --check + tar xJf automake-1.18.1.tar.xz cd automake-1.18.1 ./configure --prefix=$HOME/.local make -j$(nproc) diff --git a/native/build-config.sh b/native/build-config.sh index ebc9e3b..3f45f28 100755 --- a/native/build-config.sh +++ b/native/build-config.sh @@ -12,6 +12,16 @@ ZLIB_VERSION="1.3.2" XZ_VERSION="5.8.2" BZIP2_VERSION="1.0.8" +# SHA256 checksums for download verification +LIBARCHIVE_SHA256="d68068e74beee3a0ec0dd04aee9037d5757fcc651591a6dcf1b6d542fb15a703" +LZ4_SHA256="537512904744b35e232912055ccf8ec66d768639ff3abe5788d90d792ec5f48b" +ZSTD_SHA256="eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3" +LZO_SHA256="c0f892943208266f9b6543b3ae308fab6284c5c90e627931446fb49b4221a072" +LIBXML2_SHA256="c008bac08fd5c7b4a87f7b8a71f283fa581d80d80ff8d2efd3b26224c39bc54c" +ZLIB_SHA256="d7a0654783a4da529d1bb793b7ad9c3318020af77667bcae35f95d0e42a792f3" +XZ_SHA256="890966ec3f5d5cc151077879e157c0593500a522f413ac50ba26d22a9a145214" +BZIP2_SHA256="ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269" + # Bootlin toolchain versions (Linux only) # Bootlin stable 2025.08-1: GCC 14.3.0, musl latest, binutils 2.43.1 BOOTLIN_RELEASE="stable-2025.08-1" @@ -62,6 +72,7 @@ download_library() { local url="$1" local name="$2" local dir_name="$3" + local expected_sha256="${4:-}" # Extract archive filename from URL local archive_name="${url##*/}" @@ -101,6 +112,18 @@ download_library() { echo "Using cached ${name}..." fi + # Verify SHA256 checksum if provided + if [ -n "$expected_sha256" ]; then + echo "Verifying ${name} checksum..." + if ! echo "$expected_sha256 $cache_file" | shasum -a 256 --check --status 2>/dev/null; then + echo "ERROR: SHA256 checksum mismatch for ${name}" + echo "Expected: $expected_sha256" + echo "Got: $(shasum -a 256 "$cache_file" | cut -d' ' -f1)" + rm -f "$cache_file" + return 1 + fi + fi + # Delete any existing unpacked directory to ensure clean start rm -rf "$dir_name" @@ -162,14 +185,14 @@ download_toolchain() { # Function to download all libraries # Always unpacks fresh copies from cache download_all_libraries() { - download_library "$LIBARCHIVE_URL" "libarchive" "libarchive-${LIBARCHIVE_VERSION}" - download_library "$LZ4_URL" "lz4" "lz4-${LZ4_VERSION}" - download_library "$ZSTD_URL" "zstd" "zstd-${ZSTD_VERSION}" - download_library "$LZO_URL" "lzo" "lzo-${LZO_VERSION}" - download_library "$LIBXML2_URL" "libxml2" "libxml2-${LIBXML2_VERSION}" - download_library "$BZIP2_URL" "bzip2" "bzip2-${BZIP2_VERSION}" - download_library "$ZLIB_URL" "zlib" "zlib-${ZLIB_VERSION}" - download_library "$XZ_URL" "xz" "xz-${XZ_VERSION}" + download_library "$LIBARCHIVE_URL" "libarchive" "libarchive-${LIBARCHIVE_VERSION}" "$LIBARCHIVE_SHA256" + download_library "$LZ4_URL" "lz4" "lz4-${LZ4_VERSION}" "$LZ4_SHA256" + download_library "$ZSTD_URL" "zstd" "zstd-${ZSTD_VERSION}" "$ZSTD_SHA256" + download_library "$LZO_URL" "lzo" "lzo-${LZO_VERSION}" "$LZO_SHA256" + download_library "$LIBXML2_URL" "libxml2" "libxml2-${LIBXML2_VERSION}" "$LIBXML2_SHA256" + download_library "$BZIP2_URL" "bzip2" "bzip2-${BZIP2_VERSION}" "$BZIP2_SHA256" + download_library "$ZLIB_URL" "zlib" "zlib-${ZLIB_VERSION}" "$ZLIB_SHA256" + download_library "$XZ_URL" "xz" "xz-${XZ_VERSION}" "$XZ_SHA256" # Fix xz automake timestamp issue - touch generated files to prevent regeneration # xz 5.8.2 was built with automake 1.18.1 which may not be available on build systems diff --git a/scripts/check-native-updates.sh b/scripts/check-native-updates.sh index 53933b1..f8582a9 100755 --- a/scripts/check-native-updates.sh +++ b/scripts/check-native-updates.sh @@ -107,18 +107,52 @@ update_version() { local config_script="$REPO_ROOT/native/build-config.sh" local var_name=$(echo "$dep_name" | tr '[:lower:]-' '[:upper:]_')_VERSION + local sha_var_name=$(echo "$dep_name" | tr '[:lower:]-' '[:upper:]_')_SHA256 if [ "$DRY_RUN" = "1" ]; then echo " Would update $var_name in build-config.sh: $old_ver -> $new_ver" + echo " Would update $sha_var_name checksum" else - # Use sed to update the version variable + # Update version variable if [[ "$OSTYPE" == "darwin"* ]]; then - # macOS requires empty string after -i for in-place edit sed -i '' "s/^${var_name}=\"${old_ver}\"$/${var_name}=\"${new_ver}\"/" "$config_script" else - # Linux GNU sed sed -i "s/^${var_name}=\"${old_ver}\"$/${var_name}=\"${new_ver}\"/" "$config_script" fi + + # Compute new download URL and SHA256 + # Re-source config (which now has the new version) to get the URL + local url_var_name + url_var_name="$(echo "$dep_name" | tr '[:lower:]-' '[:upper:]_')_URL" + local new_url + new_url=$(. "$config_script" 2>/dev/null; eval echo "\$$url_var_name" 2>/dev/null || true) + + if [ -n "$new_url" ]; then + echo " Downloading new version to compute SHA256..." + local tmpfile + tmpfile=$(mktemp) + if curl -fsSL "$new_url" -o "$tmpfile" 2>/dev/null; then + local new_sha256 + new_sha256=$(shasum -a 256 "$tmpfile" | cut -d' ' -f1) + rm -f "$tmpfile" + + # Update SHA256 in config + local old_sha256 + old_sha256=$(grep "^${sha_var_name}=" "$config_script" | sed 's/^[^=]*="\([^"]*\)"/\1/') + if [ -n "$old_sha256" ]; then + if [[ "$OSTYPE" == "darwin"* ]]; then + sed -i '' "s/^${sha_var_name}=\"${old_sha256}\"$/${sha_var_name}=\"${new_sha256}\"/" "$config_script" + else + sed -i "s/^${sha_var_name}=\"${old_sha256}\"$/${sha_var_name}=\"${new_sha256}\"/" "$config_script" + fi + echo " Updated $sha_var_name: $new_sha256" + fi + else + rm -f "$tmpfile" + echo -e " ${YELLOW}Warning: Could not download new version to compute SHA256${NC}" + echo -e " ${YELLOW}Please manually update $sha_var_name in build-config.sh${NC}" + fi + fi fi } From caafc1a63e45ce71281b4fc90863bdaf2168197c Mon Sep 17 00:00:00 2001 From: James A Sutherland Date: Mon, 9 Mar 2026 10:18:09 -0500 Subject: [PATCH 2/4] fix: validate SHA256 tool availability before updating checksums Add portable sha256_check/sha256_compute helpers that work with both sha256sum (Linux) and shasum (macOS), failing explicitly if neither is available. Validate computed hashes are 64-char hex before writing to build-config.sh. Use sha256sum in build.yml since the runner is Ubuntu. --- .github/workflows/build.yml | 4 ++-- native/build-config.sh | 32 ++++++++++++++++++++++++++++++-- scripts/check-native-updates.sh | 27 +++++++++++++++++---------- 3 files changed, 49 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e799404..ab313b8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -151,7 +151,7 @@ jobs: # Download to files and verify checksums (allows HTTP mirror redirects) # Install autoconf 2.72 curl -sSL --retry 3 --retry-delay 5 https://ftpmirror.gnu.org/gnu/autoconf/autoconf-2.72.tar.xz -o autoconf-2.72.tar.xz - echo "ba885c1319578d6c94d46e9b0dceb4014caafe2490e437a0dbca3f270a223f5a autoconf-2.72.tar.xz" | shasum -a 256 --check + echo "ba885c1319578d6c94d46e9b0dceb4014caafe2490e437a0dbca3f270a223f5a autoconf-2.72.tar.xz" | sha256sum --check tar xJf autoconf-2.72.tar.xz cd autoconf-2.72 ./configure --prefix=$HOME/.local @@ -160,7 +160,7 @@ jobs: cd .. # Install automake 1.18.1 (exact version used to build xz 5.8.2) curl -sSL --retry 3 --retry-delay 5 https://ftpmirror.gnu.org/gnu/automake/automake-1.18.1.tar.xz -o automake-1.18.1.tar.xz - echo "168aa363278351b89af56684448f525a5bce5079d0b6842bd910fdd3f1646887 automake-1.18.1.tar.xz" | shasum -a 256 --check + echo "168aa363278351b89af56684448f525a5bce5079d0b6842bd910fdd3f1646887 automake-1.18.1.tar.xz" | sha256sum --check tar xJf automake-1.18.1.tar.xz cd automake-1.18.1 ./configure --prefix=$HOME/.local diff --git a/native/build-config.sh b/native/build-config.sh index 3f45f28..7de288f 100755 --- a/native/build-config.sh +++ b/native/build-config.sh @@ -48,6 +48,32 @@ BZIP2_URL="https://www.sourceware.org/pub/bzip2/bzip2-${BZIP2_VERSION}.tar.gz" ZLIB_URL="https://zlib.net/zlib-${ZLIB_VERSION}.tar.xz" XZ_URL="https://github.com/tukaani-project/xz/releases/download/v${XZ_VERSION}/xz-${XZ_VERSION}.tar.xz" +# Portable SHA256 helper (prefers sha256sum on Linux, shasum on macOS) +sha256_check() { + local expected="$1" + local file="$2" + if command -v sha256sum >/dev/null 2>&1; then + echo "$expected $file" | sha256sum --check --status + elif command -v shasum >/dev/null 2>&1; then + echo "$expected $file" | shasum -a 256 --check --status + else + echo "ERROR: No SHA256 tool found (need sha256sum or shasum)" >&2 + return 1 + fi +} + +sha256_compute() { + local file="$1" + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$file" | cut -d' ' -f1 + elif command -v shasum >/dev/null 2>&1; then + shasum -a 256 "$file" | cut -d' ' -f1 + else + echo "ERROR: No SHA256 tool found (need sha256sum or shasum)" >&2 + return 1 + fi +} + # Common build settings export PREFIX="${PREFIX:-$(pwd)/local}" @@ -115,10 +141,12 @@ download_library() { # Verify SHA256 checksum if provided if [ -n "$expected_sha256" ]; then echo "Verifying ${name} checksum..." - if ! echo "$expected_sha256 $cache_file" | shasum -a 256 --check --status 2>/dev/null; then + if ! sha256_check "$expected_sha256" "$cache_file"; then + local actual + actual=$(sha256_compute "$cache_file") || actual="(unable to compute)" echo "ERROR: SHA256 checksum mismatch for ${name}" echo "Expected: $expected_sha256" - echo "Got: $(shasum -a 256 "$cache_file" | cut -d' ' -f1)" + echo "Got: $actual" rm -f "$cache_file" return 1 fi diff --git a/scripts/check-native-updates.sh b/scripts/check-native-updates.sh index f8582a9..f05ebec 100755 --- a/scripts/check-native-updates.sh +++ b/scripts/check-native-updates.sh @@ -132,20 +132,27 @@ update_version() { local tmpfile tmpfile=$(mktemp) if curl -fsSL "$new_url" -o "$tmpfile" 2>/dev/null; then + # Use sha256_compute from build-config.sh (already sourced above) local new_sha256 - new_sha256=$(shasum -a 256 "$tmpfile" | cut -d' ' -f1) + new_sha256=$(sha256_compute "$tmpfile") || true rm -f "$tmpfile" - # Update SHA256 in config - local old_sha256 - old_sha256=$(grep "^${sha_var_name}=" "$config_script" | sed 's/^[^=]*="\([^"]*\)"/\1/') - if [ -n "$old_sha256" ]; then - if [[ "$OSTYPE" == "darwin"* ]]; then - sed -i '' "s/^${sha_var_name}=\"${old_sha256}\"$/${sha_var_name}=\"${new_sha256}\"/" "$config_script" - else - sed -i "s/^${sha_var_name}=\"${old_sha256}\"$/${sha_var_name}=\"${new_sha256}\"/" "$config_script" + # Validate we got a 64-char hex string before writing + if [[ "$new_sha256" =~ ^[0-9a-f]{64}$ ]]; then + # Update SHA256 in config + local old_sha256 + old_sha256=$(grep "^${sha_var_name}=" "$config_script" | sed 's/^[^=]*="\([^"]*\)"/\1/') + if [ -n "$old_sha256" ]; then + if [[ "$OSTYPE" == "darwin"* ]]; then + sed -i '' "s/^${sha_var_name}=\"${old_sha256}\"$/${sha_var_name}=\"${new_sha256}\"/" "$config_script" + else + sed -i "s/^${sha_var_name}=\"${old_sha256}\"$/${sha_var_name}=\"${new_sha256}\"/" "$config_script" + fi + echo " Updated $sha_var_name: $new_sha256" fi - echo " Updated $sha_var_name: $new_sha256" + else + echo -e " ${RED}ERROR: SHA256 computation failed (got: '$new_sha256')${NC}" + echo -e " ${RED}Please manually update $sha_var_name in build-config.sh${NC}" fi else rm -f "$tmpfile" From 8c3cd17ae84d7c6c290552ebb20100e63365443f Mon Sep 17 00:00:00 2001 From: James A Sutherland Date: Mon, 9 Mar 2026 10:20:38 -0500 Subject: [PATCH 3/4] fix: use portable SHA256 verification (compute and compare) macOS BSD sha256sum doesn't support --check/--status flags. Replace flag-based verification with compute-and-compare approach. Prefer shasum (available on macOS and most Linux) over sha256sum. --- .github/workflows/build.yml | 4 ++-- native/build-config.sh | 32 ++++++++++++++------------------ 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ab313b8..752540f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -151,7 +151,7 @@ jobs: # Download to files and verify checksums (allows HTTP mirror redirects) # Install autoconf 2.72 curl -sSL --retry 3 --retry-delay 5 https://ftpmirror.gnu.org/gnu/autoconf/autoconf-2.72.tar.xz -o autoconf-2.72.tar.xz - echo "ba885c1319578d6c94d46e9b0dceb4014caafe2490e437a0dbca3f270a223f5a autoconf-2.72.tar.xz" | sha256sum --check + echo "ba885c1319578d6c94d46e9b0dceb4014caafe2490e437a0dbca3f270a223f5a autoconf-2.72.tar.xz" | sha256sum -c tar xJf autoconf-2.72.tar.xz cd autoconf-2.72 ./configure --prefix=$HOME/.local @@ -160,7 +160,7 @@ jobs: cd .. # Install automake 1.18.1 (exact version used to build xz 5.8.2) curl -sSL --retry 3 --retry-delay 5 https://ftpmirror.gnu.org/gnu/automake/automake-1.18.1.tar.xz -o automake-1.18.1.tar.xz - echo "168aa363278351b89af56684448f525a5bce5079d0b6842bd910fdd3f1646887 automake-1.18.1.tar.xz" | sha256sum --check + echo "168aa363278351b89af56684448f525a5bce5079d0b6842bd910fdd3f1646887 automake-1.18.1.tar.xz" | sha256sum -c tar xJf automake-1.18.1.tar.xz cd automake-1.18.1 ./configure --prefix=$HOME/.local diff --git a/native/build-config.sh b/native/build-config.sh index 7de288f..33062eb 100755 --- a/native/build-config.sh +++ b/native/build-config.sh @@ -48,32 +48,28 @@ BZIP2_URL="https://www.sourceware.org/pub/bzip2/bzip2-${BZIP2_VERSION}.tar.gz" ZLIB_URL="https://zlib.net/zlib-${ZLIB_VERSION}.tar.xz" XZ_URL="https://github.com/tukaani-project/xz/releases/download/v${XZ_VERSION}/xz-${XZ_VERSION}.tar.xz" -# Portable SHA256 helper (prefers sha256sum on Linux, shasum on macOS) -sha256_check() { - local expected="$1" - local file="$2" - if command -v sha256sum >/dev/null 2>&1; then - echo "$expected $file" | sha256sum --check --status - elif command -v shasum >/dev/null 2>&1; then - echo "$expected $file" | shasum -a 256 --check --status - else - echo "ERROR: No SHA256 tool found (need sha256sum or shasum)" >&2 - return 1 - fi -} - +# Portable SHA256 helper (computes hash and compares directly; +# avoids --check flag which differs between GNU and BSD sha256sum) sha256_compute() { local file="$1" - if command -v sha256sum >/dev/null 2>&1; then - sha256sum "$file" | cut -d' ' -f1 - elif command -v shasum >/dev/null 2>&1; then + if command -v shasum >/dev/null 2>&1; then shasum -a 256 "$file" | cut -d' ' -f1 + elif command -v sha256sum >/dev/null 2>&1; then + sha256sum "$file" | cut -d' ' -f1 else - echo "ERROR: No SHA256 tool found (need sha256sum or shasum)" >&2 + echo "ERROR: No SHA256 tool found (need shasum or sha256sum)" >&2 return 1 fi } +sha256_check() { + local expected="$1" + local file="$2" + local actual + actual=$(sha256_compute "$file") || return 1 + [ "$expected" = "$actual" ] +} + # Common build settings export PREFIX="${PREFIX:-$(pwd)/local}" From d6ac9a27bdc18277a49760c21a036c2c40a98d5f Mon Sep 17 00:00:00 2001 From: James A Sutherland Date: Mon, 9 Mar 2026 10:25:31 -0500 Subject: [PATCH 4/4] fix: source build-config.sh in current shell for sha256_compute sha256_compute was defined in build-config.sh but only sourced in a subshell for URL extraction, making it unavailable for hash computation. Source in current shell so both the URL variables and helper functions are accessible. --- scripts/check-native-updates.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/check-native-updates.sh b/scripts/check-native-updates.sh index f05ebec..4dfdb8f 100755 --- a/scripts/check-native-updates.sh +++ b/scripts/check-native-updates.sh @@ -120,19 +120,19 @@ update_version() { sed -i "s/^${var_name}=\"${old_ver}\"$/${var_name}=\"${new_ver}\"/" "$config_script" fi - # Compute new download URL and SHA256 - # Re-source config (which now has the new version) to get the URL + # Re-source config in current shell to get updated URL and sha256_compute + . "$config_script" >/dev/null 2>&1 || true + local url_var_name url_var_name="$(echo "$dep_name" | tr '[:lower:]-' '[:upper:]_')_URL" local new_url - new_url=$(. "$config_script" 2>/dev/null; eval echo "\$$url_var_name" 2>/dev/null || true) + eval "new_url=\$$url_var_name" if [ -n "$new_url" ]; then echo " Downloading new version to compute SHA256..." local tmpfile tmpfile=$(mktemp) if curl -fsSL "$new_url" -o "$tmpfile" 2>/dev/null; then - # Use sha256_compute from build-config.sh (already sourced above) local new_sha256 new_sha256=$(sha256_compute "$tmpfile") || true rm -f "$tmpfile"