feat: two-stage CI build with cached prereqs and TensorStore FetchCon…

.gitignore

@@ -562,9 +562,14 @@ dmypy.json
 # Cython debug symbols
 cython_debug/
 
-# scratch 
+# scratch
 scratch/
 
+# CI build caches (populated by prebuild job, restored by wheel-build jobs)
+prereq_cache/
+ts_fc_cache/
+ccache_dir/
+
 build/
 build_man/
 venv/

CMakeLists.txt

@@ -36,12 +36,21 @@ set(SOURCE
            src/cpp/utilities/utilities.cpp
 )
 
+# Use ccache if available to speed up repeated TensorStore compilation in CI
+find_program(CCACHE_EXECUTABLE ccache)
+if(CCACHE_EXECUTABLE)
+    message(STATUS "ccache found: ${CCACHE_EXECUTABLE}")
+    set(CMAKE_CXX_COMPILER_LAUNCHER "${CCACHE_EXECUTABLE}" CACHE STRING "" FORCE)
+    set(CMAKE_C_COMPILER_LAUNCHER   "${CCACHE_EXECUTABLE}" CACHE STRING "" FORCE)
+endif()
+
 include(FetchContent)
 
 FetchContent_Declare(
   tensorstore
   URL "https://github.com/google/tensorstore/archive/refs/tags/v0.1.80.tar.gz"
   URL_HASH SHA256=94866de34b6139d77d30e828a50f9e8df98e7dd68e848393470879aeb50ea7bf
+  FIND_PACKAGE_ARGS
 )
 
 # Additional FetchContent_Declare calls as needed...

ci-utils/before_all_linux.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+# before_all_linux.sh
+# Replaces the inline CIBW_BEFORE_ALL_LINUX command.
+# Runs inside the manylinux container during the wheel-build job.
+# Expects:
+#   /project/prereq_cache/local_install  — populated by prebuild job (may be absent on cache miss)
+set -e
+
+cd /project
+
+# Always install NASM (required by TensorStore's aom/dav1d deps)
+curl -L https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 \
+     -o nasm.tar.bz2
+tar -xjf nasm.tar.bz2
+cd nasm-2.15.05 && ./configure && make -j"$(nproc)" && make install
+cd /project
+
+# Install ccache (CCACHE_DIR and CCACHE_MAXSIZE are set via CIBW_ENVIRONMENT_LINUX)
+dnf install -y ccache
+
+mkdir -p /tmp/argolid_bld
+
+if [ -d "/project/prereq_cache/local_install" ]; then
+    echo "==> Prereq cache hit: restoring filepattern + pybind11"
+    cp -r /project/prereq_cache/local_install /tmp/argolid_bld/local_install
+    # zlib/jpeg/png cannot be cached from /usr/local (container-internal path),
+    # so rebuild them quickly from source (~1 min total).
+    bash ci-utils/install_sys_deps_linux.sh
+else
+    echo "==> Prereq cache miss: building filepattern + pybind11 + sys deps from scratch"
+    bash ci-utils/install_prereq_linux.sh
+    cp -r /project/local_install /tmp/argolid_bld/local_install
+fi

ci-utils/before_all_macos.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+# before_all_macos.sh
+# Replaces the inline CIBW_BEFORE_ALL_MACOS commands for wheel-build jobs.
+# brew uninstalls (openssl for intel, jpeg-turbo for arm64) should be done
+# by the caller (CIBW_BEFORE_ALL inline) BEFORE invoking this script.
+# Expects:
+#   prereq_cache/local_install  — populated by prebuild job (may be absent on cache miss)
+set -e
+
+mkdir -p /tmp/argolid_bld
+
+if [ -d "prereq_cache/local_install" ]; then
+    echo "==> Prereq cache hit: restoring filepattern + pybind11"
+    cp -r prereq_cache/local_install /tmp/argolid_bld/local_install
+    # Rebuild sys deps (zlib/jpeg/png) to /usr/local — these are fast and
+    # cannot be cached across jobs since they live outside the workspace.
+    bash ci-utils/install_sys_deps_linux.sh
+else
+    echo "==> Prereq cache miss: building filepattern + pybind11 + sys deps from scratch"
+    bash ci-utils/install_prereq_linux.sh
+    cp -r local_install /tmp/argolid_bld/local_install
+fi

ci-utils/before_all_win.bat

@@ -0,0 +1,24 @@
+@echo off
+REM before_all_win.bat
+REM Replaces the inline CIBW_BEFORE_ALL_WINDOWS command for wheel-build jobs.
+REM Expects:
+REM   prereq_cache\local_install — populated by prebuild job (may be absent on cache miss)
+
+if exist prereq_cache\local_install (
+    echo =^> Prereq cache hit: restoring filepattern + pybind11
+    xcopy /E /I /y prereq_cache\local_install C:\TEMP\argolid_bld\local_install
+    REM Replicate the DLL copy that install_prereq_win.bat does for PATH
+    if exist prereq_cache\local_install\bin (
+        xcopy /E /I /y prereq_cache\local_install\bin %TEMP%\argolid\bin
+    )
+) else (
+    echo =^> Prereq cache miss: building filepattern + pybind11 from scratch
+    call ci-utils\install_prereq_win.bat
+    if errorlevel 1 exit 1
+    xcopy /E /I /y local_install C:\TEMP\argolid_bld\local_install
+    REM Explicitly copy DLLs to PATH location (don't rely on install_prereq_win.bat's
+    REM conditional copy which requires ON_GITHUB to be already set)
+    if exist local_install\bin (
+        xcopy /E /I /y local_install\bin %TEMP%\argolid\bin
+    )
+)

ci-utils/install_prereq_linux.sh

@@ -1,8 +1,16 @@
 #!/bin/bash
-# Usage: $bash install_prereq_linux.sh $INSTALL_DIR
+# Usage: bash install_prereq_linux.sh [$INSTALL_DIR] [--skip-sys-deps]
 # Default $INSTALL_DIR = ./local_install
+# --skip-sys-deps: skip building zlib, libjpeg-turbo, libpng (use when already installed)
 #
-if [ -z "$1" ]
+SKIP_SYS_DEPS=false
+for arg in "$@"; do
+    case "$arg" in
+        --skip-sys-deps) SKIP_SYS_DEPS=true ;;
+    esac
+done
+
+if [ -z "$1" ] || [ "$1" = "--skip-sys-deps" ]
 then
       echo "No path to the Argolid source location provided"
       echo "Creating local_install directory"
@@ -34,14 +42,15 @@ make install -j4
 cd ../../
 
 
+if [ "$SKIP_SYS_DEPS" != "true" ]; then
 curl -L https://github.com/madler/zlib/releases/download/v1.3.1/zlib131.zip -o zlib131.zip
 unzip zlib131.zip
 cd zlib-1.3.1
 mkdir build_man
 cd build_man
-cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=/usr/local ..  
-cmake --build . 
-cmake --build . --target install 
+cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=/usr/local ..
+cmake --build .
+cmake --build . --target install
 cd ../../
 
 curl -L https://github.com/libjpeg-turbo/libjpeg-turbo/archive/refs/tags/3.1.0.zip -o 3.1.0.zip
@@ -67,3 +76,4 @@ cd build_man
 cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=/usr/local ..
 make install -j4
 cd ../../
+fi

ci-utils/install_sys_deps_linux.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+# install_sys_deps_linux.sh
+# Builds and installs zlib, libjpeg-turbo, and libpng to /usr/local.
+# Works on Linux (manylinux) and macOS.
+# These are required by TensorStore when built with TENSORSTORE_USE_SYSTEM_* flags.
+set -e
+
+JOBS=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)
+BUILD_TMPDIR="$(mktemp -d)"
+
+# zlib
+curl -L https://github.com/madler/zlib/releases/download/v1.3.1/zlib131.zip -o "${BUILD_TMPDIR}/zlib131.zip"
+unzip -q "${BUILD_TMPDIR}/zlib131.zip" -d "${BUILD_TMPDIR}"
+mkdir -p "${BUILD_TMPDIR}/zlib-1.3.1/build_man"
+cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=/usr/local \
+      -S "${BUILD_TMPDIR}/zlib-1.3.1" -B "${BUILD_TMPDIR}/zlib-1.3.1/build_man"
+cmake --build "${BUILD_TMPDIR}/zlib-1.3.1/build_man" -j"${JOBS}"
+cmake --build "${BUILD_TMPDIR}/zlib-1.3.1/build_man" --target install
+
+# libjpeg-turbo
+curl -L https://github.com/libjpeg-turbo/libjpeg-turbo/archive/refs/tags/3.1.0.zip \
+     -o "${BUILD_TMPDIR}/3.1.0.zip"
+unzip -q "${BUILD_TMPDIR}/3.1.0.zip" -d "${BUILD_TMPDIR}"
+mkdir -p "${BUILD_TMPDIR}/libjpeg-turbo-3.1.0/build_man"
+cmake -DCMAKE_INSTALL_PREFIX=/usr/local -DENABLE_STATIC=FALSE \
+      -DCMAKE_BUILD_TYPE=Release \
+      -S "${BUILD_TMPDIR}/libjpeg-turbo-3.1.0" \
+      -B "${BUILD_TMPDIR}/libjpeg-turbo-3.1.0/build_man"
+if [[ "$OSTYPE" == "darwin"* ]]; then
+  sudo cmake --build "${BUILD_TMPDIR}/libjpeg-turbo-3.1.0/build_man" \
+             --target install -j"${JOBS}"
+else
+  cmake --build "${BUILD_TMPDIR}/libjpeg-turbo-3.1.0/build_man" \
+        --target install -j"${JOBS}"
+fi
+
+# libpng
+curl -L https://github.com/glennrp/libpng/archive/refs/tags/v1.6.53.zip \
+     -o "${BUILD_TMPDIR}/v1.6.53.zip"
+unzip -q "${BUILD_TMPDIR}/v1.6.53.zip" -d "${BUILD_TMPDIR}"
+mkdir -p "${BUILD_TMPDIR}/libpng-1.6.53/build_man"
+cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=/usr/local \
+      -S "${BUILD_TMPDIR}/libpng-1.6.53" \
+      -B "${BUILD_TMPDIR}/libpng-1.6.53/build_man"
+cmake --build "${BUILD_TMPDIR}/libpng-1.6.53/build_man" \
+      --target install -j"${JOBS}"
+
+rm -rf "${BUILD_TMPDIR}"

ci-utils/prebuild_linux.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+# prebuild_linux.sh
+# Runs inside the manylinux_2_28_x86_64 Docker container during the prebuild job.
+# Populates:
+#   /project/prereq_cache/local_install  — filepattern + pybind11 headers/libs
+#   /project/ts_fc_cache/               — TensorStore FetchContent source + build
+set -e
+
+cd /project
+
+# Install NASM (required by aom/dav1d inside TensorStore)
+curl -L https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 \
+     -o nasm.tar.bz2
+tar -xjf nasm.tar.bz2
+cd nasm-2.15.05 && ./configure && make -j"$(nproc)" && make install
+cd /project
+
+# Install ccache and point it at the workspace-mounted cache dir
+dnf install -y ccache
+mkdir -p /project/ccache_dir
+export CCACHE_DIR=/project/ccache_dir
+export CCACHE_MAXSIZE=5G
+
+# Build zlib, libjpeg-turbo, libpng to /usr/local (needed for TENSORSTORE_USE_SYSTEM_* flags)
+bash ci-utils/install_sys_deps_linux.sh
+
+# Build filepattern + pybind11 → prereq_cache/local_install
+# (--skip-sys-deps: sys deps already installed above)
+bash ci-utils/install_prereq_linux.sh ./prereq_cache/local_install --skip-sys-deps
+
+# Pre-build TensorStore using the minimal cmake project (no Python, no pybind11 needed).
+# FETCHCONTENT_BASE_DIR stores TensorStore source + build state in ts_fc_cache/
+# so wheel-build jobs can skip the expensive TensorStore download and reuse cmake state.
+cmake -S /project/ci-utils/tensorstore_prebuild \
+      -B /tmp/ts_cmake_prebuild \
+      -DFETCHCONTENT_BASE_DIR=/project/ts_fc_cache \
+      -DTENSORSTORE_USE_SYSTEM_JPEG=ON \
+      -DTENSORSTORE_USE_SYSTEM_ZLIB=ON \
+      -DTENSORSTORE_USE_SYSTEM_PNG=ON \
+      -DCMAKE_BUILD_TYPE=Release
+cmake --build /tmp/ts_cmake_prebuild -j"$(nproc)"

ci-utils/tensorstore_prebuild/CMakeLists.txt

@@ -0,0 +1,33 @@
+cmake_minimum_required(VERSION 3.24)
+project(TensorStorePrebuild)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+# Use ccache if available to speed up repeated TensorStore compilation in CI
+find_program(CCACHE_EXECUTABLE ccache)
+if(CCACHE_EXECUTABLE)
+    message(STATUS "ccache found: ${CCACHE_EXECUTABLE}")
+    set(CMAKE_CXX_COMPILER_LAUNCHER "${CCACHE_EXECUTABLE}" CACHE STRING "" FORCE)
+    set(CMAKE_C_COMPILER_LAUNCHER   "${CCACHE_EXECUTABLE}" CACHE STRING "" FORCE)
+endif()
+
+include(FetchContent)
+
+# Keep URL + SHA256 in sync with root CMakeLists.txt
+# Cache key hashes both this file and root CMakeLists.txt
+FetchContent_Declare(
+  tensorstore
+  URL "https://github.com/google/tensorstore/archive/refs/tags/v0.1.80.tar.gz"
+  URL_HASH SHA256=94866de34b6139d77d30e828a50f9e8df98e7dd68e848393470879aeb50ea7bf
+)
+
+FetchContent_MakeAvailable(tensorstore)
+
+# Executable target forces TensorStore and all its transitive deps to be compiled.
+# No Python, no pybind11, no filepattern needed.
+add_executable(ts_prebuild_marker ts_prebuild_marker.cc)
+target_link_libraries(ts_prebuild_marker PRIVATE
+  tensorstore::tensorstore
+  tensorstore::all_drivers
+)

ci-utils/tensorstore_prebuild/ts_prebuild_marker.cc

@@ -0,0 +1,6 @@
+#include "tensorstore/open.h"
+
+// Minimal stub that forces the linker to pull in TensorStore symbols.
+// This file exists solely to make ts_prebuild_marker a non-trivial executable
+// so cmake compiles all TensorStore transitive dependencies during the prebuild.
+int main() { return 0; }