diff --git a/.bazelrc b/.bazelrc index 5c70e99..333e7bb 100644 --- a/.bazelrc +++ b/.bazelrc @@ -28,13 +28,17 @@ common:windows --experimental_repository_downloader_retries=10 build --verbose_failures test --test_output=errors -# GCS remote caching config (Windows-only, active by default on Windows!) -build:windows --remote_cache=https://storage.googleapis.com/windows-cel-python-remote-cache -build:windows --google_default_credentials=true +# GCS remote caching config (Linux-only, opt-in) +build:remote-cache-linux --remote_cache=https://storage.googleapis.com/linux-cel-python-remote-cache +build:remote-cache-linux --google_default_credentials=true -# GCS remote caching config (macOS-only, active by default on macOS!) -build:macos --remote_cache=https://storage.googleapis.com/macos-cel-python-remote-cache -build:macos --google_default_credentials=true +# GCS remote caching config (Windows-only, opt-in) +build:remote-cache-windows --remote_cache=https://storage.googleapis.com/windows-cel-python-remote-cache +build:remote-cache-windows --google_default_credentials=true + +# GCS remote caching config (macOS-only, opt-in) +build:remote-cache-macos --remote_cache=https://storage.googleapis.com/macos-cel-python-remote-cache +build:remote-cache-macos --google_default_credentials=true # Silence deprecation warnings from external dependencies (Linux and macOS) build:linux --cxxopt=-Wno-deprecated-declarations diff --git a/release/kokoro/release_linux.cfg b/release/kokoro/release_linux.cfg index a42e2e3..018c022 100644 --- a/release/kokoro/release_linux.cfg +++ b/release/kokoro/release_linux.cfg @@ -5,6 +5,6 @@ build_file: "cel-python/release/kokoro/release_linux.sh" timeout_mins: 120 container_properties { - docker_image: "us-central1-docker.pkg.dev/kokoro-container-bakery/kokoro/ubuntu/ubuntu2204/ktcb:current" + docker_image: "mirror.gcr.io/library/ubuntu:24.04" docker_sibling_containers: true } diff --git a/release/kokoro/release_linux.sh b/release/kokoro/release_linux.sh index 5292ef6..ff27fa7 100755 --- a/release/kokoro/release_linux.sh +++ b/release/kokoro/release_linux.sh @@ -15,12 +15,42 @@ set -e +if ! command -v pip3 &> /dev/null || ! command -v curl &> /dev/null || ! command -v docker &> /dev/null; then + echo "Installing basic dependencies..." + apt-get update && apt-get install -y python3-pip curl + + if ! command -v docker &> /dev/null; then + echo "Installing docker CLI..." + ARCH=$(uname -m) + if [ "$ARCH" = "x86_64" ]; then + DOCKER_ARCH="x86_64" + elif [ "$ARCH" = "aarch64" ]; then + DOCKER_ARCH="aarch64" + else + echo "Unsupported arch: $ARCH" + exit 1 + fi + curl -fsSL "https://download.docker.com/linux/static/stable/${DOCKER_ARCH}/docker-24.0.7.tgz" -o docker.tgz + tar xzvf docker.tgz --strip-components=1 docker/docker + mv docker /usr/local/bin/ + rm -f docker.tgz + fi +fi + # Avoid virtualenv/pip trying to download/upgrade tools from PyPI on host export VIRTUALENV_NO_DOWNLOAD=1 export PIP_DISABLE_PIP_VERSION_CHECK=1 +export PIP_BREAK_SYSTEM_PACKAGES=1 +export PIP_DEFAULT_TIMEOUT=60 + +if [ "$(uname -m)" = "aarch64" ]; then + export CIBW_ARCHS="aarch64" +else + export CIBW_ARCHS="x86_64" +fi # Pass these environment variables to the cibuildwheel Docker container -export CIBW_ENVIRONMENT="VIRTUALENV_NO_DOWNLOAD=1 PIP_DISABLE_PIP_VERSION_CHECK=1" +export CIBW_ENVIRONMENT="VIRTUALENV_NO_DOWNLOAD=1 PIP_DISABLE_PIP_VERSION_CHECK=1 PIP_DEFAULT_TIMEOUT=120 CEL_BAZEL_FLAGS=--config=remote-cache-linux" export CIBW_DEPENDENCY_VERSIONS="latest" export CIBW_CONTAINER_ENGINE_EXTRA_ARGS="--network=host" @@ -35,7 +65,10 @@ fi # cache. In a sandboxed build environment, writing to the default cache directory # (~/.cache/pip) can encounter permission/sandbox restrictions or lead to stale # dependency resolution. Disabling the cache ensures a reliable, reproducible install. -pip install --no-cache-dir -U keyring keyrings.google-artifactregistry-auth twine cibuildwheel +pip install --no-cache-dir -U keyring keyrings.google-artifactregistry-auth twine +curl -fsSL https://github.com/pypa/cibuildwheel/archive/refs/tags/v4.1.0.tar.gz -o cibuildwheel-4.1.0.tar.gz +pip install --no-cache-dir cibuildwheel-4.1.0.tar.gz +rm -f cibuildwheel-4.1.0.tar.gz # ============================================================================== # FUTURE-PROOF RUNTIME PATCHING OF CIBUILDWHEEL @@ -203,9 +236,19 @@ rm -rf cel_expr_python/*_test.py echo "Downloading bazelisk on host..." curl -LO https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-amd64 -chmod +x bazelisk-linux-amd64 +curl -LO https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-arm64 +chmod +x bazelisk-linux-amd64 bazelisk-linux-arm64 + +echo "Downloading build dependencies on host..." +mkdir -p build_deps +pip download --no-cache-dir --only-binary=:all: --dest build_deps "setuptools>=40.8.0" "wheel" +if [ "$(uname -m)" = "aarch64" ]; then + PLATFORM_SUFFIX="aarch64" +else + PLATFORM_SUFFIX="x86_64" +fi +pip download --no-cache-dir --only-binary=:all: --dest build_deps --python-version 3.9 --platform "manylinux2014_${PLATFORM_SUFFIX}" "virtualenv" "typing-extensions>=4.13.2" -# Check if pyproject.toml exists before running sed if [ -f pyproject.toml ]; then sed -i "" "s/\$VERSION/${VERSION}/g" pyproject.toml || sed -i "s/\$VERSION/${VERSION}/g" pyproject.toml fi diff --git a/release/kokoro/release_macos.sh b/release/kokoro/release_macos.sh index 71d4805..54b92fa 100644 --- a/release/kokoro/release_macos.sh +++ b/release/kokoro/release_macos.sh @@ -65,6 +65,8 @@ if [ -f pyproject.toml ]; then sed -i "" "s/\$VERSION/${VERSION}/g" pyproject.toml || sed -i "s/\$VERSION/${VERSION}/g" pyproject.toml fi +export CEL_BAZEL_FLAGS="--config=remote-cache-macos" + echo "Running cibuildwheel: ${CIBWHEEL_BIN}" # Default CIBWHEEL_BIN if not set if [ -z "${CIBWHEEL_BIN}" ]; then diff --git a/release/kokoro/release_windows.bat b/release/kokoro/release_windows.bat index 9f95c7e..75fc8fa 100644 --- a/release/kokoro/release_windows.bat +++ b/release/kokoro/release_windows.bat @@ -16,6 +16,11 @@ setlocal enabledelayedexpansion set "RELEASE_STATUS=0" set "FETCH_RETRIES=10" set "FETCH_RETRY_DELAY_S=10" +echo --- Installing Python 3.11 via Chocolatey --- +choco install python311 -y --no-progress +if !ERRORLEVEL! NEQ 0 ( + echo WARNING: Failed to install Python 3.11 via Chocolatey. +) echo === Loading Environment Configuration === call "%~dp0set_env_windows.bat" @@ -145,6 +150,7 @@ if !FETCH_STATUS! NEQ 0 ( if exist fetch.log del fetch.log echo --- Running cibuildwheel --- +set "CEL_BAZEL_FLAGS=--config=remote-cache-windows" if "%CIBWHEEL_BIN%" == "" ( set "CIBWHEEL_BIN=!PYTHON_EXE! -m cibuildwheel" ) diff --git a/release/pyproject.toml b/release/pyproject.toml index c6b322a..d7aa211 100644 --- a/release/pyproject.toml +++ b/release/pyproject.toml @@ -44,14 +44,18 @@ test-command = "python {project}/cel_basic_test.py" build-verbosity = 1 [tool.cibuildwheel.linux] +build-frontend = { name = "pip", args = ["--no-build-isolation"] } +before-build = "pip install --no-index --find-links={project}/build_deps setuptools wheel virtualenv" +archs = ["x86_64", "aarch64"] manylinux-x86_64-image = "manylinux_2_28" +manylinux-aarch64-image = "manylinux_2_28" container-engine = "docker; disable_host_mount: True" # Google's internal Kokoro/RBE network uses a secure MITM proxy that resigns HTTPS # traffic with an internal Google CA. Since the public manylinux container does not # trust this CA, git fetches for external dependencies (like @cel-cpp) will fail # with SSL certificate errors. We disable http.sslVerify inside the container to # bypass this and allow Bazel to fetch SCM dependencies through the proxy. -before-all = "git config --global http.sslVerify false && echo 'Installing bazelisk' && cp {project}/bazelisk-linux-amd64 /usr/local/bin/bazel" +before-all = "git config --global http.sslVerify false && echo 'Installing bazelisk' && if [ $(uname -m) = 'aarch64' ]; then cp {project}/bazelisk-linux-arm64 /usr/local/bin/bazel; else cp {project}/bazelisk-linux-amd64 /usr/local/bin/bazel; fi && python3 -m pip install --no-index --find-links={project}/build_deps virtualenv" [tool.cibuildwheel.macos] archs = ["x86_64", "arm64"] diff --git a/release/setup.py b/release/setup.py index 09cfe81..e2d0962 100644 --- a/release/setup.py +++ b/release/setup.py @@ -72,6 +72,9 @@ def build_extension(self, ext): # Build with bazel # Use --compilation_mode=opt for release builds cmd = ['bazel', 'build', ext.target, '--compilation_mode=opt'] + extra_flags = os.environ.get('CEL_BAZEL_FLAGS') + if extra_flags: + cmd.extend(extra_flags.split()) if sys.platform == 'win32': self.platform_config_windows(cmd, python_version) if sys.platform == 'darwin':