Skip to content

Commit 9d5a61d

Browse files
dmitriplotnikovcopybara-github
dmitriplotnikov
authored and
copybara-github
committed
Internal change
PiperOrigin-RevId: 930640971
1 parent 76b61ae commit 9d5a61d

4 files changed

Lines changed: 180 additions & 11 deletions

File tree

‎.bazelrc‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,3 +32,7 @@ build:windows --google_default_credentials=true
3232
build:macos --remote_cache=https://storage.googleapis.com/macos-cel-python-remote-cache
3333
build:macos --google_default_credentials=true
3434

35+
# Silence deprecation warnings from external dependencies (Linux and macOS)
36+
build:linux --cxxopt=-Wno-deprecated-declarations
37+
build:macos --cxxopt=-Wno-deprecated-declarations
38+

‎release/kokoro/release_linux.cfg‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,8 @@
33

44
build_file: "cel-python/release/kokoro/release_linux.sh"
55
timeout_mins: 120
6+
7+
container_properties {
8+
docker_image: "us-central1-docker.pkg.dev/kokoro-container-bakery/kokoro/ubuntu/ubuntu2204/ktcb:current"
9+
docker_sibling_containers: true
10+
}

‎release/kokoro/release_linux.sh‎

Lines changed: 162 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,167 @@
11
#!/bin/bash
2+
# Copyright 2026 Google LLC
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# https://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
216
set -e
317

18+
# Avoid virtualenv/pip trying to download/upgrade tools from PyPI on host
19+
export VIRTUALENV_NO_DOWNLOAD=1
20+
export PIP_DISABLE_PIP_VERSION_CHECK=1
21+
22+
# Pass these environment variables to the cibuildwheel Docker container
23+
export CIBW_ENVIRONMENT="VIRTUALENV_NO_DOWNLOAD=1 PIP_DISABLE_PIP_VERSION_CHECK=1"
24+
export CIBW_DEPENDENCY_VERSIONS="latest"
25+
export CIBW_CONTAINER_ENGINE_EXTRA_ARGS="--network=host"
26+
427
# If running locally (not on Kokoro), authenticate with gcloud.
528
if [ -z "${KOKORO_BUILD_ID}" ]; then
629
if ! gcloud auth application-default print-access-token --quiet > /dev/null; then
730
gcloud auth application-default login
831
fi
932
fi
1033

11-
pip install -U keyring keyrings.google-artifactregistry-auth twine cibuildwheel
34+
# We use --no-cache-dir to force pip to download packages fresh and bypass the local
35+
# cache. In a sandboxed build environment, writing to the default cache directory
36+
# (~/.cache/pip) can encounter permission/sandbox restrictions or lead to stale
37+
# dependency resolution. Disabling the cache ensures a reliable, reproducible install.
38+
pip install --no-cache-dir -U keyring keyrings.google-artifactregistry-auth twine cibuildwheel
39+
40+
# ==============================================================================
41+
# FUTURE-PROOF RUNTIME PATCHING OF CIBUILDWHEEL
42+
# ==============================================================================
43+
# To run cibuildwheel on Google's sandboxed RBE/Kokoro infrastructure, we must:
44+
# 1. Bypass RBE's stdout proxy buffering deadlock (requires 32KB padding).
45+
# 2. Bypass RBE's stdin EOF deadlock during copy-in (requires 'docker cp'
46+
# since we use disable_host_mount: True in pyproject.toml).
47+
#
48+
# Since cibuildwheel is installed fresh from PyPI on every build (ensuring we get
49+
# the latest security and feature updates), we apply these patches at runtime.
50+
#
51+
# Why this patching strategy is future-proof and safe:
52+
# - Strict Validation: The Python patcher strictly validates that all target
53+
# code blocks exist before applying replacements. If cibuildwheel's internal
54+
# code changes in a future release, the patcher will FAIL LOUDLY and exit the
55+
# build immediately (sys.exit(1)) rather than silently running a broken,
56+
# hanging build.
57+
# - Stable Boundaries: The copy_into patch uses a robust regular expression
58+
# anchored to class method boundaries (def copy_into -> def copy_out). These
59+
# are stable, long-standing internal APIs of cibuildwheel's OCIContainer.
60+
# - Core Protocol Stability: The buffering patches target the core protocol
61+
# used to communicate with the container's persistent bash shell. This
62+
# protocol is fundamental to cibuildwheel and highly unlikely to change.
63+
# ==============================================================================
64+
OCI_PATH=$(python3 -c "import cibuildwheel.oci_container; print(cibuildwheel.oci_container.__file__)")
65+
echo "Patching cibuildwheel at $OCI_PATH..."
66+
67+
cat << 'EOF' > patch_oci.py
68+
import sys
69+
import re
70+
71+
path = sys.argv[1]
72+
with open(path, 'r') as f:
73+
content = f.read()
74+
75+
# 1. Force a 32KB flush at the end of every command execution
76+
target_write = 'printf "%04d%s\\n" $? {end_of_message}'
77+
replacement_write = 'printf "%04d%s\\n%32768s\\n" $? {end_of_message} " "'
78+
if target_write in content:
79+
content = content.replace(target_write, replacement_write)
80+
print("Patched write loop.")
81+
else:
82+
print("ERROR: Could not find write loop target in oci_container.py! The cibuildwheel version might have changed.")
83+
sys.exit(1)
84+
85+
# 2. Read and discard the 32KB padding to keep the stream clean
86+
target_read = """ # add the last line to output, without the footer
87+
output_io.write(line[0:footer_offset])
88+
output_io.flush()
89+
break"""
90+
91+
replacement_read = """ # add the last line to output, without the footer
92+
output_io.write(line[0:footer_offset])
93+
output_io.flush()
94+
# Read and discard the 32KB padding line to clear the stream!
95+
self.bash_stdout.readline()
96+
break"""
97+
98+
if target_read in content:
99+
content = content.replace(target_read, replacement_read)
100+
print("Patched read loop.")
101+
else:
102+
print("ERROR: Could not find read loop target in oci_container.py! The cibuildwheel version might have changed.")
103+
sys.exit(1)
104+
105+
# 3. Patch the entire copy_into method using a unique regex to use native 'docker cp'.
106+
# This bypasses the RBE stdin EOF deadlock when copying the project into the container.
107+
pattern = re.compile(r' def copy_into\(self,.*?\).*?:.*? def copy_out', re.DOTALL)
108+
109+
replacement_copy = """ def copy_into(self, from_path: Path, to_path: PurePath) -> None:
110+
if from_path.is_dir():
111+
self.call(["mkdir", "-p", to_path])
112+
subprocess.run(
113+
f"tar -c {self.host_tar_format} -f - . | {self.engine.name} exec -i {self.name} tar --no-same-owner -xC {shell_quote(to_path)} -f -",
114+
shell=True,
115+
check=True,
116+
cwd=from_path,
117+
)
118+
else:
119+
self.call(["mkdir", "-p", to_path.parent])
120+
# Use native docker cp to copy the file, avoiding stdin EOF deadlocks in RBE
121+
subprocess.run(
122+
[
123+
self.engine.name,
124+
"cp",
125+
str(from_path),
126+
f"{self.name}:{to_path}",
127+
],
128+
check=True,
129+
)
130+
131+
def copy_out"""
132+
133+
if pattern.search(content):
134+
content = pattern.sub(replacement_copy, content)
135+
print("Patched copy_into method using unique regex.")
136+
else:
137+
print("ERROR: Could not find copy_into method boundary in oci_container.py! The cibuildwheel version might have changed.")
138+
sys.exit(1)
139+
140+
with open(path, 'w') as f:
141+
f.write(content)
142+
143+
print("Successfully patched oci_container.py!")
144+
EOF
145+
146+
python3 patch_oci.py "$OCI_PATH"
147+
rm patch_oci.py
148+
149+
# Verify that the patched file is syntactically valid Python
150+
echo "Verifying patched oci_container.py syntax..."
151+
python3 -m py_compile "$OCI_PATH" || { echo "ERROR: Patched oci_container.py is corrupted!"; exit 1; }
152+
153+
REPO_DIR=""
154+
TMP_DIR=""
155+
cleanup() {
156+
echo "Cleaning up temporary directories..."
157+
[ -n "${REPO_DIR}" ] && rm -rf "${REPO_DIR}"
158+
[ -n "${TMP_DIR}" ] && rm -rf "${TMP_DIR}"
159+
}
160+
trap cleanup EXIT
12161

13162
REPO_DIR=$(mktemp -d)
14163
echo "Created temporary directory: ${REPO_DIR}"
15164

16-
# Ensure the temporary directory is removed on script exit
17-
trap 'echo "Cleaning up temporary directory: ${REPO_DIR}"; rm -rf "${REPO_DIR}"' EXIT
18-
19165
if [ "${DRY_RUN}" = "true" ]; then
20166
echo "[DRY RUN] Using local Kokoro clone instead of cloning main."
21167
SRC_DIR="$(cd "$(dirname "$0")/../.." && pwd)"
@@ -40,24 +186,31 @@ fi
40186
VERSION=${VERSION#v}
41187
echo "Building release for version: ${VERSION}"
42188

43-
TMP_DIR=$(mktemp -d)
189+
# Create the build directory inside the workspace volume (SRC_DIR)
190+
# instead of the ephemeral /tmp, so that the sibling container can
191+
# access it natively via volume propagation
192+
TMP_DIR="${SRC_DIR}/build_area"
193+
mkdir -p "${TMP_DIR}"
44194
echo "Build directory: ${TMP_DIR}"
45-
46-
# Add trap cleanup for TMP_DIR as well
47-
trap 'echo "Cleaning up temporary directories: ${REPO_DIR} ${TMP_DIR}"; rm -rf "${REPO_DIR}" "${TMP_DIR}"' EXIT
195+
export TMPDIR="${TMP_DIR}/tmp"
196+
mkdir -p "${TMPDIR}"
48197

49198
pushd "${TMP_DIR}"
50199

51200
cp -r "${SRC_DIR}"/{*,.*} . 2>/dev/null || true
52201
cp -r "${SRC_DIR}"/release/* . 2>/dev/null || true
53202
rm -rf cel_expr_python/*_test.py
54203

204+
echo "Downloading bazelisk on host..."
205+
curl -LO https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-amd64
206+
chmod +x bazelisk-linux-amd64
207+
55208
# Check if pyproject.toml exists before running sed
56209
if [ -f pyproject.toml ]; then
57210
sed -i "" "s/\$VERSION/${VERSION}/g" pyproject.toml || sed -i "s/\$VERSION/${VERSION}/g" pyproject.toml
58211
fi
59212

60-
echo "Running cibuildwheel: ${CIBWHEEL_BIN}"
213+
echo "Running cibuildwheel..."
61214
# Default CIBWHEEL_BIN if not set
62215
if [ -z "${CIBWHEEL_BIN}" ]; then
63216
CIBWHEEL_BIN="python3 -m cibuildwheel"

‎release/pyproject.toml‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,12 +39,19 @@ exclude = ["codelab*", "conformance*", "custom_ext*", "release*", "testing*", "w
3939

4040
[tool.cibuildwheel]
4141
build = "cp311-* cp312-* cp313-* cp314-*"
42-
skip = "*musllinux* *win32*"
42+
skip = "*musllinux* *win32* *i686*"
4343
test-command = "python {project}/cel_basic_test.py"
4444
build-verbosity = 1
4545

4646
[tool.cibuildwheel.linux]
47-
before-all = "echo 'Installing bazelisk'; curl -LO https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-amd64 && chmod +x bazelisk-linux-amd64 && mv bazelisk-linux-amd64 /usr/local/bin/bazel"
47+
manylinux-x86_64-image = "manylinux_2_28"
48+
container-engine = "docker; disable_host_mount: True"
49+
# Google's internal Kokoro/RBE network uses a secure MITM proxy that resigns HTTPS
50+
# traffic with an internal Google CA. Since the public manylinux container does not
51+
# trust this CA, git fetches for external dependencies (like @cel-cpp) will fail
52+
# with SSL certificate errors. We disable http.sslVerify inside the container to
53+
# bypass this and allow Bazel to fetch SCM dependencies through the proxy.
54+
before-all = "git config --global http.sslVerify false && echo 'Installing bazelisk' && cp {project}/bazelisk-linux-amd64 /usr/local/bin/bazel"
4855

4956
[tool.cibuildwheel.macos]
5057
before-all = "echo 'Installing bazelisk'; brew install bazelisk"

0 commit comments

Comments
 (0)