From 9b230bc5f76d9fa1e8cff71f9ff5148efb1e722d Mon Sep 17 00:00:00 2001 From: Lars Erik Wik Date: Tue, 2 Jun 2026 10:30:29 +0200 Subject: [PATCH 1/2] Add rust tool-chain and protobuf to build hosts Needs to be merged before replacing leech dependency with leech2. Signed-off-by: Lars Erik Wik --- .github/workflows/update-deps.py | 126 ++++++++++++++++++++++++++++++ .github/workflows/update-deps.yml | 4 +- ci/cfengine-build-host-setup.cf | 36 +++++++++ ci/linux-install-protobuf.sh | 40 ++++++++++ ci/linux-install-rust.sh | 100 ++++++++++++++++++++++++ 5 files changed, 304 insertions(+), 2 deletions(-) create mode 100755 ci/linux-install-protobuf.sh create mode 100755 ci/linux-install-rust.sh diff --git a/.github/workflows/update-deps.py b/.github/workflows/update-deps.py index 664515990..ee6b523f4 100644 --- a/.github/workflows/update-deps.py +++ b/.github/workflows/update-deps.py @@ -64,6 +64,16 @@ def parse_args(): action="store_true", help="update SDK 21 on build hosts (needed by Jenkins)", ) + parser.add_argument( + "--rust", + action="store_true", + help="update the Rust toolchain on build hosts (needed to build leech2)", + ) + parser.add_argument( + "--protobuf", + action="store_true", + help="update protoc on build hosts (needed to build leech2)", + ) return parser.parse_args() @@ -341,6 +351,118 @@ def update_jdk21(root): exit(1) +def update_rust(root): + base_url = "https://static.rust-lang.org/dist" + + filename = os.path.join(root, "ci/linux-install-rust.sh") + with open(filename, "r") as f: + content = f.read() + + old_version = re.search(r"version=([0-9]+\.[0-9]+\.[0-9]+)", content).group(1) + log.debug(f"Found version {old_version} in '{filename}'") + + # The stable channel manifest tells us the most recent stable release. + manifest = requests.get(f"{base_url}/channel-rust-stable.toml").text + new_version = re.search( + r"\[pkg\.rust\]\s*\nversion = \"([0-9]+\.[0-9]+\.[0-9]+)", manifest + ).group(1) + log.debug(f"Most recent stable Rust version is {new_version}") + + if old_version == new_version: + log.debug( + f"Rust toolchain is already the newest version ('{new_version}' == '{old_version}')" + ) + return + + # Each tarball has its checksum recorded right below a comment line that + # references its '.sha256' file. Refresh the version number and every SHA. + tarballs = [ + "rustc-${version}-x86_64-unknown-linux-gnu.tar.gz", + "cargo-${version}-x86_64-unknown-linux-gnu.tar.gz", + "rustc-${version}-aarch64-unknown-linux-gnu.tar.gz", + "cargo-${version}-aarch64-unknown-linux-gnu.tar.gz", + "rust-std-${version}-x86_64-unknown-linux-gnu.tar.gz", + "rust-std-${version}-aarch64-unknown-linux-gnu.tar.gz", + "rust-std-${version}-x86_64-pc-windows-gnu.tar.gz", + ] + for tarball in tarballs: + marker = re.escape(f"{tarball}.sha256") + match = re.search(marker + r"\s*\n\s*\w+=([0-9a-f]{64})", content) + old_sha = match.group(1) + + sha_file = tarball.replace("${version}", new_version) + ".sha256" + new_sha = requests.get(f"{base_url}/{sha_file}").text.split()[0] + log.debug(f"Fetched new SHA '{new_sha}' for '{sha_file}'") + + content = content.replace(old_sha, new_sha, 1) + + content = content.replace(f"version={old_version}", f"version={new_version}", 1) + + log.debug(f"Writing changes to file '{filename}'") + with open(filename, "w") as f: + f.write(content) + + if not git_commit(root, f"Updated Rust toolchain to {new_version}"): + log.error("Failed to commit changes after updating Rust toolchain") + exit(1) + + +def update_protobuf(root): + filename = os.path.join(root, "ci/linux-install-protobuf.sh") + with open(filename, "r") as f: + content = f.read() + + old_version = re.search(r"version=([0-9]+\.[0-9]+)", content).group(1) + log.debug(f"Found version {old_version} in '{filename}'") + + # protobuf releases do not ship .sha256 files, so we determine the latest + # version from the GitHub releases API and compute the checksums ourselves. + release = requests.get( + "https://api.github.com/repos/protocolbuffers/protobuf/releases/latest" + ).json() + new_version = release["tag_name"].lstrip("v") + log.debug(f"Most recent protobuf version is {new_version}") + + if old_version == new_version: + log.debug( + f"protoc is already the newest version ('{new_version}' == '{old_version}')" + ) + return + + base_url = ( + f"https://github.com/protocolbuffers/protobuf/releases/download/v{new_version}" + ) + for arch in ["linux-x86_64", "linux-aarch_64"]: + marker = re.escape(f"protoc-${{version}}-{arch}.zip") + match = re.search(marker + r"\s*\n\s*\w+=([0-9a-f]{64})", content) + old_sha = match.group(1) + + zipfile = f"protoc-{new_version}-{arch}.zip" + path = os.path.join("/tmp", zipfile) + if not os.path.exists(path): + url = f"{base_url}/{zipfile}" + log.debug(f"Fetching URL '{url}'") + urllib.request.urlretrieve(url, path) + + sha = hashlib.sha256() + with open(path, "rb") as f: + sha.update(f.read()) + new_sha = sha.hexdigest() + log.debug(f"Computed new SHA '{new_sha}' for '{zipfile}'") + + content = content.replace(old_sha, new_sha, 1) + + content = content.replace(f"version={old_version}", f"version={new_version}", 1) + + log.debug(f"Writing changes to file '{filename}'") + with open(filename, "w") as f: + f.write(content) + + if not git_commit(root, f"Updated protoc to {new_version}"): + log.error("Failed to commit changes after updating protoc") + exit(1) + + def main(): args = parse_args() loglevel = "DEBUG" if args.debug else "INFO" @@ -351,6 +473,10 @@ def main(): update_deps(args.root, args.bump, args.skip) if args.jdk21: update_jdk21(args.root) + if args.rust: + update_rust(args.root) + if args.protobuf: + update_protobuf(args.root) if __name__ == "__main__": diff --git a/.github/workflows/update-deps.yml b/.github/workflows/update-deps.yml index 5725b56e1..0a8aad812 100644 --- a/.github/workflows/update-deps.yml +++ b/.github/workflows/update-deps.yml @@ -45,8 +45,8 @@ jobs: run: | echo "COMMIT_HASH_BEFORE=$(git log -1 --format=%H)">> $GITHUB_ENV - name: Run update script - # we use only master branch for ci build host policy so only check for jdk updates in master - run: python3 /tmp/update-deps.py --debug --bump=${{ matrix.branch == 'master' && 'major' || 'minor' }} ${{ matrix.branch == 'master' && '--jdk21' || '' }} + # we use only master branch for ci build host policy so only check for jdk/rust/protobuf updates in master + run: python3 /tmp/update-deps.py --debug --bump=${{ matrix.branch == 'master' && 'major' || 'minor' }} ${{ matrix.branch == 'master' && '--jdk21 --rust --protobuf' || '' }} - name: Save commit hash after run: | echo "COMMIT_HASH_AFTER=$(git log -1 --format=%H)">> $GITHUB_ENV diff --git a/ci/cfengine-build-host-setup.cf b/ci/cfengine-build-host-setup.cf index 5bcacc288..1c236e228 100644 --- a/ci/cfengine-build-host-setup.cf +++ b/ci/cfengine-build-host-setup.cf @@ -269,6 +269,26 @@ bundle agent cfengine_build_host_setup expression => not(fileexists("/etc/cfengine-in-container.flag")), comment => "We use an explicit flag file that we control to avoid ambiguity about whether we are in a container or not."; + # Rust is build dependency for leech2 (gate on ubuntu>=20, debian>=12, redhat>=8) + ubuntu:: + "leech2_build_toolchain_host" + expression => version_compare("$(sys.os_version_major)", ">=", "20"); + + debian:: + "leech2_build_toolchain_host" + expression => version_compare("$(sys.os_version_major)", ">=", "12"); + + (redhat|centos):: + "leech2_build_toolchain_host" + expression => version_compare("$(sys.os_version_major)", ">=", "8"); + + any:: + "have_rust" + expression => fileexists("/opt/rust/bin/rustc"); + + "have_protoc" + expression => fileexists("/usr/local/bin/protoc"); + linux:: "have_tmp_mount" expression => returnszero("mount | grep '/tmp'", "useshell"); @@ -373,6 +393,22 @@ bundle agent cfengine_build_host_setup contain => in_shell, classes => results("bundle", "java"); + # leech2 build toolchain: protoc and the Rust toolchain. Both installers + # pin a version and verify the SHA256 checksum of the downloaded tarball. + leech2_build_toolchain_host.!have_protoc:: + "sh $(this.promise_dirname)/linux-install-protobuf.sh" + contain => in_shell, + comment => "Install pinned protoc; required to build the cargo-based leech2 dependency."; + + # Linux builds are native, so the installer only adds the host's own Linux + # std. Windows is the only cross-compilation target, and only MinGW build + # hosts cross-compile it, so we pass that target there alone. + leech2_build_toolchain_host.!have_rust:: + "sh $(this.promise_dirname)/linux-install-rust.sh" + args => ifelse("mingw_build_host", "x86_64-pc-windows-gnu", ""), + contain => in_shell, + comment => "Install the Rust toolchain system-wide under /opt/rust for building the cargo-based leech2 dependency."; + (redhat_7|centos_7|redhat_8|centos_8|redhat_9|redhat_10).(!have_development_tools).(yum_dnf_conf_ok):: "yum groups install -y 'Development Tools'" contain => in_shell; diff --git a/ci/linux-install-protobuf.sh b/ci/linux-install-protobuf.sh new file mode 100755 index 000000000..e3293e516 --- /dev/null +++ b/ci/linux-install-protobuf.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +set -e + +install_protobuf() { + # Install the protoc compiler "manually" from the official prebuilt + # release, verifying the SHA256 checksum of the zip. protoc is needed to + # build the cargo-based leech2 dependency. + # + # The release archives do not ship .sha256 files, so the checksums below are + # computed by us (and refreshed by the dependency update script). + cd /opt + version=35.0 + baseurl="https://github.com/protocolbuffers/protobuf/releases/download/v${version}" + + if uname -m | grep aarch64; then + arch=linux-aarch_64 + # sha256sum of protoc-${version}-linux-aarch_64.zip + sha=36b518ac14d90351cc6598228ed2bbe5afe4e357b1af470b07e0ec1609875de2 + else + arch=linux-x86_64 + # sha256sum of protoc-${version}-linux-x86_64.zip + sha=a45cda0989c17dd950db55f6fbe1e5814c50fda08e87aa422980ac1f89dddbbc + fi + + zipfile="protoc-${version}-${arch}.zip" + wget --quiet "$baseurl/$zipfile" + echo "$sha $zipfile" | sha256sum --check - + # Installs bin/protoc and include/ under /usr/local. + unzip -o "$zipfile" -d /usr/local + rm "$zipfile" + + chmod a+rx /usr/local/bin/protoc + cd - +} + +if [ "$(whoami)" = "root" ]; then + install_protobuf +else + sudo bash -c install_protobuf +fi diff --git a/ci/linux-install-rust.sh b/ci/linux-install-rust.sh new file mode 100755 index 000000000..62d3eb71c --- /dev/null +++ b/ci/linux-install-rust.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +set -e + +install_rust() { + # Install the Rust toolchain "manually" from the official standalone + # installers, verifying the SHA256 checksum of each tarball. This is the + # cargo-based build dependency needed to build leech2. + # + # We install the individual component tarballs (rustc, cargo, rust-std) + # rather than the combined "rust" archive: the combined one is ~360 MB and + # extracts to ~1.4 GB of docs/clippy/llvm-tools we never install. The build + # hosts are tight on disk, so we also delete each tarball and its extracted + # tree right after installing it to keep peak disk usage low. + # + # Linux builds are native (x86_64 packages are built on x86_64 hosts, + # aarch64 on aarch64 hosts), so we only install the host's own Linux std. + # Windows is the only cross-compilation target, and only on MinGW build + # hosts, so the caller passes "x86_64-pc-windows-gnu" as an argument there. + baseurl="https://static.rust-lang.org/dist" + version=1.96.0 + prefix=/opt/rust + extra_targets="$@" + + workdir="$(mktemp -d)" + trap 'rm -rf "$workdir"' EXIT + cd "$workdir" + + if uname -m | grep aarch64; then + host=aarch64-unknown-linux-gnu + # checksum from $baseurl/rustc-${version}-aarch64-unknown-linux-gnu.tar.gz.sha256 + rustc_sha=ba3c19a8e3a54efce3bd8d6c8ceb21173c8c64a100dd84e62fdfd8313c1ea7ed + # checksum from $baseurl/cargo-${version}-aarch64-unknown-linux-gnu.tar.gz.sha256 + cargo_sha=aff68544337c835a58ff303c47fc1ddb0a1a0bd9df332e37c8d466d8f78eaa32 + else + host=x86_64-unknown-linux-gnu + # checksum from $baseurl/rustc-${version}-x86_64-unknown-linux-gnu.tar.gz.sha256 + rustc_sha=71143d6075582b7e65233992c77e375aadbec4dfda6df2675160bf05b89410f9 + # checksum from $baseurl/cargo-${version}-x86_64-unknown-linux-gnu.tar.gz.sha256 + cargo_sha=b691a9e31b1e5498017be91155a1e7501eccf6437e7dc9ff1896e38aa1584dbf + fi + + # rust-std checksums per target. These are host-architecture independent. + # checksum from $baseurl/rust-std-${version}-x86_64-unknown-linux-gnu.tar.gz.sha256 + std_x86_64_linux_sha=36e577b66f7b2f8fc6493f97f81329e5f6e1514360d0c6c31d5d8463184e6773 + # checksum from $baseurl/rust-std-${version}-aarch64-unknown-linux-gnu.tar.gz.sha256 + std_aarch64_linux_sha=66ad5d73e79dd44b93c260ee61752abce3ce5ccb5031832beaccd1c248b88586 + # checksum from $baseurl/rust-std-${version}-x86_64-pc-windows-gnu.tar.gz.sha256 + std_x86_64_windows_sha=6951de999a0926aa8e35046017473a1912274cc34e800887eb3bfba4ddae12c9 + + # Download, verify, extract and install a single component tarball, then + # remove both the tarball and its extracted tree before moving on. + install_component() { + name="$1" + sha="$2" + tarball="$name.tar.gz" + wget --quiet "$baseurl/$tarball" + echo "$sha $tarball" | sha256sum --check - + tar xf "$tarball" + rm "$tarball" + "$name/install.sh" --prefix="$prefix" + rm -rf "$name" + } + + # Install the rust-std for a given target triple, looking up its checksum. + install_std() { + case "$1" in + x86_64-unknown-linux-gnu) sha="$std_x86_64_linux_sha" ;; + aarch64-unknown-linux-gnu) sha="$std_aarch64_linux_sha" ;; + x86_64-pc-windows-gnu) sha="$std_x86_64_windows_sha" ;; + *) + echo "No pinned checksum for rust-std target '$1'" >&2 + exit 1 + ;; + esac + install_component "rust-std-${version}-$1" "$sha" + } + + install_component "rustc-${version}-${host}" "$rustc_sha" + install_component "cargo-${version}-${host}" "$cargo_sha" + + # The host's own native std, plus any cross-compilation targets requested. + install_std "$host" + for target in $extra_targets; do + install_std "$target" + done + + tee /etc/profile.d/rust.sh << EOF +export PATH=\$PATH:$prefix/bin +EOF + + chown -R root:root "$prefix" + # Make sure it's readable by the build user. + chmod -R a+rX "$prefix" +} + +if [ "$(whoami)" = "root" ]; then + install_rust "$@" +else + sudo bash -c install_rust +fi From 1f25dbce4381b2cc0b47d799a65811696464d59a Mon Sep 17 00:00:00 2001 From: Lars Erik Wik Date: Tue, 2 Jun 2026 15:14:07 +0200 Subject: [PATCH 2/2] Install rust and protoc from fix-buildhost.sh on testing-pr builds Signed-off-by: Lars Erik Wik --- ci/fix-buildhost.sh | 30 ++++++++++++++++++++++++++++++ ci/linux-install-protobuf.sh | 9 +++++---- ci/linux-install-rust.sh | 9 +++++---- 3 files changed, 40 insertions(+), 8 deletions(-) diff --git a/ci/fix-buildhost.sh b/ci/fix-buildhost.sh index 4ab265887..5b94bcf9d 100755 --- a/ci/fix-buildhost.sh +++ b/ci/fix-buildhost.sh @@ -43,3 +43,33 @@ fi if command -v yum >/dev/null 2>/dev/null; then sudo yum erase -y openssl-devel || true fi + +# leech2 build toolchain: rust + protoc. The build-host-setup policy installs +# these when a VM is imaged; install them here too so testing-pr builds on +# not-yet-reimaged hosts (and branches that change these deps) get what they +# need without a reimage. Each call is guarded by an already-installed check, +# and gated to the same platforms as the policy (ubuntu>=20, debian>=12, +# rhel/centos>=8). +if [ -f /etc/os-release ]; then + . /etc/os-release + os_major="${VERSION_ID%%.*}" + case "$ID" in + ubuntu) min_major=20 ;; + debian) min_major=12 ;; + rhel | centos) min_major=8 ;; + *) min_major="" ;; + esac + if [ -n "$min_major" ] && [ "${os_major:-0}" -ge "$min_major" ]; then + if [ ! -x /usr/local/bin/protoc ]; then + sh "$my_dir"/linux-install-protobuf.sh + fi + if [ ! -x /opt/rust/bin/rustc ]; then + # MinGW hosts also need the Windows cross-compilation target. + if [ -f /etc/cfengine-mingw-build-host.flag ]; then + sh "$my_dir"/linux-install-rust.sh x86_64-pc-windows-gnu + else + sh "$my_dir"/linux-install-rust.sh + fi + fi + fi +fi diff --git a/ci/linux-install-protobuf.sh b/ci/linux-install-protobuf.sh index e3293e516..96c9da6c8 100755 --- a/ci/linux-install-protobuf.sh +++ b/ci/linux-install-protobuf.sh @@ -33,8 +33,9 @@ install_protobuf() { cd - } -if [ "$(whoami)" = "root" ]; then - install_protobuf -else - sudo bash -c install_protobuf +# Re-exec under sudo when not root (e.g. when sourced from fix-buildhost.sh as +# the build user). +if [ "$(id -u)" -ne 0 ]; then + exec sudo bash "$0" fi +install_protobuf diff --git a/ci/linux-install-rust.sh b/ci/linux-install-rust.sh index 62d3eb71c..041a5a657 100755 --- a/ci/linux-install-rust.sh +++ b/ci/linux-install-rust.sh @@ -93,8 +93,9 @@ EOF chmod -R a+rX "$prefix" } -if [ "$(whoami)" = "root" ]; then - install_rust "$@" -else - sudo bash -c install_rust +# Re-exec under sudo when not root (e.g. when sourced from fix-buildhost.sh as +# the build user), preserving any cross-compilation target arguments. +if [ "$(id -u)" -ne 0 ]; then + exec sudo bash "$0" "$@" fi +install_rust "$@"