ENT-14099: Add rust tool-chain and protobuf to build hosts

.github/workflows/update-deps.py

@@ -64,6 +64,16 @@ def parse_args():
         action="store_true",
         help="update SDK 21 on build hosts (needed by Jenkins)",
     )
+    parser.add_argument(
+        "--rust",
+        action="store_true",
+        help="update the Rust toolchain on build hosts (needed to build leech2)",
+    )
+    parser.add_argument(
+        "--protobuf",
+        action="store_true",
+        help="update protoc on build hosts (needed to build leech2)",
+    )
 
     return parser.parse_args()
 
@@ -341,6 +351,118 @@ def update_jdk21(root):
         exit(1)
 
 
+def update_rust(root):
+    base_url = "https://static.rust-lang.org/dist"
+
+    filename = os.path.join(root, "ci/linux-install-rust.sh")
+    with open(filename, "r") as f:
+        content = f.read()
+
+    old_version = re.search(r"version=([0-9]+\.[0-9]+\.[0-9]+)", content).group(1)
+    log.debug(f"Found version {old_version} in '{filename}'")
+
+    # The stable channel manifest tells us the most recent stable release.
+    manifest = requests.get(f"{base_url}/channel-rust-stable.toml").text
+    new_version = re.search(
+        r"\[pkg\.rust\]\s*\nversion = \"([0-9]+\.[0-9]+\.[0-9]+)", manifest
+    ).group(1)
+    log.debug(f"Most recent stable Rust version is {new_version}")
+
+    if old_version == new_version:
+        log.debug(
+            f"Rust toolchain is already the newest version ('{new_version}' == '{old_version}')"
+        )
+        return
+
+    # Each tarball has its checksum recorded right below a comment line that
+    # references its '.sha256' file. Refresh the version number and every SHA.
+    tarballs = [
+        "rustc-${version}-x86_64-unknown-linux-gnu.tar.gz",
+        "cargo-${version}-x86_64-unknown-linux-gnu.tar.gz",
+        "rustc-${version}-aarch64-unknown-linux-gnu.tar.gz",
+        "cargo-${version}-aarch64-unknown-linux-gnu.tar.gz",
+        "rust-std-${version}-x86_64-unknown-linux-gnu.tar.gz",
+        "rust-std-${version}-aarch64-unknown-linux-gnu.tar.gz",
+        "rust-std-${version}-x86_64-pc-windows-gnu.tar.gz",
+    ]
+    for tarball in tarballs:
+        marker = re.escape(f"{tarball}.sha256")
+        match = re.search(marker + r"\s*\n\s*\w+=([0-9a-f]{64})", content)
+        old_sha = match.group(1)
+
+        sha_file = tarball.replace("${version}", new_version) + ".sha256"
+        new_sha = requests.get(f"{base_url}/{sha_file}").text.split()[0]
+        log.debug(f"Fetched new SHA '{new_sha}' for '{sha_file}'")
+
+        content = content.replace(old_sha, new_sha, 1)
+
+    content = content.replace(f"version={old_version}", f"version={new_version}", 1)
+
+    log.debug(f"Writing changes to file '{filename}'")
+    with open(filename, "w") as f:
+        f.write(content)
+
+    if not git_commit(root, f"Updated Rust toolchain to {new_version}"):
+        log.error("Failed to commit changes after updating Rust toolchain")
+        exit(1)
+
+
+def update_protobuf(root):
+    filename = os.path.join(root, "ci/linux-install-protobuf.sh")
+    with open(filename, "r") as f:
+        content = f.read()
+
+    old_version = re.search(r"version=([0-9]+\.[0-9]+)", content).group(1)
+    log.debug(f"Found version {old_version} in '{filename}'")
+
+    # protobuf releases do not ship .sha256 files, so we determine the latest
+    # version from the GitHub releases API and compute the checksums ourselves.
+    release = requests.get(
+        "https://api.github.com/repos/protocolbuffers/protobuf/releases/latest"
+    ).json()
+    new_version = release["tag_name"].lstrip("v")
+    log.debug(f"Most recent protobuf version is {new_version}")
+
+    if old_version == new_version:
+        log.debug(
+            f"protoc is already the newest version ('{new_version}' == '{old_version}')"
+        )
+        return
+
+    base_url = (
+        f"https://github.com/protocolbuffers/protobuf/releases/download/v{new_version}"
+    )
+    for arch in ["linux-x86_64", "linux-aarch_64"]:
+        marker = re.escape(f"protoc-${{version}}-{arch}.zip")
+        match = re.search(marker + r"\s*\n\s*\w+=([0-9a-f]{64})", content)
+        old_sha = match.group(1)
+
+        zipfile = f"protoc-{new_version}-{arch}.zip"
+        path = os.path.join("/tmp", zipfile)
+        if not os.path.exists(path):
+            url = f"{base_url}/{zipfile}"
+            log.debug(f"Fetching URL '{url}'")
+            urllib.request.urlretrieve(url, path)
+
+        sha = hashlib.sha256()
+        with open(path, "rb") as f:
+            sha.update(f.read())
+        new_sha = sha.hexdigest()
+        log.debug(f"Computed new SHA '{new_sha}' for '{zipfile}'")
+
+        content = content.replace(old_sha, new_sha, 1)
+
+    content = content.replace(f"version={old_version}", f"version={new_version}", 1)
+
+    log.debug(f"Writing changes to file '{filename}'")
+    with open(filename, "w") as f:
+        f.write(content)
+
+    if not git_commit(root, f"Updated protoc to {new_version}"):
+        log.error("Failed to commit changes after updating protoc")
+        exit(1)
+
+
 def main():
     args = parse_args()
     loglevel = "DEBUG" if args.debug else "INFO"
@@ -351,6 +473,10 @@ def main():
     update_deps(args.root, args.bump, args.skip)
     if args.jdk21:
         update_jdk21(args.root)
+    if args.rust:
+        update_rust(args.root)
+    if args.protobuf:
+        update_protobuf(args.root)
 
 
 if __name__ == "__main__":

.github/workflows/update-deps.yml

@@ -45,8 +45,8 @@ jobs:
         run: |
           echo "COMMIT_HASH_BEFORE=$(git log -1 --format=%H)">> $GITHUB_ENV
       - name: Run update script
-        # we use only master branch for ci build host policy so only check for jdk updates in master
-        run: python3 /tmp/update-deps.py --debug --bump=${{ matrix.branch == 'master' && 'major' || 'minor' }} ${{ matrix.branch == 'master' && '--jdk21' || '' }}
+        # we use only master branch for ci build host policy so only check for jdk/rust/protobuf updates in master
+        run: python3 /tmp/update-deps.py --debug --bump=${{ matrix.branch == 'master' && 'major' || 'minor' }} ${{ matrix.branch == 'master' && '--jdk21 --rust --protobuf' || '' }}
       - name: Save commit hash after
         run: |
           echo "COMMIT_HASH_AFTER=$(git log -1 --format=%H)">> $GITHUB_ENV

ci/cfengine-build-host-setup.cf

@@ -269,6 +269,26 @@ bundle agent cfengine_build_host_setup
         expression => not(fileexists("/etc/cfengine-in-container.flag")),
         comment => "We use an explicit flag file that we control to avoid ambiguity about whether we are in a container or not.";
 
+    # Rust is build dependency for leech2 (gate on ubuntu>=20, debian>=12, redhat>=8)
+    ubuntu::
+      "leech2_build_toolchain_host"
+        expression => version_compare("$(sys.os_version_major)", ">=", "20");
+
+    debian::
+      "leech2_build_toolchain_host"
+        expression => version_compare("$(sys.os_version_major)", ">=", "12");
+
+    (redhat|centos)::
+      "leech2_build_toolchain_host"
+        expression => version_compare("$(sys.os_version_major)", ">=", "8");
+
+    any::
+      "have_rust"
+        expression => fileexists("/opt/rust/bin/rustc");
+
+      "have_protoc"
+        expression => fileexists("/usr/local/bin/protoc");
+
     linux::
       "have_tmp_mount"
         expression => returnszero("mount | grep '/tmp'", "useshell");
@@ -373,6 +393,22 @@ bundle agent cfengine_build_host_setup
         contain => in_shell,
         classes => results("bundle", "java");
 
+    # leech2 build toolchain: protoc and the Rust toolchain. Both installers
+    # pin a version and verify the SHA256 checksum of the downloaded tarball.
+    leech2_build_toolchain_host.!have_protoc::
+      "sh $(this.promise_dirname)/linux-install-protobuf.sh"
+        contain => in_shell,
+        comment => "Install pinned protoc; required to build the cargo-based leech2 dependency.";
+
+    # Linux builds are native, so the installer only adds the host's own Linux
+    # std. Windows is the only cross-compilation target, and only MinGW build
+    # hosts cross-compile it, so we pass that target there alone.
+    leech2_build_toolchain_host.!have_rust::
+      "sh $(this.promise_dirname)/linux-install-rust.sh"
+        args => ifelse("mingw_build_host", "x86_64-pc-windows-gnu", ""),
+        contain => in_shell,
+        comment => "Install the Rust toolchain system-wide under /opt/rust for building the cargo-based leech2 dependency.";
+
     (redhat_7|centos_7|redhat_8|centos_8|redhat_9|redhat_10).(!have_development_tools).(yum_dnf_conf_ok)::
       "yum groups install -y 'Development Tools'" contain => in_shell;
 

ci/fix-buildhost.sh

@@ -43,3 +43,33 @@ fi
 if command -v yum >/dev/null 2>/dev/null; then
   sudo yum erase -y openssl-devel || true
 fi
+
+# leech2 build toolchain: rust + protoc. The build-host-setup policy installs
+# these when a VM is imaged; install them here too so testing-pr builds on
+# not-yet-reimaged hosts (and branches that change these deps) get what they
+# need without a reimage. Each call is guarded by an already-installed check,
+# and gated to the same platforms as the policy (ubuntu>=20, debian>=12,
+# rhel/centos>=8).
+if [ -f /etc/os-release ]; then
+  . /etc/os-release
+  os_major="${VERSION_ID%%.*}"
+  case "$ID" in
+    ubuntu) min_major=20 ;;
+    debian) min_major=12 ;;
+    rhel | centos) min_major=8 ;;
+    *) min_major="" ;;
+  esac
+  if [ -n "$min_major" ] && [ "${os_major:-0}" -ge "$min_major" ]; then
+    if [ ! -x /usr/local/bin/protoc ]; then
+      sh "$my_dir"/linux-install-protobuf.sh
+    fi
+    if [ ! -x /opt/rust/bin/rustc ]; then
+      # MinGW hosts also need the Windows cross-compilation target.
+      if [ -f /etc/cfengine-mingw-build-host.flag ]; then
+        sh "$my_dir"/linux-install-rust.sh x86_64-pc-windows-gnu
+      else
+        sh "$my_dir"/linux-install-rust.sh
+      fi
+    fi
+  fi
+fi

ci/linux-install-protobuf.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+set -e
+
+install_protobuf() {
+  # Install the protoc compiler "manually" from the official prebuilt
+  # release, verifying the SHA256 checksum of the zip. protoc is needed to
+  # build the cargo-based leech2 dependency.
+  #
+  # The release archives do not ship .sha256 files, so the checksums below are
+  # computed by us (and refreshed by the dependency update script).
+  cd /opt
+  version=35.0
+  baseurl="https://github.com/protocolbuffers/protobuf/releases/download/v${version}"
+
+  if uname -m | grep aarch64; then
+    arch=linux-aarch_64
+    # sha256sum of protoc-${version}-linux-aarch_64.zip
+    sha=36b518ac14d90351cc6598228ed2bbe5afe4e357b1af470b07e0ec1609875de2
+  else
+    arch=linux-x86_64
+    # sha256sum of protoc-${version}-linux-x86_64.zip
+    sha=a45cda0989c17dd950db55f6fbe1e5814c50fda08e87aa422980ac1f89dddbbc
+  fi
+
+  zipfile="protoc-${version}-${arch}.zip"
+  wget --quiet "$baseurl/$zipfile"
+  echo "$sha  $zipfile" | sha256sum --check -
+  # Installs bin/protoc and include/ under /usr/local.
+  unzip -o "$zipfile" -d /usr/local
+  rm "$zipfile"
+
+  chmod a+rx /usr/local/bin/protoc
+  cd -
+}
+
+# Re-exec under sudo when not root (e.g. when sourced from fix-buildhost.sh as
+# the build user).
+if [ "$(id -u)" -ne 0 ]; then
+  exec sudo bash "$0"
+fi
+install_protobuf

ci/linux-install-rust.sh

@@ -0,0 +1,101 @@
+#!/usr/bin/env bash
+set -e
+
+install_rust() {
+  # Install the Rust toolchain "manually" from the official standalone
+  # installers, verifying the SHA256 checksum of each tarball. This is the
+  # cargo-based build dependency needed to build leech2.
+  #
+  # We install the individual component tarballs (rustc, cargo, rust-std)
+  # rather than the combined "rust" archive: the combined one is ~360 MB and
+  # extracts to ~1.4 GB of docs/clippy/llvm-tools we never install. The build
+  # hosts are tight on disk, so we also delete each tarball and its extracted
+  # tree right after installing it to keep peak disk usage low.
+  #
+  # Linux builds are native (x86_64 packages are built on x86_64 hosts,
+  # aarch64 on aarch64 hosts), so we only install the host's own Linux std.
+  # Windows is the only cross-compilation target, and only on MinGW build
+  # hosts, so the caller passes "x86_64-pc-windows-gnu" as an argument there.
+  baseurl="https://static.rust-lang.org/dist"
+  version=1.96.0
+  prefix=/opt/rust
+  extra_targets="$@"
+
+  workdir="$(mktemp -d)"
+  trap 'rm -rf "$workdir"' EXIT
+  cd "$workdir"
+
+  if uname -m | grep aarch64; then
+    host=aarch64-unknown-linux-gnu
+    # checksum from $baseurl/rustc-${version}-aarch64-unknown-linux-gnu.tar.gz.sha256
+    rustc_sha=ba3c19a8e3a54efce3bd8d6c8ceb21173c8c64a100dd84e62fdfd8313c1ea7ed
+    # checksum from $baseurl/cargo-${version}-aarch64-unknown-linux-gnu.tar.gz.sha256
+    cargo_sha=aff68544337c835a58ff303c47fc1ddb0a1a0bd9df332e37c8d466d8f78eaa32
+  else
+    host=x86_64-unknown-linux-gnu
+    # checksum from $baseurl/rustc-${version}-x86_64-unknown-linux-gnu.tar.gz.sha256
+    rustc_sha=71143d6075582b7e65233992c77e375aadbec4dfda6df2675160bf05b89410f9
+    # checksum from $baseurl/cargo-${version}-x86_64-unknown-linux-gnu.tar.gz.sha256
+    cargo_sha=b691a9e31b1e5498017be91155a1e7501eccf6437e7dc9ff1896e38aa1584dbf
+  fi
+
+  # rust-std checksums per target. These are host-architecture independent.
+  # checksum from $baseurl/rust-std-${version}-x86_64-unknown-linux-gnu.tar.gz.sha256
+  std_x86_64_linux_sha=36e577b66f7b2f8fc6493f97f81329e5f6e1514360d0c6c31d5d8463184e6773
+  # checksum from $baseurl/rust-std-${version}-aarch64-unknown-linux-gnu.tar.gz.sha256
+  std_aarch64_linux_sha=66ad5d73e79dd44b93c260ee61752abce3ce5ccb5031832beaccd1c248b88586
+  # checksum from $baseurl/rust-std-${version}-x86_64-pc-windows-gnu.tar.gz.sha256
+  std_x86_64_windows_sha=6951de999a0926aa8e35046017473a1912274cc34e800887eb3bfba4ddae12c9
+
+  # Download, verify, extract and install a single component tarball, then
+  # remove both the tarball and its extracted tree before moving on.
+  install_component() {
+    name="$1"
+    sha="$2"
+    tarball="$name.tar.gz"
+    wget --quiet "$baseurl/$tarball"
+    echo "$sha  $tarball" | sha256sum --check -
+    tar xf "$tarball"
+    rm "$tarball"
+    "$name/install.sh" --prefix="$prefix"
+    rm -rf "$name"
+  }
+
+  # Install the rust-std for a given target triple, looking up its checksum.
+  install_std() {
+    case "$1" in
+      x86_64-unknown-linux-gnu) sha="$std_x86_64_linux_sha" ;;
+      aarch64-unknown-linux-gnu) sha="$std_aarch64_linux_sha" ;;
+      x86_64-pc-windows-gnu) sha="$std_x86_64_windows_sha" ;;
+      *)
+        echo "No pinned checksum for rust-std target '$1'" >&2
+        exit 1
+        ;;
+    esac
+    install_component "rust-std-${version}-$1" "$sha"
+  }
+
+  install_component "rustc-${version}-${host}" "$rustc_sha"
+  install_component "cargo-${version}-${host}" "$cargo_sha"
+
+  # The host's own native std, plus any cross-compilation targets requested.
+  install_std "$host"
+  for target in $extra_targets; do
+    install_std "$target"
+  done
+
+  tee /etc/profile.d/rust.sh << EOF
+export PATH=\$PATH:$prefix/bin
+EOF
+
+  chown -R root:root "$prefix"
+  # Make sure it's readable by the build user.
+  chmod -R a+rX "$prefix"
+}
+
+# Re-exec under sudo when not root (e.g. when sourced from fix-buildhost.sh as
+# the build user), preserving any cross-compilation target arguments.
+if [ "$(id -u)" -ne 0 ]; then
+  exec sudo bash "$0" "$@"
+fi
+install_rust "$@"