From ad82887262dd4c2b9adf027314e136aa8f114ee8 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Sun, 22 Mar 2026 15:57:22 -0700 Subject: [PATCH 1/5] Introduce package bin crate to cross-build `pexrc`. Closes #2 --- .github/workflows/ci.yml | 45 ++----- Cargo.lock | 16 +++ Cargo.toml | 1 + build.rs | 55 +------- crates/package/Cargo.toml | 15 +++ crates/package/src/main.rs | 127 ++++++++++++++++++ crates/pexrc-build-system/src/lib.rs | 63 +++++++-- .../pexrc-build-system/src/rust_toolchain.rs | 4 +- 8 files changed, 233 insertions(+), 93 deletions(-) create mode 100644 crates/package/Cargo.toml create mode 100644 crates/package/src/main.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6cd0d7a..c29d7e4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,7 +5,6 @@ defaults: shell: bash env: PEXRC_INSTALL_TOOLS: 1 - PEXRC_PROFILE: release concurrency: group: CI-${{ github.ref }} @@ -22,34 +21,6 @@ jobs: run: | echo "This is a dummy step that will never run." - build: - name: "${{ matrix.name }} build" - needs: setup - runs-on: ${{ matrix.os }} - strategy: - matrix: - # N.B.: macos-14 is the oldest non-deprecated ARM Mac runner. - include: - - name: Linux aarch64 - os: ubuntu-22.04-arm - - name: Linux x86_64 - os: ubuntu-22.04 - - name: Mac aarch64 - os: macos-14 - - name: Mac x86_64 - os: macos-15-intel - - name: Windows aarch64 - os: windows-11-arm - - name: Windows x86_64 - os: windows-2022 - steps: - - name: Checkout Pexrc - uses: actions/checkout@v6 - - name: Build pexrc binary for all targets. - run: | - PEXRC_TARGETS=all cargo build --release - target/release/pexrc info - checks: name: "Check Formatting and Lints" needs: setup @@ -63,6 +34,18 @@ jobs: cargo +nightly fmt --check --all cargo clippy --locked --all + cross-build: + name: "Cross-build pexrc" + needs: setup + runs-on: ubuntu-22.04-arm + steps: + - name: Checkout Pexrc + uses: actions/checkout@v6 + - name: Build pexrc binary for all targets. + run: | + cargo run -p package -- --profile release --target all + target/aarch64-unknown-linux-gnu/release/pexrc info + tests: name: "${{ matrix.name }} tests" needs: setup @@ -89,13 +72,13 @@ jobs: - name: Run Tests run: | cargo test --all - uv run dev-cmd test -- -vvs + PEXRC_PROFILE=release uv run dev-cmd test -- -vvs final-status: name: Gather Final Status needs: - - build - checks + - cross-build - tests runs-on: ubuntu-24.04 steps: diff --git a/Cargo.lock b/Cargo.lock index b6f7d95..bd883cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1459,6 +1459,22 @@ version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d" +[[package]] +name = "package" +version = "0.0.0" +dependencies = [ + "anstream 1.0.0", + "anyhow", + "clap", + "clap-verbosity-flag", + "colorchoice-clap", + "env_logger", + "itertools 0.14.0", + "log", + "owo-colors", + "pexrc-build-system", +] + [[package]] name = "pathdiff" version = "0.2.3" diff --git a/Cargo.toml b/Cargo.toml index 346a70d..f9aa89c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "crates/cache", "crates/clib", "crates/interpreter", + "crates/package", "crates/pex", "crates/pexrc-build-system", "crates/pexrs", diff --git a/build.rs b/build.rs index 4d8903b..98bfce9 100644 --- a/build.rs +++ b/build.rs @@ -1,7 +1,6 @@ // Copyright 2026 Pex project contributors. // SPDX-License-Identifier: Apache-2.0 -use std::borrow::Cow; use std::ffi::OsStr; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; @@ -15,8 +14,6 @@ use pexrc_build_system::{ ClassifiedTargets, ClibConfiguration, FoundTool, - InstallDirs, - ToolInstallation, classify_targets, ensure_tools_installed, }; @@ -27,57 +24,19 @@ fn main() -> anyhow::Result<()> { env_logger::init(); let cargo: PathBuf = env::var("CARGO")?.into(); - - let target_dir: PathBuf = if let Some(custom_target_dir) = env::var_os("CARGO_TARGET_DIR") { - custom_target_dir.into() - } else { - PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()).join("target") - }; - - let install_dirs = InstallDirs::system("pexrc-dev").unwrap_or_else(|| { - let cache_base_dir = target_dir.join(".pexrc-dev"); - println!( - "cargo::warning=Failed to discover the user cache dir; using {cache_base_dir}", - cache_base_dir = cache_base_dir.display() - ); - InstallDirs::new(cache_base_dir) - }); - let cargo_manifest_contents = { let manifest_path = env::var("CARGO_MANIFEST_PATH")?; println!("cargo::rerun-if-changed={manifest_path}"); fs::read_to_string(manifest_path)? }; - - println!("cargo::rerun-if-env-changed=PEXRC_INSTALL_TOOLS"); - let install_missing_tools = env::var_os("PEXRC_INSTALL_TOOLS").unwrap_or_default() == "1"; - - let tool_installation = ensure_tools_installed( - &cargo, - &cargo_manifest_contents, - install_dirs, - install_missing_tools, - )?; - let (mut clib, glibc, found_tools) = match tool_installation { - ToolInstallation::Success(results) => results, - ToolInstallation::Failure((zig, missing_binstall_tools, tool_search_path)) => { - bail!( - "The following tools are required but are not installed: {tools}\n\ - Searched PATH: {search_path}\n\ - Re-run with PEXRC_INSTALL_TOOLS=1 to let the build script install these tools.", - tools = missing_binstall_tools - .iter() - .map(|tool| Cow::Borrowed(tool.binary_name())) - .chain( - zig.missing_version() - .iter() - .map(|version| Cow::Owned(format!("zig@{version}"))) - ) - .join(" "), - search_path = tool_search_path.display() - ); - } + let target_dir: PathBuf = if let Some(custom_target_dir) = env::var_os("CARGO_TARGET_DIR") { + custom_target_dir.into() + } else { + PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()).join("target") }; + + let (mut clib, glibc, found_tools) = + ensure_tools_installed(&cargo, &cargo_manifest_contents, &target_dir, true)?; println!("cargo::rerun-if-env-changed=PROFILE"); let profile = env::var("PROFILE")?; let clib = clib.configuration_for(&profile); diff --git a/crates/package/Cargo.toml b/crates/package/Cargo.toml new file mode 100644 index 0000000..6d800e2 --- /dev/null +++ b/crates/package/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "package" +edition = "2024" + +[dependencies] +anstream = { workspace = true } +anyhow = { workspace = true } +clap = { workspace = true } +clap-verbosity-flag = { workspace = true } +colorchoice-clap = { workspace = true } +env_logger = { workspace = true } +itertools = { workspace = true } +log = { workspace = true } +owo-colors = { workspace = true } +pexrc-build-system = { path = "../pexrc-build-system" } diff --git a/crates/package/src/main.rs b/crates/package/src/main.rs new file mode 100644 index 0000000..8f5934b --- /dev/null +++ b/crates/package/src/main.rs @@ -0,0 +1,127 @@ +// Copyright 2026 Pex project contributors. +// SPDX-License-Identifier: Apache-2.0 + +use std::path::PathBuf; +use std::process::{Command, Stdio}; +use std::{env, fs}; + +use anyhow::{anyhow, bail}; +use clap::{Parser, ValueEnum}; +use owo_colors::OwoColorize; +use pexrc_build_system::{classify_targets, ensure_tools_installed}; + +#[derive(Clone, Eq, PartialEq, ValueEnum)] +#[clap(rename_all = "kebab_case")] +enum Target { + All, + Current, +} + +/// Pexrc Packaging System. +#[derive(Parser)] +#[command(version, about, long_about = None)] +struct Cli { + #[command(flatten)] + verbosity: clap_verbosity_flag::Verbosity, + + #[command(flatten)] + color: colorchoice_clap::Color, + + #[arg(long)] + profile: Option, + + #[arg(long)] + target: Option, +} + +fn main() -> anyhow::Result<()> { + let cli = Cli::parse(); + env_logger::Builder::new() + .filter_level(cli.verbosity.into()) + .init(); + cli.color.write_global(); + + let cargo: PathBuf = env!("CARGO").into(); + let process = Command::new(&cargo) + .args(["locate-project", "--workspace", "--message-format", "plain"]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + let output = process.wait_with_output()?; + if !output.status.success() { + bail!( + "Failed to determine path to workspace Cargo.toml; process exited with {status:?}:\n\ + {stderr}", + status = output.status, + stderr = String::from_utf8_lossy(&output.stderr) + ); + } + let cargo_manifest_path: PathBuf = String::from_utf8(output.stdout)?.trim_end().into(); + let cargo_manifest_contents = fs::read_to_string(&cargo_manifest_path)?; + let cargo_manifest_dir = cargo_manifest_path.parent().ok_or_else(|| { + anyhow!( + "Failed to determine cargo project root directory from workspace Cargo.toml path: \ + {path}", + path = cargo_manifest_path.display() + ) + })?; + let target_dir: PathBuf = if let Some(custom_target_dir) = env::var_os("CARGO_TARGET_DIR") { + custom_target_dir.into() + } else { + cargo_manifest_dir.join("target") + }; + let (_, glibc, found_tools) = + ensure_tools_installed(&cargo, &cargo_manifest_contents, &target_dir, false)?; + + let rust_toolchain_contents = fs::read_to_string(cargo_manifest_dir.join("rust-toolchain"))?; + let classified_targets = classify_targets(&rust_toolchain_contents, &glibc)?; + + let profile = cli.profile.as_deref().unwrap_or("dev"); + + if cli.target.unwrap_or(Target::Current) == Target::Current { + let result = Command::new(&cargo) + .args(["build", "--profile", profile]) + .spawn()? + .wait()?; + if !result.success() { + bail!("Build via cargo build failed!"); + } + } else { + let zigbuild_targets = classified_targets.iter_zigbuild_targets(); + if zigbuild_targets.len() > 0 { + let mut command = Command::new(&cargo); + command.args(["zigbuild", "--profile", profile]); + for target in zigbuild_targets { + command.args(["--target", target]); + } + command.env("PEXRC_TARGETS", "all"); + for found_tool in &found_tools { + command.env(found_tool.env_var, &found_tool.path); + } + let result = command.spawn()?.wait()?; + if !result.success() { + bail!("Cross-build via cargo-zigbuild failed!"); + } + } + + let xwin_targets = classified_targets.iter_xwin_targets(); + if xwin_targets.len() > 0 { + let mut command = Command::new(&cargo); + command.args(["xwin", "build", "--profile", profile]); + for target in xwin_targets { + command.args(["--target", target]); + } + command.env("PEXRC_TARGETS", "all"); + for found_tool in &found_tools { + command.env(found_tool.env_var, &found_tool.path); + } + let result = command.spawn()?.wait()?; + if !result.success() { + bail!("Cross-build via cargo-xwin failed!"); + } + } + } + + anstream::println!("{}", "Build complete!".green()); + Ok(()) +} diff --git a/crates/pexrc-build-system/src/lib.rs b/crates/pexrc-build-system/src/lib.rs index 406757f..1245edc 100644 --- a/crates/pexrc-build-system/src/lib.rs +++ b/crates/pexrc-build-system/src/lib.rs @@ -6,8 +6,12 @@ mod metadata; mod rust_toolchain; mod tools; +use std::borrow::Cow; +use std::env; use std::path::{Path, PathBuf}; +use anyhow::bail; +use itertools::Itertools; pub use metadata::{Clib, ClibConfiguration, Glibc}; pub use rust_toolchain::{ClassifiedTargets, GnuLinux, Target}; use rust_toolchain::{Toolchain, parse_toolchain}; @@ -25,18 +29,6 @@ pub fn download_virtualenv( ensure_download(&metadata.build.virtualenv, &install_dirs.download_dir) } -pub fn ensure_tools_installed<'a>( - cargo: &Path, - cargo_manifest_contents: &'a str, - install_dirs: InstallDirs, - install_missing_tools: bool, -) -> anyhow::Result> { - let metadata: Metadata = parse_metadata(cargo_manifest_contents)?; - let tool_box = ToolBox::from(metadata.build); - let tool_inventory = tool_box.find_tools(install_dirs)?; - tool_inventory.ensure_tools_installed(cargo, install_missing_tools) -} - pub fn classify_targets<'a>( rust_toolchain_contents: &'a str, glibc: &'a Glibc, @@ -44,3 +36,50 @@ pub fn classify_targets<'a>( let toolchain: Toolchain = parse_toolchain(rust_toolchain_contents)?; Ok(toolchain.classify_targets(glibc)) } + +pub fn ensure_tools_installed<'a>( + cargo: &Path, + cargo_manifest_contents: &'a str, + target_dir: &Path, + is_build_script: bool, +) -> anyhow::Result<(Clib<'a>, Glibc<'a>, Vec)> { + let install_dirs = InstallDirs::system("pexrc-dev").unwrap_or_else(|| { + let cache_base_dir = target_dir.join(".pexrc-dev"); + if is_build_script { + println!( + "cargo::warning=Failed to discover the user cache dir; using {cache_base_dir}", + cache_base_dir = cache_base_dir.display() + ); + } + InstallDirs::new(cache_base_dir) + }); + + if is_build_script { + println!("cargo::rerun-if-env-changed=PEXRC_INSTALL_TOOLS"); + } + let install_missing_tools = env::var_os("PEXRC_INSTALL_TOOLS").unwrap_or_default() == "1"; + + let metadata: Metadata = parse_metadata(cargo_manifest_contents)?; + let tool_box = ToolBox::from(metadata.build); + let tool_inventory = tool_box.find_tools(install_dirs)?; + match tool_inventory.ensure_tools_installed(cargo, install_missing_tools)? { + ToolInstallation::Success(result) => Ok(result), + ToolInstallation::Failure((zig, missing_binstall_tools, tool_search_path)) => { + bail!( + "The following tools are required but are not installed: {tools}\n\ + Searched PATH: {search_path}\n\ + Re-run with PEXRC_INSTALL_TOOLS=1 to let the build script install these tools.", + tools = missing_binstall_tools + .iter() + .map(|tool| Cow::Borrowed(tool.binary_name())) + .chain( + zig.missing_version() + .iter() + .map(|version| Cow::Owned(format!("zig@{version}"))) + ) + .join(" "), + search_path = tool_search_path.display() + ); + } + } +} diff --git a/crates/pexrc-build-system/src/rust_toolchain.rs b/crates/pexrc-build-system/src/rust_toolchain.rs index 23a4028..a3519bc 100644 --- a/crates/pexrc-build-system/src/rust_toolchain.rs +++ b/crates/pexrc-build-system/src/rust_toolchain.rs @@ -103,7 +103,7 @@ impl<'a> ClassifiedTargets<'a> { } } - pub fn iter_zigbuild_targets(&'a self) -> impl Iterator { + pub fn iter_zigbuild_targets(&'a self) -> impl ExactSizeIterator { self.zigbuild_targets.iter().map(|target| { if let Target::GnuLinux(gnu_linux) = target { gnu_linux.zigbuild_target.as_str() @@ -113,7 +113,7 @@ impl<'a> ClassifiedTargets<'a> { }) } - pub fn iter_xwin_targets(&'a self) -> impl Iterator { + pub fn iter_xwin_targets(&'a self) -> impl ExactSizeIterator { self.xwin_targets.iter().map(Target::as_str) } From 53b4003321c338c83c4f025f05b4ca21a4b46fc3 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Tue, 24 Mar 2026 14:54:55 -0700 Subject: [PATCH 2/5] Debug missing env var. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c29d7e4..5079c6c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,7 +43,7 @@ jobs: uses: actions/checkout@v6 - name: Build pexrc binary for all targets. run: | - cargo run -p package -- --profile release --target all + RUST_BACKTRACE=1 cargo run -p package -- --profile release --target all target/aarch64-unknown-linux-gnu/release/pexrc info tests: From a3a035147b8481813cadcc21d0affe0e6ad2270b Mon Sep 17 00:00:00 2001 From: John Sirois Date: Tue, 24 Mar 2026 15:10:53 -0700 Subject: [PATCH 3/5] Fix package: get target triple via target-lexicon. --- .github/workflows/ci.yml | 2 +- Cargo.lock | 1 + crates/pexrc-build-system/Cargo.toml | 1 + crates/pexrc-build-system/src/rust_toolchain.rs | 4 ++-- crates/pexrc-build-system/src/tools.rs | 5 +++-- 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5079c6c..c29d7e4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,7 +43,7 @@ jobs: uses: actions/checkout@v6 - name: Build pexrc binary for all targets. run: | - RUST_BACKTRACE=1 cargo run -p package -- --profile release --target all + cargo run -p package -- --profile release --target all target/aarch64-unknown-linux-gnu/release/pexrc info tests: diff --git a/Cargo.lock b/Cargo.lock index bd883cf..a752ee7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1598,6 +1598,7 @@ dependencies = [ "strum", "strum_macros", "tar", + "target-lexicon", "tempfile", "toml", "which", diff --git a/crates/pexrc-build-system/Cargo.toml b/crates/pexrc-build-system/Cargo.toml index 8294a5c..2d17bec 100644 --- a/crates/pexrc-build-system/Cargo.toml +++ b/crates/pexrc-build-system/Cargo.toml @@ -20,6 +20,7 @@ sha2 = { workspace = true } strum = { workspace = true } strum_macros = { workspace = true } tar = { workspace = true } +target-lexicon = { workspace = true } toml = { workspace = true } tempfile = { workspace = true } xz2 = { workspace = true } diff --git a/crates/pexrc-build-system/src/rust_toolchain.rs b/crates/pexrc-build-system/src/rust_toolchain.rs index a3519bc..43e8917 100644 --- a/crates/pexrc-build-system/src/rust_toolchain.rs +++ b/crates/pexrc-build-system/src/rust_toolchain.rs @@ -2,10 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 use std::borrow::Cow; -use std::env; use itertools::Itertools; use serde::Deserialize; +use target_lexicon::HOST; use crate::metadata::Glibc; @@ -91,7 +91,7 @@ impl<'a> ClassifiedTargets<'a> { } pub fn is_just_current(&'a self) -> anyhow::Result> { - let current_target = env::var("TARGET")?; + let current_target = HOST.to_string(); let mut all_targets_iter = self.iter_all_targets(); if let Some(target) = all_targets_iter.next() && target.as_str() == current_target diff --git a/crates/pexrc-build-system/src/tools.rs b/crates/pexrc-build-system/src/tools.rs index 5dc5e9a..33a9b1e 100644 --- a/crates/pexrc-build-system/src/tools.rs +++ b/crates/pexrc-build-system/src/tools.rs @@ -13,6 +13,7 @@ use const_format::concatcp; use fs_err::File; use strum::{EnumCount, IntoEnumIterator}; use strum_macros::{EnumCount, EnumIter}; +use target_lexicon::HOST; use which::which_in_global; use crate::downloads::ensure_download; @@ -317,8 +318,8 @@ fn binstall( { eprintln!("Found cargo-binstall at {exe}", exe = exe.display()); } else { - let target = env::var("TARGET")?; - if let Some(download) = cargo_binstall.download_for(&target)? { + let current_target = HOST.to_string(); + if let Some(download) = cargo_binstall.download_for(¤t_target)? { let cargo_binstall = ensure_download(&download, &install_dirs.download_dir)? .join(CARGO_BINSTALL_FILE_NAME); let cargo_binstall_fp = File::open(&cargo_binstall)?; From 423e61bfef14a0072b462acbf32278f97f7cea15 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Tue, 24 Mar 2026 17:52:36 -0700 Subject: [PATCH 4/5] Split up cross-build into 3 chunks of 4 targets. --- .github/workflows/ci.yml | 29 +++- Cargo.lock | 2 - Cargo.toml | 2 +- crates/package/Cargo.toml | 2 - crates/package/src/main.rs | 153 +++++++++++++----- crates/pexrc-build-system/src/lib.rs | 5 + .../pexrc-build-system/src/rust_toolchain.rs | 4 + 7 files changed, 142 insertions(+), 55 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c29d7e4..1433fbf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,8 +18,7 @@ jobs: steps: - name: Noop if: false - run: | - echo "This is a dummy step that will never run." + run: echo "This is a dummy step that will never run." checks: name: "Check Formatting and Lints" @@ -38,13 +37,31 @@ jobs: name: "Cross-build pexrc" needs: setup runs-on: ubuntu-22.04-arm + # N.B.: We break these up just to save wall time; they all can be built on 1 machine in ~40 + # minutes. + strategy: + matrix: + include: + - targets: >- + --target aarch64-unknown-linux-gnu + --target aarch64-unknown-linux-musl + --target armv7-unknown-linux-gnueabihf + --target powerpc64le-unknown-linux-gnu + - targets: >- + --target riscv64gc-unknown-linux-gnu + --target s390x-unknown-linux-gnu + --target x86_64-unknown-linux-gnu + --target x86_64-unknown-linux-musl + - targets: >- + --target aarch64-apple-darwin + --target x86_64-apple-darwin + --target aarch64-pc-windows-gnullvm + --target x86_64-pc-windows-gnu steps: - name: Checkout Pexrc uses: actions/checkout@v6 - name: Build pexrc binary for all targets. - run: | - cargo run -p package -- --profile release --target all - target/aarch64-unknown-linux-gnu/release/pexrc info + run: cargo run -p package -- --profile release ${{ matrix.targets }} tests: name: "${{ matrix.name }} tests" @@ -80,7 +97,7 @@ jobs: - checks - cross-build - tests - runs-on: ubuntu-24.04 + runs-on: ubuntu-slim steps: - name: Check Non-Success if: | diff --git a/Cargo.lock b/Cargo.lock index a752ee7..95165f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1469,8 +1469,6 @@ dependencies = [ "clap-verbosity-flag", "colorchoice-clap", "env_logger", - "itertools 0.14.0", - "log", "owo-colors", "pexrc-build-system", ] diff --git a/Cargo.toml b/Cargo.toml index f9aa89c..6f4ee8f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -99,7 +99,7 @@ anyhow = "1.0" base64 = "0.22" bstr = "1.12" build-target = "0.8" -clap = { version = "4.5", features = ["derive"] } +clap = { version = "4.5", features = ["derive", "string"] } clap-verbosity-flag = "3.0" colorchoice-clap = "1.0" const_format = { version = "0.2", features = ["rust_1_64"] } diff --git a/crates/package/Cargo.toml b/crates/package/Cargo.toml index 6d800e2..6aba29d 100644 --- a/crates/package/Cargo.toml +++ b/crates/package/Cargo.toml @@ -9,7 +9,5 @@ clap = { workspace = true } clap-verbosity-flag = { workspace = true } colorchoice-clap = { workspace = true } env_logger = { workspace = true } -itertools = { workspace = true } -log = { workspace = true } owo-colors = { workspace = true } pexrc-build-system = { path = "../pexrc-build-system" } diff --git a/crates/package/src/main.rs b/crates/package/src/main.rs index 8f5934b..7a8a977 100644 --- a/crates/package/src/main.rs +++ b/crates/package/src/main.rs @@ -1,21 +1,74 @@ // Copyright 2026 Pex project contributors. // SPDX-License-Identifier: Apache-2.0 -use std::path::PathBuf; +use std::collections::HashSet; +use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; +use std::string::ToString; +use std::sync::LazyLock; use std::{env, fs}; use anyhow::{anyhow, bail}; -use clap::{Parser, ValueEnum}; +use clap::builder::Str; +use clap::{ArgAction, Parser}; use owo_colors::OwoColorize; -use pexrc_build_system::{classify_targets, ensure_tools_installed}; +use pexrc_build_system::{all_targets, classify_targets, ensure_tools_installed}; -#[derive(Clone, Eq, PartialEq, ValueEnum)] -#[clap(rename_all = "kebab_case")] -enum Target { - All, - Current, -} +static CARGO: LazyLock = LazyLock::new(|| env!("CARGO").into()); + +static CARGO_MANIFEST_PATH: LazyLock> = LazyLock::new(|| { + let process = Command::new(CARGO.as_path()) + .args(["locate-project", "--workspace", "--message-format", "plain"]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + let output = process.wait_with_output()?; + if !output.status.success() { + bail!( + "Failed to determine path to workspace Cargo.toml; process exited with {status:?}:\n\ + {stderr}", + status = output.status, + stderr = String::from_utf8_lossy(&output.stderr) + ); + } + Ok(String::from_utf8(output.stdout)?.trim_end().into()) +}); + +static CARGO_MANIFEST_DIR: LazyLock> = + LazyLock::new(|| match CARGO_MANIFEST_PATH.as_ref() { + Ok(cargo_manifest_path) => cargo_manifest_path.parent().ok_or_else(|| { + anyhow!( + "Failed to determine cargo project root directory from workspace Cargo.toml \ + path: {path}", + path = cargo_manifest_path.display() + ) + }), + Err(err) => bail!("Failed to determine cargo workspace root dir: {err}"), + }); + +static ALL_TARGETS: LazyLock = LazyLock::new(|| "all".to_string()); + +static AVAILABLE_TARGETS: LazyLock> = LazyLock::new(|| { + let mut available_targets = vec![Str::from(ALL_TARGETS.as_str())]; + let cargo_manifest_dir = match CARGO_MANIFEST_DIR.as_ref() { + Ok(manifest_dir) => manifest_dir, + Err(err) => panic!("Failed to determine cargo workspace root dir: {err}"), + }; + let rust_toolchain_contents = + match fs::read_to_string(cargo_manifest_dir.join("rust-toolchain")) { + Ok(contents) => contents, + Err(err) => panic!("Failed to read rust-toolchain file: {err}"), + }; + match all_targets(&rust_toolchain_contents) { + Ok(targets) => { + for target in targets { + available_targets.push(Str::from(target)) + } + } + Err(err) => panic!("Failed to parse rust-toolchain file: {err}"), + } + available_targets +}); /// Pexrc Packaging System. #[derive(Parser)] @@ -30,8 +83,10 @@ struct Cli { #[arg(long)] profile: Option, - #[arg(long)] - target: Option, + #[arg(long = "target")] + #[arg(action=ArgAction::Append)] + #[arg(value_parser=clap::builder::PossibleValuesParser::new(AVAILABLE_TARGETS.iter()))] + targets: Vec, } fn main() -> anyhow::Result<()> { @@ -41,45 +96,31 @@ fn main() -> anyhow::Result<()> { .init(); cli.color.write_global(); - let cargo: PathBuf = env!("CARGO").into(); - let process = Command::new(&cargo) - .args(["locate-project", "--workspace", "--message-format", "plain"]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()?; - let output = process.wait_with_output()?; - if !output.status.success() { - bail!( - "Failed to determine path to workspace Cargo.toml; process exited with {status:?}:\n\ - {stderr}", - status = output.status, - stderr = String::from_utf8_lossy(&output.stderr) - ); - } - let cargo_manifest_path: PathBuf = String::from_utf8(output.stdout)?.trim_end().into(); - let cargo_manifest_contents = fs::read_to_string(&cargo_manifest_path)?; - let cargo_manifest_dir = cargo_manifest_path.parent().ok_or_else(|| { - anyhow!( - "Failed to determine cargo project root directory from workspace Cargo.toml path: \ - {path}", - path = cargo_manifest_path.display() - ) - })?; + let cargo = CARGO.as_path(); + let cargo_manifest_path = match CARGO_MANIFEST_PATH.as_ref() { + Ok(manifest_path) => manifest_path, + Err(err) => bail!("{err}"), + }; + let cargo_manifest_contents = fs::read_to_string(cargo_manifest_path)?; + let cargo_manifest_dir = match CARGO_MANIFEST_DIR.as_ref() { + Ok(manifest_dir) => manifest_dir, + Err(err) => bail!("{err}"), + }; let target_dir: PathBuf = if let Some(custom_target_dir) = env::var_os("CARGO_TARGET_DIR") { custom_target_dir.into() } else { cargo_manifest_dir.join("target") }; let (_, glibc, found_tools) = - ensure_tools_installed(&cargo, &cargo_manifest_contents, &target_dir, false)?; + ensure_tools_installed(cargo, &cargo_manifest_contents, &target_dir, false)?; let rust_toolchain_contents = fs::read_to_string(cargo_manifest_dir.join("rust-toolchain"))?; let classified_targets = classify_targets(&rust_toolchain_contents, &glibc)?; let profile = cli.profile.as_deref().unwrap_or("dev"); - if cli.target.unwrap_or(Target::Current) == Target::Current { - let result = Command::new(&cargo) + if cli.targets.is_empty() { + let result = Command::new(cargo) .args(["build", "--profile", profile]) .spawn()? .wait()?; @@ -87,9 +128,30 @@ fn main() -> anyhow::Result<()> { bail!("Build via cargo build failed!"); } } else { - let zigbuild_targets = classified_targets.iter_zigbuild_targets(); - if zigbuild_targets.len() > 0 { - let mut command = Command::new(&cargo); + let targeted: HashSet = if cli.targets.contains(&ALL_TARGETS) { + AVAILABLE_TARGETS.iter().map(Str::to_string).collect() + } else { + cli.targets.into_iter().collect() + }; + let zigbuild_targets = classified_targets + .iter_zigbuild_targets() + .filter(|target| { + // Strip the `.{glibc-version}` suffix from `*-gnu.{glibc-version}` targets. + // TODO: Encode classified targets such that we don't need to use string parsing + // here to undo earlier string concatenation of the glibc-version when classifying + // the targets. + let target = if target.contains("-gnu.") + && let Some(target) = target.splitn(2, ".").take(1).next() + { + target + } else { + target + }; + targeted.contains(target) + }) + .collect::>(); + if !zigbuild_targets.is_empty() { + let mut command = Command::new(cargo); command.args(["zigbuild", "--profile", profile]); for target in zigbuild_targets { command.args(["--target", target]); @@ -104,9 +166,12 @@ fn main() -> anyhow::Result<()> { } } - let xwin_targets = classified_targets.iter_xwin_targets(); - if xwin_targets.len() > 0 { - let mut command = Command::new(&cargo); + let xwin_targets = classified_targets + .iter_xwin_targets() + .filter(|target| targeted.contains(*target)) + .collect::>(); + if !xwin_targets.is_empty() { + let mut command = Command::new(cargo); command.args(["xwin", "build", "--profile", profile]); for target in xwin_targets { command.args(["--target", target]); diff --git a/crates/pexrc-build-system/src/lib.rs b/crates/pexrc-build-system/src/lib.rs index 1245edc..243bb26 100644 --- a/crates/pexrc-build-system/src/lib.rs +++ b/crates/pexrc-build-system/src/lib.rs @@ -29,6 +29,11 @@ pub fn download_virtualenv( ensure_download(&metadata.build.virtualenv, &install_dirs.download_dir) } +pub fn all_targets(rust_toolchain_contents: &str) -> anyhow::Result> { + let toolchain: Toolchain = parse_toolchain(rust_toolchain_contents)?; + Ok(toolchain.into_targets()) +} + pub fn classify_targets<'a>( rust_toolchain_contents: &'a str, glibc: &'a Glibc, diff --git a/crates/pexrc-build-system/src/rust_toolchain.rs b/crates/pexrc-build-system/src/rust_toolchain.rs index 43e8917..ebddc9d 100644 --- a/crates/pexrc-build-system/src/rust_toolchain.rs +++ b/crates/pexrc-build-system/src/rust_toolchain.rs @@ -129,6 +129,10 @@ pub(crate) struct Toolchain<'a> { } impl<'a> Toolchain<'a> { + pub(crate) fn into_targets(self) -> Vec { + self.targets.into_iter().map(str::to_string).collect() + } + pub(crate) fn classify_targets(&self, glibc: &'a Glibc<'a>) -> ClassifiedTargets<'a> { ClassifiedTargets::parse(self.targets.iter().copied(), glibc) } From 9edfffefdd199f9226edf752dad532d181138ec9 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Tue, 24 Mar 2026 19:03:00 -0700 Subject: [PATCH 5/5] Shard crossbuild more. --- .github/workflows/ci.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1433fbf..2f678fb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,23 +38,26 @@ jobs: needs: setup runs-on: ubuntu-22.04-arm # N.B.: We break these up just to save wall time; they all can be built on 1 machine in ~40 - # minutes. + # minutes; this gets us to a ~20 minute long-pole. strategy: matrix: include: - targets: >- - --target aarch64-unknown-linux-gnu - --target aarch64-unknown-linux-musl - --target armv7-unknown-linux-gnueabihf + --target aarch64-unknown-linux-gnu + --target aarch64-unknown-linux-musl + - targets: >- + --target armv7-unknown-linux-gnueabihf --target powerpc64le-unknown-linux-gnu - targets: >- --target riscv64gc-unknown-linux-gnu --target s390x-unknown-linux-gnu + - targets: >- --target x86_64-unknown-linux-gnu --target x86_64-unknown-linux-musl - targets: >- --target aarch64-apple-darwin --target x86_64-apple-darwin + - targets: >- --target aarch64-pc-windows-gnullvm --target x86_64-pc-windows-gnu steps: