diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6cd0d7a..2f678fb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,7 +5,6 @@ defaults: shell: bash env: PEXRC_INSTALL_TOOLS: 1 - PEXRC_PROFILE: release concurrency: group: CI-${{ github.ref }} @@ -19,36 +18,7 @@ jobs: steps: - name: Noop if: false - run: | - echo "This is a dummy step that will never run." - - build: - name: "${{ matrix.name }} build" - needs: setup - runs-on: ${{ matrix.os }} - strategy: - matrix: - # N.B.: macos-14 is the oldest non-deprecated ARM Mac runner. - include: - - name: Linux aarch64 - os: ubuntu-22.04-arm - - name: Linux x86_64 - os: ubuntu-22.04 - - name: Mac aarch64 - os: macos-14 - - name: Mac x86_64 - os: macos-15-intel - - name: Windows aarch64 - os: windows-11-arm - - name: Windows x86_64 - os: windows-2022 - steps: - - name: Checkout Pexrc - uses: actions/checkout@v6 - - name: Build pexrc binary for all targets. - run: | - PEXRC_TARGETS=all cargo build --release - target/release/pexrc info + run: echo "This is a dummy step that will never run." checks: name: "Check Formatting and Lints" @@ -63,6 +33,39 @@ jobs: cargo +nightly fmt --check --all cargo clippy --locked --all + cross-build: + name: "Cross-build pexrc" + needs: setup + runs-on: ubuntu-22.04-arm + # N.B.: We break these up just to save wall time; they all can be built on 1 machine in ~40 + # minutes; this gets us to a ~20 minute long-pole. + strategy: + matrix: + include: + - targets: >- + --target aarch64-unknown-linux-gnu + --target aarch64-unknown-linux-musl + - targets: >- + --target armv7-unknown-linux-gnueabihf + --target powerpc64le-unknown-linux-gnu + - targets: >- + --target riscv64gc-unknown-linux-gnu + --target s390x-unknown-linux-gnu + - targets: >- + --target x86_64-unknown-linux-gnu + --target x86_64-unknown-linux-musl + - targets: >- + --target aarch64-apple-darwin + --target x86_64-apple-darwin + - targets: >- + --target aarch64-pc-windows-gnullvm + --target x86_64-pc-windows-gnu + steps: + - name: Checkout Pexrc + uses: actions/checkout@v6 + - name: Build pexrc binary for all targets. + run: cargo run -p package -- --profile release ${{ matrix.targets }} + tests: name: "${{ matrix.name }} tests" needs: setup @@ -89,15 +92,15 @@ jobs: - name: Run Tests run: | cargo test --all - uv run dev-cmd test -- -vvs + PEXRC_PROFILE=release uv run dev-cmd test -- -vvs final-status: name: Gather Final Status needs: - - build - checks + - cross-build - tests - runs-on: ubuntu-24.04 + runs-on: ubuntu-slim steps: - name: Check Non-Success if: | diff --git a/Cargo.lock b/Cargo.lock index b6f7d95..95165f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1459,6 +1459,20 @@ version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d" +[[package]] +name = "package" +version = "0.0.0" +dependencies = [ + "anstream 1.0.0", + "anyhow", + "clap", + "clap-verbosity-flag", + "colorchoice-clap", + "env_logger", + "owo-colors", + "pexrc-build-system", +] + [[package]] name = "pathdiff" version = "0.2.3" @@ -1582,6 +1596,7 @@ dependencies = [ "strum", "strum_macros", "tar", + "target-lexicon", "tempfile", "toml", "which", diff --git a/Cargo.toml b/Cargo.toml index 346a70d..6f4ee8f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "crates/cache", "crates/clib", "crates/interpreter", + "crates/package", "crates/pex", "crates/pexrc-build-system", "crates/pexrs", @@ -98,7 +99,7 @@ anyhow = "1.0" base64 = "0.22" bstr = "1.12" build-target = "0.8" -clap = { version = "4.5", features = ["derive"] } +clap = { version = "4.5", features = ["derive", "string"] } clap-verbosity-flag = "3.0" colorchoice-clap = "1.0" const_format = { version = "0.2", features = ["rust_1_64"] } diff --git a/build.rs b/build.rs index 4d8903b..98bfce9 100644 --- a/build.rs +++ b/build.rs @@ -1,7 +1,6 @@ // Copyright 2026 Pex project contributors. // SPDX-License-Identifier: Apache-2.0 -use std::borrow::Cow; use std::ffi::OsStr; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; @@ -15,8 +14,6 @@ use pexrc_build_system::{ ClassifiedTargets, ClibConfiguration, FoundTool, - InstallDirs, - ToolInstallation, classify_targets, ensure_tools_installed, }; @@ -27,57 +24,19 @@ fn main() -> anyhow::Result<()> { env_logger::init(); let cargo: PathBuf = env::var("CARGO")?.into(); - - let target_dir: PathBuf = if let Some(custom_target_dir) = env::var_os("CARGO_TARGET_DIR") { - custom_target_dir.into() - } else { - PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()).join("target") - }; - - let install_dirs = InstallDirs::system("pexrc-dev").unwrap_or_else(|| { - let cache_base_dir = target_dir.join(".pexrc-dev"); - println!( - "cargo::warning=Failed to discover the user cache dir; using {cache_base_dir}", - cache_base_dir = cache_base_dir.display() - ); - InstallDirs::new(cache_base_dir) - }); - let cargo_manifest_contents = { let manifest_path = env::var("CARGO_MANIFEST_PATH")?; println!("cargo::rerun-if-changed={manifest_path}"); fs::read_to_string(manifest_path)? }; - - println!("cargo::rerun-if-env-changed=PEXRC_INSTALL_TOOLS"); - let install_missing_tools = env::var_os("PEXRC_INSTALL_TOOLS").unwrap_or_default() == "1"; - - let tool_installation = ensure_tools_installed( - &cargo, - &cargo_manifest_contents, - install_dirs, - install_missing_tools, - )?; - let (mut clib, glibc, found_tools) = match tool_installation { - ToolInstallation::Success(results) => results, - ToolInstallation::Failure((zig, missing_binstall_tools, tool_search_path)) => { - bail!( - "The following tools are required but are not installed: {tools}\n\ - Searched PATH: {search_path}\n\ - Re-run with PEXRC_INSTALL_TOOLS=1 to let the build script install these tools.", - tools = missing_binstall_tools - .iter() - .map(|tool| Cow::Borrowed(tool.binary_name())) - .chain( - zig.missing_version() - .iter() - .map(|version| Cow::Owned(format!("zig@{version}"))) - ) - .join(" "), - search_path = tool_search_path.display() - ); - } + let target_dir: PathBuf = if let Some(custom_target_dir) = env::var_os("CARGO_TARGET_DIR") { + custom_target_dir.into() + } else { + PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()).join("target") }; + + let (mut clib, glibc, found_tools) = + ensure_tools_installed(&cargo, &cargo_manifest_contents, &target_dir, true)?; println!("cargo::rerun-if-env-changed=PROFILE"); let profile = env::var("PROFILE")?; let clib = clib.configuration_for(&profile); diff --git a/crates/package/Cargo.toml b/crates/package/Cargo.toml new file mode 100644 index 0000000..6aba29d --- /dev/null +++ b/crates/package/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "package" +edition = "2024" + +[dependencies] +anstream = { workspace = true } +anyhow = { workspace = true } +clap = { workspace = true } +clap-verbosity-flag = { workspace = true } +colorchoice-clap = { workspace = true } +env_logger = { workspace = true } +owo-colors = { workspace = true } +pexrc-build-system = { path = "../pexrc-build-system" } diff --git a/crates/package/src/main.rs b/crates/package/src/main.rs new file mode 100644 index 0000000..7a8a977 --- /dev/null +++ b/crates/package/src/main.rs @@ -0,0 +1,192 @@ +// Copyright 2026 Pex project contributors. +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::HashSet; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::string::ToString; +use std::sync::LazyLock; +use std::{env, fs}; + +use anyhow::{anyhow, bail}; +use clap::builder::Str; +use clap::{ArgAction, Parser}; +use owo_colors::OwoColorize; +use pexrc_build_system::{all_targets, classify_targets, ensure_tools_installed}; + +static CARGO: LazyLock = LazyLock::new(|| env!("CARGO").into()); + +static CARGO_MANIFEST_PATH: LazyLock> = LazyLock::new(|| { + let process = Command::new(CARGO.as_path()) + .args(["locate-project", "--workspace", "--message-format", "plain"]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + let output = process.wait_with_output()?; + if !output.status.success() { + bail!( + "Failed to determine path to workspace Cargo.toml; process exited with {status:?}:\n\ + {stderr}", + status = output.status, + stderr = String::from_utf8_lossy(&output.stderr) + ); + } + Ok(String::from_utf8(output.stdout)?.trim_end().into()) +}); + +static CARGO_MANIFEST_DIR: LazyLock> = + LazyLock::new(|| match CARGO_MANIFEST_PATH.as_ref() { + Ok(cargo_manifest_path) => cargo_manifest_path.parent().ok_or_else(|| { + anyhow!( + "Failed to determine cargo project root directory from workspace Cargo.toml \ + path: {path}", + path = cargo_manifest_path.display() + ) + }), + Err(err) => bail!("Failed to determine cargo workspace root dir: {err}"), + }); + +static ALL_TARGETS: LazyLock = LazyLock::new(|| "all".to_string()); + +static AVAILABLE_TARGETS: LazyLock> = LazyLock::new(|| { + let mut available_targets = vec![Str::from(ALL_TARGETS.as_str())]; + let cargo_manifest_dir = match CARGO_MANIFEST_DIR.as_ref() { + Ok(manifest_dir) => manifest_dir, + Err(err) => panic!("Failed to determine cargo workspace root dir: {err}"), + }; + let rust_toolchain_contents = + match fs::read_to_string(cargo_manifest_dir.join("rust-toolchain")) { + Ok(contents) => contents, + Err(err) => panic!("Failed to read rust-toolchain file: {err}"), + }; + match all_targets(&rust_toolchain_contents) { + Ok(targets) => { + for target in targets { + available_targets.push(Str::from(target)) + } + } + Err(err) => panic!("Failed to parse rust-toolchain file: {err}"), + } + available_targets +}); + +/// Pexrc Packaging System. +#[derive(Parser)] +#[command(version, about, long_about = None)] +struct Cli { + #[command(flatten)] + verbosity: clap_verbosity_flag::Verbosity, + + #[command(flatten)] + color: colorchoice_clap::Color, + + #[arg(long)] + profile: Option, + + #[arg(long = "target")] + #[arg(action=ArgAction::Append)] + #[arg(value_parser=clap::builder::PossibleValuesParser::new(AVAILABLE_TARGETS.iter()))] + targets: Vec, +} + +fn main() -> anyhow::Result<()> { + let cli = Cli::parse(); + env_logger::Builder::new() + .filter_level(cli.verbosity.into()) + .init(); + cli.color.write_global(); + + let cargo = CARGO.as_path(); + let cargo_manifest_path = match CARGO_MANIFEST_PATH.as_ref() { + Ok(manifest_path) => manifest_path, + Err(err) => bail!("{err}"), + }; + let cargo_manifest_contents = fs::read_to_string(cargo_manifest_path)?; + let cargo_manifest_dir = match CARGO_MANIFEST_DIR.as_ref() { + Ok(manifest_dir) => manifest_dir, + Err(err) => bail!("{err}"), + }; + let target_dir: PathBuf = if let Some(custom_target_dir) = env::var_os("CARGO_TARGET_DIR") { + custom_target_dir.into() + } else { + cargo_manifest_dir.join("target") + }; + let (_, glibc, found_tools) = + ensure_tools_installed(cargo, &cargo_manifest_contents, &target_dir, false)?; + + let rust_toolchain_contents = fs::read_to_string(cargo_manifest_dir.join("rust-toolchain"))?; + let classified_targets = classify_targets(&rust_toolchain_contents, &glibc)?; + + let profile = cli.profile.as_deref().unwrap_or("dev"); + + if cli.targets.is_empty() { + let result = Command::new(cargo) + .args(["build", "--profile", profile]) + .spawn()? + .wait()?; + if !result.success() { + bail!("Build via cargo build failed!"); + } + } else { + let targeted: HashSet = if cli.targets.contains(&ALL_TARGETS) { + AVAILABLE_TARGETS.iter().map(Str::to_string).collect() + } else { + cli.targets.into_iter().collect() + }; + let zigbuild_targets = classified_targets + .iter_zigbuild_targets() + .filter(|target| { + // Strip the `.{glibc-version}` suffix from `*-gnu.{glibc-version}` targets. + // TODO: Encode classified targets such that we don't need to use string parsing + // here to undo earlier string concatenation of the glibc-version when classifying + // the targets. + let target = if target.contains("-gnu.") + && let Some(target) = target.splitn(2, ".").take(1).next() + { + target + } else { + target + }; + targeted.contains(target) + }) + .collect::>(); + if !zigbuild_targets.is_empty() { + let mut command = Command::new(cargo); + command.args(["zigbuild", "--profile", profile]); + for target in zigbuild_targets { + command.args(["--target", target]); + } + command.env("PEXRC_TARGETS", "all"); + for found_tool in &found_tools { + command.env(found_tool.env_var, &found_tool.path); + } + let result = command.spawn()?.wait()?; + if !result.success() { + bail!("Cross-build via cargo-zigbuild failed!"); + } + } + + let xwin_targets = classified_targets + .iter_xwin_targets() + .filter(|target| targeted.contains(*target)) + .collect::>(); + if !xwin_targets.is_empty() { + let mut command = Command::new(cargo); + command.args(["xwin", "build", "--profile", profile]); + for target in xwin_targets { + command.args(["--target", target]); + } + command.env("PEXRC_TARGETS", "all"); + for found_tool in &found_tools { + command.env(found_tool.env_var, &found_tool.path); + } + let result = command.spawn()?.wait()?; + if !result.success() { + bail!("Cross-build via cargo-xwin failed!"); + } + } + } + + anstream::println!("{}", "Build complete!".green()); + Ok(()) +} diff --git a/crates/pexrc-build-system/Cargo.toml b/crates/pexrc-build-system/Cargo.toml index 8294a5c..2d17bec 100644 --- a/crates/pexrc-build-system/Cargo.toml +++ b/crates/pexrc-build-system/Cargo.toml @@ -20,6 +20,7 @@ sha2 = { workspace = true } strum = { workspace = true } strum_macros = { workspace = true } tar = { workspace = true } +target-lexicon = { workspace = true } toml = { workspace = true } tempfile = { workspace = true } xz2 = { workspace = true } diff --git a/crates/pexrc-build-system/src/lib.rs b/crates/pexrc-build-system/src/lib.rs index 406757f..243bb26 100644 --- a/crates/pexrc-build-system/src/lib.rs +++ b/crates/pexrc-build-system/src/lib.rs @@ -6,8 +6,12 @@ mod metadata; mod rust_toolchain; mod tools; +use std::borrow::Cow; +use std::env; use std::path::{Path, PathBuf}; +use anyhow::bail; +use itertools::Itertools; pub use metadata::{Clib, ClibConfiguration, Glibc}; pub use rust_toolchain::{ClassifiedTargets, GnuLinux, Target}; use rust_toolchain::{Toolchain, parse_toolchain}; @@ -25,16 +29,9 @@ pub fn download_virtualenv( ensure_download(&metadata.build.virtualenv, &install_dirs.download_dir) } -pub fn ensure_tools_installed<'a>( - cargo: &Path, - cargo_manifest_contents: &'a str, - install_dirs: InstallDirs, - install_missing_tools: bool, -) -> anyhow::Result> { - let metadata: Metadata = parse_metadata(cargo_manifest_contents)?; - let tool_box = ToolBox::from(metadata.build); - let tool_inventory = tool_box.find_tools(install_dirs)?; - tool_inventory.ensure_tools_installed(cargo, install_missing_tools) +pub fn all_targets(rust_toolchain_contents: &str) -> anyhow::Result> { + let toolchain: Toolchain = parse_toolchain(rust_toolchain_contents)?; + Ok(toolchain.into_targets()) } pub fn classify_targets<'a>( @@ -44,3 +41,50 @@ pub fn classify_targets<'a>( let toolchain: Toolchain = parse_toolchain(rust_toolchain_contents)?; Ok(toolchain.classify_targets(glibc)) } + +pub fn ensure_tools_installed<'a>( + cargo: &Path, + cargo_manifest_contents: &'a str, + target_dir: &Path, + is_build_script: bool, +) -> anyhow::Result<(Clib<'a>, Glibc<'a>, Vec)> { + let install_dirs = InstallDirs::system("pexrc-dev").unwrap_or_else(|| { + let cache_base_dir = target_dir.join(".pexrc-dev"); + if is_build_script { + println!( + "cargo::warning=Failed to discover the user cache dir; using {cache_base_dir}", + cache_base_dir = cache_base_dir.display() + ); + } + InstallDirs::new(cache_base_dir) + }); + + if is_build_script { + println!("cargo::rerun-if-env-changed=PEXRC_INSTALL_TOOLS"); + } + let install_missing_tools = env::var_os("PEXRC_INSTALL_TOOLS").unwrap_or_default() == "1"; + + let metadata: Metadata = parse_metadata(cargo_manifest_contents)?; + let tool_box = ToolBox::from(metadata.build); + let tool_inventory = tool_box.find_tools(install_dirs)?; + match tool_inventory.ensure_tools_installed(cargo, install_missing_tools)? { + ToolInstallation::Success(result) => Ok(result), + ToolInstallation::Failure((zig, missing_binstall_tools, tool_search_path)) => { + bail!( + "The following tools are required but are not installed: {tools}\n\ + Searched PATH: {search_path}\n\ + Re-run with PEXRC_INSTALL_TOOLS=1 to let the build script install these tools.", + tools = missing_binstall_tools + .iter() + .map(|tool| Cow::Borrowed(tool.binary_name())) + .chain( + zig.missing_version() + .iter() + .map(|version| Cow::Owned(format!("zig@{version}"))) + ) + .join(" "), + search_path = tool_search_path.display() + ); + } + } +} diff --git a/crates/pexrc-build-system/src/rust_toolchain.rs b/crates/pexrc-build-system/src/rust_toolchain.rs index 23a4028..ebddc9d 100644 --- a/crates/pexrc-build-system/src/rust_toolchain.rs +++ b/crates/pexrc-build-system/src/rust_toolchain.rs @@ -2,10 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 use std::borrow::Cow; -use std::env; use itertools::Itertools; use serde::Deserialize; +use target_lexicon::HOST; use crate::metadata::Glibc; @@ -91,7 +91,7 @@ impl<'a> ClassifiedTargets<'a> { } pub fn is_just_current(&'a self) -> anyhow::Result> { - let current_target = env::var("TARGET")?; + let current_target = HOST.to_string(); let mut all_targets_iter = self.iter_all_targets(); if let Some(target) = all_targets_iter.next() && target.as_str() == current_target @@ -103,7 +103,7 @@ impl<'a> ClassifiedTargets<'a> { } } - pub fn iter_zigbuild_targets(&'a self) -> impl Iterator { + pub fn iter_zigbuild_targets(&'a self) -> impl ExactSizeIterator { self.zigbuild_targets.iter().map(|target| { if let Target::GnuLinux(gnu_linux) = target { gnu_linux.zigbuild_target.as_str() @@ -113,7 +113,7 @@ impl<'a> ClassifiedTargets<'a> { }) } - pub fn iter_xwin_targets(&'a self) -> impl Iterator { + pub fn iter_xwin_targets(&'a self) -> impl ExactSizeIterator { self.xwin_targets.iter().map(Target::as_str) } @@ -129,6 +129,10 @@ pub(crate) struct Toolchain<'a> { } impl<'a> Toolchain<'a> { + pub(crate) fn into_targets(self) -> Vec { + self.targets.into_iter().map(str::to_string).collect() + } + pub(crate) fn classify_targets(&self, glibc: &'a Glibc<'a>) -> ClassifiedTargets<'a> { ClassifiedTargets::parse(self.targets.iter().copied(), glibc) } diff --git a/crates/pexrc-build-system/src/tools.rs b/crates/pexrc-build-system/src/tools.rs index 5dc5e9a..33a9b1e 100644 --- a/crates/pexrc-build-system/src/tools.rs +++ b/crates/pexrc-build-system/src/tools.rs @@ -13,6 +13,7 @@ use const_format::concatcp; use fs_err::File; use strum::{EnumCount, IntoEnumIterator}; use strum_macros::{EnumCount, EnumIter}; +use target_lexicon::HOST; use which::which_in_global; use crate::downloads::ensure_download; @@ -317,8 +318,8 @@ fn binstall( { eprintln!("Found cargo-binstall at {exe}", exe = exe.display()); } else { - let target = env::var("TARGET")?; - if let Some(download) = cargo_binstall.download_for(&target)? { + let current_target = HOST.to_string(); + if let Some(download) = cargo_binstall.download_for(¤t_target)? { let cargo_binstall = ensure_download(&download, &install_dirs.download_dir)? .join(CARGO_BINSTALL_FILE_NAME); let cargo_binstall_fp = File::open(&cargo_binstall)?;