diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2f678fb..d83ab0d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,6 +10,7 @@ concurrency: group: CI-${{ github.ref }} # Queue on all branches and tags, but only cancel overlapping PR burns. cancel-in-progress: ${{ github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/tags/') }} + jobs: setup: name: Check GitHub Organization @@ -36,7 +37,7 @@ jobs: cross-build: name: "Cross-build pexrc" needs: setup - runs-on: ubuntu-22.04-arm + runs-on: ubuntu-24.04-arm # N.B.: We break these up just to save wall time; they all can be built on 1 machine in ~40 # minutes; this gets us to a ~20 minute long-pole. strategy: @@ -64,7 +65,7 @@ jobs: - name: Checkout Pexrc uses: actions/checkout@v6 - name: Build pexrc binary for all targets. - run: cargo run -p package -- --profile release ${{ matrix.targets }} + run: cargo run -p package -- --profile release -o dist ${{ matrix.targets }} tests: name: "${{ matrix.name }} tests" diff --git a/.gitignore b/.gitignore index 3da87f4..91929ab 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Rust build /target/ +/dist/ # Python "build" *.py[cdo] @@ -8,4 +9,3 @@ __pycache__/ /.pytest_cache/ /.ruff_cache/ /.venv/ -/python/pexrc/__pex__/ \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 95165f8..86af8aa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -225,6 +225,7 @@ dependencies = [ "base64", "dirs", "fs-err", + "hex", "logging_timer", "sha2", "tempfile", @@ -1465,12 +1466,16 @@ version = "0.0.0" dependencies = [ "anstream 1.0.0", "anyhow", + "cache", "clap", "clap-verbosity-flag", "colorchoice-clap", "env_logger", + "fs-err", "owo-colors", "pexrc-build-system", + "platform", + "sha2", ] [[package]] diff --git a/build.rs b/build.rs index 98bfce9..f2584be 100644 --- a/build.rs +++ b/build.rs @@ -14,6 +14,7 @@ use pexrc_build_system::{ ClassifiedTargets, ClibConfiguration, FoundTool, + Target, classify_targets, ensure_tools_installed, }; @@ -91,14 +92,14 @@ fn main() -> anyhow::Result<()> { &["zigbuild", "--target-dir", tgt_arg], clib.profile, &found_tools, - targets.iter_zigbuild_targets(), + targets.iter_zigbuild_targets().map(Target::zigbuild_target), )?; custom_cargo_build( &cargo, &["xwin", "build", "--target-dir", tgt_arg], clib.profile, &found_tools, - targets.iter_xwin_targets(), + targets.iter_xwin_targets().map(Target::as_str), )?; collect_clibs(&targets, &tgt_path, clib, &clibs_dir, true) } else { @@ -182,7 +183,7 @@ fn collect_clibs<'a>( } let mut dst = File::create(clibs_dir.join(format!( "{target}.{clib_name}", - target = target.python_identifier() + target = target.simplified_target_triple() )))?; if compress { let encoder = zstd::Encoder::new(dst, clib.compression_level)?; diff --git a/crates/boot/src/boot.py b/crates/boot/src/boot.py index 0833622..6e00b95 100644 --- a/crates/boot/src/boot.py +++ b/crates/boot/src/boot.py @@ -99,15 +99,11 @@ def current(cls): def __init__( self, name, - vendor, # type: str lib_extension, # type: str lib_prefix="", # type: str - alt_os=None, # type: Optional[str] ): # type: (...) -> None self.name = name - self._os = alt_os or name - self._vendor = vendor self._lib_prefix = lib_prefix self._lib_extension = lib_extension @@ -117,9 +113,7 @@ def target_triple( abi=None, # type: Optional[ABI] ): # type: (...) -> str - return "{arch}-{vendor}-{os}{abi}".format( - arch=arch, vendor=self._vendor, os=self._os, abi="-" + abi.name if abi else "" - ) + return "{arch}-{os}{abi}".format(arch=arch, os=self.name, abi="-" + abi.name if abi else "") def library_file_name(self, lib_name): # type: (str) -> str @@ -132,11 +126,9 @@ def __str__(self): return self.name -LINUX = OperatingSystem("linux", vendor="unknown", lib_prefix="lib", lib_extension="so") -MACOS = OperatingSystem( - "macos", vendor="apple", alt_os="darwin", lib_prefix="lib", lib_extension="dylib" -) -WINDOWS = OperatingSystem("windows", vendor="pc", lib_extension="dll") +LINUX = OperatingSystem("linux", lib_prefix="lib", lib_extension="so") +MACOS = OperatingSystem("macos", lib_prefix="lib", lib_extension="dylib") +WINDOWS = OperatingSystem("windows", lib_extension="dll") CURRENT_OS = OperatingSystem.current() diff --git a/crates/cache/Cargo.toml b/crates/cache/Cargo.toml index 99c1737..ebb7415 100644 --- a/crates/cache/Cargo.toml +++ b/crates/cache/Cargo.toml @@ -10,6 +10,7 @@ anyhow = { workspace = true } base64 = { workspace = true } dirs = { workspace = true } fs-err = { workspace = true } +hex = { workspace = true } logging_timer = { workspace = true } sha2 = { workspace = true } tempfile = { workspace = true } \ No newline at end of file diff --git a/crates/cache/src/fingerprint.rs b/crates/cache/src/fingerprint.rs index f48f6a4..790a9a8 100644 --- a/crates/cache/src/fingerprint.rs +++ b/crates/cache/src/fingerprint.rs @@ -25,6 +25,11 @@ impl Fingerprint { pub fn base64_digest(&self) -> String { URL_SAFE_NO_PAD.encode(&self.0) } + + #[time("debug", "Fingerprint.{}")] + pub fn hex_digest(&self) -> String { + hex::encode(&self.0) + } } impl Display for Fingerprint { diff --git a/crates/package/Cargo.toml b/crates/package/Cargo.toml index 6aba29d..406887e 100644 --- a/crates/package/Cargo.toml +++ b/crates/package/Cargo.toml @@ -5,9 +5,13 @@ edition = "2024" [dependencies] anstream = { workspace = true } anyhow = { workspace = true } +cache = { path = "../cache" } clap = { workspace = true } clap-verbosity-flag = { workspace = true } colorchoice-clap = { workspace = true } env_logger = { workspace = true } +fs-err = { workspace = true } owo-colors = { workspace = true } pexrc-build-system = { path = "../pexrc-build-system" } +platform = { path = "../platform" } +sha2 = { workspace = true } diff --git a/crates/package/src/main.rs b/crates/package/src/main.rs index 7a8a977..7a6eb21 100644 --- a/crates/package/src/main.rs +++ b/crates/package/src/main.rs @@ -6,13 +6,17 @@ use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use std::string::ToString; use std::sync::LazyLock; -use std::{env, fs}; +use std::{cmp, env, io}; use anyhow::{anyhow, bail}; +use cache::Fingerprint; use clap::builder::Str; use clap::{ArgAction, Parser}; +use fs_err as fs; +use fs_err::File; use owo_colors::OwoColorize; -use pexrc_build_system::{all_targets, classify_targets, ensure_tools_installed}; +use pexrc_build_system::{Target, all_targets, classify_targets, ensure_tools_installed}; +use sha2::{Digest, Sha256}; static CARGO: LazyLock = LazyLock::new(|| env!("CARGO").into()); @@ -87,6 +91,9 @@ struct Cli { #[arg(action=ArgAction::Append)] #[arg(value_parser=clap::builder::PossibleValuesParser::new(AVAILABLE_TARGETS.iter()))] targets: Vec, + + #[arg(short = 'o', long)] + dist_dir: Option, } fn main() -> anyhow::Result<()> { @@ -118,43 +125,51 @@ fn main() -> anyhow::Result<()> { let classified_targets = classify_targets(&rust_toolchain_contents, &glibc)?; let profile = cli.profile.as_deref().unwrap_or("dev"); + let (profile_dir_name, profile_target_suffix) = if profile == "dev" { + ("debug", Some("-debug")) + } else { + (profile, None) + }; - if cli.targets.is_empty() { + let built = if cli.targets.is_empty() { let result = Command::new(cargo) .args(["build", "--profile", profile]) + .env("PEXRC_TARGETS", "all") .spawn()? .wait()?; if !result.success() { bail!("Build via cargo build failed!"); } + let current_target = Target::current(&glibc); + vec![( + target_dir + .join(profile_dir_name) + .join(current_target.binary_name("pexrc", None).as_ref()), + current_target.fully_qualified_binary_name("pexrc", profile_target_suffix), + )] } else { let targeted: HashSet = if cli.targets.contains(&ALL_TARGETS) { AVAILABLE_TARGETS.iter().map(Str::to_string).collect() } else { cli.targets.into_iter().collect() }; + let mut built: Vec<(PathBuf, String)> = Vec::with_capacity(targeted.len()); let zigbuild_targets = classified_targets .iter_zigbuild_targets() - .filter(|target| { - // Strip the `.{glibc-version}` suffix from `*-gnu.{glibc-version}` targets. - // TODO: Encode classified targets such that we don't need to use string parsing - // here to undo earlier string concatenation of the glibc-version when classifying - // the targets. - let target = if target.contains("-gnu.") - && let Some(target) = target.splitn(2, ".").take(1).next() - { - target - } else { - target - }; - targeted.contains(target) - }) + .filter(|target| targeted.contains(target.as_str())) .collect::>(); if !zigbuild_targets.is_empty() { let mut command = Command::new(cargo); command.args(["zigbuild", "--profile", profile]); for target in zigbuild_targets { - command.args(["--target", target]); + command.args(["--target", target.zigbuild_target()]); + built.push(( + target_dir + .join(target.as_str()) + .join(profile_dir_name) + .join(target.binary_name("pexrc", None).as_ref()), + target.fully_qualified_binary_name("pexrc", profile_target_suffix), + )); } command.env("PEXRC_TARGETS", "all"); for found_tool in &found_tools { @@ -168,13 +183,20 @@ fn main() -> anyhow::Result<()> { let xwin_targets = classified_targets .iter_xwin_targets() - .filter(|target| targeted.contains(*target)) + .filter(|target| targeted.contains(target.as_str())) .collect::>(); if !xwin_targets.is_empty() { let mut command = Command::new(cargo); command.args(["xwin", "build", "--profile", profile]); for target in xwin_targets { - command.args(["--target", target]); + command.args(["--target", target.as_str()]); + built.push(( + target_dir + .join(target.as_str()) + .join(profile_dir_name) + .join(target.binary_name("pexrc", None).as_ref()), + target.fully_qualified_binary_name("pexrc", profile_target_suffix), + )); } command.env("PEXRC_TARGETS", "all"); for found_tool in &found_tools { @@ -185,8 +207,53 @@ fn main() -> anyhow::Result<()> { bail!("Cross-build via cargo-xwin failed!"); } } - } + built + }; - anstream::println!("{}", "Build complete!".green()); + if let Some(dist_dir) = cli.dist_dir { + let mut max_width = 0; + for (_, dst_file_name) in &built { + max_width = cmp::max(max_width, dst_file_name.len()); + } + fs::create_dir_all(&dist_dir)?; + let count = built.len(); + anstream::println!( + "Built {count} {binaries} to {dist_dir}:", + binaries = if count == 1 { "binary" } else { "binaries" }, + dist_dir = dist_dir.display() + ); + let dist_dir = dist_dir.canonicalize()?; + for (idx, (src, dst_file_name)) in built.iter().enumerate() { + let dst = dist_dir.join(dst_file_name); + if dst.exists() { + fs::remove_file(&dst)?; + } + let (size, fingerprint) = hash_file(src)?; + platform::link_or_copy(src, &dst)?; + fs::write( + dst.with_added_extension("sha256"), + format!( + "{hex_digest} *{dst_file_name}", + hex_digest = fingerprint.hex_digest() + ), + )?; + anstream::println!( + "{idx:>3}. {path} {pad}{size:<9} bytes {alg}:{fingerprint}", + idx = (idx + 1).yellow(), + path = dst_file_name.blue(), + pad = " ".repeat(max_width - dst_file_name.len()), + alg = "sha256-base64".green(), + fingerprint = fingerprint.base64_digest().green(), + ) + } + } else { + anstream::println!("{}", "Build complete!".green()); + } Ok(()) } + +fn hash_file(path: &Path) -> anyhow::Result<(u64, Fingerprint)> { + let mut digest = Sha256::new(); + let size = io::copy(&mut File::open(path)?, &mut digest)?; + Ok((size, Fingerprint::new(digest))) +} diff --git a/crates/pexrc-build-system/src/rust_toolchain.rs b/crates/pexrc-build-system/src/rust_toolchain.rs index ebddc9d..161b03d 100644 --- a/crates/pexrc-build-system/src/rust_toolchain.rs +++ b/crates/pexrc-build-system/src/rust_toolchain.rs @@ -2,8 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 use std::borrow::Cow; +use std::sync::LazyLock; -use itertools::Itertools; use serde::Deserialize; use target_lexicon::HOST; @@ -21,7 +21,36 @@ pub enum Target<'a> { Windows(&'a str), } +pub static CURRENT_TARGET: LazyLock = LazyLock::new(|| HOST.to_string()); + impl<'a> Target<'a> { + pub fn current(glibc: &'a Glibc<'a>) -> Self { + Self::classify(CURRENT_TARGET.as_str(), glibc) + } + + pub fn classify(target: &'a str, glibc: &'a Glibc<'a>) -> Target<'a> { + if target.contains("-apple-") { + Target::Apple(target) + } else if target.contains("-linux-") { + if target.ends_with("-gnu") { + let zigbuild_target = format!( + "{target}.{glibc_version}", + glibc_version = glibc.version(target) + ); + Target::GnuLinux(GnuLinux { + target, + zigbuild_target, + }) + } else { + Target::Unix(target) + } + } else if target.contains("-windows-") { + Target::Windows(target) + } else { + panic!("The build system does not know how to handle") + } + } + pub fn as_str(&self) -> &str { match self { Target::Apple(target) | Target::Unix(target) | Target::Windows(target) => target, @@ -29,17 +58,10 @@ impl<'a> Target<'a> { } } - pub fn python_identifier(&self) -> Cow<'_, str> { + pub fn zigbuild_target(&self) -> &str { match self { - Target::Windows(target) => { - // N.B.: The last element of the "target-triple" (the 4th element) is msvc or gnu - // depending on whether this was a native build or cross-build. Either way, the dll - // can be loaded by the host Python interpreter; so we store the dll without the - // C-lib component. - Cow::Owned(target.split("-").take(3).join("-")) - } - Target::Apple(target) | Target::Unix(target) => Cow::Borrowed(target), - Target::GnuLinux(linux) => Cow::Borrowed(linux.target), + Target::Apple(target) | Target::Unix(target) | Target::Windows(target) => target, + Target::GnuLinux(linux) => &linux.zigbuild_target, } } @@ -50,6 +72,62 @@ impl<'a> Target<'a> { Target::Windows(_) => format!("{lib_name}.dll"), } } + + pub fn binary_name(&self, binary_name: &'a str, exe_suffix: Option<&str>) -> Cow<'a, str> { + match self { + Target::Windows(_) => Cow::Owned(format!( + "{binary_name}{suffix}.exe", + suffix = exe_suffix.unwrap_or_default() + )), + _ => { + if let Some(suffix) = exe_suffix { + Cow::Owned(format!("{binary_name}{suffix}")) + } else { + Cow::Borrowed(binary_name) + } + } + } + } + + fn arch(&self) -> &'a str { + let target = match self { + Target::Apple(target) | Target::Unix(target) | Target::Windows(target) => target, + Target::GnuLinux(linux) => linux.target, + }; + target + .split("-") + .next() + .expect("Target triples always have a leading arch component.") + } + + pub fn simplified_target_triple(&self) -> Cow<'a, str> { + match self { + Target::Apple(_target) => Cow::Owned(format!("{arch}-macos", arch = self.arch())), + Target::GnuLinux(GnuLinux { target, .. }) | Target::Unix(target) => { + if target.contains("-unknown-") { + Cow::Owned(target.replace("-unknown", "")) + } else { + Cow::Borrowed(target) + } + } + Target::Windows(_target) => Cow::Owned(format!("{arch}-windows", arch = self.arch())), + } + } + + pub fn fully_qualified_binary_name( + &self, + binary_name: &str, + target_suffix: Option<&str>, + ) -> String { + let triple = self.simplified_target_triple(); + let target_suffix = target_suffix.unwrap_or_default(); + match self { + Target::Windows(_target) => { + format!("{binary_name}-{triple}{target_suffix}.exe") + } + _ => format!("{binary_name}-{triple}{target_suffix}"), + } + } } pub struct ClassifiedTargets<'a> { @@ -60,28 +138,7 @@ pub struct ClassifiedTargets<'a> { impl<'a> ClassifiedTargets<'a> { pub fn parse(targets: impl Iterator, glibc: &'a Glibc<'a>) -> Self { let (xwin_targets, zigbuild_targets) = targets - .map(|target| { - if target.contains("-apple-") { - Target::Apple(target) - } else if target.contains("-linux-") { - if target.ends_with("-gnu") { - let zigbuild_target = format!( - "{target}.{glibc_version}", - glibc_version = glibc.version(target) - ); - Target::GnuLinux(GnuLinux { - target, - zigbuild_target, - }) - } else { - Target::Unix(target) - } - } else if target.contains("-windows-") { - Target::Windows(target) - } else { - panic!("The build system does not know how to handle") - } - }) + .map(|target| Target::classify(target, glibc)) .partition::, _>(|target| matches!(target, Target::Windows(_))); Self { // TODO: Resolve cargo xwin build issues or delete the cargo-xwin code paths. @@ -103,18 +160,12 @@ impl<'a> ClassifiedTargets<'a> { } } - pub fn iter_zigbuild_targets(&'a self) -> impl ExactSizeIterator { - self.zigbuild_targets.iter().map(|target| { - if let Target::GnuLinux(gnu_linux) = target { - gnu_linux.zigbuild_target.as_str() - } else { - target.as_str() - } - }) + pub fn iter_zigbuild_targets(&'a self) -> impl ExactSizeIterator> { + self.zigbuild_targets.iter() } - pub fn iter_xwin_targets(&'a self) -> impl ExactSizeIterator { - self.xwin_targets.iter().map(Target::as_str) + pub fn iter_xwin_targets(&'a self) -> impl ExactSizeIterator> { + self.xwin_targets.iter() } pub fn iter_all_targets(&'a self) -> impl Iterator> { diff --git a/crates/platform/src/lib.rs b/crates/platform/src/lib.rs index 3ddbf50..a7c52a4 100644 --- a/crates/platform/src/lib.rs +++ b/crates/platform/src/lib.rs @@ -10,10 +10,11 @@ mod windows; use std::path::Path; use anyhow::anyhow; +use fs_err as fs; #[cfg(unix)] -pub use unix::{is_executable, link_or_copy, mark_executable, path_as_bytes}; +pub use unix::{is_executable, mark_executable, path_as_bytes, symlink_or_link_or_copy}; #[cfg(windows)] -pub use windows::{is_executable, link_or_copy, mark_executable, path_as_bytes}; +pub use windows::{is_executable, mark_executable, path_as_bytes, symlink_or_link_or_copy}; pub fn path_as_str(path: &Path) -> anyhow::Result<&str> { path.to_str().ok_or_else(|| { @@ -23,3 +24,9 @@ pub fn path_as_str(path: &Path) -> anyhow::Result<&str> { ) }) } + +pub fn link_or_copy(src: impl AsRef, dst: impl AsRef) -> anyhow::Result<()> { + fs::hard_link(&src, &dst) + .or_else(|_| fs::copy(src, dst).map(|_| ())) + .map_err(anyhow::Error::new) +} diff --git a/crates/platform/src/unix.rs b/crates/platform/src/unix.rs index 79d060d..516683a 100644 --- a/crates/platform/src/unix.rs +++ b/crates/platform/src/unix.rs @@ -12,7 +12,7 @@ use nix::errno::Errno; use nix::unistd; use nix::unistd::AccessFlags; -pub fn link_or_copy( +pub fn symlink_or_link_or_copy( src: impl AsRef, dst: impl AsRef, relative: bool, diff --git a/crates/platform/src/windows.rs b/crates/platform/src/windows.rs index 769e608..fa5ba4d 100644 --- a/crates/platform/src/windows.rs +++ b/crates/platform/src/windows.rs @@ -7,14 +7,12 @@ use std::path::Path; use fs_err as fs; use is_executable::IsExecutable; -pub fn link_or_copy( +pub fn symlink_or_link_or_copy( src: impl AsRef, dst: impl AsRef, _relative: bool, ) -> anyhow::Result<()> { - fs::hard_link(&src, &dst) - .or_else(|_| fs::copy(src, dst).map(|_| ())) - .map_err(anyhow::Error::new) + crate::link_or_copy(src, dst) } pub fn is_executable(path: impl AsRef) -> anyhow::Result { diff --git a/crates/venv/src/venv_pex.rs b/crates/venv/src/venv_pex.rs index 2a22918..dd6049d 100644 --- a/crates/venv/src/venv_pex.rs +++ b/crates/venv/src/venv_pex.rs @@ -12,7 +12,7 @@ use indexmap::{IndexMap, IndexSet}; use log::warn; use logging_timer::time; use pex::{BinPath, InheritPath, Pex, PexInfo}; -use platform::{link_or_copy, mark_executable, path_as_bytes, path_as_str}; +use platform::{mark_executable, path_as_bytes, path_as_str, symlink_or_link_or_copy}; use rayon::iter::{IntoParallelIterator, ParallelIterator}; use resources::{Resources, VenvPexReplScript, VenvPexScript}; use zip::ZipArchive; @@ -263,7 +263,7 @@ if __name__ == "__main__": ) )?; mark_executable(main_py_fp.file_mut())?; - link_or_copy(&main_py, venv.prefix().join("pex"), true) + symlink_or_link_or_copy(&main_py, venv.prefix().join("pex"), true) } fn write_repl<'a>( diff --git a/crates/venv/src/virtualenv.rs b/crates/venv/src/virtualenv.rs index 68b15f6..81d1109 100644 --- a/crates/venv/src/virtualenv.rs +++ b/crates/venv/src/virtualenv.rs @@ -12,7 +12,7 @@ use anyhow::{anyhow, bail}; use const_format::concatcp; use interpreter::Interpreter; use logging_timer::time; -use platform::link_or_copy; +use platform::symlink_or_link_or_copy; use resources::{InterpreterIdentificationScript, Resources, VendoredVirtualenvScript}; use target_lexicon::{HOST, OperatingSystem}; @@ -129,7 +129,7 @@ fn create_pep_405_venv<'a>( )?; let scripts_dir = path.join(SCRIPTS_DIR); fs::create_dir_all(&scripts_dir)?; - link_or_copy( + symlink_or_link_or_copy( &base_interpreter.realpath, scripts_dir.join(PYTHON_EXE), false, diff --git a/src/commands/info.rs b/src/commands/info.rs index 8032556..7c2d0fe 100644 --- a/src/commands/info.rs +++ b/src/commands/info.rs @@ -30,7 +30,7 @@ pub fn display() -> anyhow::Result<()> { digest.write_all(clib.contents())?; let fingerprint = Fingerprint::new(digest); anstream::println!( - "{idx:>3}. {path} {pad}{size:<7} bytes {alg}:{fingerprint}", + "{idx:>3}. {path} {pad}{size:<8} bytes {alg}:{fingerprint}", idx = (idx + 1).yellow(), path = path.blue(), pad = " ".repeat(max_width - path.len()),