From ce435fad78d6deeac71054828988da555af2c466 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 9 Jun 2026 17:10:41 -0400 Subject: [PATCH 1/7] refactor: add shared build runner and Builder interface Adds cibuildwheel/platforms/runner.py: a Builder ABC (generic over the wheel path type, since Linux wheels live in the container as PurePosixPath) whose methods are the platform-specific build steps, and run_builds(), a single flat loop that owns everything identical across platforms: logging, compatible-wheel reuse, built/repaired wheel validation, the test gate, output bookkeeping, and cleanup. HostBuilder provides the step bodies shared by the five non-Linux platforms. run_before_all() and fatal_on_called_process_error() replace the per-platform copies of the before_all hook and the CalledProcessError-to-FatalError wrapper. No platform uses this yet; subsequent commits port them one at a time. Assisted-by: ClaudeCode:claude-fable-5 --- cibuildwheel/platforms/runner.py | 271 ++++++++++++++++++++++++++++ unit_test/runner_test.py | 293 +++++++++++++++++++++++++++++++ 2 files changed, 564 insertions(+) create mode 100644 cibuildwheel/platforms/runner.py create mode 100644 unit_test/runner_test.py diff --git a/cibuildwheel/platforms/runner.py b/cibuildwheel/platforms/runner.py new file mode 100644 index 000000000..7b9a092b4 --- /dev/null +++ b/cibuildwheel/platforms/runner.py @@ -0,0 +1,271 @@ +""" +The shared build loop, used by every platform module. + +Each platform defines a Builder subclass whose methods implement the +platform-specific steps; run_builds() drives those steps for each build +identifier, in a fixed order: + + setup -> before_build -> build_wheel -> repair_wheel -> audit_wheel + -> test_wheel -> move_to_output -> cleanup + +run_builds() also owns everything that's identical across platforms: logging, +reuse of compatible wheels, validation of the built/repaired wheels, the test +gate, and moving wheels to the output directory. +""" + +from __future__ import annotations + +import contextlib +import os +import shutil +import subprocess +from abc import ABC, abstractmethod +from pathlib import Path, PurePath +from typing import Generic, TypeVar + +from cibuildwheel import errors +from cibuildwheel.audit import run_audit +from cibuildwheel.logger import log +from cibuildwheel.util import resources +from cibuildwheel.util.cmd import shell +from cibuildwheel.util.file import copy_test_sources, move_file +from cibuildwheel.util.helpers import prepare_command +from cibuildwheel.util.packaging import find_compatible_wheel + +TYPE_CHECKING = False +if TYPE_CHECKING: + from collections.abc import Callable, Generator, Mapping, Sequence + + from cibuildwheel.options import BuildOptions, Options + from cibuildwheel.typing import GenericPythonConfiguration + +PathT = TypeVar("PathT", bound=PurePath) + + +class Builder(ABC, Generic[PathT]): + """ + The platform-specific steps to build one wheel (one build identifier). + + Constructing a Builder must not do any work; all work happens in the step + methods, which run_builds() calls in a fixed order. PathT is the type of + the paths the wheels live at while building: Path on the host platforms, + PurePosixPath inside the Linux container. + """ + + identifier: str + build_options: BuildOptions + + @abstractmethod + def setup(self) -> None: + """Install Python and prepare the build environment, storing any state + the later steps need (e.g. self.env) on the instance. Logs its own + log.step()s and may leave the last one open.""" + + @abstractmethod + def before_build(self) -> None: + """Run the user's before-build command. Only called if one is set.""" + + @abstractmethod + def build_wheel(self) -> PathT: + """Build the wheel with the configured frontend and return its path.""" + + @abstractmethod + def repair_wheel(self, built_wheel: PathT) -> list[PathT]: + """Run the user's repair command (or, if none is set, move the wheel + as-is) and return the contents of the repaired-wheel directory. The + runner validates that exactly one wheel was produced.""" + + @abstractmethod + def audit_wheel(self, repaired_wheel: PathT) -> None: + """Run the audit command, if one is configured.""" + + @abstractmethod + def test_wheel(self, repaired_wheel: PathT) -> None: + """Test the wheel: set up a test environment, run before-test, install + the wheel and its test requirements, and run the test command. Only + called if a test command is set and the test selector matches. Skip + conditions that can only be determined at build time (e.g. an arch + that can't be tested on this machine) live here too.""" + + @abstractmethod + def move_to_output(self, repaired_wheel: PathT) -> PathT: + """Move the wheel to the output directory, returning the path that + later builds can reuse it from (and install it from, in tests).""" + + @abstractmethod + def cleanup(self) -> None: + """Remove this identifier's temporary files.""" + + +class HostBuilder(Builder[Path]): + """A Builder whose wheels live on the host filesystem; implements the + steps that are common to every platform except Linux (which builds inside + a container).""" + + env: dict[str, str] # set by setup() in each subclass + + def __init__( + self, + *, + identifier: str, + build_options: BuildOptions, + tmp_dir: Path, + session_tmp_dir: Path, + ) -> None: + self.identifier = identifier + self.build_options = build_options + # per-identifier scratch space, removed by cleanup() + self.tmp_dir = tmp_dir + # shared across identifiers; run_audit() reuses its venv between calls + self.session_tmp_dir = session_tmp_dir + self.built_wheel_dir = tmp_dir / "built_wheel" + self.repaired_wheel_dir = tmp_dir / "repaired_wheel" + + def before_build(self) -> None: + assert self.build_options.before_build is not None + before_build_prepared = prepare_command( + self.build_options.before_build, + project=".", + package=self.build_options.package_dir, + ) + shell(before_build_prepared, env=self.env) + + def repair_wheel(self, built_wheel: Path) -> list[Path]: + self.repaired_wheel_dir.mkdir(exist_ok=True) + if self.build_options.repair_command: + repair_command_prepared = prepare_command( + self.build_options.repair_command, + wheel=built_wheel, + dest_dir=self.repaired_wheel_dir, + package=self.build_options.package_dir, + project=".", + ) + shell(repair_command_prepared, env=self.env) + else: + shutil.move(str(built_wheel), self.repaired_wheel_dir) + return list(self.repaired_wheel_dir.glob("*.whl")) + + def audit_wheel(self, repaired_wheel: Path) -> None: + run_audit( + tmp_dir=self.session_tmp_dir, build_options=self.build_options, wheel=repaired_wheel + ) + + def move_to_output(self, repaired_wheel: Path) -> Path: + output_wheel = self.build_options.output_dir / repaired_wheel.name + moved_wheel = move_file(repaired_wheel, output_wheel) + if moved_wheel != output_wheel.resolve(): + log.warning(f"{repaired_wheel} was moved to {moved_wheel} instead of {output_wheel}") + return output_wheel + + def cleanup(self) -> None: + # ignore_errors: occasionally Windows fails to unlink a file, and we + # don't want to abort a build because of a leftover temp dir + shutil.rmtree(self.tmp_dir, ignore_errors=True) + + +def run_builds(builders: Sequence[Builder[PathT]]) -> None: + """Build a wheel for each builder, in order.""" + built_wheels: list[PathT] = [] + + for builder in builders: + build_options = builder.build_options + log.build_start(builder.identifier) + + builder.setup() + + compatible_wheel = find_compatible_wheel(built_wheels, builder.identifier) + if compatible_wheel is not None: + log.step_end() + print( + f"\nFound previously built wheel {compatible_wheel.name}, that's compatible with " + f"{builder.identifier}. Skipping build step..." + ) + repaired_wheel = compatible_wheel + else: + if build_options.before_build: + log.step("Running before_build...") + builder.before_build() + + log.step("Building wheel...") + built_wheel = builder.build_wheel() + if built_wheel.name.endswith("none-any.whl"): + raise errors.NonPlatformWheelError() + + if build_options.repair_command: + log.step("Repairing wheel...") + repaired_wheels = builder.repair_wheel(built_wheel) + match repaired_wheels: + case [only_wheel]: + repaired_wheel = only_wheel + case []: + raise errors.RepairStepProducedNoWheelError() + case many_wheels: + raise errors.RepairStepProducedMultipleWheelsError( + [wheel.name for wheel in many_wheels] + ) + + if repaired_wheel.name in {wheel.name for wheel in built_wheels}: + raise errors.AlreadyBuiltWheelError(repaired_wheel.name) + log.step_end() + + builder.audit_wheel(repaired_wheel) + + if build_options.test_command and build_options.test_selector(builder.identifier): + builder.test_wheel(repaired_wheel) + + output_wheel: Path | None = None + if compatible_wheel is None: + tracked_wheel = builder.move_to_output(repaired_wheel) + built_wheels.append(tracked_wheel) + # on Linux, the wheel only arrives at this host path when the + # container exits; the summary reads the file lazily, so that's fine + output_wheel = build_options.output_dir / tracked_wheel.name + + builder.cleanup() + log.build_end(output_wheel) + + +def run_before_all( + options: Options, + python_configurations: Sequence[GenericPythonConfiguration], + *, + env_defaults: Mapping[str, str] | None = None, +) -> None: + """Run the user's before-all command on the host, if one is set.""" + before_all_options = options.build_options(python_configurations[0].identifier) + if not before_all_options.before_all: + return + + log.step("Running before_all...") + env = before_all_options.environment.as_dictionary(prev_environment=os.environ) + for name, value in (env_defaults or {}).items(): + env.setdefault(name, value) + before_all_prepared = prepare_command( + before_all_options.before_all, project=".", package=before_all_options.package_dir + ) + shell(before_all_prepared, env=env) + + +@contextlib.contextmanager +def fatal_on_called_process_error( + troubleshoot: Callable[[subprocess.CalledProcessError], None] | None = None, +) -> Generator[None, None, None]: + """Turn a failed command anywhere in the build into a FatalError.""" + try: + yield + except subprocess.CalledProcessError as error: + if troubleshoot is not None: + troubleshoot(error) + msg = f"Command {error.cmd} failed with code {error.returncode}. {error.stdout or ''}" + raise errors.FatalError(msg) from error + + +def prepare_test_cwd(test_cwd: Path, test_sources: list[str]) -> None: + """Populate the directory the test command runs in: the user's test + sources if configured, otherwise a sentinel test that fails with a + helpful message if the user's test command assumes the project dir.""" + test_cwd.mkdir(exist_ok=True) + if test_sources: + copy_test_sources(test_sources, Path.cwd(), test_cwd) + else: + (test_cwd / "test_fail.py").write_text(resources.TEST_FAIL_CWD_FILE.read_text()) diff --git a/unit_test/runner_test.py b/unit_test/runner_test.py new file mode 100644 index 000000000..b29da8268 --- /dev/null +++ b/unit_test/runner_test.py @@ -0,0 +1,293 @@ +from __future__ import annotations + +import dataclasses +import subprocess +from pathlib import Path + +import pytest + +import cibuildwheel.options +import cibuildwheel.selector +from cibuildwheel import errors +from cibuildwheel.environment import parse_environment +from cibuildwheel.frontend import BuildFrontendConfig +from cibuildwheel.oci_container import OCIContainerEngineConfig +from cibuildwheel.options import BuildOptions, GlobalOptions +from cibuildwheel.platforms import runner +from cibuildwheel.selector import BuildSelector +from cibuildwheel.util.packaging import DependencyConstraints + + +@pytest.fixture +def build_options(tmp_path: Path) -> BuildOptions: + return BuildOptions( + globals=GlobalOptions( + package_dir=Path(), + output_dir=tmp_path / "output", + build_selector=BuildSelector(build_config="*", skip_config=""), + test_selector=cibuildwheel.selector.TestSelector(skip_config=""), + architectures=set(), + allow_empty=False, + ), + environment=parse_environment(""), + before_all="", + before_build=None, + xbuild_tools=None, + xbuild_files={}, + repair_command="", + manylinux_images=None, + musllinux_images=None, + dependency_constraints=DependencyConstraints.latest(), + test_command=None, + before_test=None, + test_sources=[], + test_requires=[], + test_extras="", + test_groups=[], + test_environment=parse_environment(""), + test_runtime=cibuildwheel.options.TestRuntimeConfig(), + audit_requires=[], + audit_command=[], + build_verbosity=0, + build_frontend=BuildFrontendConfig(name="build"), + config_settings="", + container_engine=OCIContainerEngineConfig(name="docker"), + pyodide_version=None, + ) + + +class FakeBuilder(runner.Builder[Path]): + """Records the steps the runner invokes; touches no real files.""" + + def __init__( + self, + *, + identifier: str, + build_options: BuildOptions, + calls: list[tuple[str, str]], + built_wheel_name: str = "spam-0.1.0-cp311-cp311-macosx_11_0_arm64.whl", + repaired_wheel_names: tuple[str, ...] | None = None, + ) -> None: + self.identifier = identifier + self.build_options = build_options + self.calls = calls + self.built_wheel_name = built_wheel_name + self.repaired_wheel_names = repaired_wheel_names + + def record(self, step: str) -> None: + self.calls.append((self.identifier, step)) + + def setup(self) -> None: + self.record("setup") + + def before_build(self) -> None: + self.record("before_build") + + def build_wheel(self) -> Path: + self.record("build_wheel") + return Path("/fake/built") / self.built_wheel_name + + def repair_wheel(self, built_wheel: Path) -> list[Path]: + self.record("repair_wheel") + names = ( + self.repaired_wheel_names + if self.repaired_wheel_names is not None + else (built_wheel.name,) + ) + return [Path("/fake/repaired") / name for name in names] + + def audit_wheel(self, repaired_wheel: Path) -> None: # noqa: ARG002 + self.record("audit_wheel") + + def test_wheel(self, repaired_wheel: Path) -> None: + self.record(f"test_wheel:{repaired_wheel.name}") + + def move_to_output(self, repaired_wheel: Path) -> Path: + self.record("move_to_output") + return self.build_options.output_dir / repaired_wheel.name + + def cleanup(self) -> None: + self.record("cleanup") + + +def test_step_order(build_options: BuildOptions) -> None: + build_options = dataclasses.replace( + build_options, before_build="make deps", test_command="pytest" + ) + calls: list[tuple[str, str]] = [] + wheel_name = "spam-0.1.0-cp311-cp311-macosx_11_0_arm64.whl" + builder = FakeBuilder( + identifier="cp311-macosx_arm64", + build_options=build_options, + calls=calls, + built_wheel_name=wheel_name, + ) + + runner.run_builds([builder]) + + assert calls == [ + ("cp311-macosx_arm64", "setup"), + ("cp311-macosx_arm64", "before_build"), + ("cp311-macosx_arm64", "build_wheel"), + ("cp311-macosx_arm64", "repair_wheel"), + ("cp311-macosx_arm64", "audit_wheel"), + ("cp311-macosx_arm64", f"test_wheel:{wheel_name}"), + ("cp311-macosx_arm64", "move_to_output"), + ("cp311-macosx_arm64", "cleanup"), + ] + + +def test_no_before_build_no_test(build_options: BuildOptions) -> None: + calls: list[tuple[str, str]] = [] + builder = FakeBuilder(identifier="cp311-macosx_arm64", build_options=build_options, calls=calls) + + runner.run_builds([builder]) + + steps = [step for _, step in calls] + assert "before_build" not in steps + assert not any(step.startswith("test_wheel") for step in steps) + assert steps == [ + "setup", + "build_wheel", + "repair_wheel", + "audit_wheel", + "move_to_output", + "cleanup", + ] + + +def test_compatible_wheel_reuse(build_options: BuildOptions) -> None: + build_options = dataclasses.replace(build_options, test_command="pytest") + calls: list[tuple[str, str]] = [] + abi3_wheel_name = "spam-0.1.0-cp311-abi3-macosx_11_0_arm64.whl" + builders = [ + FakeBuilder( + identifier="cp311-macosx_arm64", + build_options=build_options, + calls=calls, + built_wheel_name=abi3_wheel_name, + ), + FakeBuilder( + identifier="cp312-macosx_arm64", + build_options=build_options, + calls=calls, + ), + ] + + runner.run_builds(builders) + + second = [step for identifier, step in calls if identifier == "cp312-macosx_arm64"] + # build/repair/audit/move are skipped, but the reused wheel is still tested + assert second == ["setup", f"test_wheel:{abi3_wheel_name}", "cleanup"] + + +def test_repair_produced_no_wheel(build_options: BuildOptions) -> None: + builder = FakeBuilder( + identifier="cp311-macosx_arm64", + build_options=build_options, + calls=[], + repaired_wheel_names=(), + ) + + with pytest.raises(errors.RepairStepProducedNoWheelError): + runner.run_builds([builder]) + + +def test_repair_produced_multiple_wheels(build_options: BuildOptions) -> None: + builder = FakeBuilder( + identifier="cp311-macosx_arm64", + build_options=build_options, + calls=[], + repaired_wheel_names=( + "spam-0.1.0-cp311-cp311-macosx_11_0_arm64.whl", + "ham-0.1.0-cp311-cp311-macosx_11_0_arm64.whl", + ), + ) + + with pytest.raises(errors.RepairStepProducedMultipleWheelsError): + runner.run_builds([builder]) + + +def test_already_built_wheel(build_options: BuildOptions) -> None: + # two identifiers producing the same (non-reusable) wheel name + calls: list[tuple[str, str]] = [] + wheel_name = "spam-0.1.0-cp311-cp311-macosx_11_0_arm64.whl" + builders = [ + FakeBuilder( + identifier="cp311-macosx_arm64", + build_options=build_options, + calls=calls, + built_wheel_name=wheel_name, + ), + FakeBuilder( + identifier="cp311-macosx_x86_64", + build_options=build_options, + calls=calls, + built_wheel_name=wheel_name, + ), + ] + + with pytest.raises(errors.AlreadyBuiltWheelError): + runner.run_builds(builders) + + +def test_none_any_wheel_rejected(build_options: BuildOptions) -> None: + builder = FakeBuilder( + identifier="cp311-macosx_arm64", + build_options=build_options, + calls=[], + built_wheel_name="spam-0.1.0-py3-none-any.whl", + ) + + with pytest.raises(errors.NonPlatformWheelError): + runner.run_builds([builder]) + + +def test_test_skipped_by_selector(build_options: BuildOptions) -> None: + build_options = dataclasses.replace( + build_options, + globals=dataclasses.replace( + build_options.globals, test_selector=cibuildwheel.selector.TestSelector(skip_config="*") + ), + test_command="pytest", + ) + calls: list[tuple[str, str]] = [] + builder = FakeBuilder(identifier="cp311-macosx_arm64", build_options=build_options, calls=calls) + + runner.run_builds([builder]) + + assert not any(step.startswith("test_wheel") for _, step in calls) + + +def test_fatal_on_called_process_error() -> None: + with ( + pytest.raises(errors.FatalError, match="failed with code 42"), + runner.fatal_on_called_process_error(), + ): + raise subprocess.CalledProcessError(42, ["false"]) + + +def test_fatal_on_called_process_error_troubleshoot() -> None: + seen: list[subprocess.CalledProcessError] = [] + with pytest.raises(errors.FatalError), runner.fatal_on_called_process_error(seen.append): + raise subprocess.CalledProcessError(1, ["false"]) + assert len(seen) == 1 + + +def test_prepare_test_cwd_sentinel(tmp_path: Path) -> None: + test_cwd = tmp_path / "test_cwd" + runner.prepare_test_cwd(test_cwd, []) + assert "Please specify a path to your tests" in (test_cwd / "test_fail.py").read_text() + + +def test_prepare_test_cwd_sources(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + project = tmp_path / "project" + (project / "tests").mkdir(parents=True) + (project / "tests" / "test_spam.py").write_text("def test_spam(): pass\n") + monkeypatch.chdir(project) + + test_cwd = tmp_path / "test_cwd" + runner.prepare_test_cwd(test_cwd, ["tests"]) + + assert (test_cwd / "tests" / "test_spam.py").exists() + assert not (test_cwd / "test_fail.py").exists() From 2e0d8cc58e7f8f68e934a72200226dd2db14d2d1 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 9 Jun 2026 17:14:06 -0400 Subject: [PATCH 2/7] refactor: port pyodide to the shared build runner The per-wheel loop in build() is replaced by a PyodideBuilder driven by runner.run_builds(). Behavior changes: - a compatible-wheel (abi3) reuse no longer crashes with a NameError (the old loop assigned built_wheel but read repaired_wheel; it was unreachable in practice since pyodide wheels are never reused) - a repair command producing zero wheels now raises RepairStepProducedNoWheelError instead of a raw StopIteration, and producing several is now an error, like on Linux - before_test no longer receives the undocumented {wheel} placeholder; test_command now does, matching the other platforms - the per-identifier temp dir is now removed after each build Assisted-by: ClaudeCode:claude-fable-5 --- cibuildwheel/platforms/pyodide.py | 386 +++++++++++++----------------- 1 file changed, 166 insertions(+), 220 deletions(-) diff --git a/cibuildwheel/platforms/pyodide.py b/cibuildwheel/platforms/pyodide.py index b4f7cd1b7..c99fb920d 100644 --- a/cibuildwheel/platforms/pyodide.py +++ b/cibuildwheel/platforms/pyodide.py @@ -4,8 +4,6 @@ import functools import json import os -import shutil -import subprocess import sys import tomllib import typing @@ -17,22 +15,20 @@ from cibuildwheel import errors from cibuildwheel.architecture import Architecture -from cibuildwheel.audit import run_audit from cibuildwheel.frontend import get_build_frontend_extra_flags, prepare_config_settings from cibuildwheel.logger import log +from cibuildwheel.platforms import runner from cibuildwheel.util import resources from cibuildwheel.util.cmd import call, shell from cibuildwheel.util.file import ( CIBW_CACHE_PATH, - copy_test_sources, download, extract_tar, extract_zip, - move_file, remove_on_error, ) from cibuildwheel.util.helpers import prepare_command, unwrap, unwrap_preserving_paragraphs -from cibuildwheel.util.packaging import find_compatible_wheel, get_pip_version +from cibuildwheel.util.packaging import get_pip_version from cibuildwheel.util.python_build_standalone import ( PythonBuildStandaloneError, create_python_build_standalone_environment, @@ -44,7 +40,7 @@ from collections.abc import Set from cibuildwheel.environment import ParsedEnvironment - from cibuildwheel.options import Options + from cibuildwheel.options import BuildOptions, Options from cibuildwheel.selector import BuildSelector IS_WIN: Final[bool] = sys.platform.startswith("win") @@ -345,235 +341,185 @@ def get_python_configurations( return [c for c in all_python_configurations() if build_selector(c.identifier)] -def build(options: Options, tmp_path: Path) -> None: - python_configurations = get_python_configurations( - options.globals.build_selector, options.globals.architectures - ) +class PyodideBuilder(runner.HostBuilder): + def __init__( + self, + *, + config: PythonConfiguration, + build_options: BuildOptions, + tmp_dir: Path, + session_tmp_dir: Path, + ) -> None: + super().__init__( + identifier=config.identifier, + build_options=build_options, + tmp_dir=tmp_dir, + session_tmp_dir=session_tmp_dir, + ) + self.config = config - if not python_configurations: - return + def setup(self) -> None: + build_options = self.build_options - try: - before_all_options_identifier = python_configurations[0].identifier - before_all_options = options.build_options(before_all_options_identifier) - - if before_all_options.before_all: - log.step("Running before_all...") - env = before_all_options.environment.as_dictionary(prev_environment=os.environ) - before_all_prepared = prepare_command( - before_all_options.before_all, project=".", package=before_all_options.package_dir - ) - shell(before_all_prepared, env=env) + if build_options.build_frontend.name == "pip": + msg = "The pyodide platform doesn't support pip frontend" + raise errors.FatalError(msg) + + self.tmp_dir.mkdir() + self.built_wheel_dir.mkdir() - built_wheels: list[Path] = [] + constraints_path = build_options.dependency_constraints.get_for_python_version( + version=self.config.version, variant="pyodide", tmp_dir=self.tmp_dir + ) - for config in python_configurations: - build_options = options.build_options(config.identifier) - build_frontend = build_options.build_frontend + env = setup_python( + tmp=self.tmp_dir / "build", + python_configuration=self.config, + constraints_path=constraints_path, + environment=build_options.environment, + user_pyodide_version=build_options.pyodide_version, + ) + env["CIBUILDWHEEL_BUILD_IDENTIFIER"] = self.identifier + self.pip_version = get_pip_version(env) + + # The Pyodide command line runner mounts all directories in the host + # filesystem into the Pyodide file system, except for the custom + # file systems /dev, /lib, /proc, and /tmp. Mounting the mount + # points for alternate file systems causes some mysterious failure + # of the process (it just quits without any clear error). + # + # Because of this, by default Pyodide can't see anything under /tmp. + # This environment variable tells it also to mount our temp + # directory. + oldmounts = "" + extra_mounts = [str(self.tmp_dir)] + if Path.cwd().is_relative_to("/tmp"): + extra_mounts.append(str(Path.cwd())) + + if "_PYODIDE_EXTRA_MOUNTS" in env: + oldmounts = env["_PYODIDE_EXTRA_MOUNTS"] + ":" + env["_PYODIDE_EXTRA_MOUNTS"] = oldmounts + ":".join(extra_mounts) + + self.env = env + + def build_wheel(self) -> Path: + build_options = self.build_options + + extra_flags = get_build_frontend_extra_flags( + build_options.build_frontend, + build_options.build_verbosity, + prepare_config_settings( + build_options.config_settings, + project=".", + package=build_options.package_dir, + ), + ) - if build_frontend.name == "pip": - msg = "The pyodide platform doesn't support pip frontend" - raise errors.FatalError(msg) + call( + "pyodide", + "build", + build_options.package_dir, + f"--outdir={self.built_wheel_dir}", + *extra_flags, + env=self.env, + ) + return next(self.built_wheel_dir.glob("*.whl")) - log.build_start(config.identifier) + def test_wheel(self, repaired_wheel: Path) -> None: + build_options = self.build_options + assert build_options.test_command is not None - identifier_tmp_dir = tmp_path / config.identifier + log.step("Testing wheel...") - built_wheel_dir = identifier_tmp_dir / "built_wheel" - repaired_wheel_dir = identifier_tmp_dir / "repaired_wheel" - identifier_tmp_dir.mkdir() - built_wheel_dir.mkdir() - repaired_wheel_dir.mkdir() + # set up a virtual environment to install and test from, to make sure + # there are no dependencies that were pulled in at build time. + venv_dir = self.tmp_dir / "venv-test" - constraints_path = build_options.dependency_constraints.get_for_python_version( - version=config.version, variant="pyodide", tmp_dir=identifier_tmp_dir - ) + virtualenv_env = self.env.copy() + virtualenv_env["PATH"] = os.pathsep.join( + [ + str(ensure_node(self.config.node_version)), + virtualenv_env["PATH"], + ] + ) + + # pyodide venv uses virtualenv under the hood + # use the pip embedded with virtualenv & disable network updates + virtualenv_create_env = virtualenv_env.copy() + virtualenv_create_env["VIRTUALENV_PIP"] = self.pip_version + virtualenv_create_env["VIRTUALENV_NO_PERIODIC_UPDATE"] = "1" + + call("pyodide", "venv", venv_dir, env=virtualenv_create_env) + + virtualenv_env["PATH"] = os.pathsep.join( + [ + str(venv_dir / "bin"), + virtualenv_env["PATH"], + ] + ) + virtualenv_env["VIRTUAL_ENV"] = str(venv_dir) - env = setup_python( - tmp=identifier_tmp_dir / "build", - python_configuration=config, - constraints_path=constraints_path, - environment=build_options.environment, - user_pyodide_version=build_options.pyodide_version, + virtualenv_env = build_options.test_environment.as_dictionary( + prev_environment=virtualenv_env + ) + + # check that we are using the Python from the virtual environment + call("which", "python", env=virtualenv_env) + + if build_options.before_test: + before_test_prepared = prepare_command( + build_options.before_test, + project=".", + package=build_options.package_dir, ) - env["CIBUILDWHEEL_BUILD_IDENTIFIER"] = config.identifier - pip_version = get_pip_version(env) - # The Pyodide command line runner mounts all directories in the host - # filesystem into the Pyodide file system, except for the custom - # file systems /dev, /lib, /proc, and /tmp. Mounting the mount - # points for alternate file systems causes some mysterious failure - # of the process (it just quits without any clear error). - # - # Because of this, by default Pyodide can't see anything under /tmp. - # This environment variable tells it also to mount our temp - # directory. - oldmounts = "" - extra_mounts = [str(identifier_tmp_dir)] - if Path.cwd().is_relative_to("/tmp"): - extra_mounts.append(str(Path.cwd())) - - if "_PYODIDE_EXTRA_MOUNTS" in env: - oldmounts = env["_PYODIDE_EXTRA_MOUNTS"] + ":" - env["_PYODIDE_EXTRA_MOUNTS"] = oldmounts + ":".join(extra_mounts) - - compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) - if compatible_wheel: - log.step_end() - print( - f"\nFound previously built wheel {compatible_wheel.name}, that's compatible with {config.identifier}. Skipping build step..." - ) - built_wheel = compatible_wheel - else: - if build_options.before_build: - log.step("Running before_build...") - before_build_prepared = prepare_command( - build_options.before_build, project=".", package=build_options.package_dir - ) - shell(before_build_prepared, env=env) - - log.step("Building wheel...") - - extra_flags = get_build_frontend_extra_flags( - build_frontend, - build_options.build_verbosity, - prepare_config_settings( - build_options.config_settings, - project=".", - package=build_options.package_dir, - ), - ) + shell(before_test_prepared, env=virtualenv_env) + + # install the wheel + call( + "pip", + "install", + f"{repaired_wheel}{build_options.test_extras}", + env=virtualenv_env, + ) - call( - "pyodide", - "build", - build_options.package_dir, - f"--outdir={built_wheel_dir}", - *extra_flags, - env=env, - ) - built_wheel = next(built_wheel_dir.glob("*.whl")) - - if built_wheel.name.endswith("none-any.whl"): - raise errors.NonPlatformWheelError() - - if build_options.repair_command: - log.step("Repairing wheel...") - - repair_command_prepared = prepare_command( - build_options.repair_command, - wheel=built_wheel, - dest_dir=repaired_wheel_dir, - package=build_options.package_dir, - project=".", - ) - shell(repair_command_prepared, env=env) - log.step_end() - else: - shutil.move(str(built_wheel), repaired_wheel_dir) - - repaired_wheel = next(repaired_wheel_dir.glob("*.whl")) - - if repaired_wheel.name in {wheel.name for wheel in built_wheels}: - raise errors.AlreadyBuiltWheelError(repaired_wheel.name) - - run_audit(tmp_dir=tmp_path, build_options=build_options, wheel=repaired_wheel) - - if build_options.test_command and build_options.test_selector(config.identifier): - log.step("Testing wheel...") - - venv_dir = identifier_tmp_dir / "venv-test" - # set up a virtual environment to install and test from, to make sure - # there are no dependencies that were pulled in at build time. - - virtualenv_env = env.copy() - virtualenv_env["PATH"] = os.pathsep.join( - [ - str(ensure_node(config.node_version)), - virtualenv_env["PATH"], - ] - ) + # test the wheel + if build_options.test_requires: + call("pip", "install", *build_options.test_requires, env=virtualenv_env) + + # run the tests from a temp dir, with an absolute path in the command + # (this ensures that Python runs the tests against the installed wheel + # and not the repo code) + test_command_prepared = prepare_command( + build_options.test_command, + project=Path.cwd(), + package=build_options.package_dir.resolve(), + wheel=repaired_wheel, + ) - # pyodide venv uses virtualenv under the hood - # use the pip embedded with virtualenv & disable network updates - virtualenv_create_env = virtualenv_env.copy() - virtualenv_create_env["VIRTUALENV_PIP"] = pip_version - virtualenv_create_env["VIRTUALENV_NO_PERIODIC_UPDATE"] = "1" + test_cwd = self.tmp_dir / "test_cwd" + runner.prepare_test_cwd(test_cwd, build_options.test_sources) - call("pyodide", "venv", venv_dir, env=virtualenv_create_env) + shell(test_command_prepared, cwd=test_cwd, env=virtualenv_env) - virtualenv_env["PATH"] = os.pathsep.join( - [ - str(venv_dir / "bin"), - virtualenv_env["PATH"], - ] - ) - virtualenv_env["VIRTUAL_ENV"] = str(venv_dir) - virtualenv_env = build_options.test_environment.as_dictionary( - prev_environment=virtualenv_env - ) +def build(options: Options, tmp_path: Path) -> None: + python_configurations = get_python_configurations( + options.globals.build_selector, options.globals.architectures + ) - # check that we are using the Python from the virtual environment - call("which", "python", env=virtualenv_env) - - if build_options.before_test: - before_test_prepared = prepare_command( - build_options.before_test, - project=".", - package=build_options.package_dir, - wheel=repaired_wheel, - ) - shell(before_test_prepared, env=virtualenv_env) - - # install the wheel - call( - "pip", - "install", - f"{repaired_wheel}{build_options.test_extras}", - env=virtualenv_env, - ) + if not python_configurations: + return - # test the wheel - if build_options.test_requires: - call("pip", "install", *build_options.test_requires, env=virtualenv_env) - - # run the tests from a temp dir, with an absolute path in the command - # (this ensures that Python runs the tests against the installed wheel - # and not the repo code) - test_command_prepared = prepare_command( - build_options.test_command, - project=Path.cwd(), - package=build_options.package_dir.resolve(), + with runner.fatal_on_called_process_error(): + runner.run_before_all(options, python_configurations) + runner.run_builds( + [ + PyodideBuilder( + config=config, + build_options=options.build_options(config.identifier), + tmp_dir=tmp_path / config.identifier, + session_tmp_dir=tmp_path, ) - - test_cwd = identifier_tmp_dir / "test_cwd" - test_cwd.mkdir(exist_ok=True) - - if build_options.test_sources: - copy_test_sources( - build_options.test_sources, - Path.cwd(), - test_cwd, - ) - else: - # Use the test_fail.py file to raise a nice error if the user - # tries to run tests in the cwd - (test_cwd / "test_fail.py").write_text(resources.TEST_FAIL_CWD_FILE.read_text()) - - shell(test_command_prepared, cwd=test_cwd, env=virtualenv_env) - - # we're all done here; move it to output (overwrite existing) - output_wheel: Path | None = None - if compatible_wheel is None: - output_wheel = build_options.output_dir.joinpath(repaired_wheel.name) - moved_wheel = move_file(repaired_wheel, output_wheel) - if moved_wheel != output_wheel.resolve(): - log.warning( - f"{repaired_wheel} was moved to {moved_wheel} instead of {output_wheel}" - ) - built_wheels.append(output_wheel) - log.build_end(output_wheel) - - except subprocess.CalledProcessError as error: - msg = f"Command {error.cmd} failed with code {error.returncode}. {error.stdout or ''}" - raise errors.FatalError(msg) from error + for config in python_configurations + ] + ) From a494d0b3e4909a512c8ca49f571b3cda6e41e555 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 9 Jun 2026 17:16:44 -0400 Subject: [PATCH 3/7] refactor: port macos to the shared build runner The per-wheel loop becomes a MacOSBuilder; the per-architecture test loop (universal2 testing both halves, Rosetta emulation, arch-specific test-skip identifiers) stays inside test_wheel(), unchanged. Behavior changes: - a repair command producing several wheels is now an error, as on Linux - a missing uv now raises FatalError instead of AssertionError - the per-identifier temp dir is removed with ignore_errors, like Windows Assisted-by: ClaudeCode:claude-fable-5 --- cibuildwheel/platforms/macos.py | 592 +++++++++++++++----------------- 1 file changed, 277 insertions(+), 315 deletions(-) diff --git a/cibuildwheel/platforms/macos.py b/cibuildwheel/platforms/macos.py index c2aa23df2..ef5a891cc 100644 --- a/cibuildwheel/platforms/macos.py +++ b/cibuildwheel/platforms/macos.py @@ -7,7 +7,6 @@ import platform import re import shutil -import subprocess import sys import typing from pathlib import Path @@ -17,7 +16,6 @@ from packaging.version import Version from cibuildwheel import errors -from cibuildwheel.audit import run_audit from cibuildwheel.ci import detect_ci_provider from cibuildwheel.frontend import ( BuildFrontendName, @@ -25,17 +23,16 @@ prepare_config_settings, ) from cibuildwheel.logger import log +from cibuildwheel.platforms import runner from cibuildwheel.util import resources from cibuildwheel.util.cmd import call, shell from cibuildwheel.util.file import ( CIBW_CACHE_PATH, - copy_test_sources, download, - move_file, remove_on_error, ) from cibuildwheel.util.helpers import prepare_command, unwrap -from cibuildwheel.util.packaging import find_compatible_wheel, get_pip_version +from cibuildwheel.util.packaging import get_pip_version from cibuildwheel.venv import constraint_flags, find_uv, target_marker_env, virtualenv TYPE_CHECKING = False @@ -45,7 +42,7 @@ from cibuildwheel.architecture import Architecture from cibuildwheel.environment import ParsedEnvironment - from cibuildwheel.options import Options + from cibuildwheel.options import BuildOptions, Options from cibuildwheel.selector import BuildSelector @@ -433,344 +430,309 @@ def setup_python( return base_python, env -def build(options: Options, tmp_path: Path) -> None: - python_configurations = get_python_configurations( - options.globals.build_selector, options.globals.architectures - ) +class MacOSBuilder(runner.HostBuilder): + def __init__( + self, + *, + config: PythonConfiguration, + build_options: BuildOptions, + tmp_dir: Path, + session_tmp_dir: Path, + ) -> None: + super().__init__( + identifier=config.identifier, + build_options=build_options, + tmp_dir=tmp_dir, + session_tmp_dir=session_tmp_dir, + ) + self.config = config - if not python_configurations: - return + @property + def config_is_arm64(self) -> bool: + return self.identifier.endswith("arm64") - try: - before_all_options_identifier = python_configurations[0].identifier - before_all_options = options.build_options(before_all_options_identifier) + @property + def config_is_universal2(self) -> bool: + return self.identifier.endswith("universal2") - if before_all_options.before_all: - log.step("Running before_all...") - env = before_all_options.environment.as_dictionary(prev_environment=os.environ) - env.setdefault("MACOSX_DEPLOYMENT_TARGET", "10.9") - before_all_prepared = prepare_command( - before_all_options.before_all, project=".", package=before_all_options.package_dir - ) - shell(before_all_prepared, env=env) - - built_wheels: list[Path] = [] - - for config in python_configurations: - build_options = options.build_options(config.identifier) - build_frontend = build_options.build_frontend - use_uv = build_frontend.name in {"build[uv]", "uv"} - uv_path = find_uv() - if use_uv and uv_path is None: - msg = "uv not found" - raise AssertionError(msg) - pip = ["pip"] if not use_uv else [str(uv_path), "pip"] - log.build_start(config.identifier) - - identifier_tmp_dir = tmp_path / config.identifier - identifier_tmp_dir.mkdir() - built_wheel_dir = identifier_tmp_dir / "built_wheel" - repaired_wheel_dir = identifier_tmp_dir / "repaired_wheel" - - config_is_arm64 = config.identifier.endswith("arm64") - config_is_universal2 = config.identifier.endswith("universal2") - - constraints_path = build_options.dependency_constraints.get_for_python_version( - version=config.version, tmp_dir=identifier_tmp_dir - ) + def setup(self) -> None: + build_options = self.build_options + build_frontend = build_options.build_frontend - base_python, env = setup_python( - identifier_tmp_dir / "build", - config, - constraints_path, - build_options.environment, - build_frontend.name, - ) - env["CIBUILDWHEEL_BUILD_IDENTIFIER"] = config.identifier - pip_version = None if use_uv else get_pip_version(env) - - compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) - if compatible_wheel: - log.step_end() - print( - f"\nFound previously built wheel {compatible_wheel.name}, that's compatible with {config.identifier}. Skipping build step..." + self.use_uv = build_frontend.name in {"build[uv]", "uv"} + self.uv_path = find_uv() + if self.use_uv and self.uv_path is None: + msg = "uv not found" + raise errors.FatalError(msg) + self.pip: list[str] = ["pip"] if not self.use_uv else [str(self.uv_path), "pip"] + + self.tmp_dir.mkdir() + + constraints_path = build_options.dependency_constraints.get_for_python_version( + version=self.config.version, tmp_dir=self.tmp_dir + ) + + self.base_python, env = setup_python( + self.tmp_dir / "build", + self.config, + constraints_path, + build_options.environment, + build_frontend.name, + ) + env["CIBUILDWHEEL_BUILD_IDENTIFIER"] = self.identifier + self.pip_version = None if self.use_uv else get_pip_version(env) + self.env = env + + def build_wheel(self) -> Path: + build_options = self.build_options + build_frontend = build_options.build_frontend + + self.built_wheel_dir.mkdir() + + extra_flags = get_build_frontend_extra_flags( + build_frontend, + build_options.build_verbosity, + prepare_config_settings( + build_options.config_settings, + project=".", + package=build_options.package_dir, + ), + ) + + build_env = self.env.copy() + + match build_frontend.name: + case "pip": + # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org + # see https://github.com/pypa/cibuildwheel/pull/369 + call( + "python", + "-m", + "pip", + "wheel", + build_options.package_dir.resolve(), + f"--wheel-dir={self.built_wheel_dir}", + "--no-deps", + *extra_flags, + env=build_env, ) - repaired_wheel = compatible_wheel - else: - if build_options.before_build: - log.step("Running before_build...") - before_build_prepared = prepare_command( - build_options.before_build, project=".", package=build_options.package_dir - ) - shell(before_build_prepared, env=env) - - log.step("Building wheel...") - built_wheel_dir.mkdir() - - extra_flags = get_build_frontend_extra_flags( - build_frontend, - build_options.build_verbosity, - prepare_config_settings( - build_options.config_settings, - project=".", - package=build_options.package_dir, - ), + case "build" | "build[uv]": + if self.use_uv and "--no-isolation" not in extra_flags and "-n" not in extra_flags: + extra_flags.append("--installer=uv") + call( + "python", + "-m", + "build", + build_options.package_dir, + "--wheel", + f"--outdir={self.built_wheel_dir}", + *extra_flags, + env=build_env, ) + case "uv": + assert self.uv_path is not None + call( + self.uv_path, + "build", + f"--python={self.base_python}", + build_options.package_dir, + "--wheel", + f"--out-dir={self.built_wheel_dir}", + *extra_flags, + env=build_env, + ) + case _: + assert_never(build_frontend) - build_env = env.copy() - - match build_frontend.name: - case "pip": - # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org - # see https://github.com/pypa/cibuildwheel/pull/369 - call( - "python", - "-m", - "pip", - "wheel", - build_options.package_dir.resolve(), - f"--wheel-dir={built_wheel_dir}", - "--no-deps", - *extra_flags, - env=build_env, - ) - case "build" | "build[uv]": - if ( - use_uv - and "--no-isolation" not in extra_flags - and "-n" not in extra_flags - ): - extra_flags.append("--installer=uv") - call( - "python", - "-m", - "build", - build_options.package_dir, - "--wheel", - f"--outdir={built_wheel_dir}", - *extra_flags, - env=build_env, - ) - case "uv": - assert uv_path is not None - call( - uv_path, - "build", - f"--python={base_python}", - build_options.package_dir, - "--wheel", - f"--out-dir={built_wheel_dir}", - *extra_flags, - env=build_env, - ) - case _: - assert_never(build_frontend) - - built_wheel = next(built_wheel_dir.glob("*.whl")) + return next(self.built_wheel_dir.glob("*.whl")) - repaired_wheel_dir.mkdir() + def repair_wheel(self, built_wheel: Path) -> list[Path]: + build_options = self.build_options + self.repaired_wheel_dir.mkdir(exist_ok=True) - if built_wheel.name.endswith("none-any.whl"): - raise errors.NonPlatformWheelError() + if build_options.repair_command: + if self.config_is_universal2: + delocate_archs = "x86_64,arm64" + elif self.config_is_arm64: + delocate_archs = "arm64" + else: + delocate_archs = "x86_64" + + repair_command_prepared = prepare_command( + build_options.repair_command, + wheel=built_wheel, + dest_dir=self.repaired_wheel_dir, + delocate_archs=delocate_archs, + package=build_options.package_dir, + project=".", + ) + shell(repair_command_prepared, env=self.env) + else: + shutil.move(str(built_wheel), self.repaired_wheel_dir) - if build_options.repair_command: - log.step("Repairing wheel...") + return list(self.repaired_wheel_dir.glob("*.whl")) - if config_is_universal2: - delocate_archs = "x86_64,arm64" - elif config_is_arm64: - delocate_archs = "arm64" - else: - delocate_archs = "x86_64" + def test_wheel(self, repaired_wheel: Path) -> None: + build_options = self.build_options + assert build_options.test_command is not None - repair_command_prepared = prepare_command( - build_options.repair_command, - wheel=built_wheel, - dest_dir=repaired_wheel_dir, - delocate_archs=delocate_archs, - package=build_options.package_dir, - project=".", - ) - shell(repair_command_prepared, env=env) - else: - shutil.move(str(built_wheel), repaired_wheel_dir) + machine_arch = platform.machine() + testing_archs: list[Literal["x86_64", "arm64"]] - try: - repaired_wheel = next(repaired_wheel_dir.glob("*.whl")) - except StopIteration: - raise errors.RepairStepProducedNoWheelError() from None + if self.config_is_arm64: + testing_archs = ["arm64"] + elif self.config_is_universal2: + testing_archs = ["x86_64", "arm64"] + else: + testing_archs = ["x86_64"] - if repaired_wheel.name in {wheel.name for wheel in built_wheels}: - raise errors.AlreadyBuiltWheelError(repaired_wheel.name) + for testing_arch in testing_archs: + if self.config_is_universal2: + arch_specific_identifier = f"{self.identifier}:{testing_arch}" + if not build_options.test_selector(arch_specific_identifier): + continue - log.step_end() + if machine_arch == "x86_64" and testing_arch == "arm64": + if self.config_is_arm64: + log.warning( + unwrap( + """ + While arm64 wheels can be built on x86_64, they cannot be + tested. Consider building arm64 wheels natively, if your CI + provider offers this. To silence this warning, set + `CIBW_TEST_SKIP: "*-macosx_arm64"`. + """ + ) + ) + elif self.config_is_universal2: + log.warning( + unwrap( + """ + While universal2 wheels can be built on x86_64, the arm64 part + of the wheel cannot be tested on x86_64. Consider building + universal2 wheels on an arm64 runner, if your CI provider offers + this. Notably, an arm64 runner can also test the x86_64 part of + the wheel, through Rosetta emulation. To silence this warning, + set `CIBW_TEST_SKIP: "*-macosx_universal2:arm64"`. + """ + ) + ) + else: + msg = "unreachable" + raise RuntimeError(msg) - run_audit(tmp_dir=tmp_path, build_options=build_options, wheel=repaired_wheel) + # skip this test + continue - if build_options.test_command and build_options.test_selector(config.identifier): - machine_arch = platform.machine() - testing_archs: list[Literal["x86_64", "arm64"]] + log.step( + "Testing wheel..." + if testing_arch == machine_arch + else f"Testing wheel on {testing_arch}..." + ) - if config_is_arm64: - testing_archs = ["arm64"] - elif config_is_universal2: - testing_archs = ["x86_64", "arm64"] + arch_prefix = [] + uv_arch_args = [] + if testing_arch != machine_arch: + if machine_arch == "arm64" and testing_arch == "x86_64": + # rosetta2 will provide the emulation with just the arch prefix. + arch_prefix = ["arch", "-x86_64"] + uv_arch_args = ["--python-platform", "x86_64-apple-darwin"] else: - testing_archs = ["x86_64"] - - for testing_arch in testing_archs: - if config_is_universal2: - arch_specific_identifier = f"{config.identifier}:{testing_arch}" - if not build_options.test_selector(arch_specific_identifier): - continue - - if machine_arch == "x86_64" and testing_arch == "arm64": - if config_is_arm64: - log.warning( - unwrap( - """ - While arm64 wheels can be built on x86_64, they cannot be - tested. Consider building arm64 wheels natively, if your CI - provider offers this. To silence this warning, set - `CIBW_TEST_SKIP: "*-macosx_arm64"`. - """ - ) - ) - elif config_is_universal2: - log.warning( - unwrap( - """ - While universal2 wheels can be built on x86_64, the arm64 part - of the wheel cannot be tested on x86_64. Consider building - universal2 wheels on an arm64 runner, if your CI provider offers - this. Notably, an arm64 runner can also test the x86_64 part of - the wheel, through Rosetta emulation. To silence this warning, - set `CIBW_TEST_SKIP: "*-macosx_universal2:arm64"`. - """ - ) - ) - else: - msg = "unreachable" - raise RuntimeError(msg) - - # skip this test - continue - - log.step( - "Testing wheel..." - if testing_arch == machine_arch - else f"Testing wheel on {testing_arch}..." - ) + msg = f"don't know how to emulate {testing_arch} on {machine_arch}" + raise RuntimeError(msg) + + # define a custom 'call' function that adds the arch prefix each time + call_with_arch = functools.partial(call, *arch_prefix) + shell_with_arch = functools.partial(call, *arch_prefix, "/bin/sh", "-c") + + # set up a virtual environment to install and test from, to make sure + # there are no dependencies that were pulled in at build time. + venv_dir = self.tmp_dir / f"venv-test-{testing_arch}" + virtualenv_env = virtualenv( + self.config.version, + self.base_python, + venv_dir, + None, + use_uv=self.use_uv, + env=self.env, + pip_version=self.pip_version, + ) + if self.use_uv: + pip_install = functools.partial(call, *self.pip, "install", *uv_arch_args) + else: + pip_install = functools.partial(call_with_arch, *self.pip, "install") - arch_prefix = [] - uv_arch_args = [] - if testing_arch != machine_arch: - if machine_arch == "arm64" and testing_arch == "x86_64": - # rosetta2 will provide the emulation with just the arch prefix. - arch_prefix = ["arch", "-x86_64"] - uv_arch_args = ["--python-platform", "x86_64-apple-darwin"] - else: - msg = f"don't know how to emulate {testing_arch} on {machine_arch}" - raise RuntimeError(msg) - - # define a custom 'call' function that adds the arch prefix each time - call_with_arch = functools.partial(call, *arch_prefix) - shell_with_arch = functools.partial(call, *arch_prefix, "/bin/sh", "-c") - - # set up a virtual environment to install and test from, to make sure - # there are no dependencies that were pulled in at build time. - venv_dir = identifier_tmp_dir / f"venv-test-{testing_arch}" - virtualenv_env = virtualenv( - config.version, - base_python, - venv_dir, - None, - use_uv=use_uv, - env=env, - pip_version=pip_version, - ) - if use_uv: - pip_install = functools.partial(call, *pip, "install", *uv_arch_args) - else: - pip_install = functools.partial(call_with_arch, *pip, "install") + virtualenv_env["MACOSX_DEPLOYMENT_TARGET"] = get_test_macosx_deployment_target() - virtualenv_env["MACOSX_DEPLOYMENT_TARGET"] = get_test_macosx_deployment_target() + virtualenv_env = build_options.test_environment.as_dictionary( + prev_environment=virtualenv_env + ) - virtualenv_env = build_options.test_environment.as_dictionary( - prev_environment=virtualenv_env - ) + # check that we are using the Python from the virtual environment + call_with_arch("which", "python", env=virtualenv_env) - # check that we are using the Python from the virtual environment - call_with_arch("which", "python", env=virtualenv_env) + if build_options.before_test: + before_test_prepared = prepare_command( + build_options.before_test, + project=".", + package=build_options.package_dir, + ) + shell_with_arch(before_test_prepared, env=virtualenv_env) - if build_options.before_test: - before_test_prepared = prepare_command( - build_options.before_test, - project=".", - package=build_options.package_dir, - ) - shell_with_arch(before_test_prepared, env=virtualenv_env) + # install the wheel + pip_install( + f"{repaired_wheel}{build_options.test_extras}", + env=virtualenv_env, + ) - # install the wheel - pip_install( - f"{repaired_wheel}{build_options.test_extras}", - env=virtualenv_env, - ) + # test the wheel + if build_options.test_requires: + pip_install( + *build_options.test_requires, + env=virtualenv_env, + ) - # test the wheel - if build_options.test_requires: - pip_install( - *build_options.test_requires, - env=virtualenv_env, - ) + # run the tests from a temp dir, with an absolute path in the command + # (this ensures that Python runs the tests against the installed wheel + # and not the repo code) + test_command_prepared = prepare_command( + build_options.test_command, + project=Path.cwd(), + package=build_options.package_dir.resolve(), + wheel=repaired_wheel, + ) - # run the tests from a temp dir, with an absolute path in the command - # (this ensures that Python runs the tests against the installed wheel - # and not the repo code) - test_command_prepared = prepare_command( - build_options.test_command, - project=Path.cwd(), - package=build_options.package_dir.resolve(), - wheel=repaired_wheel, - ) + # only populate test_cwd if it doesn't already exist - it + # may have been created during a previous `testing_arch` + test_cwd = self.tmp_dir / "test_cwd" + if not test_cwd.exists(): + runner.prepare_test_cwd(test_cwd, build_options.test_sources) - test_cwd = identifier_tmp_dir / "test_cwd" - - if build_options.test_sources: - # only create test_cwd if it doesn't already exist - it - # may have been created during a previous `testing_arch` - if not test_cwd.exists(): - test_cwd.mkdir() - copy_test_sources( - build_options.test_sources, - Path.cwd(), - test_cwd, - ) - else: - # Use the test_fail.py file to raise a nice error if the user - # tries to run tests in the cwd - test_cwd.mkdir(exist_ok=True) - (test_cwd / "test_fail.py").write_text( - resources.TEST_FAIL_CWD_FILE.read_text() - ) + shell_with_arch(test_command_prepared, cwd=test_cwd, env=virtualenv_env) - shell_with_arch(test_command_prepared, cwd=test_cwd, env=virtualenv_env) - # we're all done here; move it to output (overwrite existing) - output_wheel = None - if compatible_wheel is None: - output_wheel = build_options.output_dir.joinpath(repaired_wheel.name) - moved_wheel = move_file(repaired_wheel, output_wheel) - if moved_wheel != output_wheel.resolve(): - log.warning( - f"{repaired_wheel} was moved to {moved_wheel} instead of {output_wheel}" - ) - built_wheels.append(output_wheel) +def build(options: Options, tmp_path: Path) -> None: + python_configurations = get_python_configurations( + options.globals.build_selector, options.globals.architectures + ) - # clean up - shutil.rmtree(identifier_tmp_dir) + if not python_configurations: + return - log.build_end(output_wheel) - except subprocess.CalledProcessError as error: - msg = f"Command {error.cmd} failed with code {error.returncode}. {error.stdout or ''}" - raise errors.FatalError(msg) from error + with runner.fatal_on_called_process_error(): + runner.run_before_all( + options, + python_configurations, + env_defaults={"MACOSX_DEPLOYMENT_TARGET": "10.9"}, + ) + runner.run_builds( + [ + MacOSBuilder( + config=config, + build_options=options.build_options(config.identifier), + tmp_dir=tmp_path / config.identifier, + session_tmp_dir=tmp_path, + ) + for config in python_configurations + ] + ) From 9ad37e4affdd9c85fb8c15b0b645994c76db56b9 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 9 Jun 2026 17:18:49 -0400 Subject: [PATCH 4/7] refactor: port windows to the shared build runner The per-wheel loop becomes a WindowsBuilder. The ARM64-on-non-ARM64 test skip moves inside test_wheel(), so its warning now only appears when a test command is actually configured. A repair command producing several wheels is now an error, as on Linux. Assisted-by: ClaudeCode:claude-fable-5 --- cibuildwheel/platforms/windows.py | 453 +++++++++++++----------------- 1 file changed, 193 insertions(+), 260 deletions(-) diff --git a/cibuildwheel/platforms/windows.py b/cibuildwheel/platforms/windows.py index dc0a2977b..2c25258e3 100644 --- a/cibuildwheel/platforms/windows.py +++ b/cibuildwheel/platforms/windows.py @@ -1,10 +1,7 @@ from __future__ import annotations import dataclasses -import os import platform as platform_module -import shutil -import subprocess import textwrap from functools import cache from pathlib import Path @@ -14,25 +11,23 @@ from cibuildwheel import errors from cibuildwheel.architecture import Architecture -from cibuildwheel.audit import run_audit from cibuildwheel.frontend import ( BuildFrontendName, get_build_frontend_extra_flags, prepare_config_settings, ) from cibuildwheel.logger import log +from cibuildwheel.platforms import runner from cibuildwheel.util import resources from cibuildwheel.util.cmd import call, shell from cibuildwheel.util.file import ( CIBW_CACHE_PATH, - copy_test_sources, download, extract_zip, - move_file, remove_on_error, ) from cibuildwheel.util.helpers import prepare_command, unwrap -from cibuildwheel.util.packaging import find_compatible_wheel, get_pip_version +from cibuildwheel.util.packaging import get_pip_version from cibuildwheel.venv import constraint_flags, find_uv, target_marker_env, virtualenv TYPE_CHECKING = False @@ -40,7 +35,7 @@ from collections.abc import MutableMapping, Sequence, Set from cibuildwheel.environment import ParsedEnvironment - from cibuildwheel.options import Options + from cibuildwheel.options import BuildOptions, Options from cibuildwheel.selector import BuildSelector @@ -392,270 +387,208 @@ def setup_python( return base_python, env -def build(options: Options, tmp_path: Path) -> None: - python_configurations = get_python_configurations( - options.globals.build_selector, options.globals.architectures - ) - - if not python_configurations: - return +class WindowsBuilder(runner.HostBuilder): + def __init__( + self, + *, + config: PythonConfiguration, + build_options: BuildOptions, + tmp_dir: Path, + session_tmp_dir: Path, + ) -> None: + super().__init__( + identifier=config.identifier, + build_options=build_options, + tmp_dir=tmp_dir, + session_tmp_dir=session_tmp_dir, + ) + self.config = config - uv_path = find_uv() + def setup(self) -> None: + build_options = self.build_options + build_frontend = build_options.build_frontend - try: - before_all_options_identifier = python_configurations[0].identifier - before_all_options = options.build_options(before_all_options_identifier) + self.use_uv = build_frontend.name in {"build[uv]", "uv"} + self.uv_path = find_uv() - if before_all_options.before_all: - log.step("Running before_all...") - env = before_all_options.environment.as_dictionary(prev_environment=os.environ) - before_all_prepared = prepare_command( - before_all_options.before_all, project=".", package=options.globals.package_dir - ) - shell(before_all_prepared, env=env) + self.tmp_dir.mkdir() - built_wheels: list[Path] = [] + constraints_path = build_options.dependency_constraints.get_for_python_version( + version=self.config.version, tmp_dir=self.tmp_dir + ) - for config in python_configurations: - build_options = options.build_options(config.identifier) - build_frontend = build_options.build_frontend - use_uv = build_frontend.name in {"build[uv]", "uv"} - log.build_start(config.identifier) + # install Python + self.base_python, env = setup_python( + self.tmp_dir / "build", + self.config, + constraints_path, + build_options.environment, + build_frontend.name, + ) + env["CIBUILDWHEEL_BUILD_IDENTIFIER"] = self.identifier + self.pip_version = None if self.use_uv else get_pip_version(env) + self.env = env + + def build_wheel(self) -> Path: + build_options = self.build_options + build_frontend = build_options.build_frontend + + self.built_wheel_dir.mkdir() + + extra_flags = get_build_frontend_extra_flags( + build_frontend, + build_options.build_verbosity, + prepare_config_settings( + build_options.config_settings, + project=".", + package=build_options.package_dir, + ), + ) - identifier_tmp_dir = tmp_path / config.identifier - identifier_tmp_dir.mkdir() - built_wheel_dir = identifier_tmp_dir / "built_wheel" - repaired_wheel_dir = identifier_tmp_dir / "repaired_wheel" + match build_frontend.name: + case "pip": + # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org + # see https://github.com/pypa/cibuildwheel/pull/369 + call( + "python", + "-m", + "pip", + "wheel", + build_options.package_dir.resolve(), + f"--wheel-dir={self.built_wheel_dir}", + "--no-deps", + *extra_flags, + env=self.env, + ) + case "build" | "build[uv]": + if self.use_uv and "--no-isolation" not in extra_flags and "-n" not in extra_flags: + extra_flags.append("--installer=uv") - constraints_path = build_options.dependency_constraints.get_for_python_version( - version=config.version, - tmp_dir=identifier_tmp_dir, + call( + "python", + "-m", + "build", + build_options.package_dir, + "--wheel", + f"--outdir={self.built_wheel_dir}", + *extra_flags, + env=self.env, + ) + case "uv": + assert self.uv_path is not None + call( + self.uv_path, + "build", + f"--python={self.base_python}", + build_options.package_dir, + "--wheel", + f"--out-dir={self.built_wheel_dir}", + *extra_flags, + env=self.env, + ) + case _: + assert_never(build_frontend) + + return next(self.built_wheel_dir.glob("*.whl")) + + def test_wheel(self, repaired_wheel: Path) -> None: + build_options = self.build_options + assert build_options.test_command is not None + + if self.config.arch == "ARM64" != platform_module.machine(): + log.warning( + unwrap( + """ + While arm64 wheels can be built on other platforms, they cannot + be tested. An arm64 runner is required. To silence this warning, + set `CIBW_TEST_SKIP: "*-win_arm64"`. + """ + ) ) + # skip this test + return + + log.step("Testing wheel...") + # set up a virtual environment to install and test from, to make sure + # there are no dependencies that were pulled in at build time. + venv_dir = self.tmp_dir / "venv-test" + virtualenv_env = virtualenv( + self.config.version, + self.base_python, + venv_dir, + None, + use_uv=self.use_uv, + env=self.env, + pip_version=self.pip_version, + ) - # install Python - base_python, env = setup_python( - identifier_tmp_dir / "build", - config, - constraints_path, - build_options.environment, - build_frontend.name, + virtualenv_env = build_options.test_environment.as_dictionary( + prev_environment=virtualenv_env + ) + + # check that we are using the Python from the virtual environment + call("where", "python", env=virtualenv_env) + + if build_options.before_test: + before_test_prepared = prepare_command( + build_options.before_test, + project=".", + package=build_options.package_dir, ) - env["CIBUILDWHEEL_BUILD_IDENTIFIER"] = config.identifier - pip_version = None if use_uv else get_pip_version(env) - - compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) - if compatible_wheel: - log.step_end() - print( - f"\nFound previously built wheel {compatible_wheel.name}, that's compatible with {config.identifier}. Skipping build step..." - ) - repaired_wheel = compatible_wheel - else: - # run the before_build command - if build_options.before_build: - log.step("Running before_build...") - before_build_prepared = prepare_command( - build_options.before_build, - project=".", - package=options.globals.package_dir, - ) - shell(before_build_prepared, env=env) - - log.step("Building wheel...") - built_wheel_dir.mkdir() - - extra_flags = get_build_frontend_extra_flags( - build_frontend, - build_options.build_verbosity, - prepare_config_settings( - build_options.config_settings, - project=".", - package=options.globals.package_dir, - ), - ) + shell(before_test_prepared, env=virtualenv_env) + + pip: Sequence[Path | str] + if self.use_uv: + assert self.uv_path is not None + pip = [self.uv_path, "pip"] + else: + pip = ["pip"] + + # install the wheel + call( + *pip, + "install", + str(repaired_wheel) + build_options.test_extras, + env=virtualenv_env, + ) - match build_frontend.name: - case "pip": - # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org - # see https://github.com/pypa/cibuildwheel/pull/369 - call( - "python", - "-m", - "pip", - "wheel", - options.globals.package_dir.resolve(), - f"--wheel-dir={built_wheel_dir}", - "--no-deps", - *extra_flags, - env=env, - ) - case "build" | "build[uv]": - if ( - use_uv - and "--no-isolation" not in extra_flags - and "-n" not in extra_flags - ): - extra_flags.append("--installer=uv") - - call( - "python", - "-m", - "build", - build_options.package_dir, - "--wheel", - f"--outdir={built_wheel_dir}", - *extra_flags, - env=env, - ) - case "uv": - assert uv_path is not None - call( - uv_path, - "build", - f"--python={base_python}", - build_options.package_dir, - "--wheel", - f"--out-dir={built_wheel_dir}", - *extra_flags, - env=env, - ) - case _: - assert_never(build_frontend) - - built_wheel = next(built_wheel_dir.glob("*.whl")) - - # repair the wheel - repaired_wheel_dir.mkdir() - - if built_wheel.name.endswith("none-any.whl"): - raise errors.NonPlatformWheelError() - - if build_options.repair_command: - log.step("Repairing wheel...") - repair_command_prepared = prepare_command( - build_options.repair_command, - wheel=built_wheel, - dest_dir=repaired_wheel_dir, - package=build_options.package_dir, - project=".", - ) - shell(repair_command_prepared, env=env) - else: - shutil.move(str(built_wheel), repaired_wheel_dir) - - try: - repaired_wheel = next(repaired_wheel_dir.glob("*.whl")) - except StopIteration: - raise errors.RepairStepProducedNoWheelError() from None - - if repaired_wheel.name in {wheel.name for wheel in built_wheels}: - raise errors.AlreadyBuiltWheelError(repaired_wheel.name) - - run_audit(tmp_dir=tmp_path, build_options=build_options, wheel=repaired_wheel) - - test_selected = options.globals.test_selector(config.identifier) - if test_selected and config.arch == "ARM64" != platform_module.machine(): - log.warning( - unwrap( - """ - While arm64 wheels can be built on other platforms, they cannot - be tested. An arm64 runner is required. To silence this warning, - set `CIBW_TEST_SKIP: "*-win_arm64"`. - """ - ) - ) - # skip this test - elif test_selected and build_options.test_command: - log.step("Testing wheel...") - # set up a virtual environment to install and test from, to make sure - # there are no dependencies that were pulled in at build time. - venv_dir = identifier_tmp_dir / "venv-test" - virtualenv_env = virtualenv( - config.version, - base_python, - venv_dir, - None, - use_uv=use_uv, - env=env, - pip_version=pip_version, - ) + # test the wheel + if build_options.test_requires: + call(*pip, "install", *build_options.test_requires, env=virtualenv_env) + + # run the tests from a temp dir, with an absolute path in the command + # (this ensures that Python runs the tests against the installed wheel + # and not the repo code) + test_cwd = self.tmp_dir / "test_cwd" + runner.prepare_test_cwd(test_cwd, build_options.test_sources) + + test_command_prepared = prepare_command( + build_options.test_command, + project=Path.cwd(), + package=build_options.package_dir.resolve(), + wheel=repaired_wheel, + ) + shell(test_command_prepared, cwd=test_cwd, env=virtualenv_env) - virtualenv_env = build_options.test_environment.as_dictionary( - prev_environment=virtualenv_env - ) - # check that we are using the Python from the virtual environment - call("where", "python", env=virtualenv_env) - - if build_options.before_test: - before_test_prepared = prepare_command( - build_options.before_test, - project=".", - package=build_options.package_dir, - ) - shell(before_test_prepared, env=virtualenv_env) - - pip: Sequence[Path | str] - if use_uv: - assert uv_path is not None - pip = [uv_path, "pip"] - else: - pip = ["pip"] - - # install the wheel - call( - *pip, - "install", - str(repaired_wheel) + build_options.test_extras, - env=virtualenv_env, - ) +def build(options: Options, tmp_path: Path) -> None: + python_configurations = get_python_configurations( + options.globals.build_selector, options.globals.architectures + ) - # test the wheel - if build_options.test_requires: - call(*pip, "install", *build_options.test_requires, env=virtualenv_env) - - # run the tests from a temp dir, with an absolute path in the command - # (this ensures that Python runs the tests against the installed wheel - # and not the repo code) - test_cwd = identifier_tmp_dir / "test_cwd" - test_cwd.mkdir() - - if build_options.test_sources: - copy_test_sources( - build_options.test_sources, - Path.cwd(), - test_cwd, - ) - else: - # Use the test_fail.py file to raise a nice error if the user - # tries to run tests in the cwd - (test_cwd / "test_fail.py").write_text(resources.TEST_FAIL_CWD_FILE.read_text()) - - test_command_prepared = prepare_command( - build_options.test_command, - project=Path.cwd(), - package=options.globals.package_dir.resolve(), - wheel=repaired_wheel, + if not python_configurations: + return + + with runner.fatal_on_called_process_error(): + runner.run_before_all(options, python_configurations) + runner.run_builds( + [ + WindowsBuilder( + config=config, + build_options=options.build_options(config.identifier), + tmp_dir=tmp_path / config.identifier, + session_tmp_dir=tmp_path, ) - shell(test_command_prepared, cwd=test_cwd, env=virtualenv_env) - - # we're all done here; move it to output (remove if already exists) - output_wheel = None - if compatible_wheel is None: - output_wheel = build_options.output_dir.joinpath(repaired_wheel.name) - moved_wheel = move_file(repaired_wheel, output_wheel) - if moved_wheel != output_wheel.resolve(): - log.warning( - f"{repaired_wheel} was moved to {moved_wheel} instead of {output_wheel}" - ) - built_wheels.append(output_wheel) - - # clean up - # (we ignore errors because occasionally Windows fails to unlink a file and we - # don't want to abort a build because of that) - shutil.rmtree(identifier_tmp_dir, ignore_errors=True) - - log.build_end(output_wheel) - except subprocess.CalledProcessError as error: - msg = f"Command {error.cmd} failed with code {error.returncode}. {error.stdout or ''}" - raise errors.FatalError(msg) from error + for config in python_configurations + ] + ) From a349ee4a2eda5b713e3e69a79143fd6e9c6eb48d Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 9 Jun 2026 17:21:17 -0400 Subject: [PATCH 5/7] refactor: port ios to the shared build runner The per-wheel loop becomes an IOSBuilder; the testbed-based test flow, the non-simulator/non-native skip steps, and the python-module test command validation stay inside test_wheel(). Behavior changes: - a test-suite failure now raises FatalError instead of calling sys.exit(1) directly (same exit code, consistent error reporting) - a repair command producing several wheels is now an error, as on Linux - the compatible-wheel reuse message now matches the other platforms - the per-identifier temp dir is removed with ignore_errors Assisted-by: ClaudeCode:claude-fable-5 --- cibuildwheel/platforms/ios.py | 583 ++++++++++++++++------------------ 1 file changed, 265 insertions(+), 318 deletions(-) diff --git a/cibuildwheel/platforms/ios.py b/cibuildwheel/platforms/ios.py index 751f3616e..a12f54eed 100644 --- a/cibuildwheel/platforms/ios.py +++ b/cibuildwheel/platforms/ios.py @@ -15,13 +15,13 @@ from packaging.version import Version from cibuildwheel import errors -from cibuildwheel.audit import run_audit from cibuildwheel.frontend import ( BuildFrontendName, get_build_frontend_extra_flags, prepare_config_settings, ) from cibuildwheel.logger import log +from cibuildwheel.platforms import runner from cibuildwheel.platforms.macos import install_cpython as install_build_cpython from cibuildwheel.util import resources from cibuildwheel.util.cmd import call, shell, split_command @@ -29,11 +29,9 @@ CIBW_CACHE_PATH, copy_test_sources, download, - move_file, remove_on_error, ) from cibuildwheel.util.helpers import prepare_command, unwrap_preserving_paragraphs -from cibuildwheel.util.packaging import find_compatible_wheel from cibuildwheel.venv import constraint_flags, virtualenv TYPE_CHECKING = False @@ -42,7 +40,7 @@ from cibuildwheel.architecture import Architecture from cibuildwheel.environment import ParsedEnvironment - from cibuildwheel.options import Options + from cibuildwheel.options import BuildOptions, Options from cibuildwheel.selector import BuildSelector @@ -438,339 +436,288 @@ def setup_python( return target_install_path, env -def build(options: Options, tmp_path: Path) -> None: - if sys.platform != "darwin": - msg = "iOS binaries can only be built on macOS" - raise errors.FatalError(msg) +class IOSBuilder(runner.HostBuilder): + def __init__( + self, + *, + config: PythonConfiguration, + build_options: BuildOptions, + tmp_dir: Path, + session_tmp_dir: Path, + ) -> None: + super().__init__( + identifier=config.identifier, + build_options=build_options, + tmp_dir=tmp_dir, + session_tmp_dir=session_tmp_dir, + ) + self.config = config - python_configurations = get_python_configurations( - build_selector=options.globals.build_selector, - architectures=options.globals.architectures, - ) + def setup(self) -> None: + build_options = self.build_options + build_frontend = build_options.build_frontend - if not python_configurations: - return + # uv doesn't support iOS + # Not using set because mypy can't narrow it + if build_frontend.name == "build[uv]" or build_frontend.name == "uv": # noqa: PLR1714 + msg = "uv doesn't support iOS" + raise errors.FatalError(msg) - try: - before_all_options_identifier = python_configurations[0].identifier - before_all_options = options.build_options(before_all_options_identifier) + self.tmp_dir.mkdir() - if before_all_options.before_all: - log.step("Running before_all...") - env = before_all_options.environment.as_dictionary(prev_environment=os.environ) - env.setdefault("IPHONEOS_DEPLOYMENT_TARGET", "13.0") - before_all_prepared = prepare_command( - before_all_options.before_all, - project=".", - package=before_all_options.package_dir, - ) - shell(before_all_prepared, env=env) + constraints_path = build_options.dependency_constraints.get_for_python_version( + version=self.config.version, tmp_dir=self.tmp_dir + ) + + self.target_install_path, env = setup_python( + self.tmp_dir / "build", + python_configuration=self.config, + dependency_constraint=constraints_path, + environment=build_options.environment, + build_frontend=build_frontend.name, + xbuild_tools=build_options.xbuild_tools, + ) + env["CIBUILDWHEEL_BUILD_IDENTIFIER"] = self.identifier + self.env = env + + def build_wheel(self) -> Path: + build_options = self.build_options + build_frontend = build_options.build_frontend + + self.built_wheel_dir.mkdir() - built_wheels: list[Path] = [] + extra_flags = get_build_frontend_extra_flags( + build_frontend, + build_options.build_verbosity, + prepare_config_settings( + build_options.config_settings, + project=".", + package=build_options.package_dir, + ), + ) - for config in python_configurations: - build_options = options.build_options(config.identifier) - build_frontend = build_options.build_frontend - # uv doesn't support iOS - # Not using set because mypy can't narrow it - if build_frontend.name == "build[uv]" or build_frontend.name == "uv": # noqa: PLR1714 - msg = "uv doesn't support iOS" + match build_frontend.name: + case "pip": + # Path.resolve() is needed. Without it pip wheel may try to + # fetch package from pypi.org. See + # https://github.com/pypa/cibuildwheel/pull/369 + call( + "python", + "-m", + "pip", + "wheel", + build_options.package_dir.resolve(), + f"--wheel-dir={self.built_wheel_dir}", + "--no-deps", + *extra_flags, + env=self.env, + ) + case "build": + call( + "python", + "-m", + "build", + build_options.package_dir, + "--wheel", + f"--outdir={self.built_wheel_dir}", + *extra_flags, + env=self.env, + ) + case _: + # the uv frontends were rejected in setup() + msg = f"Unsupported build frontend {build_frontend.name!r} for iOS" raise errors.FatalError(msg) - log.build_start(config.identifier) + return next(self.built_wheel_dir.glob("*.whl")) + + def test_wheel(self, repaired_wheel: Path) -> None: + build_options = self.build_options + assert build_options.test_command is not None - identifier_tmp_dir = tmp_path / config.identifier - identifier_tmp_dir.mkdir() - built_wheel_dir = identifier_tmp_dir / "built_wheel" - repaired_wheel_dir = identifier_tmp_dir / "repaired_wheel" + if not self.config.is_simulator: + log.step("Skipping tests on non-simulator SDK") + return + if self.config.arch != os.uname().machine: + log.step("Skipping tests on non-native simulator architecture") + return - constraints_path = build_options.dependency_constraints.get_for_python_version( - version=config.version, tmp_dir=identifier_tmp_dir + test_env = build_options.test_environment.as_dictionary(prev_environment=self.env) + + if build_options.before_test: + before_test_prepared = prepare_command( + build_options.before_test, + project=".", + package=build_options.package_dir, ) + shell(before_test_prepared, env=test_env) + + log.step("Setting up test harness...") + # Clone the testbed project into the build directory + testbed_path = self.tmp_dir / "testbed" + call( + "python", + self.target_install_path / "testbed", + "clone", + testbed_path, + env=test_env, + ) + + testbed_app_path = testbed_path / "iOSTestbed" / "app" - target_install_path, env = setup_python( - identifier_tmp_dir / "build", - python_configuration=config, - dependency_constraint=constraints_path, - environment=build_options.environment, - build_frontend=build_frontend.name, - xbuild_tools=build_options.xbuild_tools, + # Copy the test sources to the testbed app + if build_options.test_sources: + copy_test_sources( + build_options.test_sources, + Path.cwd(), + testbed_app_path, ) - env["CIBUILDWHEEL_BUILD_IDENTIFIER"] = config.identifier - - compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) - if compatible_wheel: - log.step_end() - print( - f"\nFound previously built wheel {compatible_wheel.name} " - f"that is compatible with {config.identifier}. " - "Skipping build step..." - ) - test_wheel = compatible_wheel - else: - if build_options.before_build: - log.step("Running before_build...") - before_build_prepared = prepare_command( - build_options.before_build, - project=".", - package=build_options.package_dir, - ) - shell(before_build_prepared, env=env) - - log.step("Building wheel...") - built_wheel_dir.mkdir() - - extra_flags = get_build_frontend_extra_flags( - build_frontend, - build_options.build_verbosity, - prepare_config_settings( - build_options.config_settings, - project=".", - package=build_options.package_dir, - ), - ) + else: + (testbed_app_path / "test_fail.py").write_text(resources.TEST_FAIL_CWD_FILE.read_text()) - match build_frontend.name: - case "pip": - # Path.resolve() is needed. Without it pip wheel may try to - # fetch package from pypi.org. See - # https://github.com/pypa/cibuildwheel/pull/369 - call( - "python", - "-m", - "pip", - "wheel", - build_options.package_dir.resolve(), - f"--wheel-dir={built_wheel_dir}", - "--no-deps", - *extra_flags, - env=env, - ) - case "build": - call( - "python", - "-m", - "build", - build_options.package_dir, - "--wheel", - f"--outdir={built_wheel_dir}", - *extra_flags, - env=env, - ) - case _: - assert_never(build_frontend) - - built_wheel = next(built_wheel_dir.glob("*.whl")) - - if built_wheel.name.endswith("none-any.whl"): - raise errors.NonPlatformWheelError() - - repaired_wheel_dir.mkdir() - if build_options.repair_command: - log.step("Repairing wheel...") - - repair_command_prepared = prepare_command( - build_options.repair_command, - wheel=built_wheel, - dest_dir=repaired_wheel_dir, - package=build_options.package_dir, - project=".", - ) - shell(repair_command_prepared, env=env) - else: - shutil.move(str(built_wheel), repaired_wheel_dir) - - try: - repaired_wheel = next(repaired_wheel_dir.glob("*.whl")) - except StopIteration: - raise errors.RepairStepProducedNoWheelError() from None - - if repaired_wheel.name in {wheel.name for wheel in built_wheels}: - raise errors.AlreadyBuiltWheelError(repaired_wheel.name) - - log.step_end() - - run_audit(tmp_dir=tmp_path, build_options=build_options, wheel=repaired_wheel) - - test_wheel = repaired_wheel - - if build_options.test_command and build_options.test_selector(config.identifier): - if not config.is_simulator: - log.step("Skipping tests on non-simulator SDK") - elif config.arch != os.uname().machine: - log.step("Skipping tests on non-native simulator architecture") - else: - test_env = build_options.test_environment.as_dictionary(prev_environment=env) - - if build_options.before_test: - before_test_prepared = prepare_command( - build_options.before_test, - project=".", - package=build_options.package_dir, - ) - shell(before_test_prepared, env=test_env) - - log.step("Setting up test harness...") - # Clone the testbed project into the build directory - testbed_path = identifier_tmp_dir / "testbed" - call( - "python", - target_install_path / "testbed", - "clone", - testbed_path, - env=test_env, - ) - - testbed_app_path = testbed_path / "iOSTestbed" / "app" - - # Copy the test sources to the testbed app - if build_options.test_sources: - copy_test_sources( - build_options.test_sources, - Path.cwd(), - testbed_app_path, - ) - else: - (testbed_app_path / "test_fail.py").write_text( - resources.TEST_FAIL_CWD_FILE.read_text() - ) + log.step("Installing test requirements...") + # Install the compiled wheel (with any test extras), plus + # the test requirements. Use the --platform tag to force + # the installation of iOS wheels; this requires the use of + # --only-binary=:all: + ios_version = test_env["IPHONEOS_DEPLOYMENT_TARGET"] + platform_tag = f"ios_{ios_version.replace('.', '_')}_{self.config.arch}_{self.config.sdk}" - log.step("Installing test requirements...") - # Install the compiled wheel (with any test extras), plus - # the test requirements. Use the --platform tag to force - # the installation of iOS wheels; this requires the use of - # --only-binary=:all: - ios_version = test_env["IPHONEOS_DEPLOYMENT_TARGET"] - platform_tag = f"ios_{ios_version.replace('.', '_')}_{config.arch}_{config.sdk}" - - call( - "python", - "-m", - "pip", - "install", - "--only-binary=:all:", - "--platform", - platform_tag, - "--target", - testbed_path / "iOSTestbed" / "app_packages", - f"{test_wheel}{build_options.test_extras}", - *build_options.test_requires, - env=test_env, - ) - - log.step("Running test suite...") - - # iOS doesn't support placeholders in the test command, - # because the source dir isn't visible on the simulator. - if ( - "{project}" in build_options.test_command - or "{package}" in build_options.test_command - ): - msg = unwrap_preserving_paragraphs( - f""" - iOS tests configured with a test command that uses the "{{project}}" or - "{{package}}" placeholder. iOS tests cannot use placeholders, because the - source directory is not visible on the simulator. + call( + "python", + "-m", + "pip", + "install", + "--only-binary=:all:", + "--platform", + platform_tag, + "--target", + testbed_path / "iOSTestbed" / "app_packages", + f"{repaired_wheel}{build_options.test_extras}", + *build_options.test_requires, + env=test_env, + ) + + log.step("Running test suite...") + + # iOS doesn't support placeholders in the test command, + # because the source dir isn't visible on the simulator. + if "{project}" in build_options.test_command or "{package}" in build_options.test_command: + msg = unwrap_preserving_paragraphs( + f""" + iOS tests configured with a test command that uses the "{{project}}" or + "{{package}}" placeholder. iOS tests cannot use placeholders, because the + source directory is not visible on the simulator. - In addition, iOS tests must run as a Python module, so the test command - must begin with 'python -m'. + In addition, iOS tests must run as a Python module, so the test command + must begin with 'python -m'. + + Test command: {build_options.test_command!r} + """ + ) + raise errors.FatalError(msg) + + test_command_list = shlex.split(build_options.test_command) + try: + for test_command_parts in split_command(test_command_list): + match test_command_parts: + case ["python", "-m", *rest]: + final_command = rest + case ["pytest", *rest]: + # pytest works exactly the same as a module, so we + # can just run it as a module. + msg = unwrap_preserving_paragraphs(f""" + iOS tests configured with a test command which doesn't start + with 'python -m'. iOS tests must execute python modules - other + entrypoints are not supported. + + cibuildwheel will try to execute it as if it started with + 'python -m'. If this works, all you need to do is add that to + your test command. Test command: {build_options.test_command!r} + """) + log.warning(msg) + final_command = ["pytest", *rest] + case _: + msg = unwrap_preserving_paragraphs( + f""" + iOS tests configured with a test command which doesn't start + with 'python -m'. iOS tests must execute python modules - other + entrypoints are not supported. + + Test command: {build_options.test_command!r} """ ) raise errors.FatalError(msg) - test_command_list = shlex.split(build_options.test_command) - try: - for test_command_parts in split_command(test_command_list): - match test_command_parts: - case ["python", "-m", *rest]: - final_command = rest - case ["pytest", *rest]: - # pytest works exactly the same as a module, so we - # can just run it as a module. - msg = unwrap_preserving_paragraphs(f""" - iOS tests configured with a test command which doesn't start - with 'python -m'. iOS tests must execute python modules - other - entrypoints are not supported. - - cibuildwheel will try to execute it as if it started with - 'python -m'. If this works, all you need to do is add that to - your test command. - - Test command: {build_options.test_command!r} - """) - log.warning(msg) - final_command = ["pytest", *rest] - case _: - msg = unwrap_preserving_paragraphs( - f""" - iOS tests configured with a test command which doesn't start - with 'python -m'. iOS tests must execute python modules - other - entrypoints are not supported. - - Test command: {build_options.test_command!r} - """ - ) - raise errors.FatalError(msg) - - test_runtime_args = build_options.test_runtime.args - - # 2025-10: The GitHub Actions macos-15 runner has a known issue where - # the default simulator won't start due to a disk performance issue; - # see https://github.com/actions/runner-images/issues/12777 for details. - # In the meantime, if it looks like we're running on a GitHub Actions - # macos-15 runner, use a simulator that is known to work, unless the - # user explicitly specifies a simulator. - os_version, _, arch = platform.mac_ver() - if ( - "GITHUB_ACTIONS" in os.environ - and os_version.startswith("15.") - and arch == "arm64" - and not any( - arg.startswith("--simulator") for arg in test_runtime_args - ) - ): - test_runtime_args = [ - "--simulator", - "iPhone 16e,OS=18.5", - *test_runtime_args, - ] - - call( - "python", - testbed_path, - "run", - *(["--verbose"] if build_options.build_verbosity > 0 else []), - *test_runtime_args, - "--", - *final_command, - env=test_env, - ) - except subprocess.CalledProcessError: - # catches the first test command failure in the loop, - # implementing short-circuiting - log.step_end(success=False) - log.error(f"Test suite failed on {config.identifier}") - sys.exit(1) - - log.step_end() - - # We're all done here; move it to output (overwrite existing) - output_wheel: Path | None = None - if compatible_wheel is None: - output_wheel = build_options.output_dir.joinpath(repaired_wheel.name) - moved_wheel = move_file(repaired_wheel, output_wheel) - if moved_wheel != output_wheel.resolve(): - log.warning( - f"{repaired_wheel} was moved to {moved_wheel} instead of {output_wheel}" - ) - built_wheels.append(output_wheel) - - # Clean up - shutil.rmtree(identifier_tmp_dir) - - log.build_end(output_wheel) - except subprocess.CalledProcessError as error: - msg = f"Command {error.cmd} failed with code {error.returncode}. {error.stdout or ''}" - raise errors.FatalError(msg) from error + test_runtime_args = build_options.test_runtime.args + + # 2025-10: The GitHub Actions macos-15 runner has a known issue where + # the default simulator won't start due to a disk performance issue; + # see https://github.com/actions/runner-images/issues/12777 for details. + # In the meantime, if it looks like we're running on a GitHub Actions + # macos-15 runner, use a simulator that is known to work, unless the + # user explicitly specifies a simulator. + os_version, _, arch = platform.mac_ver() + if ( + "GITHUB_ACTIONS" in os.environ + and os_version.startswith("15.") + and arch == "arm64" + and not any(arg.startswith("--simulator") for arg in test_runtime_args) + ): + test_runtime_args = [ + "--simulator", + "iPhone 16e,OS=18.5", + *test_runtime_args, + ] + + call( + "python", + testbed_path, + "run", + *(["--verbose"] if build_options.build_verbosity > 0 else []), + *test_runtime_args, + "--", + *final_command, + env=test_env, + ) + except subprocess.CalledProcessError: + # catches the first test command failure in the loop, + # implementing short-circuiting + log.step_end(success=False) + msg = f"Test suite failed on {self.identifier}" + raise errors.FatalError(msg) from None + + log.step_end() + + +def build(options: Options, tmp_path: Path) -> None: + if sys.platform != "darwin": + msg = "iOS binaries can only be built on macOS" + raise errors.FatalError(msg) + + python_configurations = get_python_configurations( + build_selector=options.globals.build_selector, + architectures=options.globals.architectures, + ) + + if not python_configurations: + return + + with runner.fatal_on_called_process_error(): + runner.run_before_all( + options, + python_configurations, + env_defaults={"IPHONEOS_DEPLOYMENT_TARGET": "13.0"}, + ) + runner.run_builds( + [ + IOSBuilder( + config=config, + build_options=options.build_options(config.identifier), + tmp_dir=tmp_path / config.identifier, + session_tmp_dir=tmp_path, + ) + for config in python_configurations + ] + ) From c7723a404aa0b4cbf792a72606390a8db7e55047 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 9 Jun 2026 17:24:32 -0400 Subject: [PATCH 6/7] refactor: port android to the shared build runner The BuildState dataclass and its step functions fold into an AndroidBuilder; setup_target_python/setup_env and the other environment helpers are unchanged. Behavior changes: - test-environment (CIBW_TEST_ENVIRONMENT_ANDROID) is now applied to the test steps (before-test, wheel install, testbed invocation); it was documented but previously ignored on Android - a wheel name colliding with an earlier build now raises AlreadyBuiltWheelError, like every other platform - the redundant post-repair none-any.whl check is dropped (the built wheel is still checked, like every other platform) - the compatible-wheel reuse message now matches the other platforms, and the moved-elsewhere warning from move_file is now emitted - a missing uv now raises FatalError instead of AssertionError Assisted-by: ClaudeCode:claude-fable-5 --- cibuildwheel/platforms/android.py | 562 ++++++++++++++---------------- 1 file changed, 256 insertions(+), 306 deletions(-) diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index 52773e058..fb92b23fb 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -5,7 +5,6 @@ import re import shlex import shutil -import subprocess import sys from dataclasses import dataclass from pathlib import Path @@ -21,24 +20,21 @@ from cibuildwheel import errors, platforms # pylint: disable=cyclic-import from cibuildwheel.architecture import Architecture, arch_synonym -from cibuildwheel.audit import run_audit from cibuildwheel.frontend import ( get_build_frontend_extra_flags, parse_config_settings, prepare_config_settings, ) from cibuildwheel.logger import log +from cibuildwheel.platforms import runner from cibuildwheel.util import resources from cibuildwheel.util.cmd import call, shell from cibuildwheel.util.file import ( CIBW_CACHE_PATH, - copy_test_sources, download, - move_file, remove_on_error, ) from cibuildwheel.util.helpers import prepare_command -from cibuildwheel.util.packaging import find_compatible_wheel from cibuildwheel.util.python_build_standalone import create_python_build_standalone_environment from cibuildwheel.venv import constraint_flags, find_uv, virtualenv @@ -101,88 +97,6 @@ def shell_prepared(command: str, *, build_options: BuildOptions, env: dict[str, ) -def before_all(options: Options, python_configurations: list[PythonConfiguration]) -> None: - before_all_options = options.build_options(python_configurations[0].identifier) - if before_all_options.before_all: - log.step("Running before_all...") - shell_prepared( - before_all_options.before_all, - build_options=before_all_options, - env=before_all_options.environment.as_dictionary(os.environ), - ) - - -@dataclass(frozen=True) -class BuildState: - config: PythonConfiguration - options: BuildOptions - build_path: Path - python_dir: Path - build_env: dict[str, str] - android_env: dict[str, str] - - -def build(options: Options, tmp_path: Path) -> None: - if "ANDROID_HOME" not in os.environ: - msg = ( - "ANDROID_HOME environment variable is not set. For instructions, see " - "https://cibuildwheel.pypa.io/en/stable/platforms/#android" - ) - raise errors.FatalError(msg) - - configs = get_python_configurations( - options.globals.build_selector, options.globals.architectures - ) - if not configs: - return - - try: - before_all(options, configs) - - built_wheels: list[Path] = [] - for config in configs: - log.build_start(config.identifier) - build_options = options.build_options(config.identifier) - build_path = tmp_path / config.identifier - build_path.mkdir() - python_dir = setup_target_python(config, build_path) - build_env, android_env = setup_env(config, build_options, build_path, python_dir) - - state = BuildState( - config, build_options, build_path, python_dir, build_env, android_env - ) - setup_xbuild_files(state) - - compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) - if compatible_wheel: - print( - f"\nFound previously built wheel {compatible_wheel.name} that is " - f"compatible with {config.identifier}. Skipping build step..." - ) - repaired_wheel = compatible_wheel - else: - before_build(state) - built_wheel = build_wheel(state) - repaired_wheel = repair_wheel(state, built_wheel) - run_audit(tmp_dir=tmp_path, build_options=build_options, wheel=repaired_wheel) - - test_wheel(state, repaired_wheel) - - output_wheel: Path | None = None - if compatible_wheel is None: - output_wheel = move_file( - repaired_wheel, build_options.output_dir / repaired_wheel.name - ) - built_wheels.append(output_wheel) - - shutil.rmtree(build_path) - log.build_end(output_wheel) - - except subprocess.CalledProcessError as error: - msg = f"Command {error.cmd} failed with code {error.returncode}. {error.stdout or ''}" - raise errors.FatalError(msg) from error - - def setup_target_python(config: PythonConfiguration, build_path: Path) -> Path: log.step("Installing target Python...") python_tgz = CIBW_CACHE_PATH / config.url.rpartition("/")[-1] @@ -486,41 +400,6 @@ def setup_fortran(env: dict[str, str]) -> None: env["FC"] = str(shim_out) -def setup_xbuild_files(state: BuildState) -> None: - _, pip = find_pip(state.options) - xbf_dir = state.build_path / "xbuild_files" - xbf_dir.mkdir() - - for requirement in call(*pip, "freeze", env=state.build_env, capture_stdout=True).splitlines(): - name, _, _ = requirement.strip().partition("==") - xbuild_files = state.options.xbuild_files.get(canonicalize_name(name), []) - if xbuild_files: - log.step(f"Installing xbuild-files for {name}...") - pip_install_android(state, xbf_dir, "--no-deps", requirement) - for xbf in xbuild_files: - if (xbf_dir / xbf).exists(): - shutil.copy( - xbf_dir / xbf, - find_site_packages(state.build_env) / xbf, - ) - else: - log.warning(f"{xbf_dir / xbf} does not exist") - - -def pip_install_android(state: BuildState, target: Path, *args: PathOrStr) -> None: - use_uv, pip = find_pip(state.options) - call( - *pip, - "install", - "--only-binary=:all:", - *(["--python-platform", android_triplet(state.config.identifier)] if use_uv else []), - "--target", - target, - *args, - env=state.android_env, - ) - - def find_site_packages(env: dict[str, str]) -> Path: return glob1(Path(env["VIRTUAL_ENV"]), "lib/python*/site-packages") @@ -538,216 +417,287 @@ def find_pip(build_options: BuildOptions) -> tuple[bool, list[str]]: uv_path = find_uv() if use_uv and uv_path is None: msg = "uv not found" - raise AssertionError(msg) + raise errors.FatalError(msg) pip = ["pip"] if not use_uv else [str(uv_path), "pip"] return use_uv, pip -def before_build(state: BuildState) -> None: - if state.options.before_build: - log.step("Running before_build...") - shell_prepared( - state.options.before_build, - build_options=state.options, - env=state.android_env, +class AndroidBuilder(runner.HostBuilder): + def __init__( + self, + *, + config: PythonConfiguration, + build_options: BuildOptions, + tmp_dir: Path, + session_tmp_dir: Path, + ) -> None: + super().__init__( + identifier=config.identifier, + build_options=build_options, + tmp_dir=tmp_dir, + session_tmp_dir=session_tmp_dir, ) + self.config = config + def setup(self) -> None: + self.tmp_dir.mkdir() + self.python_dir = setup_target_python(self.config, self.tmp_dir) + self.build_env, self.android_env = setup_env( + self.config, self.build_options, self.tmp_dir, self.python_dir + ) + self.setup_xbuild_files() + + def setup_xbuild_files(self) -> None: + _, pip = find_pip(self.build_options) + xbf_dir = self.tmp_dir / "xbuild_files" + xbf_dir.mkdir() + + for requirement in call( + *pip, "freeze", env=self.build_env, capture_stdout=True + ).splitlines(): + name, _, _ = requirement.strip().partition("==") + xbuild_files = self.build_options.xbuild_files.get(canonicalize_name(name), []) + if xbuild_files: + log.step(f"Installing xbuild-files for {name}...") + self.pip_install_android(self.android_env, xbf_dir, "--no-deps", requirement) + for xbf in xbuild_files: + if (xbf_dir / xbf).exists(): + shutil.copy( + xbf_dir / xbf, + find_site_packages(self.build_env) / xbf, + ) + else: + log.warning(f"{xbf_dir / xbf} does not exist") + + def pip_install_android(self, env: dict[str, str], target: Path, *args: PathOrStr) -> None: + use_uv, pip = find_pip(self.build_options) + call( + *pip, + "install", + "--only-binary=:all:", + *(["--python-platform", android_triplet(self.identifier)] if use_uv else []), + "--target", + target, + *args, + env=env, + ) -def build_wheel(state: BuildState) -> Path: - log.step("Building wheel...") - built_wheel_dir = state.build_path / "built_wheel" - match state.options.build_frontend.name: - case "build" | "build[uv]": - call( - "python", - "-m", - "build", - state.options.package_dir, - "--wheel", - "--no-isolation", - "--skip-dependency-check", - f"--outdir={built_wheel_dir}", - *get_build_frontend_extra_flags( - state.options.build_frontend, - state.options.build_verbosity, - prepare_config_settings( - state.options.config_settings, - project=".", - package=state.options.package_dir, + def before_build(self) -> None: + assert self.build_options.before_build is not None + shell_prepared( + self.build_options.before_build, + build_options=self.build_options, + env=self.android_env, + ) + + def build_wheel(self) -> Path: + build_options = self.build_options + match build_options.build_frontend.name: + case "build" | "build[uv]": + call( + "python", + "-m", + "build", + build_options.package_dir, + "--wheel", + "--no-isolation", + "--skip-dependency-check", + f"--outdir={self.built_wheel_dir}", + *get_build_frontend_extra_flags( + build_options.build_frontend, + build_options.build_verbosity, + prepare_config_settings( + build_options.config_settings, + project=".", + package=build_options.package_dir, + ), ), - ), - env=state.android_env, - ) - case "uv": - uv_path = find_uv() - assert uv_path is not None - call( - uv_path, - "build", - state.options.package_dir, - "--wheel", - "--no-build-isolation", - f"--out-dir={built_wheel_dir}", - *get_build_frontend_extra_flags( - state.options.build_frontend, - state.options.build_verbosity, - prepare_config_settings( - state.options.config_settings, - project=".", - package=state.options.package_dir, + env=self.android_env, + ) + case "uv": + uv_path = find_uv() + assert uv_path is not None + call( + uv_path, + "build", + build_options.package_dir, + "--wheel", + "--no-build-isolation", + f"--out-dir={self.built_wheel_dir}", + *get_build_frontend_extra_flags( + build_options.build_frontend, + build_options.build_verbosity, + prepare_config_settings( + build_options.config_settings, + project=".", + package=build_options.package_dir, + ), ), + env=self.android_env, + ) + case x: + msg = f"Android requires the build frontend to be 'build' or 'uv', not {x!r}" + raise errors.FatalError(msg) + + return glob1(self.built_wheel_dir, "*.whl") + + def repair_wheel(self, built_wheel: Path) -> list[Path]: + self.repaired_wheel_dir.mkdir() + + if self.build_options.repair_command: + # Tell auditwheel the locations of compiler libraries. + toolchain = Path(self.android_env["CC"]).parent.parent + triplet = android_triplet(self.identifier) + ldpaths = ":".join( + str(glob1(toolchain, pattern)) + for pattern in [ + f"lib/clang/*/lib/linux/{triplet.split('-')[0]}", # libomp + f"sysroot/usr/lib/{triplet}", # libc++_shared + ] + ) + shell( + prepare_command( + self.build_options.repair_command, + ldpaths=ldpaths, + wheel=built_wheel, + dest_dir=self.repaired_wheel_dir, + package=self.build_options.package_dir, + project=".", ), - env=state.android_env, + env=self.build_env, ) - case x: - msg = f"Android requires the build frontend to be 'build' or 'uv', not {x!r}" - raise errors.FatalError(msg) + else: + shutil.move(built_wheel, self.repaired_wheel_dir) - built_wheel = glob1(built_wheel_dir, "*.whl") - if built_wheel.name.endswith("none-any.whl"): - raise errors.NonPlatformWheelError() - return built_wheel - - -def repair_wheel(state: BuildState, built_wheel: Path) -> Path: - log.step("Repairing wheel...") - repaired_wheel_dir = state.build_path / "repaired_wheel" - repaired_wheel_dir.mkdir() - - if state.options.repair_command: - # Tell auditwheel the locations of compiler libraries. - toolchain = Path(state.android_env["CC"]).parent.parent - triplet = android_triplet(state.config.identifier) - ldpaths = ":".join( - str(glob1(toolchain, pattern)) - for pattern in [ - f"lib/clang/*/lib/linux/{triplet.split('-')[0]}", # libomp - f"sysroot/usr/lib/{triplet}", # libc++_shared - ] - ) - shell( - prepare_command( - state.options.repair_command, - ldpaths=ldpaths, - wheel=built_wheel, - dest_dir=repaired_wheel_dir, - package=state.options.package_dir, - project=".", - ), - env=state.build_env, + return list(self.repaired_wheel_dir.glob("*.whl")) + + def test_wheel(self, repaired_wheel: Path) -> None: + build_options = self.build_options + test_command = build_options.test_command + assert test_command is not None + + log.step("Testing wheel...") + native_arch = arch_synonym(platform.machine(), platforms.native_platform(), "android") + if self.config.arch != native_arch: + log.warning( + f"Skipping tests for {self.config.arch}, as the build machine only " + f"supports {native_arch}" + ) + return + + # The test steps run in the environments below; the testbed doesn't + # automatically propagate arbitrary variables into the emulator process. + android_test_env = build_options.test_environment.as_dictionary( + prev_environment=self.android_env ) - else: - shutil.move(built_wheel, repaired_wheel_dir) - - repaired_wheels = list(repaired_wheel_dir.glob("*.whl")) - if len(repaired_wheels) == 0: - raise errors.RepairStepProducedNoWheelError() - if len(repaired_wheels) != 1: - raise errors.RepairStepProducedMultipleWheelsError( - [rw.name for rw in repaired_wheels], + build_test_env = build_options.test_environment.as_dictionary( + prev_environment=self.build_env ) - repaired_wheel = repaired_wheels[0] - if repaired_wheel.name.endswith("none-any.whl"): - raise errors.NonPlatformWheelError() - return repaired_wheel + if build_options.before_test: + shell_prepared( + build_options.before_test, + build_options=build_options, + env=android_test_env, + ) + # Install the wheel and test-requires. + site_packages_dir = self.tmp_dir / "site-packages" + site_packages_dir.mkdir() + self.pip_install_android( + android_test_env, + site_packages_dir, + f"{repaired_wheel}{build_options.test_extras}", + *build_options.test_requires, + ) -def test_wheel(state: BuildState, wheel: Path) -> None: - test_command = state.options.test_command - if not (test_command and state.options.test_selector(state.config.identifier)): - return + # Copy test-sources. + cwd_dir = self.tmp_dir / "cwd" + runner.prepare_test_cwd(cwd_dir, build_options.test_sources) - log.step("Testing wheel...") - native_arch = arch_synonym(platform.machine(), platforms.native_platform(), "android") - if state.config.arch != native_arch: - log.warning( - f"Skipping tests for {state.config.arch}, as the build machine only " - f"supports {native_arch}" - ) - return + # Android doesn't support placeholders in the test command. + if any(("{" + placeholder + "}") in test_command for placeholder in ("project", "package")): + msg = ( + f"Test command {test_command!r} with a " + "'{project}' or '{package}' placeholder is not supported on Android, " + "because the source directory is not visible on the emulator." + ) + raise errors.FatalError(msg) - if state.options.before_test: - shell_prepared( - state.options.before_test, - build_options=state.options, - env=state.android_env, - ) + # Parse test-command. + test_args = shlex.split(test_command) + if test_args[0] in {"python", "python3"} and any(arg in test_args for arg in ("-c", "-m")): + # Forward the args to the CPython testbed script. We require '-c' or '-m' + # to be in the command, because without those flags, the testbed script + # will prepend '-m test', which will run Python's own test suite. + del test_args[0] + elif test_args[0] == "pytest": + # We transform some commands into the `python -m` form, but this is deprecated. + msg = ( + f"Test command {test_command!r} is not supported on Android. " + "cibuildwheel will try to execute it as if it started with 'python -m'. " + "If this works, all you need to do is add that to your test command." + ) + log.warning(msg) + test_args.insert(0, "-m") + else: + msg = ( + f"Test command {test_command!r} is not supported on Android. " + f"Command must begin with 'python' or 'python3', and contain '-m' or '-c'." + ) + raise errors.FatalError(msg) - # Install the wheel and test-requires. - site_packages_dir = state.build_path / "site-packages" - site_packages_dir.mkdir() - pip_install_android( - state, - site_packages_dir, - f"{wheel}{state.options.test_extras}", - *state.options.test_requires, - ) + # By default, run on a testbed managed emulator running the newest supported + # Android version. However, if the user specifies a --managed or --connected + # test execution argument, that argument takes precedence. + test_runtime_args = build_options.test_runtime.args - # Copy test-sources. - cwd_dir = state.build_path / "cwd" - cwd_dir.mkdir() - if state.options.test_sources: - copy_test_sources(state.options.test_sources, Path.cwd(), cwd_dir) - else: - (cwd_dir / "test_fail.py").write_text( - resources.TEST_FAIL_CWD_FILE.read_text(), + if any(arg.startswith(("--managed", "--connected")) for arg in test_runtime_args): + emulator_args = [] + else: + emulator_args = ["--managed", "maxVersion"] + + # Run the test app. + call( + self.python_dir / "android.py", + "test", + "--site-packages", + site_packages_dir, + "--cwd", + cwd_dir, + *emulator_args, + *(["-v"] if build_options.build_verbosity > 0 else []), + *test_runtime_args, + "--", + *test_args, + env=build_test_env, ) - # Android doesn't support placeholders in the test command. - if any(("{" + placeholder + "}") in test_command for placeholder in ("project", "package")): - msg = ( - f"Test command {test_command!r} with a " - "'{project}' or '{package}' placeholder is not supported on Android, " - "because the source directory is not visible on the emulator." - ) - raise errors.FatalError(msg) - # Parse test-command. - test_args = shlex.split(test_command) - if test_args[0] in {"python", "python3"} and any(arg in test_args for arg in ("-c", "-m")): - # Forward the args to the CPython testbed script. We require '-c' or '-m' - # to be in the command, because without those flags, the testbed script - # will prepend '-m test', which will run Python's own test suite. - del test_args[0] - elif test_args[0] == "pytest": - # We transform some commands into the `python -m` form, but this is deprecated. - msg = ( - f"Test command {test_command!r} is not supported on Android. " - "cibuildwheel will try to execute it as if it started with 'python -m'. " - "If this works, all you need to do is add that to your test command." - ) - log.warning(msg) - test_args.insert(0, "-m") - else: +def build(options: Options, tmp_path: Path) -> None: + if "ANDROID_HOME" not in os.environ: msg = ( - f"Test command {test_command!r} is not supported on Android. " - f"Command must begin with 'python' or 'python3', and contain '-m' or '-c'." + "ANDROID_HOME environment variable is not set. For instructions, see " + "https://cibuildwheel.pypa.io/en/stable/platforms/#android" ) raise errors.FatalError(msg) - # By default, run on a testbed managed emulator running the newest supported - # Android version. However, if the user specifies a --managed or --connected - # test execution argument, that argument takes precedence. - test_runtime_args = state.options.test_runtime.args - - if any(arg.startswith(("--managed", "--connected")) for arg in test_runtime_args): - emulator_args = [] - else: - emulator_args = ["--managed", "maxVersion"] - - # Run the test app. - call( - state.python_dir / "android.py", - "test", - "--site-packages", - site_packages_dir, - "--cwd", - cwd_dir, - *emulator_args, - *(["-v"] if state.options.build_verbosity > 0 else []), - *test_runtime_args, - "--", - *test_args, - env=state.build_env, + configs = get_python_configurations( + options.globals.build_selector, options.globals.architectures ) + if not configs: + return + + with runner.fatal_on_called_process_error(): + runner.run_before_all(options, configs) + runner.run_builds( + [ + AndroidBuilder( + config=config, + build_options=options.build_options(config.identifier), + tmp_dir=tmp_path / config.identifier, + session_tmp_dir=tmp_path, + ) + for config in configs + ] + ) From 39fdbba981d806d979cce83cdb567725def4fd06 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 9 Jun 2026 17:28:00 -0400 Subject: [PATCH 7/7] refactor: port linux to the shared build runner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-config loop in build_in_container() becomes a LinuxBuilder (generic over PurePosixPath — wheels stay container-side, and the existing copy-out and host-side audit behavior is preserved as overridden steps). build() keeps the container grouping, engine check, and OCIContainer lifecycle; the CalledProcessError wrapper keeps its per-container scope and troubleshoot() hook via fatal_on_called_process_error. build_in_container() keeps its signature, since tests mock it. Also fixes a latent bug: the host-side per-identifier temp dir was never created, so dependency-versions with packages: would have crashed writing constraints.txt on Linux. Assisted-by: ClaudeCode:claude-fable-5 --- cibuildwheel/platforms/linux.py | 559 +++++++++++++++++--------------- 1 file changed, 297 insertions(+), 262 deletions(-) diff --git a/cibuildwheel/platforms/linux.py b/cibuildwheel/platforms/linux.py index f97f6a6a1..c9d654baf 100644 --- a/cibuildwheel/platforms/linux.py +++ b/cibuildwheel/platforms/linux.py @@ -2,6 +2,7 @@ import contextlib import dataclasses +import functools import shutil import subprocess import sys @@ -16,10 +17,10 @@ from cibuildwheel.frontend import get_build_frontend_extra_flags, prepare_config_settings from cibuildwheel.logger import log from cibuildwheel.oci_container import OCIContainer, OCIContainerEngineConfig, OCIPlatform +from cibuildwheel.platforms import runner from cibuildwheel.util import resources from cibuildwheel.util.file import copy_test_sources from cibuildwheel.util.helpers import prepare_command, unwrap -from cibuildwheel.util.packaging import find_compatible_wheel TYPE_CHECKING = False if TYPE_CHECKING: @@ -169,56 +170,47 @@ def check_all_python_exist( raise errors.FatalError(message) -def build_in_container( - *, - options: Options, - platform_configs: Sequence[PythonConfiguration], - container: OCIContainer, - container_project_path: PurePath, - container_package_dir: PurePath, - local_tmp_dir: Path, -) -> None: - container_output_dir = PurePosixPath("/output") - - check_all_python_exist(platform_configs=platform_configs, container=container) - - log.step("Copying project into container...") - container.copy_into(Path.cwd(), container_project_path) - - before_all_options_identifier = platform_configs[0].identifier - before_all_options = options.build_options(before_all_options_identifier) - - if before_all_options.before_all: - log.step("Running before_all...") - - env = container.get_environment() - env["PATH"] = f"/opt/python/cp39-cp39/bin:{env['PATH']}" - env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" - env["PIP_ROOT_USER_ACTION"] = "ignore" - env = before_all_options.environment.as_dictionary( - env, executor=container.environment_executor - ) - - before_all_prepared = prepare_command( - before_all_options.before_all, - project=container_project_path, - package=container_package_dir, - ) - container.call(["sh", "-c", before_all_prepared], env=env) - - built_wheels: list[PurePosixPath] = [] - - for config in platform_configs: - log.build_start(config.identifier) - local_identifier_tmp_dir = local_tmp_dir / config.identifier - build_options = options.build_options(config.identifier) - build_frontend = build_options.build_frontend - use_uv = build_frontend.name in {"build[uv]", "uv"} - pip = ["uv", "pip"] if use_uv else ["pip"] +class LinuxBuilder(runner.Builder[PurePosixPath]): + """A Builder that runs every step inside an OCI container. Wheel paths + are container-side paths; the wheels are copied back to the host once, + after every build in the container has finished.""" + + def __init__( + self, + *, + config: PythonConfiguration, + build_options: BuildOptions, + container: OCIContainer, + container_project_path: PurePath, + container_package_dir: PurePath, + local_tmp_dir: Path, + ) -> None: + self.identifier = config.identifier + self.build_options = build_options + self.config = config + self.container = container + self.container_project_path = container_project_path + self.container_package_dir = container_package_dir + # host-side scratch space, shared across identifiers + self.local_tmp_dir = local_tmp_dir + self.container_output_dir = PurePosixPath("/output") + container_tmp_dir = PurePosixPath("/tmp/cibuildwheel") + self.built_wheel_dir = container_tmp_dir / "built_wheel" + self.repaired_wheel_dir = container_tmp_dir / "repaired_wheel" + + def setup(self) -> None: + container = self.container + config = self.config + build_options = self.build_options + + self.use_uv = build_options.build_frontend.name in {"build[uv]", "uv"} + self.pip: list[str] = ["uv", "pip"] if self.use_uv else ["pip"] log.step("Setting up build environment...") - dependency_constraint_flags: list[PathOrStr] = [] + self.dependency_constraint_flags: list[PathOrStr] = [] + local_identifier_tmp_dir = self.local_tmp_dir / self.identifier + local_identifier_tmp_dir.mkdir(parents=True, exist_ok=True) local_constraints_file = build_options.dependency_constraints.get_for_python_version( version=config.version, tmp_dir=local_identifier_tmp_dir, @@ -226,261 +218,309 @@ def build_in_container( if local_constraints_file: container_constraints_file = PurePosixPath("/constraints.txt") container.copy_into(local_constraints_file, container_constraints_file) - dependency_constraint_flags = ["-c", container_constraints_file] + self.dependency_constraint_flags = ["-c", container_constraints_file] env = container.get_environment() env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" env["PIP_ROOT_USER_ACTION"] = "ignore" # put this config's python top of the list - python_bin = config.path / "bin" - env["PATH"] = f"{python_bin}:{env['PATH']}" + self.python_bin = config.path / "bin" + env["PATH"] = f"{self.python_bin}:{env['PATH']}" env = build_options.environment.as_dictionary(env, executor=container.environment_executor) - env["CIBUILDWHEEL_BUILD_IDENTIFIER"] = config.identifier + env["CIBUILDWHEEL_BUILD_IDENTIFIER"] = self.identifier # check config python is still on PATH which_python = container.call(["which", "python"], env=env, capture_output=True).strip() - if PurePosixPath(which_python) != python_bin / "python": + if PurePosixPath(which_python) != self.python_bin / "python": msg = "python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it." raise errors.FatalError(msg) container.call(["python", "-V", "-V"], env=env) - if use_uv: + if self.use_uv: which_uv = container.call(["which", "uv"], env=env, capture_output=True).strip() if not which_uv: msg = "uv not found on PATH. You must use a supported manylinux or musllinux environment with uv." raise errors.FatalError(msg) else: which_pip = container.call(["which", "pip"], env=env, capture_output=True).strip() - if PurePosixPath(which_pip) != python_bin / "pip": + if PurePosixPath(which_pip) != self.python_bin / "pip": msg = "pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it." raise errors.FatalError(msg) - compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) - if compatible_wheel: - log.step_end() - print( - f"\nFound previously built wheel {compatible_wheel.name}, that's compatible with {config.identifier}. Skipping build step..." - ) - repaired_wheel = compatible_wheel - else: - if build_options.before_build: - log.step("Running before_build...") - before_build_prepared = prepare_command( - build_options.before_build, - project=container_project_path, - package=container_package_dir, - ) - before_build_env = env.copy() - if use_uv: - # On Linux, no virtualenv is created for the build environment - # (unlike macOS/Windows, where one is set up before before_build - # runs). uv requires either an active venv or an explicit Python - # target to install packages. Pin UV_PYTHON to the exact interpreter - # for this build so that `uv pip install` works in before_build - # without requiring users to pass --system. - before_build_env["UV_PYTHON"] = str(python_bin / "python") - container.call(["sh", "-c", before_build_prepared], env=before_build_env) - - log.step("Building wheel...") - - temp_dir = PurePosixPath("/tmp/cibuildwheel") - built_wheel_dir = temp_dir / "built_wheel" - container.call(["rm", "-rf", built_wheel_dir]) - container.call(["mkdir", "-p", built_wheel_dir]) - - extra_flags = get_build_frontend_extra_flags( - build_frontend, - build_options.build_verbosity, - prepare_config_settings( - build_options.config_settings, - project=container_project_path, - package=container_package_dir, - ), - ) + self.env = env + + def before_build(self) -> None: + build_options = self.build_options + assert build_options.before_build is not None + + before_build_prepared = prepare_command( + build_options.before_build, + project=self.container_project_path, + package=self.container_package_dir, + ) + before_build_env = self.env.copy() + if self.use_uv: + # On Linux, no virtualenv is created for the build environment + # (unlike macOS/Windows, where one is set up before before_build + # runs). uv requires either an active venv or an explicit Python + # target to install packages. Pin UV_PYTHON to the exact interpreter + # for this build so that `uv pip install` works in before_build + # without requiring users to pass --system. + before_build_env["UV_PYTHON"] = str(self.python_bin / "python") + self.container.call(["sh", "-c", before_build_prepared], env=before_build_env) + + def build_wheel(self) -> PurePosixPath: + container = self.container + build_options = self.build_options + build_frontend = build_options.build_frontend + + container.call(["rm", "-rf", self.built_wheel_dir]) + container.call(["mkdir", "-p", self.built_wheel_dir]) + + extra_flags = get_build_frontend_extra_flags( + build_frontend, + build_options.build_verbosity, + prepare_config_settings( + build_options.config_settings, + project=self.container_project_path, + package=self.container_package_dir, + ), + ) - match build_frontend.name: - case "pip": - container.call( - [ - "python", - "-m", - "pip", - "wheel", - container_package_dir, - f"--wheel-dir={built_wheel_dir}", - "--no-deps", - *extra_flags, - ], - env=env, - ) - case "build" | "build[uv]": - if use_uv and "--no-isolation" not in extra_flags and "-n" not in extra_flags: - extra_flags += ["--installer=uv"] - container.call( - [ - "python", - "-m", - "build", - container_package_dir, - "--wheel", - f"--outdir={built_wheel_dir}", - *extra_flags, - ], - env=env, - ) - case "uv": - container.call( - [ - "uv", - "build", - f"--python={python_bin / 'python'}", - container_package_dir, - "--wheel", - f"--out-dir={built_wheel_dir}", - *extra_flags, - ], - env=env, - ) - case _: - assert_never(build_frontend) - - built_wheel = container.glob(built_wheel_dir, "*.whl")[0] - - repaired_wheel_dir = temp_dir / "repaired_wheel" - container.call(["rm", "-rf", repaired_wheel_dir]) - container.call(["mkdir", "-p", repaired_wheel_dir]) - - if built_wheel.name.endswith("none-any.whl"): - raise errors.NonPlatformWheelError() - - if build_options.repair_command: - log.step("Repairing wheel...") - repair_command_prepared = prepare_command( - build_options.repair_command, - wheel=built_wheel, - dest_dir=repaired_wheel_dir, - package=container_package_dir, - project=container_project_path, + match build_frontend.name: + case "pip": + container.call( + [ + "python", + "-m", + "pip", + "wheel", + self.container_package_dir, + f"--wheel-dir={self.built_wheel_dir}", + "--no-deps", + *extra_flags, + ], + env=self.env, + ) + case "build" | "build[uv]": + if self.use_uv and "--no-isolation" not in extra_flags and "-n" not in extra_flags: + extra_flags += ["--installer=uv"] + container.call( + [ + "python", + "-m", + "build", + self.container_package_dir, + "--wheel", + f"--outdir={self.built_wheel_dir}", + *extra_flags, + ], + env=self.env, ) - container.call(["sh", "-c", repair_command_prepared], env=env) - else: - container.call(["mv", built_wheel, repaired_wheel_dir]) - - match container.glob(repaired_wheel_dir, "*.whl"): - case []: - raise errors.RepairStepProducedNoWheelError() - case [repaired_wheel]: - pass - case too_many: - raise errors.RepairStepProducedMultipleWheelsError([p.name for p in too_many]) - - if repaired_wheel.name in {wheel.name for wheel in built_wheels}: - raise errors.AlreadyBuiltWheelError(repaired_wheel.name) - - log.step_end() - - if needs_audit(build_options.audit_command, repaired_wheel.name): - local_abi3audit_dir = local_identifier_tmp_dir / "audit" - local_abi3audit_dir.mkdir(parents=True, exist_ok=True) - try: - container.copy_out(repaired_wheel_dir, local_abi3audit_dir) - local_wheel = local_abi3audit_dir / repaired_wheel.name - run_audit(tmp_dir=local_tmp_dir, build_options=build_options, wheel=local_wheel) - finally: - shutil.rmtree(local_abi3audit_dir, ignore_errors=True) - - if build_options.test_command and build_options.test_selector(config.identifier): - log.step("Testing wheel...") - - # set up a virtual environment to install and test from, to make sure - # there are no dependencies that were pulled in at build time. - if not use_uv: + case "uv": container.call( - ["pip", "install", "virtualenv", *dependency_constraint_flags], env=env + [ + "uv", + "build", + f"--python={self.python_bin / 'python'}", + self.container_package_dir, + "--wheel", + f"--out-dir={self.built_wheel_dir}", + *extra_flags, + ], + env=self.env, ) + case _: + assert_never(build_frontend) + + return container.glob(self.built_wheel_dir, "*.whl")[0] - testing_temp_dir = PurePosixPath( - container.call(["mktemp", "-d"], capture_output=True).strip() + def repair_wheel(self, built_wheel: PurePosixPath) -> list[PurePosixPath]: + container = self.container + build_options = self.build_options + + container.call(["rm", "-rf", self.repaired_wheel_dir]) + container.call(["mkdir", "-p", self.repaired_wheel_dir]) + + if build_options.repair_command: + repair_command_prepared = prepare_command( + build_options.repair_command, + wheel=built_wheel, + dest_dir=self.repaired_wheel_dir, + package=self.container_package_dir, + project=self.container_project_path, ) - venv_dir = testing_temp_dir / "venv" - - if use_uv: - container.call(["uv", "venv", venv_dir, "--python", python_bin / "python"], env=env) - else: - # Use embedded dependencies from virtualenv to ensure determinism - venv_args = ["--no-periodic-update", "--pip=embed", "--no-setuptools"] - if "38" in config.identifier: - venv_args.append("--no-wheel") - container.call(["python", "-m", "virtualenv", *venv_args, venv_dir], env=env) - - virtualenv_env = env.copy() - virtualenv_env["PATH"] = f"{venv_dir / 'bin'}:{virtualenv_env['PATH']}" - virtualenv_env["VIRTUAL_ENV"] = str(venv_dir) - virtualenv_env = build_options.test_environment.as_dictionary( - prev_environment=virtualenv_env + container.call(["sh", "-c", repair_command_prepared], env=self.env) + else: + container.call(["mv", built_wheel, self.repaired_wheel_dir]) + + return container.glob(self.repaired_wheel_dir, "*.whl") + + def audit_wheel(self, repaired_wheel: PurePosixPath) -> None: + build_options = self.build_options + if not needs_audit(build_options.audit_command, repaired_wheel.name): + return + + # the wheel lives in the container, but the audit tools run on the + # host; copy it out to a temporary directory first + local_abi3audit_dir = self.local_tmp_dir / self.identifier / "audit" + local_abi3audit_dir.mkdir(parents=True, exist_ok=True) + try: + self.container.copy_out(self.repaired_wheel_dir, local_abi3audit_dir) + local_wheel = local_abi3audit_dir / repaired_wheel.name + run_audit(tmp_dir=self.local_tmp_dir, build_options=build_options, wheel=local_wheel) + finally: + shutil.rmtree(local_abi3audit_dir, ignore_errors=True) + + def test_wheel(self, repaired_wheel: PurePosixPath) -> None: + container = self.container + build_options = self.build_options + assert build_options.test_command is not None + + log.step("Testing wheel...") + + # set up a virtual environment to install and test from, to make sure + # there are no dependencies that were pulled in at build time. + if not self.use_uv: + container.call( + ["pip", "install", "virtualenv", *self.dependency_constraint_flags], env=self.env ) - if build_options.before_test: - before_test_prepared = prepare_command( - build_options.before_test, - project=container_project_path, - package=container_package_dir, - ) - container.call(["sh", "-c", before_test_prepared], env=virtualenv_env) + testing_temp_dir = PurePosixPath( + container.call(["mktemp", "-d"], capture_output=True).strip() + ) + venv_dir = testing_temp_dir / "venv" - # Install the wheel we just built + if self.use_uv: container.call( - [*pip, "install", str(repaired_wheel) + build_options.test_extras], - env=virtualenv_env, + ["uv", "venv", venv_dir, "--python", self.python_bin / "python"], env=self.env ) + else: + # Use embedded dependencies from virtualenv to ensure determinism + venv_args = ["--no-periodic-update", "--pip=embed", "--no-setuptools"] + if "38" in self.identifier: + venv_args.append("--no-wheel") + container.call(["python", "-m", "virtualenv", *venv_args, venv_dir], env=self.env) + + virtualenv_env = self.env.copy() + virtualenv_env["PATH"] = f"{venv_dir / 'bin'}:{virtualenv_env['PATH']}" + virtualenv_env["VIRTUAL_ENV"] = str(venv_dir) + virtualenv_env = build_options.test_environment.as_dictionary( + prev_environment=virtualenv_env + ) + + if build_options.before_test: + before_test_prepared = prepare_command( + build_options.before_test, + project=self.container_project_path, + package=self.container_package_dir, + ) + container.call(["sh", "-c", before_test_prepared], env=virtualenv_env) - # Install any requirements to run the tests - if build_options.test_requires: - container.call([*pip, "install", *build_options.test_requires], env=virtualenv_env) + # Install the wheel we just built + container.call( + [*self.pip, "install", str(repaired_wheel) + build_options.test_extras], + env=virtualenv_env, + ) + + # Install any requirements to run the tests + if build_options.test_requires: + container.call([*self.pip, "install", *build_options.test_requires], env=virtualenv_env) + + # Run the tests from a different directory + test_command_prepared = prepare_command( + build_options.test_command, + project=self.container_project_path, + package=self.container_package_dir, + wheel=repaired_wheel, + ) - # Run the tests from a different directory - test_command_prepared = prepare_command( - build_options.test_command, - project=container_project_path, - package=container_package_dir, - wheel=repaired_wheel, + test_cwd = testing_temp_dir / "test_cwd" + container.call(["mkdir", "-p", test_cwd]) + + if build_options.test_sources: + copy_test_sources( + build_options.test_sources, + Path.cwd(), + test_cwd, + copy_into=container.copy_into, ) + else: + # Use the test_fail.py file to raise a nice error if the user + # tries to run tests in the cwd + container.copy_into(resources.TEST_FAIL_CWD_FILE, test_cwd / "test_fail.py") - test_cwd = testing_temp_dir / "test_cwd" - container.call(["mkdir", "-p", test_cwd]) + container.call(["sh", "-c", test_command_prepared], cwd=test_cwd, env=virtualenv_env) - if build_options.test_sources: - copy_test_sources( - build_options.test_sources, - Path.cwd(), - test_cwd, - copy_into=container.copy_into, - ) - else: - # Use the test_fail.py file to raise a nice error if the user - # tries to run tests in the cwd - container.copy_into(resources.TEST_FAIL_CWD_FILE, test_cwd / "test_fail.py") + # clean up test environment + container.call(["rm", "-rf", testing_temp_dir]) + + def move_to_output(self, repaired_wheel: PurePosixPath) -> PurePosixPath: + self.container.call(["mkdir", "-p", self.container_output_dir]) + self.container.call(["mv", repaired_wheel, self.container_output_dir]) + return self.container_output_dir / repaired_wheel.name + + def cleanup(self) -> None: + # nothing to do: the container is removed when the build step ends, + # and the host temp dir is removed at the end of the run + pass + + +def build_in_container( + *, + options: Options, + platform_configs: Sequence[PythonConfiguration], + container: OCIContainer, + container_project_path: PurePath, + container_package_dir: PurePath, + local_tmp_dir: Path, +) -> None: + check_all_python_exist(platform_configs=platform_configs, container=container) - container.call(["sh", "-c", test_command_prepared], cwd=test_cwd, env=virtualenv_env) + log.step("Copying project into container...") + container.copy_into(Path.cwd(), container_project_path) - # clean up test environment - container.call(["rm", "-rf", testing_temp_dir]) + # before_all runs once per container, not once per session + before_all_options_identifier = platform_configs[0].identifier + before_all_options = options.build_options(before_all_options_identifier) - # move repaired wheel to output - output_wheel: Path | None = None - if compatible_wheel is None: - container.call(["mkdir", "-p", container_output_dir]) - container.call(["mv", repaired_wheel, container_output_dir]) - built_wheels.append(container_output_dir / repaired_wheel.name) - output_wheel = options.globals.output_dir / repaired_wheel.name + if before_all_options.before_all: + log.step("Running before_all...") - log.build_end(output_wheel) + env = container.get_environment() + env["PATH"] = f"/opt/python/cp39-cp39/bin:{env['PATH']}" + env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" + env["PIP_ROOT_USER_ACTION"] = "ignore" + env = before_all_options.environment.as_dictionary( + env, executor=container.environment_executor + ) + + before_all_prepared = prepare_command( + before_all_options.before_all, + project=container_project_path, + package=container_package_dir, + ) + container.call(["sh", "-c", before_all_prepared], env=env) + + runner.run_builds( + [ + LinuxBuilder( + config=config, + build_options=options.build_options(config.identifier), + container=container, + container_project_path=container_project_path, + container_package_dir=container_package_dir, + local_tmp_dir=local_tmp_dir, + ) + for config in platform_configs + ] + ) log.step("Copying wheels back to host...") # copy the output back into the host - container.copy_out(container_output_dir, options.globals.output_dir) + container.copy_out(PurePosixPath("/output"), options.globals.output_dir) log.step_end() @@ -518,7 +558,7 @@ def build(options: Options, tmp_path: Path) -> None: ) raise errors.ConfigurationError(msg) from error - try: + with runner.fatal_on_called_process_error(functools.partial(troubleshoot, options)): ids_to_build = [x.identifier for x in build_step.platform_configs] log.step(f"Starting container image {build_step.container_image}...") @@ -540,11 +580,6 @@ def build(options: Options, tmp_path: Path) -> None: local_tmp_dir=tmp_path, ) - except subprocess.CalledProcessError as error: - troubleshoot(options, error) - msg = f"Command {error.cmd} failed with code {error.returncode}. {error.stdout or ''}" - raise errors.FatalError(msg) from error - def _matches_prepared_command(error_cmd: Sequence[str], command_template: str) -> bool: if len(error_cmd) < 3 or error_cmd[0:2] != ["sh", "-c"]: