unix: support building Linux x86-64 from macOS aarch64

Docker on aarch64 macOS will automagically virtualize x86-64 containers
if containers are spawned with `platform=linux/amd64`. Performance of
spawned containers is a bit slower than native, but not horrible. This
functionality means it is viable to develop Linux x86-64 from modern
Apple hardware.

This commit teaches the build system to support cross-compiling
Linux x86-64 from macOS aarch64.

Implementing this wasn't too difficult: we need to pass `platform` into
Docker's APIs for building and creating containers. We need to teach
code to resolve the effective host platform when this scenario is
detected. And we need to advertise support for cross-compiling in the
`targets.yml` file.

In case you are wondering, yes, a similar solution could be employed for
Linux too by using emulation. But this requires Docker be configured to
support emulation, which isn't common. Rosetta on macOS "just works" and
is therefore the lowest hanging fruit to implement.

Author: indygreg

cpython-unix/build-main.py

@@ -142,7 +142,14 @@ def main():
     effective_host_platform = host_platform
     if building_linux_from_macos:
         if host_platform == "macos_arm64":
-            effective_host_platform = "linux_aarch64"
+            if target_triple.startswith("aarch64"):
+                effective_host_platform = "linux_aarch64"
+            elif target_triple.startswith("x86_64"):
+                effective_host_platform = "linux_x86_64"
+            else:
+                raise Exception(
+                    f"unsupported macOS cross-compile: {host_platform} -> {target_triple}"
+                )
         else:
             raise Exception(f"Unhandled macOS platform: {host_platform}")
         print(

cpython-unix/build.py

@@ -25,14 +25,20 @@
     meets_python_minimum_version,
     parse_setup_line,
 )
-from pythonbuild.docker import build_docker_image, get_image, write_dockerfiles
+from pythonbuild.docker import (
+    build_docker_image,
+    docker_platform_from_host_platform,
+    get_image,
+    write_dockerfiles,
+)
 from pythonbuild.downloads import DOWNLOADS
 from pythonbuild.logging import log, set_logger
 from pythonbuild.utils import (
     add_env_common,
     add_licenses_to_extension_entry,
     clang_toolchain,
     create_tar_from_directory,
+    current_host_platform,
     download_entry,
     get_target_settings,
     get_targets,
@@ -98,21 +104,33 @@ def add_target_env(env, build_platform, target_triple, build_env, build_options)
         extra_target_ldflags.append("--rtlib=compiler-rt")
 
     if build_platform.startswith("linux_"):
-        machine = platform.machine()
+        # autoconf is not aware of microarch triples. Normalize those out:
+        # we force targeting via -march CFLAG.
+        env["TARGET_TRIPLE"] = (
+            target_triple.replace("x86_64_v2-", "x86_64-")
+            .replace("x86_64_v3-", "x86_64-")
+            .replace("x86_64_v4-", "x86_64-")
+        )
 
-        # arm64 allows building for Linux on a macOS host using Docker
-        if machine == "aarch64" or machine == "arm64":
-            env["BUILD_TRIPLE"] = "aarch64-unknown-linux-gnu"
-            env["TARGET_TRIPLE"] = target_triple
-        elif machine == "x86_64":
-            env["BUILD_TRIPLE"] = "x86_64-unknown-linux-gnu"
-            env["TARGET_TRIPLE"] = (
-                target_triple.replace("x86_64_v2-", "x86_64-")
-                .replace("x86_64_v3-", "x86_64-")
-                .replace("x86_64_v4-", "x86_64-")
-            )
+        # On macOS, we support building Linux in a virtualized container that
+        # always matches the target platform. Set build/host triple to whatever
+        # we're building.
+        #
+        # Note: we always use the *-gnu triple otherwise autoconf can have
+        # trouble reasoning about cross-compiling since its detected triple from
+        # our build environment is always GNU based.
+        if current_host_platform().startswith("macos_"):
+            env["BUILD_TRIPLE"] = env["TARGET_TRIPLE"].replace("-musl", "-gnu")
         else:
-            raise Exception("unhandled Linux machine value: %s" % machine)
+            # Otherwise assume the container environment matches the machine
+            # type of the current process.
+            host_machine = platform.machine()
+            if host_machine == "aarch64" or host_machine == "arm64":
+                env["BUILD_TRIPLE"] = "aarch64-unknown-linux-gnu"
+            elif host_machine == "x86_64":
+                env["BUILD_TRIPLE"] = "x86_64-unknown-linux-gnu"
+            else:
+                raise Exception("unhandled Linux machine value: %s" % host_machine)
 
         # This will make x86_64_v2, etc count as cross-compiling. This is
         # semantically correct, since the current machine may not support
@@ -960,16 +978,6 @@ def main():
     DOWNLOADS_PATH.mkdir(exist_ok=True)
     (BUILD / "logs").mkdir(exist_ok=True)
 
-    if os.environ.get("PYBUILD_NO_DOCKER"):
-        client = None
-    else:
-        try:
-            client = docker.from_env(timeout=600)
-            client.ping()
-        except Exception as e:
-            print("unable to connect to Docker: %s" % e, file=sys.stderr)
-            return 1
-
     # Note these arguments must be synced with `build-main.py`
     parser = argparse.ArgumentParser()
     parser.add_argument(
@@ -1031,6 +1039,17 @@ def main():
 
     settings = get_target_settings(TARGETS_CONFIG, target_triple)
 
+    if os.environ.get("PYBUILD_NO_DOCKER"):
+        client = None
+    else:
+        try:
+            client = docker.from_env(timeout=600)
+            client.ping()
+            client._pbs_platform = docker_platform_from_host_platform(host_platform)
+        except Exception as e:
+            print("unable to connect to Docker: %s" % e, file=sys.stderr)
+            return 1
+
     if args.action == "dockerfiles":
         log_name = "dockerfiles"
     elif args.action == "makefiles":

cpython-unix/targets.yml

@@ -556,6 +556,7 @@ x86_64-apple-darwin:
 x86_64-unknown-linux-gnu:
   host_platforms:
     - linux_x86_64
+    - macos_arm64
   pythons_supported:
     - '3.10'
     - '3.11'
@@ -605,6 +606,7 @@ x86_64-unknown-linux-gnu:
 x86_64_v2-unknown-linux-gnu:
   host_platforms:
     - linux_x86_64
+    - macos_arm64
   pythons_supported:
     - '3.10'
     - '3.11'
@@ -655,6 +657,7 @@ x86_64_v2-unknown-linux-gnu:
 x86_64_v3-unknown-linux-gnu:
   host_platforms:
     - linux_x86_64
+    - macos_arm64
   pythons_supported:
     - '3.10'
     - '3.11'
@@ -705,6 +708,9 @@ x86_64_v3-unknown-linux-gnu:
 x86_64_v4-unknown-linux-gnu:
   host_platforms:
     - linux_x86_64
+    # Rosetta doesn't support AVX-512. So we cannot run x86-64-v4 binaries
+    # under Rosetta. But they can build correctly.
+    - macos_arm64
   pythons_supported:
     - '3.10'
     - '3.11'
@@ -755,6 +761,9 @@ x86_64_v4-unknown-linux-gnu:
 x86_64-unknown-linux-musl:
   host_platforms:
     - linux_x86_64
+    # Rosetta doesn't support AVX-512. So we cannot run x86-64-v4 binaries
+    # under Rosetta. But they can build correctly.
+    - macos_arm64
   pythons_supported:
     - '3.10'
     - '3.11'
@@ -802,6 +811,7 @@ x86_64-unknown-linux-musl:
 x86_64_v2-unknown-linux-musl:
   host_platforms:
     - linux_x86_64
+    - macos_arm64
   pythons_supported:
     - '3.10'
     - '3.11'
@@ -850,6 +860,7 @@ x86_64_v2-unknown-linux-musl:
 x86_64_v3-unknown-linux-musl:
   host_platforms:
     - linux_x86_64
+    - macos_arm64
   pythons_supported:
     - '3.10'
     - '3.11'
@@ -898,6 +909,7 @@ x86_64_v3-unknown-linux-musl:
 x86_64_v4-unknown-linux-musl:
   host_platforms:
     - linux_x86_64
+    - macos_arm64
   pythons_supported:
     - '3.10'
     - '3.11'

pythonbuild/buildenv.py

@@ -266,7 +266,10 @@ def find_output_files(self, base_path, pattern):
 def build_environment(client, image):
     if client is not None:
         container = client.containers.run(
-            image, command=["/bin/sleep", "86400"], detach=True
+            image,
+            command=["/bin/sleep", "86400"],
+            detach=True,
+            platform=client._pbs_platform,
         )
         td = None
         context = ContainerContext(container)

pythonbuild/docker.py

@@ -2,12 +2,12 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-import contextlib
 import io
 import operator
 import os
 import pathlib
 import tarfile
+import typing
 
 import docker  # type: ignore
 import jinja2
@@ -29,16 +29,31 @@ def write_dockerfiles(source_dir: pathlib.Path, dest_dir: pathlib.Path):
         write_if_different(dest_dir / f, data.encode("utf-8"))
 
 
+def docker_platform_from_host_platform(host_platform: str) -> typing.Optional[str]:
+    """Convert a PBS host platform to a Docker platform."""
+    if host_platform == "linux_x86_64":
+        return "linux/amd64"
+    elif host_platform == "linux_aarch64":
+        return "linux/arm64"
+    else:
+        return None
+
+
 def build_docker_image(
     client, image_data: bytes, image_dir: pathlib.Path, name, host_platform
 ):
     image_path = image_dir / f"image-{name}.{host_platform}"
 
-    return ensure_docker_image(client, io.BytesIO(image_data), image_path=image_path)
+    return ensure_docker_image(
+        client,
+        io.BytesIO(image_data),
+        image_path=image_path,
+        platform=docker_platform_from_host_platform(host_platform),
+    )
 
 
-def ensure_docker_image(client, fh, image_path=None):
-    res = client.api.build(fileobj=fh, decode=True)
+def ensure_docker_image(client, fh, image_path=None, platform=None):
+    res = client.api.build(fileobj=fh, decode=True, platform=platform)
 
     image = None
 
@@ -111,18 +126,6 @@ def copy_file_to_container(path, container, container_path, archive_path=None):
     container.put_archive(container_path, buf.getvalue())
 
 
-@contextlib.contextmanager
-def run_container(client, image):
-    container = client.containers.run(
-        image, command=["/bin/sleep", "86400"], detach=True
-    )
-    try:
-        yield container
-    finally:
-        container.stop(timeout=0)
-        container.remove()
-
-
 def container_exec(container, command, user="build", environment=None):
     # docker-py's exec_run() won't return the exit code. So we reinvent the
     # wheel.