diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1a12a3e..3d94a4f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,8 @@ jobs: runs-on-namespace: namespace-profile-macos-8x14 steps: - uses: actions/checkout@v6 + with: + fetch-depth: 0 - name: Select Xcode 16 if: runner.os == 'macOS' run: | @@ -62,12 +64,10 @@ jobs: fail-fast: false matrix: include: - - image: debian:bookworm-slim + - image: debian:trixie-slim runs-on: ubuntu-latest - image: ubuntu:24.04 runs-on: ubuntu-latest - - image: ubuntu:20.04 - runs-on: ubuntu-latest - image: fedora:41 runs-on: ubuntu-latest - image: archlinux:latest @@ -76,7 +76,7 @@ jobs: runs-on: ubuntu-latest - image: ghcr.io/void-linux/void-glibc:latest runs-on: ubuntu-latest - - image: debian:bookworm-slim + - image: debian:trixie-slim runs-on: ubuntu-24.04-arm # TODO: add musl test steps: @@ -92,8 +92,6 @@ jobs: if: github.event_name == 'push' && github.ref == 'refs/heads/master' needs: [build, test] runs-on: ubuntu-latest - permissions: - contents: write steps: - uses: actions/checkout@v6 - uses: actions/download-artifact@v4 @@ -101,16 +99,8 @@ jobs: pattern: wheels-* merge-multiple: true path: dist - - uses: actions/cache/restore@v4 - with: - key: build-cache-v2-${{ matrix.platform }}-${{ hashFiles('build.sh', 'setup.sh', '*/pyproject.toml', '*/setup.py', '*/build.sh') }} - restore-keys: build-cache-v2-${{ matrix.platform }}- - path: | - .uv-cache - */*/install - */*/toolchain - */*/bin - - name: Publish wheels and shims + - name: Publish wheels to PyPI env: - GH_TOKEN: ${{ github.token }} - run: ./release.sh --publish-only + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_PAT }} + run: ./release.sh diff --git a/README.md b/README.md index 8736a7b..00de26d 100644 --- a/README.md +++ b/README.md @@ -40,15 +40,23 @@ contributions welcome for other platforms! ## usage +pre-built wheels are published to PyPI under the `comma-deps-` prefix. the import name +is unchanged (e.g. `import capnproto`), only the distribution name is prefixed. +packages require Python 3.12 or newer. + ```python dependencies = [ - # use per-package release branches for pre-built wheels - "capnproto @ git+https://github.com/commaai/dependencies.git@release-capnproto#subdirectory=capnproto", - "ffmpeg @ git+https://github.com/commaai/dependencies.git@release-ffmpeg#subdirectory=ffmpeg", + "comma-deps-capnproto>=1.0.1,<1.0.2", + "comma-deps-ffmpeg>=7.1.0,<7.1.1", +] +``` + +to build from source instead, point at the master branch of this repo: - # use the master branch to build the package on pip install - "capnproto @ git+https://github.com/commaai/dependencies.git@master#subdirectory=capnproto", - "ffmpeg @ git+https://github.com/commaai/dependencies.git@master#subdirectory=ffmpeg", +```python +dependencies = [ + "comma-deps-capnproto @ git+https://github.com/commaai/dependencies.git@master#subdirectory=capnproto", + "comma-deps-ffmpeg @ git+https://github.com/commaai/dependencies.git@master#subdirectory=ffmpeg", ] ``` @@ -56,6 +64,12 @@ dependencies = [ to add a new package: * start a new top-level directory as a new package -* `./test.sh` tests the building of all packages -* on pushes to `master`, wheels are built for our target platforms and pushed to a GitHub release -* each `release-` branch contains a single shim package, so old lockfiles keep resolving even as new packages are added +* `./build.sh` builds all packages, `./test_wheels_in_image.sh` tests the built wheels in a range of distros +* on pushes to `master`, wheels are built for our target platforms, tested, and published to PyPI + +> [!NOTE] +> PyPI does not allow overwriting an uploaded file. each build publishes +> versions as `.postN`, where `N` is `git rev-list --count HEAD` +> for the commit being built. package `pyproject.toml` files keep the upstream +> base version; `./build.sh` applies the `.postN` suffix only while building +> wheels. existing `.postM` suffixes are replaced by the build suffix. diff --git a/_shim_setup.py b/_shim_setup.py deleted file mode 100644 index 8b58b1c..0000000 --- a/_shim_setup.py +++ /dev/null @@ -1,107 +0,0 @@ -"""Shim setup.py: downloads pre-built wheels from GitHub Releases at install time.""" -import os -import platform -import time -import zipfile -from io import BytesIO -from urllib.error import URLError -from urllib.request import urlopen - -try: - import tomllib -except ImportError: - import tomli as tomllib - -from setuptools import setup -from setuptools.command.build_py import build_py - -_HERE = os.path.dirname(os.path.abspath(__file__)) -with open(os.path.join(_HERE, "pyproject.toml"), "rb") as _f: - _cfg = tomllib.load(_f) - -REPO_URL = _cfg["tool"]["shim"]["repo_url"] -TAG = _cfg["tool"]["shim"]["tag"] -DATADIR = _cfg["tool"]["shim"]["datadir"] -VERSION = _cfg["project"]["version"] -MODULE = _cfg["project"]["name"].replace("-", "_") - -# all top-level packages bundled in this wheel (e.g. acados + casadi). -# `packages` may be a flat list or a dict with find.include patterns. -_pkgs = _cfg.get("tool", {}).get("setuptools", {}).get("packages", [f"{MODULE}*"]) -if isinstance(_pkgs, list): - _INCLUDE = _pkgs -elif isinstance(_pkgs, dict): - _INCLUDE = _pkgs.get("find", {}).get("include", [f"{MODULE}*"]) -else: - _INCLUDE = [f"{MODULE}*"] -TOP_PACKAGES = sorted({p.rstrip("*").rstrip("/") for p in _INCLUDE if p.rstrip("*").rstrip("/")}) - -PLATFORM_MAP = { - ("Linux", "x86_64"): "linux_x86_64", - ("Linux", "aarch64"): "linux_aarch64", - ("Darwin", "arm64"): "macosx_11_0_arm64", -} - - -class InstallPrebuilt(build_py): - def run(self): - module_dir = os.path.join(_HERE, MODULE) - data_dir = os.path.join(module_dir, DATADIR) - - if not os.path.exists(os.path.join(data_dir, "bin")): - key = (platform.system(), platform.machine()) - plat = PLATFORM_MAP.get(key) - if plat is None: - raise RuntimeError(f"unsupported platform: {key}") - - whl_names = [ - f"{MODULE}-{VERSION}-py3-none-{plat}.whl", - f"{MODULE}-{VERSION}-py3-none-any.whl", - ] - - raw = None - for whl_name in whl_names: - url = f"{REPO_URL}/releases/download/{TAG}/{whl_name}" - print(f"Downloading {url} ...") - for attempt in range(3): - try: - raw = urlopen(url, timeout=60).read() - break - except (URLError, OSError) as e: - if attempt == 2: - if whl_name == whl_names[-1]: - raise - break - wait = 2 ** attempt - print(f"Download failed ({e}), retrying in {wait}s ...") - time.sleep(wait) - if raw is not None: - break - - print("Extracting wheel ...") - with zipfile.ZipFile(BytesIO(raw)) as zf: - purelib_data = f"{MODULE}-{VERSION}.data/purelib/" - for info in zf.infolist(): - if info.is_dir(): - continue - name = info.filename - if name.startswith(purelib_data): - rel = name[len(purelib_data):] - else: - rel = name - if "/" not in rel: - continue - top = rel.split("/", 1)[0] - if top not in TOP_PACKAGES: - continue - dest = os.path.join(_HERE, rel) - os.makedirs(os.path.dirname(dest), exist_ok=True) - with open(dest, "wb") as f: - f.write(zf.read(info)) - if info.external_attr >> 16 & 0o111: - os.chmod(dest, 0o755) - - super().run() - - -setup(cmdclass={"build_py": InstallPrebuilt}) diff --git a/acados/pyproject.toml b/acados/pyproject.toml index 4625284..ee30c66 100644 --- a/acados/pyproject.toml +++ b/acados/pyproject.toml @@ -3,10 +3,10 @@ requires = ["setuptools>=64", "wheel"] build-backend = "setuptools.build_meta" [project] -name = "acados" +name = "comma-deps-acados" version = "0.2.2" description = "acados solvers + Python interface, with a slimmed casadi vendored alongside" -requires-python = ">=3.8" +requires-python = ">=3.12" dependencies = [ # the vendored casadi.py shim imports numpy at module load time "numpy", diff --git a/bootstrap-icons/pyproject.toml b/bootstrap-icons/pyproject.toml index 67c7fbe..0b4a6e7 100644 --- a/bootstrap-icons/pyproject.toml +++ b/bootstrap-icons/pyproject.toml @@ -3,10 +3,10 @@ requires = ["setuptools>=64", "wheel", "fonttools"] build-backend = "setuptools.build_meta" [project] -name = "bootstrap-icons" +name = "comma-deps-bootstrap-icons" version = "1.10.5.0" description = "Bootstrap Icons SVG sprite and TTF font" -requires-python = ">=3.8" +requires-python = ">=3.12" [tool.setuptools.packages.find] include = ["bootstrap_icons*"] diff --git a/build.sh b/build.sh index 1e32c7e..b3de1fe 100755 --- a/build.sh +++ b/build.sh @@ -62,12 +62,23 @@ fi export CMAKE_C_COMPILER_LAUNCHER=ccache export CMAKE_CXX_COMPILER_LAUNCHER=ccache +restore_build_versions() { + python3 build_versions.py restore */pyproject.toml +} + +trap restore_build_versions EXIT + echo "Building workspace packages into dist" START_SECS=$SECONDS mkdir -p dist/ rm -rf dist/* + +POST_N="${DEPENDENCIES_POST_VERSION_N:-$(git rev-list --count HEAD)}" +echo "Applying .post$POST_N package versions for this build" +python3 build_versions.py apply "$POST_N" */pyproject.toml uv build --all-packages --wheel --out-dir dist --no-create-gitignore +restore_build_versions if [[ -n "${BUILD_SH_IN_MANYLINUX:-}" ]]; then VENV_DIR="$ROOT_DIR/.venv-manylinux" diff --git a/build_versions.py b/build_versions.py new file mode 100644 index 0000000..c1b9304 --- /dev/null +++ b/build_versions.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +import os +import re +import shutil +import sys + +BACKUP_SUFFIX = ".build-version.bak" +VERSION_RE = re.compile(r'^(?P\s*version\s*=\s*")(?P[^"]+)(?P".*)$') + + +def post_version(version: str, post_n: str) -> str: + base = re.sub(r"\.post\d+$", "", version) + if not re.fullmatch(r"\d+(?:\.\d+)*(?:(?:a|b|rc)\d+)?", base): + raise ValueError(f"unsupported version: {version}") + return f"{base}.post{post_n}" + + +def apply(post_n: str, paths: list[str]) -> None: + if not post_n.isdigit(): + raise ValueError("post number must be a non-negative integer") + + for path in paths: + shutil.copy2(path, path + BACKUP_SUFFIX) + lines = open(path, encoding="utf-8").read().splitlines(keepends=True) + in_project = changed = False + + for i, line in enumerate(lines): + stripped = line.strip() + if stripped.startswith("[") and stripped.endswith("]"): + in_project = stripped == "[project]" + if in_project and (match := VERSION_RE.match(line)): + lines[i] = f"{match['prefix']}{post_version(match['version'], post_n)}{match['suffix']}\n" + changed = True + + if not changed: + raise RuntimeError(f"no [project] version found in {path}") + + open(path, "w", encoding="utf-8").writelines(lines) + + +def restore(paths: list[str]) -> None: + for path in paths: + backup = path + BACKUP_SUFFIX + if os.path.exists(backup): + os.replace(backup, path) + + +if __name__ == "__main__": + if len(sys.argv) < 3 or sys.argv[1] not in {"apply", "restore"}: + raise SystemExit("usage: build_versions.py apply POST_N FILE... | restore FILE...") + if sys.argv[1] == "apply": + apply(sys.argv[2], sys.argv[3:]) + else: + restore(sys.argv[2:]) diff --git a/bzip2/pyproject.toml b/bzip2/pyproject.toml index b7bbaa6..2eabed7 100644 --- a/bzip2/pyproject.toml +++ b/bzip2/pyproject.toml @@ -3,10 +3,10 @@ requires = ["setuptools>=64", "wheel"] build-backend = "setuptools.build_meta" [project] -name = "bzip2" +name = "comma-deps-bzip2" version = "1.0.8" description = "bzip2 compression library (static build)" -requires-python = ">=3.8" +requires-python = ">=3.12" [tool.setuptools.packages.find] include = ["bzip2*"] diff --git a/capnproto/capnproto/__init__.py b/capnproto/capnproto/__init__.py index b654748..d530002 100644 --- a/capnproto/capnproto/__init__.py +++ b/capnproto/capnproto/__init__.py @@ -1,4 +1,5 @@ import os +import tempfile import sys DIR = os.path.join(os.path.dirname(__file__), "install") @@ -42,3 +43,9 @@ def smoketest(): subprocess.run([capnp, "--version"], check=True, env=env) subprocess.run([capnpc, "--version"], check=True, env=env) subprocess.run([capnpc_cpp, "--version"], check=True, env=env) + with tempfile.TemporaryDirectory() as temp_dir: + schema = os.path.join(temp_dir, "test.capnp") + with open(schema, "w") as f: + f.write("@0xd12f9faaa6e3fdec; struct Test { value @0 :UInt32; }\n") + env["PWD"] = temp_dir + subprocess.run([capnp, "compile", "-oc++", schema], cwd=temp_dir, check=True, env=env) diff --git a/capnproto/pyproject.toml b/capnproto/pyproject.toml index 3bea931..0e3f4b8 100644 --- a/capnproto/pyproject.toml +++ b/capnproto/pyproject.toml @@ -3,15 +3,14 @@ requires = ["setuptools>=64", "wheel"] build-backend = "setuptools.build_meta" [project] -name = "capnproto" +name = "comma-deps-capnproto" version = "1.0.1" description = "Cap'n Proto serialization library and compiler" -requires-python = ">=3.8" +requires-python = ">=3.12" [project.scripts] capnp = "capnproto:_run_capnp" capnpc = "capnproto:_run_capnpc" -"capnpc-c++" = "capnproto:_run_capnpc_cpp" [tool.setuptools.packages.find] include = ["capnproto*"] diff --git a/catch2/pyproject.toml b/catch2/pyproject.toml index 2c8f6f8..1cca2cc 100644 --- a/catch2/pyproject.toml +++ b/catch2/pyproject.toml @@ -3,10 +3,10 @@ requires = ["setuptools>=64", "wheel"] build-backend = "setuptools.build_meta" [project] -name = "catch2" +name = "comma-deps-catch2" version = "2.13.10" description = "Catch2 C++ test framework headers" -requires-python = ">=3.8" +requires-python = ">=3.12" [tool.setuptools.packages.find] include = ["catch2*"] diff --git a/cppcheck/pyproject.toml b/cppcheck/pyproject.toml index 9b6c5b3..87cd758 100644 --- a/cppcheck/pyproject.toml +++ b/cppcheck/pyproject.toml @@ -3,10 +3,10 @@ requires = ["setuptools>=64", "wheel"] build-backend = "setuptools.build_meta" [project] -name = "cppcheck" +name = "comma-deps-cppcheck" version = "2.21.0" description = "Cppcheck static analysis tool" -requires-python = ">=3.8" +requires-python = ">=3.12" [project.scripts] cppcheck = "cppcheck:_run" diff --git a/eigen/pyproject.toml b/eigen/pyproject.toml index 8928294..5dffcd9 100644 --- a/eigen/pyproject.toml +++ b/eigen/pyproject.toml @@ -3,10 +3,10 @@ requires = ["setuptools>=64", "wheel"] build-backend = "setuptools.build_meta" [project] -name = "eigen" +name = "comma-deps-eigen" version = "3.4.0" description = "Eigen C++ linear algebra headers" -requires-python = ">=3.8" +requires-python = ">=3.12" [tool.setuptools.packages.find] include = ["eigen*"] diff --git a/ffmpeg/build.sh b/ffmpeg/build.sh index 48f2fb8..87b23fa 100755 --- a/ffmpeg/build.sh +++ b/ffmpeg/build.sh @@ -16,6 +16,8 @@ NJOBS="$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 2)" CC="ccache ${CC:-cc}" PREFIX="$DIR/build/prefix" mkdir -p "$DIR/build" +rm -rf "$PREFIX" +mkdir -p "$PREFIX" # --- Build zlib (static) --- if [ ! -d "zlib-src/.git" ]; then @@ -111,6 +113,7 @@ if [ ! -d "ffmpeg-src/.git" ]; then fi git -C ffmpeg-src fetch --depth 1 origin "n${FFMPEG_VERSION}" git -C ffmpeg-src checkout --force FETCH_HEAD +git -C ffmpeg-src clean -xffd cd ffmpeg-src diff --git a/ffmpeg/pyproject.toml b/ffmpeg/pyproject.toml index 5ac7fa2..dc9a469 100644 --- a/ffmpeg/pyproject.toml +++ b/ffmpeg/pyproject.toml @@ -3,10 +3,10 @@ requires = ["setuptools>=64", "wheel"] build-backend = "setuptools.build_meta" [project] -name = "ffmpeg" +name = "comma-deps-ffmpeg" version = "7.1.0" description = "FFmpeg media tools and libraries (minimal build for openpilot)" -requires-python = ">=3.8" +requires-python = ">=3.12" [project.scripts] ffmpeg = "ffmpeg:_run_ffmpeg" diff --git a/ffmpeg/setup.py b/ffmpeg/setup.py index e6abc30..4bcc055 100644 --- a/ffmpeg/setup.py +++ b/ffmpeg/setup.py @@ -1,5 +1,6 @@ import os import platform +import shutil import subprocess from setuptools.command.build_py import build_py @@ -18,6 +19,7 @@ def run(self): build_script = os.path.join(pkg_dir, "build.sh") subprocess.check_call(["bash", build_script], cwd=pkg_dir) + shutil.rmtree(os.path.join(self.build_lib, "ffmpeg"), ignore_errors=True) super().run() diff --git a/gcc-arm-none-eabi/pyproject.toml b/gcc-arm-none-eabi/pyproject.toml index efd348a..d48a3f7 100644 --- a/gcc-arm-none-eabi/pyproject.toml +++ b/gcc-arm-none-eabi/pyproject.toml @@ -3,10 +3,10 @@ requires = ["setuptools>=64", "wheel"] build-backend = "setuptools.build_meta" [project] -name = "gcc-arm-none-eabi" +name = "comma-deps-gcc-arm-none-eabi" version = "13.2.1" description = "ARM GCC toolchain for bare-metal targets" -requires-python = ">=3.8" +requires-python = ">=3.12" [project.scripts] arm-none-eabi-gcc = "gcc_arm_none_eabi:_run_gcc" diff --git a/git-lfs/pyproject.toml b/git-lfs/pyproject.toml index 293de1e..bf3ce18 100644 --- a/git-lfs/pyproject.toml +++ b/git-lfs/pyproject.toml @@ -3,10 +3,10 @@ requires = ["setuptools>=64", "wheel"] build-backend = "setuptools.build_meta" [project] -name = "git-lfs" +name = "comma-deps-git-lfs" version = "3.6.1" description = "Git Large File Storage extension" -requires-python = ">=3.8" +requires-python = ">=3.12" [project.scripts] git-lfs = "git_lfs:_run" diff --git a/imgui/pyproject.toml b/imgui/pyproject.toml index e33eaf9..6c6d20b 100644 --- a/imgui/pyproject.toml +++ b/imgui/pyproject.toml @@ -3,10 +3,10 @@ requires = ["setuptools>=64", "wheel"] build-backend = "setuptools.build_meta" [project] -name = "imgui" +name = "comma-deps-imgui" version = "1.92.7" description = "Dear ImGui + ImPlot + GLFW 3.4 (static)" -requires-python = ">=3.8" +requires-python = ">=3.12" [tool.setuptools] packages = ["imgui"] diff --git a/json11/pyproject.toml b/json11/pyproject.toml index ecf0377..557ab6d 100644 --- a/json11/pyproject.toml +++ b/json11/pyproject.toml @@ -3,10 +3,10 @@ requires = ["setuptools>=64", "wheel"] build-backend = "setuptools.build_meta" [project] -name = "json11" +name = "comma-deps-json11" version = "20170411.0" description = "json11 JSON parser library (static build)" -requires-python = ">=3.8" +requires-python = ">=3.12" [tool.setuptools.packages.find] include = ["json11*"] diff --git a/libjpeg/pyproject.toml b/libjpeg/pyproject.toml index c6ea4a1..d785ea5 100644 --- a/libjpeg/pyproject.toml +++ b/libjpeg/pyproject.toml @@ -3,10 +3,10 @@ requires = ["setuptools>=64", "wheel"] build-backend = "setuptools.build_meta" [project] -name = "libjpeg" +name = "comma-deps-libjpeg" version = "3.1.0" description = "libjpeg-turbo JPEG library (static build)" -requires-python = ">=3.8" +requires-python = ">=3.12" [tool.setuptools.packages.find] include = ["libjpeg*"] diff --git a/libusb/pyproject.toml b/libusb/pyproject.toml index 9b17ab1..38c5b99 100644 --- a/libusb/pyproject.toml +++ b/libusb/pyproject.toml @@ -3,10 +3,10 @@ requires = ["setuptools>=64", "wheel"] build-backend = "setuptools.build_meta" [project] -name = "libusb" +name = "comma-deps-libusb" version = "1.0.29" description = "libusb USB device access library (static build)" -requires-python = ">=3.8" +requires-python = ">=3.12" [tool.setuptools.packages.find] include = ["libusb*"] diff --git a/libyuv/pyproject.toml b/libyuv/pyproject.toml index 59facd7..a974ef4 100644 --- a/libyuv/pyproject.toml +++ b/libyuv/pyproject.toml @@ -3,10 +3,10 @@ requires = ["setuptools>=64", "wheel"] build-backend = "setuptools.build_meta" [project] -name = "libyuv" +name = "comma-deps-libyuv" version = "1922.0" description = "libyuv image processing library (static build)" -requires-python = ">=3.8" +requires-python = ">=3.12" [tool.setuptools.packages.find] include = ["libyuv*"] diff --git a/ncurses/pyproject.toml b/ncurses/pyproject.toml index 300bce7..8220037 100644 --- a/ncurses/pyproject.toml +++ b/ncurses/pyproject.toml @@ -3,10 +3,10 @@ requires = ["setuptools>=64", "wheel"] build-backend = "setuptools.build_meta" [project] -name = "ncurses" +name = "comma-deps-ncurses" version = "6.5" description = "ncurses terminal library (static build)" -requires-python = ">=3.8" +requires-python = ">=3.12" [tool.setuptools.packages.find] include = ["ncurses*"] diff --git a/raylib/pyproject.toml b/raylib/pyproject.toml index 2a7f4b9..f7a4312 100644 --- a/raylib/pyproject.toml +++ b/raylib/pyproject.toml @@ -3,10 +3,10 @@ requires = ["setuptools>=70.1", "cffi>=1.17.1"] build-backend = "setuptools.build_meta" [project] -name = "raylib" +name = "comma-deps-raylib" version = "6.0.0.0" description = "raylib + pyray Python bindings (commaai fork)" -requires-python = ">=3.8" +requires-python = ">=3.12" dependencies = ["cffi>=1.17.1"] [tool.setuptools] diff --git a/release.sh b/release.sh index ad08fce..12e8c72 100755 --- a/release.sh +++ b/release.sh @@ -4,174 +4,55 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" cd "$ROOT_DIR" -REPO=commaai/dependencies +echo "Publishing wheels to PyPI" -echo -echo "Publishing wheels to GitHub Releases ($REPO)" +if ! command -v uvx >/dev/null 2>&1; then + UV_BIN_DIR="$HOME/.local/bin" + mkdir -p "$UV_BIN_DIR" + curl -LsSf https://astral.sh/uv/install.sh | env UV_UNMANAGED_INSTALL="$UV_BIN_DIR" sh + export PATH="$UV_BIN_DIR:$PATH" +fi shopt -s nullglob -for toml in */pyproject.toml; do - pkg="$(dirname "$toml")" - module="${pkg//-/_}" - version="$(python3 -c "import tomllib; print(tomllib.load(open('$toml', 'rb'))['project']['version'])")" - tag="${pkg}/v${version}" - - wheels=("dist/${module}-${version}-"*.whl) - if [[ ${#wheels[@]} -eq 0 ]]; then - echo "missing wheel for $pkg ($module-$version) in dist" >&2 - exit 1 - fi - - echo "[$pkg] Uploading ${#wheels[@]} wheel(s) to $tag" - - gh release create "$tag" "${wheels[@]}" --repo "$REPO" --title "$pkg v$version" --notes "Platform wheels for $pkg $version" 2>/dev/null || - gh release upload "$tag" "${wheels[@]}" --repo "$REPO" --clobber -done +wheels=(dist/*.whl) shopt -u nullglob -TOKEN="$(gh auth token 2>/dev/null)" || { echo "set GH_TOKEN to publish shim branches" >&2; exit 1; } - -TMP_DIR="$(mktemp -d)" -python3 - "$TMP_DIR" "$REPO" <<'PY' -import json -import pathlib -import shutil -import tomllib -import sys - -tmp_dir = pathlib.Path(sys.argv[1]) -repo = sys.argv[2] -repo_url = f"https://github.com/{repo}" -shim_setup = pathlib.Path("_shim_setup.py") - -for toml in sorted(pathlib.Path(".").glob("*/pyproject.toml")): - pkg = toml.parent.name - module = pkg.replace("-", "_") - data = tomllib.load(toml.open("rb")) - version = str(data["project"]["version"]) - tag = f"{pkg}/v{version}" - description = data["project"]["description"] - patterns = data.get("tool", {}).get("setuptools", {}).get("package-data", {}).get(module, [""]) - datadir = patterns[0].split("/", 1)[0] if patterns and patterns[0] else "" - scripts = data.get("project", {}).get("scripts", {}) or {} - - repo_dir = tmp_dir / pkg - pkg_dir = repo_dir / pkg - mod_dir = pkg_dir / module - mod_dir.mkdir(parents=True, exist_ok=True) - - # copy all .py files from the main module - src_mod = pathlib.Path(pkg) / module - for py_file in src_mod.glob("*.py"): - shutil.copy2(py_file, mod_dir / py_file.name) - - # copy extra packages (e.g. pyray for raylib, slim casadi for acados) - # binaries (.so/.dylib) are fetched from the wheel at install time, so the - # shim repo only carries Python sources to keep it small - def _ignore_binaries(_dir, names): - return [n for n in names if n.endswith((".so", ".dylib", ".a")) or ".so." in n or n == "__pycache__"] +if [[ ${#wheels[@]} -eq 0 ]]; then + echo "no wheels in dist/" >&2 + exit 1 +fi - # `packages` is either a flat list (["acados", "casadi"]) or a dict with - # `find.include` patterns (["acados*", "casadi*"]) — handle both - pkgs_val = data.get("tool", {}).get("setuptools", {}).get("packages", []) - if isinstance(pkgs_val, list): - include_patterns = list(pkgs_val) - elif isinstance(pkgs_val, dict): - include_patterns = pkgs_val.get("find", {}).get("include", []) - else: - include_patterns = [] - extra_packages = [] - for pattern in include_patterns: - p = pattern.rstrip("*") - if p and p != module and p != f"{module}/": - src_extra = pathlib.Path(pkg) / p - dst_extra = pkg_dir / p - if src_extra.is_dir(): - shutil.copytree(src_extra, dst_extra, dirs_exist_ok=True, ignore=_ignore_binaries) - else: - # source not in the workspace (typically built lazily by build.sh and - # not cached into the publish job). Drop a placeholder __init__.py so - # setuptools.packages.find picks it up; the shim's setup.py overwrites - # it from the wheel at install time. - dst_extra.mkdir(parents=True, exist_ok=True) - (dst_extra / "__init__.py").write_text("") - extra_packages.append(pattern) +echo "Found ${#wheels[@]} wheel(s):" +printf ' %s\n' "${wheels[@]}" - shutil.copy2(shim_setup, pkg_dir / "setup.py") +upload_dir="$(mktemp -d)" +trap 'rm -rf "$upload_dir"' EXIT +cp "${wheels[@]}" "$upload_dir"/ - deps = data.get("project", {}).get("dependencies", []) +retag_linux_wheels() { + local from_platform="$1" + local to_platform="$2" - lines = [ - "[build-system]", - 'requires = ["setuptools>=64", "wheel", \'tomli; python_version < \"3.11\"\']', - 'build-backend = "setuptools.build_meta"', - "", - "[project]", - f'name = "{pkg}"', - f'version = "{version}"', - f"description = {json.dumps(description + ' (pre-built)')}", - 'requires-python = ">=3.8"', - ] + shopt -s nullglob + local platform_wheels=("$upload_dir"/*-"$from_platform".whl) + shopt -u nullglob - if deps: - lines.append(f"dependencies = {json.dumps(deps)}") - - if scripts: - lines += ["", "[project.scripts]"] - for name, target in scripts.items(): - lines.append(f'"{name}" = {json.dumps(target)}') - - # propagate the workspace's package-data so the shim wheel knows to ship - # everything the upstream wheel ships (e.g. acados_template/, casadi/*.so). - workspace_pkgdata = data.get("tool", {}).get("setuptools", {}).get("package-data", {}) - - # extra_packages may be plain names (["casadi"]) or globs (["casadi*"]). - # Normalise to plain package names for both `packages` and `package-data`. - extra_pkg_names = [p.rstrip("*").rstrip("/") for p in extra_packages] - - lines += [ - "", - "[tool.setuptools]", - f"packages = {json.dumps([module] + extra_pkg_names)}", - "", - "[tool.setuptools.package-data]", - ] - module_data = sorted(set(workspace_pkgdata.get(module, [f"{datadir}/**/*"]) + ["*.so"])) - lines.append(f"{module} = {json.dumps(module_data)}") - for name in extra_pkg_names: - extra_data = workspace_pkgdata.get(name, ["**/*"]) - lines.append(f"{name} = {json.dumps(list(extra_data))}") - - lines += [ - "", - "[tool.shim]", - f'repo_url = "{repo_url}"', - f'tag = "{tag}"', - f'datadir = "{datadir}"', - ] + if [[ ${#platform_wheels[@]} -gt 0 ]]; then + uvx --from wheel wheel tags --remove --platform-tag "$to_platform" "${platform_wheels[@]}" + fi +} - (pkg_dir / "pyproject.toml").write_text("\n".join(lines) + "\n") -PY +# PyPI rejects generic linux_* binary wheel tags. The packages are built in +# manylinux_2_28 containers in CI, so only rewrite the upload copies. +retag_linux_wheels linux_x86_64 manylinux_2_28_x86_64 +retag_linux_wheels linux_aarch64 manylinux_2_28_aarch64 +retag_linux_wheels linux_arm64 manylinux_2_28_aarch64 shopt -s nullglob -for repo_dir in "$TMP_DIR"/*; do - [[ -d "$repo_dir" ]] || continue - - pkg="$(basename "$repo_dir")" - branch="release-$pkg" - - echo "[$pkg] Publishing shim branch $branch" - - ( - cd "$repo_dir" - git init - git checkout -b "$branch" - git add "$pkg" - git -c user.name="github-actions[bot]" -c user.email="github-actions[bot]@users.noreply.github.com" commit -m "update $pkg shim" - git remote add origin "https://x-access-token:${TOKEN}@github.com/${REPO}.git" - git push -f origin "$branch" - ) -done +upload_wheels=("$upload_dir"/*.whl) shopt -u nullglob -rm -rf "$TMP_DIR" +echo "Uploading ${#upload_wheels[@]} PyPI wheel(s):" +printf ' %s\n' "${upload_wheels[@]}" + +uvx twine upload --skip-existing "${upload_wheels[@]}" diff --git a/xvfb/build.sh b/xvfb/build.sh index 8913736..45dd9b7 100755 --- a/xvfb/build.sh +++ b/xvfb/build.sh @@ -43,9 +43,11 @@ fi rm -rf "$INSTALL_DIR" mkdir -p "$INSTALL_DIR"/{bin,lib,share/X11/xkb} -# Xvfb may live in /usr/bin (RPM/Debian) — just locate it. -XVFB_SRC="$(command -v Xvfb || true)" -XKBCOMP_SRC="$(command -v xkbcomp || true)" +# Prefer package-manager binaries; PATH may contain Python wrappers. +XVFB_SRC="/usr/bin/Xvfb" +[[ -x "$XVFB_SRC" ]] || XVFB_SRC="$(command -v Xvfb || true)" +XKBCOMP_SRC="/usr/bin/xkbcomp" +[[ -x "$XKBCOMP_SRC" ]] || XKBCOMP_SRC="$(command -v xkbcomp || true)" if [[ -z "$XVFB_SRC" || -z "$XKBCOMP_SRC" ]]; then echo "xvfb: Xvfb or xkbcomp not found after install" >&2 exit 1 diff --git a/xvfb/pyproject.toml b/xvfb/pyproject.toml index 3e810fa..fe7f909 100644 --- a/xvfb/pyproject.toml +++ b/xvfb/pyproject.toml @@ -3,10 +3,10 @@ requires = ["setuptools>=64", "wheel"] build-backend = "setuptools.build_meta" [project] -name = "xvfb" +name = "comma-deps-xvfb" version = "1.20.11.post1" description = "Xvfb (X virtual framebuffer) headless X server" -requires-python = ">=3.8" +requires-python = ">=3.12" [project.scripts] Xvfb = "xvfb:_run_xvfb" diff --git a/zeromq/pyproject.toml b/zeromq/pyproject.toml index ecbda65..4323905 100644 --- a/zeromq/pyproject.toml +++ b/zeromq/pyproject.toml @@ -3,10 +3,10 @@ requires = ["setuptools>=64", "wheel"] build-backend = "setuptools.build_meta" [project] -name = "zeromq" +name = "comma-deps-zeromq" version = "4.3.5" description = "ZeroMQ messaging library (static build)" -requires-python = ">=3.8" +requires-python = ">=3.12" [tool.setuptools.packages.find] include = ["zeromq*"] diff --git a/zstd/pyproject.toml b/zstd/pyproject.toml index f84702a..0c08d03 100644 --- a/zstd/pyproject.toml +++ b/zstd/pyproject.toml @@ -3,10 +3,10 @@ requires = ["setuptools>=64", "wheel"] build-backend = "setuptools.build_meta" [project] -name = "zstd" +name = "comma-deps-zstd" version = "1.5.6" description = "Zstandard compression library (static build)" -requires-python = ">=3.8" +requires-python = ">=3.12" [tool.setuptools.packages.find] include = ["zstd*"]