try pypi publish

.github/workflows/ci.yml

@@ -24,6 +24,8 @@ jobs:
             runs-on-namespace: namespace-profile-macos-8x14
     steps:
       - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
       - name: Select Xcode 16
         if: runner.os == 'macOS'
         run: |
@@ -62,12 +64,10 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - image: debian:bookworm-slim
+          - image: debian:trixie-slim
             runs-on: ubuntu-latest
           - image: ubuntu:24.04
             runs-on: ubuntu-latest
-          - image: ubuntu:20.04
-            runs-on: ubuntu-latest
           - image: fedora:41
             runs-on: ubuntu-latest
           - image: archlinux:latest
@@ -76,7 +76,7 @@ jobs:
             runs-on: ubuntu-latest
           - image: ghcr.io/void-linux/void-glibc:latest
             runs-on: ubuntu-latest
-          - image: debian:bookworm-slim
+          - image: debian:trixie-slim
             runs-on: ubuntu-24.04-arm
           # TODO: add musl test
     steps:
@@ -92,25 +92,15 @@ jobs:
     if: github.event_name == 'push' && github.ref == 'refs/heads/master'
     needs: [build, test]
     runs-on: ubuntu-latest
-    permissions:
-      contents: write
     steps:
       - uses: actions/checkout@v6
       - uses: actions/download-artifact@v4
         with:
           pattern: wheels-*
           merge-multiple: true
           path: dist
-      - uses: actions/cache/restore@v4
-        with:
-          key: build-cache-v2-${{ matrix.platform }}-${{ hashFiles('build.sh', 'setup.sh', '*/pyproject.toml', '*/setup.py', '*/build.sh') }}
-          restore-keys: build-cache-v2-${{ matrix.platform }}-
-          path: |
-            .uv-cache
-            */*/install
-            */*/toolchain
-            */*/bin
-      - name: Publish wheels and shims
+      - name: Publish wheels to PyPI
         env:
-          GH_TOKEN: ${{ github.token }}
-        run: ./release.sh --publish-only
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_PAT }}
+        run: ./release.sh

README.md

@@ -40,22 +40,36 @@ contributions welcome for other platforms!
 
 ## usage
 
+pre-built wheels are published to PyPI under the `comma-deps-` prefix. the import name
+is unchanged (e.g. `import capnproto`), only the distribution name is prefixed.
+packages require Python 3.12 or newer.
+
 ```python
 dependencies = [
-  # use per-package release branches for pre-built wheels
-  "capnproto @ git+https://github.com/commaai/dependencies.git@release-capnproto#subdirectory=capnproto",
-  "ffmpeg @ git+https://github.com/commaai/dependencies.git@release-ffmpeg#subdirectory=ffmpeg",
+  "comma-deps-capnproto>=1.0.1,<1.0.2",
+  "comma-deps-ffmpeg>=7.1.0,<7.1.1",
+]
+```
+
+to build from source instead, point at the master branch of this repo:
 
-  # use the master branch to build the package on pip install
-  "capnproto @ git+https://github.com/commaai/dependencies.git@master#subdirectory=capnproto",
-  "ffmpeg @ git+https://github.com/commaai/dependencies.git@master#subdirectory=ffmpeg",
+```python
+dependencies = [
+  "comma-deps-capnproto @ git+https://github.com/commaai/dependencies.git@master#subdirectory=capnproto",
+  "comma-deps-ffmpeg @ git+https://github.com/commaai/dependencies.git@master#subdirectory=ffmpeg",
 ]
 ```
 
 ## workflow
 
 to add a new package:
 * start a new top-level directory as a new package
-* `./test.sh` tests the building of all packages
-*  on pushes to `master`, wheels are built for our target platforms and pushed to a GitHub release
-*  each `release-<package>` branch contains a single shim package, so old lockfiles keep resolving even as new packages are added
+* `./build.sh` builds all packages, `./test_wheels_in_image.sh` tests the built wheels in a range of distros
+*  on pushes to `master`, wheels are built for our target platforms, tested, and published to PyPI
+
+> [!NOTE]
+> PyPI does not allow overwriting an uploaded file. each build publishes
+> versions as `<package version>.postN`, where `N` is `git rev-list --count HEAD`
+> for the commit being built. package `pyproject.toml` files keep the upstream
+> base version; `./build.sh` applies the `.postN` suffix only while building
+> wheels. existing `.postM` suffixes are replaced by the build suffix.

acados/pyproject.toml

@@ -3,10 +3,10 @@ requires = ["setuptools>=64", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [project]
-name = "acados"
+name = "comma-deps-acados"
 version = "0.2.2"
 description = "acados solvers + Python interface, with a slimmed casadi vendored alongside"
-requires-python = ">=3.8"
+requires-python = ">=3.12"
 dependencies = [
   # the vendored casadi.py shim imports numpy at module load time
   "numpy",

bootstrap-icons/pyproject.toml

@@ -3,10 +3,10 @@ requires = ["setuptools>=64", "wheel", "fonttools"]
 build-backend = "setuptools.build_meta"
 
 [project]
-name = "bootstrap-icons"
+name = "comma-deps-bootstrap-icons"
 version = "1.10.5.0"
 description = "Bootstrap Icons SVG sprite and TTF font"
-requires-python = ">=3.8"
+requires-python = ">=3.12"
 
 [tool.setuptools.packages.find]
 include = ["bootstrap_icons*"]

build.sh

@@ -62,12 +62,23 @@ fi
 export CMAKE_C_COMPILER_LAUNCHER=ccache
 export CMAKE_CXX_COMPILER_LAUNCHER=ccache
 
+restore_build_versions() {
+  python3 build_versions.py restore */pyproject.toml
+}
+
+trap restore_build_versions EXIT
+
 echo "Building workspace packages into dist"
 START_SECS=$SECONDS
 
 mkdir -p dist/
 rm -rf dist/*
+
+POST_N="${DEPENDENCIES_POST_VERSION_N:-$(git rev-list --count HEAD)}"
+echo "Applying .post$POST_N package versions for this build"
+python3 build_versions.py apply "$POST_N" */pyproject.toml
 uv build --all-packages --wheel --out-dir dist --no-create-gitignore
+restore_build_versions
 
 if [[ -n "${BUILD_SH_IN_MANYLINUX:-}" ]]; then
   VENV_DIR="$ROOT_DIR/.venv-manylinux"

build_versions.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+import os
+import re
+import shutil
+import sys
+
+BACKUP_SUFFIX = ".build-version.bak"
+VERSION_RE = re.compile(r'^(?P<prefix>\s*version\s*=\s*")(?P<version>[^"]+)(?P<suffix>".*)$')
+
+
+def post_version(version: str, post_n: str) -> str:
+  base = re.sub(r"\.post\d+$", "", version)
+  if not re.fullmatch(r"\d+(?:\.\d+)*(?:(?:a|b|rc)\d+)?", base):
+    raise ValueError(f"unsupported version: {version}")
+  return f"{base}.post{post_n}"
+
+
+def apply(post_n: str, paths: list[str]) -> None:
+  if not post_n.isdigit():
+    raise ValueError("post number must be a non-negative integer")
+
+  for path in paths:
+    shutil.copy2(path, path + BACKUP_SUFFIX)
+    lines = open(path, encoding="utf-8").read().splitlines(keepends=True)
+    in_project = changed = False
+
+    for i, line in enumerate(lines):
+      stripped = line.strip()
+      if stripped.startswith("[") and stripped.endswith("]"):
+        in_project = stripped == "[project]"
+      if in_project and (match := VERSION_RE.match(line)):
+        lines[i] = f"{match['prefix']}{post_version(match['version'], post_n)}{match['suffix']}\n"
+        changed = True
+
+    if not changed:
+      raise RuntimeError(f"no [project] version found in {path}")
+
+    open(path, "w", encoding="utf-8").writelines(lines)
+
+
+def restore(paths: list[str]) -> None:
+  for path in paths:
+    backup = path + BACKUP_SUFFIX
+    if os.path.exists(backup):
+      os.replace(backup, path)
+
+
+if __name__ == "__main__":
+  if len(sys.argv) < 3 or sys.argv[1] not in {"apply", "restore"}:
+    raise SystemExit("usage: build_versions.py apply POST_N FILE... | restore FILE...")
+  if sys.argv[1] == "apply":
+    apply(sys.argv[2], sys.argv[3:])
+  else:
+    restore(sys.argv[2:])

bzip2/pyproject.toml

@@ -3,10 +3,10 @@ requires = ["setuptools>=64", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [project]
-name = "bzip2"
+name = "comma-deps-bzip2"
 version = "1.0.8"
 description = "bzip2 compression library (static build)"
-requires-python = ">=3.8"
+requires-python = ">=3.12"
 
 [tool.setuptools.packages.find]
 include = ["bzip2*"]

capnproto/capnproto/__init__.py

@@ -1,4 +1,5 @@
 import os
+import tempfile
 import sys
 
 DIR = os.path.join(os.path.dirname(__file__), "install")
@@ -42,3 +43,9 @@ def smoketest():
   subprocess.run([capnp, "--version"], check=True, env=env)
   subprocess.run([capnpc, "--version"], check=True, env=env)
   subprocess.run([capnpc_cpp, "--version"], check=True, env=env)
+  with tempfile.TemporaryDirectory() as temp_dir:
+    schema = os.path.join(temp_dir, "test.capnp")
+    with open(schema, "w") as f:
+      f.write("@0xd12f9faaa6e3fdec; struct Test { value @0 :UInt32; }\n")
+    env["PWD"] = temp_dir
+    subprocess.run([capnp, "compile", "-oc++", schema], cwd=temp_dir, check=True, env=env)

capnproto/pyproject.toml

@@ -3,15 +3,14 @@ requires = ["setuptools>=64", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [project]
-name = "capnproto"
+name = "comma-deps-capnproto"
 version = "1.0.1"
 description = "Cap'n Proto serialization library and compiler"
-requires-python = ">=3.8"
+requires-python = ">=3.12"
 
 [project.scripts]
 capnp = "capnproto:_run_capnp"
 capnpc = "capnproto:_run_capnpc"
-"capnpc-c++" = "capnproto:_run_capnpc_cpp"
 
 [tool.setuptools.packages.find]
 include = ["capnproto*"]

catch2/pyproject.toml

@@ -3,10 +3,10 @@ requires = ["setuptools>=64", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [project]
-name = "catch2"
+name = "comma-deps-catch2"
 version = "2.13.10"
 description = "Catch2 C++ test framework headers"
-requires-python = ">=3.8"
+requires-python = ">=3.12"
 
 [tool.setuptools.packages.find]
 include = ["catch2*"]

cppcheck/pyproject.toml

@@ -3,10 +3,10 @@ requires = ["setuptools>=64", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [project]
-name = "cppcheck"
+name = "comma-deps-cppcheck"
 version = "2.21.0"
 description = "Cppcheck static analysis tool"
-requires-python = ">=3.8"
+requires-python = ">=3.12"
 
 [project.scripts]
 cppcheck = "cppcheck:_run"