feat: pypi publishing and automation

.github/workflows/ci.yaml

@@ -2,54 +2,32 @@ name: CI
 
 on:
   push:
+    branches: [main]
+    tags: ['*']
   pull_request:
 
 permissions:
-  contents: write
+  contents: read
 
 jobs:
+  variants:
+    name: Read variant matrix from firmware/manifest.toml
+    runs-on: ubuntu-24.04
+    outputs:
+      matrix: ${{ steps.read.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v6
+      - uses: astral-sh/setup-uv@v8.1.0
+      - id: read
+        working-directory: pypi
+        run: uv run python -m tools.emit_ci_matrix >> "$GITHUB_OUTPUT"
+
   build:
-    name: Build ${{ matrix.config.board_name }} (${{ matrix.config.conf_name }})
+    needs: variants
+    name: Build ${{ matrix.board_name }} (${{ matrix.conf_name }})
     strategy:
       fail-fast: false
-      matrix:
-        config:
-          - board: nrf52840dk/nrf52840
-            board_name: nrf52840dk
-            path: ../zephyr/samples/bluetooth/hci_usb
-            conf_name: default
-            cmake_extra_args: ""
-            sysbuild: false
-            hex_file: zephyr/zephyr.hex
-          - board: nrf52840dk/nrf52840
-            board_name: nrf52840dk
-            path: ../zephyr/samples/bluetooth/hci_usb
-            conf_name: usbd_next
-            cmake_extra_args: "-DCONF_FILE=usbd_next_prj.conf"
-            sysbuild: false
-            hex_file: zephyr/zephyr.hex
-          - board: nrf52840dk/nrf52840
-            board_name: nrf52840dk
-            path: ../zephyr/samples/bluetooth/hci_usb
-            conf_name: ACL_502
-            cmake_extra_args: "-DEXTRA_CONF_FILE=$PWD/hci_usb/ACL_502.conf"
-            sysbuild: false
-            hex_file: zephyr/zephyr.hex
-          - board: nrf5340dk/nrf5340/cpuapp
-            board_name: nrf5340dk
-            path: ../zephyr/samples/bluetooth/hci_usb
-            conf_name: default
-            cmake_extra_args: ""
-            sysbuild: true
-            hex_file: merged.hex
-          - board: nrf5340dk/nrf5340/cpuapp
-            board_name: nrf5340dk
-            path: ../zephyr/samples/bluetooth/hci_usb
-            conf_name: ACL_502
-            cmake_extra_args: "-DEXTRA_CONF_FILE=$PWD/hci_usb/ACL_502_host.conf -Dhci_ipc_EXTRA_CONF_FILE=$PWD/hci_usb/ACL_502_netcore.conf"
-            sysbuild: true
-            hex_file: merged.hex
-
+      matrix: ${{ fromJson(needs.variants.outputs.matrix) }}
     runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v6
@@ -65,22 +43,22 @@ jobs:
         with:
           app-path: working_directory
           toolchains: arm-zephyr-eabi
-          ccache-cache-key: ${{ matrix.config.board }}-${{ matrix.config.conf_name }}-ccache
+          ccache-cache-key: ${{ matrix.board }}-${{ matrix.conf_name }}-ccache
 
       - working-directory: working_directory
         run: ./setup.sh
 
       - working-directory: working_directory
         run: >-
-          west build -b ${{ matrix.config.board }} ${{ matrix.config.path }}
-          ${{ matrix.config.sysbuild && '--sysbuild' || '' }}
-          -- ${{ matrix.config.cmake_extra_args }}
+          west build -b ${{ matrix.board }} ${{ matrix.path }}
+          ${{ matrix.sysbuild && '--sysbuild' || '' }}
+          -- ${{ matrix.cmake_extra_args }}
 
       - name: >-
           Merge hex files (workaround for
           https://github.com/zephyrproject-rtos/zephyr/discussions/102647,
           https://github.com/zephyrproject-rtos/zephyr/pull/83085)
-        if: matrix.config.sysbuild
+        if: matrix.sysbuild
         working-directory: working_directory
         run: >-
           python ../zephyr/scripts/build/mergehex.py
@@ -89,10 +67,11 @@ jobs:
           -o build/merged.hex
 
       - run: |
-          zephyr_version=$(west list zephyr --format='{revision}')
-          app_name=$(basename "${{ matrix.config.path }}")
-          fw_name="zephyr_${app_name}_${{ matrix.config.conf_name }}_${zephyr_version}_${{ matrix.config.board_name }}.hex"
-          cp "build/${{ matrix.config.hex_file }}" "build/${fw_name}"
+          zephyr_version=$(west list zephyr --format='{revision}') # e.g. v4.4.0
+          version_no_v="${zephyr_version#v}"
+          app_name=$(basename "${{ matrix.path }}")
+          fw_name="zephyr_${version_no_v}_${app_name}_${{ matrix.board_name }}_${{ matrix.conf_name }}.hex"
+          cp "build/${{ matrix.hex_file }}" "build/${fw_name}"
           echo "name=${fw_name}" >> "$GITHUB_OUTPUT"
         id: fw
         working-directory: working_directory
@@ -107,6 +86,8 @@ jobs:
     needs: build
     if: github.ref == 'refs/heads/main' && github.event_name == 'push'
     runs-on: ubuntu-24.04
+    permissions:
+      contents: write
     steps:
       - uses: actions/checkout@v6
 
@@ -126,3 +107,86 @@ jobs:
           make_latest: true
           target_commitish: ${{ github.sha }}
           files: artifacts/**/*.hex
+
+  camas_check:
+    name: Lint / mypy / pytest (camas)
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+      checks: write # GitHubChecks effect: per-leaf PR check entries
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: astral-sh/setup-uv@v8.1.0
+
+      - working-directory: pypi
+        run: uv sync
+
+      - name: camas check
+        working-directory: pypi
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          uv run camas check --effects='(
+            Summary(SummaryOptions(Fixed(100))),
+            GitHubChecks(GitHubChecksOptions(
+              sha="${{ github.event.pull_request.head.sha || github.sha }}",
+            )),
+          )'
+
+  publish_pypi:
+    needs: [build, camas_check]
+    if: github.ref_type == 'tag' && github.ref_name != 'latest'
+    runs-on: ubuntu-24.04
+    environment: pypi # optional manual-approval gate; remove this line if you skipped step C of the setup checklist
+    permissions:
+      contents: write # create / upload to the tag's GitHub release
+      id-token: write # Sigstore signing of PEP 740 attestations
+      attestations: write # PEP 740 attestation upload
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0 # hatch-vcs needs full tag history
+
+      - uses: astral-sh/setup-uv@v8.1.0
+
+      - uses: actions/download-artifact@v8
+        with:
+          path: artifacts
+          merge-multiple: true
+
+      - name: Resolve PyPI package version from git tag
+        id: ver
+        run: |
+          # Tag is bare semver (no 'v' prefix, per the project's SemVer-strict convention).
+          echo "version=${GITHUB_REF_NAME}" >> "$GITHUB_OUTPUT"
+          echo "Resolved PyPI version: ${GITHUB_REF_NAME}"
+
+      - name: Build wheels (verifies SHA against firmware/manifest.toml)
+        working-directory: pypi
+        run: |
+          uv run python -m tools.build_fw_packages \
+            --artifacts ../artifacts \
+            --out ../dist \
+            --version "${{ steps.ver.outputs.version }}" \
+            --manifest firmware/manifest.toml
+
+      - name: twine check
+        run: uv tool run --from twine twine check dist/*
+
+      - uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}
+          attestations: true
+          packages-dir: dist/
+
+      - name: Create / update the tag's GitHub release
+        run: |
+          gh release create "${GITHUB_REF_NAME}" \
+            --title "${GITHUB_REF_NAME}" \
+            --notes "Built from ${GITHUB_SHA}. PyPI version ${{ steps.ver.outputs.version }}." \
+            dist/*.whl dist/*.tar.gz artifacts/*.hex \
+          || gh release upload "${GITHUB_REF_NAME}" \
+            dist/*.whl dist/*.tar.gz artifacts/*.hex --clobber
+        env:
+          GH_TOKEN: ${{ github.token }}

.gitignore

@@ -1,2 +1,3 @@
 .venv
-build*
+build
+build-*

pypi/.gitignore

@@ -0,0 +1,4 @@
+__pycache__
+*.pyc
+dist
+*.egg-info

pypi/.pythonversion

@@ -0,0 +1 @@
+3.14

pypi/README.md

@@ -0,0 +1,5 @@
+# `pypi/`
+
+Tooling that publishes the Zephyr HCI firmware to PyPI.
+
+See [`camas`](https://pypi.org/project/camas/) for tasks: `uv run camas`.

pypi/firmware/manifest.toml

@@ -0,0 +1,52 @@
+zephyr_revision = "v4.4.0"
+app = "hci_usb"
+
+[[firmware]]
+board = "nrf52840dk"
+options = "default"
+board_path = "nrf52840dk/nrf52840"
+sample_path = "bluetooth/hci_usb"
+cmake_args = ""
+sysbuild = false
+hex_file = "zephyr/zephyr.hex"
+sha256 = "2c1796e8e1574f74080b0c6b14699612247f58d47f4b1b569c83d7559a8166c0"
+
+[[firmware]]
+board = "nrf52840dk"
+options = "acl_502"
+board_path = "nrf52840dk/nrf52840"
+sample_path = "bluetooth/hci_usb"
+cmake_args = "-DEXTRA_CONF_FILE={workspace}/hci_usb/ACL_502.conf"
+sysbuild = false
+hex_file = "zephyr/zephyr.hex"
+sha256 = "ae23562e9b8030f68972e0a196f9f9680a191622897ee19cabc31cb11ed4a01d"
+
+[[firmware]]
+board = "nrf52840dk"
+options = "legacy"
+board_path = "nrf52840dk/nrf52840"
+sample_path = "subsys/usb/legacy/hci_usb"
+cmake_args = ""
+sysbuild = false
+hex_file = "zephyr/zephyr.hex"
+sha256 = "954230a39c2f6a426e824a0db1eaa8aa98e8f6b15228ae94c9901815fd334b80"
+
+[[firmware]]
+board = "nrf5340dk"
+options = "default"
+board_path = "nrf5340dk/nrf5340/cpuapp"
+sample_path = "bluetooth/hci_usb"
+cmake_args = ""
+sysbuild = true
+hex_file = "merged.hex"
+sha256 = "6ebe341f018ccb6a8b7040001b86dbb7866edf32420736fdc1c3562c1aa979cf"
+
+[[firmware]]
+board = "nrf5340dk"
+options = "acl_502"
+board_path = "nrf5340dk/nrf5340/cpuapp"
+sample_path = "bluetooth/hci_usb"
+cmake_args = "-DEXTRA_CONF_FILE={workspace}/hci_usb/ACL_502_host.conf -Dhci_ipc_EXTRA_CONF_FILE={workspace}/hci_usb/ACL_502_netcore.conf"
+sysbuild = true
+hex_file = "merged.hex"
+sha256 = "43dbde167996bc9ed9beb295d9259157351a911e7cef9aa2c1dbd3812eecc48f"

pypi/pyproject.toml

@@ -0,0 +1,55 @@
+[project]
+name = "zephyr_hci_tools"
+description = "Build tooling for the intercreate/zephyr-hci PyPI packages. Not published."
+requires-python = ">=3.14"
+dynamic = ["version"]
+dependencies = ["hatchling", "hatch-vcs", "tomli-w"]
+
+[build-system]
+requires = ["hatchling", "hatch-vcs"]
+build-backend = "hatchling.build"
+
+[tool.hatch.version]
+source = "vcs"
+
+[tool.hatch.version.raw-options]
+root = ".."
+local_scheme = "no-local-version"
+
+[tool.hatch.build.targets.wheel]
+packages = ["tools"]
+
+[dependency-groups]
+dev = ["camas[github_checks]", "mypy", "ruff", "pytest", "twine", "build"]
+
+[tool.mypy]
+strict = true
+python_version = "3.14"
+files = ["tools", "tests", "tasks.py"]
+warn_unreachable = true
+enable_error_code = [
+"redundant-expr",
+"truthy-bool",
+"ignore-without-code",
+"unused-awaitable",
+]
+
+[tool.ruff]
+target-version = "py314"
+line-length = 120
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "B", "UP", "SIM", "RUF", "PL"]
+ignore = ["PLR0913"]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/*" = ["PLR2004"] # magic-value comparisons are fine in assertions
+
+[tool.ruff.format]
+quote-style = "double"
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+markers = ["slow: end-to-end tests that build venvs and install wheels (slow)"]
+
+# Tasks live in tasks.py (camas: 'tasks.py wins over pyproject.toml').

pypi/tasks.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+import tomllib
+from pathlib import Path
+
+from camas import Parallel, Sequential, Task
+
+_PYPI: Path = Path(__file__).resolve().parent
+_MANIFEST: Path = _PYPI / "firmware" / "manifest.toml"
+
+
+def _load_variant_tags() -> tuple[str, ...]:
+    data = tomllib.loads(_MANIFEST.read_text())
+    return tuple(f"{e['board']}:{e['options']}" for e in data["firmware"])
+
+
+_VARIANT_TAGS: tuple[str, ...] = _load_variant_tags()
+
+build = Parallel(
+    Task("python -m tools.build_firmware --variant {VARIANT}", cwd=_PYPI),
+    matrix={"VARIANT": _VARIANT_TAGS},
+    name="build",
+)
+
+_refresh_manifest = Task(
+    "python -m tools.update_manifest --artifacts ../build-hexes",
+    name="refresh_manifest",
+    cwd=_PYPI,
+)
+
+update = Sequential(build, _refresh_manifest, name="update")
+
+update_manifest = Task("python -m tools.update_manifest", cwd=_PYPI)
+
+wheels = Task(
+    "python -m tools.build_fw_packages --artifacts ../build-hexes --out ../dist --version 0.0.0.dev0",
+    name="wheels",
+    cwd=_PYPI,
+)
+
+format_check = Task("ruff format --check .", cwd=_PYPI)
+lint = Task("ruff check .", cwd=_PYPI)
+mypy = Task("mypy tools tests tasks.py", cwd=_PYPI)
+test = Task("pytest", cwd=_PYPI)
+check = Parallel(format_check, lint, mypy, test, name="check")