diff --git a/.github/actions/compute-version/action.yml b/.github/actions/compute-version/action.yml new file mode 100644 index 0000000..8cc34a1 --- /dev/null +++ b/.github/actions/compute-version/action.yml @@ -0,0 +1,25 @@ +name: Compute release version +description: > + Compute the next OE5XRX linux-image release version (YYYY.MM.DD-HH[a-z], UTC + hour, letter suffix on collision) from existing git tags. Mirrors the FW + one-button-release pattern. + +outputs: + version: + description: The computed release version. + value: ${{ steps.compute.outputs.version }} + +runs: + using: composite + steps: + - name: Fetch tags + shell: bash + run: git fetch --tags --quiet + + - name: Compute + id: compute + shell: bash + run: | + version="$(scripts/compute-release-version.sh)" + echo "Computed release version: ${version}" + echo "version=${version}" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/boot-ota-pr.yml b/.github/workflows/boot-ota-pr.yml new file mode 100644 index 0000000..2971454 --- /dev/null +++ b/.github/workflows/boot-ota-pr.yml @@ -0,0 +1,71 @@ +name: Boot/OTA (boot-critical PRs) + +# Shift-left: on PRs that touch boot/OTA-critical paths, build the x86 image and +# run BOTH the boot smoke (T1) and the cross-build OTA cycle (T2) — slot A = the +# latest published release, slot B = this PR's build. The release gate runs the +# same test again as the publish barrier. If no release exists yet (bootstrap), +# T2 skips gracefully and only T1 runs. + +# Draft PRs do NOT trigger the (expensive) Hetzner build + QEMU test — so +# Copilot-loop / WIP pushes on a draft don't re-run everything. Mark the PR +# ready for review to run it (that also fires `ready_for_review`). +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + paths: + - 'meta-oe5xrx-remotestation/wic/**' + - 'meta-oe5xrx-remotestation/recipes-bsp/**' + - 'meta-oe5xrx-remotestation/recipes-core/ab-layout/**' + - 'meta-oe5xrx-remotestation/recipes-core/images/**' + - 'meta-oe5xrx-remotestation/recipes-core/station-agent/**' + - 'tests/ota-integration/**' + - '.github/workflows/boot-ota-test.yml' + - '.github/workflows/boot-ota-pr.yml' + +permissions: + contents: read + actions: read # download-artifact from the build job in the same run + +concurrency: + group: boot-ota-pr-${{ github.ref }} + cancel-in-progress: true + +jobs: + resolve-slot-a: + name: Resolve slot A (last release) + if: ${{ !github.event.pull_request.draft }} + runs-on: ubuntu-latest + outputs: + tag: ${{ steps.last.outputs.tag }} + steps: + - name: Latest published release + id: last + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + tag=$(gh release list -R "$REPO" -L 1 --exclude-drafts --exclude-pre-releases --json tagName -q '.[0].tagName' || true) + echo "Slot A (last release): ${tag:-}" + echo "tag=$tag" >> "$GITHUB_OUTPUT" + + build-x64: + name: Build qemux86-64 (dev) + if: ${{ !github.event.pull_request.draft }} + uses: ./.github/workflows/build.yml + with: + machine: qemux86-64 + release_tag: "" + secrets: inherit + + test: + name: Boot + cross-build OTA (T1+T2) + if: ${{ !github.event.pull_request.draft }} + needs: [build-x64, resolve-slot-a] + uses: ./.github/workflows/boot-ota-test.yml + with: + machine: qemux86-64 + new_artifact: yocto-image-qemux86-64 + expected_tag: dev + last_release_tag: ${{ needs.resolve-slot-a.outputs.tag }} + secrets: inherit diff --git a/.github/workflows/boot-ota-spike.yml b/.github/workflows/boot-ota-spike.yml new file mode 100644 index 0000000..ec4fa5f --- /dev/null +++ b/.github/workflows/boot-ota-spike.yml @@ -0,0 +1,89 @@ +name: Boot/OTA Feasibility Spike + +# One-off decision aid: does a GH-hosted ubuntu-latest runner boot the image +# under TCG (no KVM) in acceptable wall-clock and disk? Boots the latest (or a +# given) release via the T1 harness and reports timing + RAM + disk headroom. +# If this is too slow / disk-tight, flip boot-ota-test.yml's runner_label to a +# self-hosted Hetzner label. + +on: + workflow_dispatch: + inputs: + release_tag: + description: Release to boot (empty → latest). + type: string + default: "" + +permissions: + contents: read # gh release download + +jobs: + spike: + name: TCG boot timing + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v6 + + - name: Host capacity (before) + run: | + echo "== nproc =="; nproc + echo "== free -m =="; free -m + echo "== df -h / =="; df -h / + echo "== /dev/kvm present? =="; ls -l /dev/kvm || echo "no KVM (expected → TCG)" + + - name: Install QEMU + OVMF + deps + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends qemu-system-x86 ovmf grub-common grub2-common python3-pip + sudo python3 -m pip install --break-system-packages -r tests/ota-integration/requirements.txt + + - name: Resolve release tag + id: tag + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + REQ_TAG: ${{ inputs.release_tag }} + run: | + set -euo pipefail + tag="$REQ_TAG" + if [ -z "$tag" ]; then + tag=$(gh release list -R "$REPO" -L 1 --exclude-drafts --exclude-pre-releases --json tagName -q '.[0].tagName') + fi + echo "tag=$tag" >> "$GITHUB_OUTPUT" + + - name: Download release wic + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + TAG: ${{ steps.tag.outputs.tag }} + run: | + set -euo pipefail + gh release download "$TAG" -R "$REPO" \ + --pattern "*qemux86-64*.wic.bz2" \ + --pattern "*qemux86-64*.wic.xz" \ + --pattern "*qemux86-64*.wic.gz" \ + --pattern "*qemux86-64*.wic" \ + -D spike + + - name: Timed TCG boot (T1) + env: + OTA_IT_WIC: spike + OTA_IT_EXPECTED_TAG: ${{ steps.tag.outputs.tag }} + run: | + set -euo pipefail + shopt -s nullglob + cands=(spike/*.wic.bz2 spike/*.wic.xz spike/*.wic.gz spike/*.wic) + OTA_IT_WIC="${cands[0]:-}" + [ -n "$OTA_IT_WIC" ] || { echo "no wic asset downloaded" >&2; exit 1; } + export OTA_IT_WIC + start=$(date +%s) + sudo -E env "PATH=$PATH" \ + python3 -m pytest tests/ota-integration/test_boot.py -m qemu -v + end=$(date +%s) + echo "T1 boot wall-clock: $((end - start))s" + + - name: Host capacity (after) + run: | + echo "== free -m =="; free -m + echo "== df -h / =="; df -h / diff --git a/.github/workflows/boot-ota-test.yml b/.github/workflows/boot-ota-test.yml new file mode 100644 index 0000000..dbd74ef --- /dev/null +++ b/.github/workflows/boot-ota-test.yml @@ -0,0 +1,98 @@ +name: Boot & OTA Test + +# Reusable boot + cross-build OTA integration test. Boots the built image in +# QEMU (TCG), drives the real station-agent against a dummy OTA server, and +# asserts a serial banner+login AND an agent check-in/commit. Called by the +# boot-critical PR workflow (T1) and by the release gate (T1+T2). +# +# Runner is a parameter: default GH-hosted ubuntu-latest (TCG). If a feasibility +# spike shows GH-hosted is insufficient, pass a self-hosted Hetzner label. + +on: + workflow_call: + inputs: + machine: + type: string + default: qemux86-64 + runner_label: + type: string + default: ubuntu-latest + new_artifact: + description: Artifact name of the build-under-test wic (same run). + type: string + required: true + expected_tag: + description: Version stamped into the build-under-test (banner + os-release). + type: string + required: true + last_release_tag: + description: Previous release tag = slot A for T2 (empty → T1 only). + type: string + default: "" + +jobs: + test: + name: Boot & OTA (${{ inputs.machine }}) + runs-on: ${{ inputs.runner_label }} + timeout-minutes: 60 + steps: + - uses: actions/checkout@v6 + + - name: Install QEMU + OVMF + deps + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends qemu-system-x86 ovmf grub-common grub2-common python3-pip + sudo python3 -m pip install --break-system-packages -r tests/ota-integration/requirements.txt + + - name: Download build-under-test artifact + uses: actions/download-artifact@v8 + with: + name: ${{ inputs.new_artifact }} + path: it-artifacts + + - name: Locate build-under-test wic + id: wic + run: | + set -euo pipefail + shopt -s nullglob + # Multiple patterns via an array so a non-matching glob can't fail the + # step under `set -e` when another pattern does match. + cands=(it-artifacts/*.wic.bz2 it-artifacts/*.wic.xz it-artifacts/*.wic.gz it-artifacts/*.rootfs.wic it-artifacts/*.wic) + wic="${cands[0]:-}" + [ -n "$wic" ] || { echo "no wic artifact found" >&2; exit 1; } + echo "wic=$wic" >> "$GITHUB_OUTPUT" + + - name: Fetch last release (slot A) + id: slota + if: inputs.last_release_tag != '' + env: + GH_TOKEN: ${{ github.token }} + LAST_RELEASE_TAG: ${{ inputs.last_release_tag }} + MACHINE: ${{ inputs.machine }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + # Accept the same compression set release.yml may publish, in the same + # extension order; sidecars (.sha256/.bundle) don't match these globs. + gh release download "$LAST_RELEASE_TAG" -R "$REPO" \ + --pattern "*${MACHINE}*.wic.bz2" \ + --pattern "*${MACHINE}*.wic.xz" \ + --pattern "*${MACHINE}*.wic.gz" \ + --pattern "*${MACHINE}*.wic" \ + -D slota + shopt -s nullglob + cands=(slota/*.wic.bz2 slota/*.wic.xz slota/*.wic.gz slota/*.wic) + wic="${cands[0]:-}" + # Fail fast: last_release_tag was set, so a missing slot-A asset must + # NOT silently empty the output and make T2 skip (a false-green gate). + [ -n "$wic" ] || { echo "last_release_tag='$LAST_RELEASE_TAG' set but no slot-A wic asset was downloaded" >&2; exit 1; } + echo "wic=$wic" >> "$GITHUB_OUTPUT" + + - name: Run boot + OTA integration tests + env: + OTA_IT_WIC: ${{ steps.wic.outputs.wic }} + OTA_IT_LAST_RELEASE_WIC: ${{ steps.slota.outputs.wic }} + OTA_IT_EXPECTED_TAG: ${{ inputs.expected_tag }} + run: | + sudo -E env "PATH=$PATH" \ + python3 -m pytest tests/ota-integration -m qemu -v diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9f5ddb2..3ffc49f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,6 +36,13 @@ jobs: run: | find meta-oe5xrx-remotestation -name '*.sh' -print0 \ | xargs -0 -r shellcheck -e SC1091 -e SC2039 + # New scripts added by the boot/OTA test work (lint explicitly rather + # than the whole scripts/ dir, which carries pre-existing intentional + # warnings, e.g. run-qemu.sh's word-splitting ${CPU_FLAGS}). + shellcheck -e SC1091 -e SC2039 \ + scripts/l0a-fstab-uuid-lint.sh \ + scripts/compute-release-version.sh \ + scripts/release.sh - name: Lint YAML run: | @@ -53,6 +60,16 @@ jobs: # udev is installed in the "Install kas + linters" step above. run: udevadm verify meta-oe5xrx-remotestation/recipes-core/oe5xrx-slot-udev/files/90-oe5xrx-slots.rules + - name: L0a — static fstab UUID-mount guard + # Fast counterpart to the L0b ROOTFS_POSTPROCESS assertion: flag any + # wks --use-uuid mount without --no-fstab-update (the #37 bug class). + run: scripts/l0a-fstab-uuid-lint.sh . + + - name: OTA-integration unit tests + run: | + pip install --quiet -r tests/ota-integration/requirements.txt + python -m pytest tests/ota-integration -m unit -q + sim-harness: runs-on: ubuntu-latest steps: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c402292..d5bf4d9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,39 +1,36 @@ name: Release -# Runs on release-tag push. Builds BOTH images (x86-64 + Raspberry Pi), -# checksums them, signs the bundles with cosign keyless (Sigstore + OIDC), -# and publishes a GitHub Release with everything attached. +# One-button release (workflow_dispatch). The workflow computes its own next +# version (YYYY.MM.DD-HH[a-z], UTC hour, letter suffix on collision), builds +# BOTH images, GATES publication on the boot + cross-build OTA integration +# test (x86), signs the bundles with cosign keyless, and publishes a GitHub +# Release. The action creates the tag from the computed version — there is no +# tag-push trigger anymore (mirrors FW-RemoteStation). # -# Tag format is YYYY.MM.DD-HH (UTC), with an optional single lowercase -# letter [a-z] suffix for same-hour hotfixes. See scripts/release.sh. -# -# The legacy v* tags (v1-alpha ... v1-delta) stay on GitHub as -# historical releases — run-qemu.sh can still fetch them — but pushing -# a new v* tag will NOT trigger a build. Only the timestamp format is -# accepted going forward. +# dry_run=true: run preflight + version + build + gate, but do NOT sign, tag, +# or publish (use it to validate the pipeline from a feature branch). + on: - push: - tags: - # YYYY.MM.DD-HH (no suffix) — the common case - - '[0-9][0-9][0-9][0-9].[0-9][0-9].[0-9][0-9]-[0-9][0-9]' - # YYYY.MM.DD-HH[a-z] — same-hour hotfix - - '[0-9][0-9][0-9][0-9].[0-9][0-9].[0-9][0-9]-[0-9][0-9][a-z]' + workflow_dispatch: + inputs: + dry_run: + description: "Build + gate only; do not sign/tag/publish" + type: boolean + default: false permissions: - contents: write # create release, upload assets + contents: write # create release + tag, upload assets id-token: write # cosign keyless OIDC token actions: read # download-artifact from called workflows concurrency: - group: release-${{ github.ref }} + group: release cancel-in-progress: false jobs: - # Cheap preflight gate: makes the build fail within seconds (no Yocto - # spin-up) if the station-agent recipe still uses ${AUTOREV}. On a - # release tag we require a pinned SHA so the image is reproducible — - # otherwise a later rebuild of the same tag could silently pick up a - # different agent revision. See scripts/pin-station-agent.sh. + # Cheap preflight gate: fails within seconds (no Yocto spin-up) if the + # station-agent recipe still uses ${AUTOREV}, or FM artifacts are unpinned / + # unsigned. A release must be reproducible. preflight: name: Preflight — require pinned SRCREV runs-on: ubuntu-latest @@ -57,20 +54,11 @@ jobs: fi echo "Found: $line" if printf '%s' "$line" | grep -qF '${AUTOREV}'; then - echo "::error file=$recipe::SRCREV is \${AUTOREV} on a release tag — run scripts/pin-station-agent.sh before tagging" + echo "::error file=$recipe::SRCREV is \${AUTOREV} — run scripts/pin-station-agent.sh before releasing" exit 1 fi - # Strip the `SRCREV =` prefix, then peel off surrounding - # quotes — accept both " and ' since BitBake allows either. - # Using sed -n+p so a non-match yields empty output and the - # 40-char regex check below surfaces a clear error rather - # than the whole unparsed line. raw=$(printf '%s\n' "$line" | sed -nE 's/^[[:space:]]*SRCREV[[:space:]]*=[[:space:]]*(.*)$/\1/p') sha=$(printf '%s' "$raw" | sed -E 's/^"([^"]*)".*$/\1/; s/^'"'"'([^'"'"']*)'"'"'.*$/\1/') - # Git accepts mixed-case hex for commit IDs (the Yocto fetcher - # doesn't care either). Reject anything that isn't exactly 40 - # hex chars, but allow upper/lowercase — otherwise a human-pasted - # SHA from GitHub's UI could fail preflight despite being valid. if ! printf '%s' "$sha" | grep -Eq '^[0-9a-fA-F]{40}$'; then echo "::error file=$recipe::SRCREV value '$sha' is not a full 40-char SHA" exit 1 @@ -83,8 +71,6 @@ jobs: - name: Fail if FM artifacts are unpinned or unsigned run: | set -euo pipefail - # FW-RemoteStation signs releases from refs/heads/main (workflow_dispatch); - # pin the identity to that exact ref+workflow rather than a wildcard @refs/.+ . id_re='^https://github.com/OE5XRX/FW-RemoteStation/\.github/workflows/release\.yml@refs/heads/main$' issuer='https://token.actions.githubusercontent.com' recipes=" @@ -116,29 +102,84 @@ jobs: rm -rf "$tmp" done + # Compute the next version + resolve the previous release (slot A for the + # gate's cross-build OTA test). Since nothing is tagged yet, the latest + # existing release IS the previous one. + version: + name: Compute version + runs-on: ubuntu-latest + outputs: + version: ${{ steps.cv.outputs.version }} + last_release: ${{ steps.last.outputs.tag }} + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Guard — real releases only on the default branch + if: ${{ !inputs.dry_run }} + env: + REF: ${{ github.ref_name }} + DEFAULT: ${{ github.event.repository.default_branch }} + run: | + if [ "$REF" != "$DEFAULT" ]; then + echo "::error::real release must run on '$DEFAULT'; use dry_run=true on a feature branch" >&2 + exit 1 + fi + + - name: Compute next version + id: cv + uses: ./.github/actions/compute-version + + - name: Resolve previous release (slot A) + id: last + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + tag=$(gh release list -R "$REPO" -L 1 --exclude-drafts --exclude-pre-releases --json tagName -q '.[0].tagName' || true) + echo "Previous release: ${tag:-}" + echo "tag=$tag" >> "$GITHUB_OUTPUT" + build-x64: name: Build qemux86-64 - needs: preflight + needs: [preflight, version] uses: ./.github/workflows/build.yml with: machine: qemux86-64 - release_tag: ${{ github.ref_name }} + release_tag: ${{ needs.version.outputs.version }} secrets: inherit build-rpi: name: Build raspberrypi4-64 # Each target has its own cache volume, so the two builds run in - # parallel. Both still gate on preflight. - needs: preflight + # parallel. Both still gate on preflight + version. + needs: [preflight, version] uses: ./.github/workflows/build.yml with: machine: raspberrypi4-64 - release_tag: ${{ github.ref_name }} + release_tag: ${{ needs.version.outputs.version }} + secrets: inherit + + gate: + name: Boot & OTA gate (x64) + # The publish gate: boot the freshly built x86 image and run the + # cross-build OTA cycle (slot A = previous release). Publication depends + # on this passing — the two shipped OTA bugs (#36, #37) would fail here. + needs: [version, build-x64] + uses: ./.github/workflows/boot-ota-test.yml + with: + machine: qemux86-64 + new_artifact: yocto-image-qemux86-64 + expected_tag: ${{ needs.version.outputs.version }} + last_release_tag: ${{ needs.version.outputs.last_release }} secrets: inherit release: name: Sign & Publish Release - needs: [build-x64, build-rpi] + needs: [version, build-x64, build-rpi, gate] + if: ${{ !inputs.dry_run }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 @@ -160,13 +201,11 @@ jobs: - name: Package, checksum, sign env: - TAG: ${{ github.ref_name }} + TAG: ${{ needs.version.outputs.version }} run: | set -euo pipefail mkdir -p release for machine in qemux86-64 raspberrypi4-64; do - # Prefer compressed wic, fall back to raw wic. CI uploads both - # if the image recipe emits them — we only ship one. src="" srcext="" for ext in wic.bz2 wic.xz wic.gz wic; do @@ -186,9 +225,6 @@ jobs: cp "${src}" "${dest}" (cd release && sha256sum "$(basename "${dest}")" > "$(basename "${dest}").sha256") - # Keyless signing — binds the signature to this repo + workflow + - # tag via the GitHub Actions OIDC token. Verifiable later with - # `cosign verify-blob` (see SECURITY.md for the exact command). cosign sign-blob --yes \ --bundle "${dest}.bundle" \ "${dest}" @@ -197,10 +233,12 @@ jobs: - name: Create GitHub Release env: - TAG: ${{ github.ref_name }} + TAG: ${{ needs.version.outputs.version }} REPO: ${{ github.repository }} uses: softprops/action-gh-release@v3 with: + tag_name: ${{ needs.version.outputs.version }} + target_commitish: ${{ github.sha }} files: | release/oe5xrx-* generate_release_notes: true @@ -209,7 +247,8 @@ jobs: body: | ## OE5XRX Linux Image ${{ env.TAG }} - Pre-built Yocto images for both supported targets. + Pre-built Yocto images for both supported targets. Published only + after passing the boot + cross-build OTA integration gate. | Target | File | |---|---| diff --git a/docs/superpowers/plans/2026-07-11-boot-ota-integration-test.md b/docs/superpowers/plans/2026-07-11-boot-ota-integration-test.md new file mode 100644 index 0000000..cbbb96a --- /dev/null +++ b/docs/superpowers/plans/2026-07-11-boot-ota-integration-test.md @@ -0,0 +1,489 @@ +# Boot & OTA Integration Test — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** A QEMU boot + cross-build A/B OTA integration test for the OE5XRX Linux image that would have caught the two shipped OTA bugs (#36 fs-label, #37 ESP-by-UUID), plus build-time guards and a reworked one-button release gated on the test. + +**Architecture:** A standalone stdlib **dummy OTA server** speaks the agent's real protocol; a **`Target`** abstraction (QemuTarget now, Cm4Target future) boots the image and reaches its console; **pytest+pexpect** tests assert both a serial banner+login AND an agent check-in/commit. Cheap static/build guards catch the UUID-mount class. The release workflow becomes `workflow_dispatch` with a self-computed version, gated on the test. + +**Tech Stack:** Python 3 (stdlib `http.server`, `bz2`, `hashlib`; pytest; pexpect; pyyaml; `cryptography` for Ed25519), QEMU (`qemu-system-x86_64`, OVMF, TCG), GitHub Actions (reusable workflows + composite action), BitBake/wic (Yocto scarthgap). + +## Global Constraints + +- **Repo:** all work in `/home/pbuchegger/OE5XRX/linux-image` on branch `test/boot-ota-integration` (already checked out; spec already committed). +- **Test package root:** `tests/ota-integration/` (mirrors existing `tests/sim-harness/`). +- **No new heavy deps:** dummy server = stdlib only. Harness may use `pytest`, `pexpect`, `pyyaml`, `cryptography` (all pip-installable in CI). +- **Condition-based waits only** in the harness (pexpect `expect`, never fixed `sleep`), generous TCG timeouts. +- **Image stays unmodified** — only the data partition carries test config (`Target.seed_config`). +- **Runner is a parameter** — `boot-ota-test.yml` takes `runner_label`, default GH-hosted `ubuntu-latest`. +- **Cross-build is mandatory for T2** — slot A = last published release, slot B = the build under test. Never same-build (would hide the UUID bug). +- **Commit style:** end commit messages with `Co-Authored-By: Claude Opus 4.8 `. Do NOT merge to main (user reviews the PR). + +## Protocol Reference (ground truth — the dummy server MUST match) + +Agent lives in `station-manager/station_agent/`; contract verified against it and `station-manager/apps/`. + +**Auth (agent → server, every request):** headers `Authorization: DeviceKey `, `X-Device-Signature: `, `X-Device-Timestamp: `. **The agent never validates server responses beyond `status_code`.** → the dummy IGNORES all auth headers and just returns the bodies below. + +**Endpoints (all under `/api/v1/`):** + +1. `POST /heartbeat/` — req body has `hostname, os_version (=PRETTY_NAME), uptime, ip_address, agent_version, module_versions, inventory, timestamp`. Resp: **200** `{"status":"ok"}`. Agent reads only the status code. +2. `POST /deployments/check/` — req `{"current_version": ""}`. Resp: **204** = no update; or **200** with: + ```json + {"deployment_result_id": 1, "deployment_id": 1, "deployment_result_status": "pending", + "target_tag": "2026.07.11-15", "checksum_sha256": "<64hex>", "size_bytes": 12345, + "download_url": "/api/v1/deployments/1/download/"} + ``` + `deployment_result_id` is used in the status path; `deployment_id` in the download_url; agent uses `download_url` **verbatim**. To trigger the post-reboot branch, return `deployment_result_status` = `"rebooting"` or `"verifying"` (agent then skips download and goes to verify+commit). +3. `GET ` — serve the **rootfs bz2** (see payload note) with correct `Content-Length`. Simplest faithful behavior: **ignore Range, always 200 full body**; the agent restarts-from-zero if it sent a Range. Content-Type `application/x-bzip2`. +4. `POST /deployments//status/` — req `{"status": "", "error_message": "<...>"}`, `` ∈ {downloading, installing, rebooting, verifying, failed, rolled_back}. Resp **200** `{"status":"ok"}`. +5. `POST /deployments/commit/` — req `{"version": ""}`. Resp **200** `{"status":"ok"}`. The `version` MUST equal the advertised `target_tag`. + +**OTA success sequence:** heartbeat(s) → check(200, status=pending) → status(downloading) → GET download → status(installing) → status(rebooting) → **reboot** → heartbeat → check(200, status=rebooting) → status(verifying) → commit(version=target_tag). Failure emits `failed`/`rolled_back` on `/status/`. + +**Checksum:** `checksum_sha256` in the check response = SHA-256 of the **compressed bz2 bytes** the agent downloads (verified before decompression). Empty string = agent skips verification. + +**Timers:** agent `heartbeat_interval` (min 10s), `ota_check_interval` = "check every N heartbeats" (default 5). Test config sets `heartbeat_interval: 10`, `ota_check_interval: 1` → OTA check every ~10 s. + +## Payload note (OTA download content) + +The build produces `*.rootfs.wic` and `*.rootfs.wic.bz2` (IMAGE_FSTYPES = `wic wic.bz2`), **no standalone rootfs ext4**. The server, at ImageRelease import, extracts the **root_a partition** (an ext4 rootfs) from the wic and serves it bz2-compressed; the agent decompresses that into `root_`. The harness must mirror this: **extract partition 2 (`root_a`, the ext4 rootfs) from the build's wic, bz2-compress it, and compute its sha256** — that (compressed) blob + sha256 is what the dummy serves and advertises. (Partition order from `oe5xrx-remotestation-ab-x64.wks.in`: 1=`efi`, 2=`root_a`, 3=`root_b`, 4=`data`.) + +## grubenv / A/B state (x86) + +grubenv lives at `/boot/EFI/BOOT/grubenv` (ESP, PARTLABEL `efi`, partition 1). Vars: `boot_part` (a|b), `bootcount`, `upgrade_available` (1=trial,0=committed), `bootlimit` (3). Committed default: `boot_part=a bootcount=0 upgrade_available=0 bootlimit=3`. GRUB increments bootcount every boot and, if `upgrade_available=1 && bootcount>bootlimit`, swaps `boot_part` and clears the trial (rollback). Read/reset from the host by mounting partition 1 (FAT) and rewriting `EFI/BOOT/grubenv`, or via `grub-editenv` in-guest. + +## QEMU invocation (mirror `scripts/run-qemu.sh`, headless + pty serial) + +``` +qemu-system-x86_64 [-enable-kvm if /dev/kvm] -cpu IvyBridge -machine q35 \ + -m 1024 -smp 2 -nographic \ + -serial pty (harness reads the reported /dev/pts/N; NOT mon:stdio) \ + -drive if=pflash,format=raw,readonly=on,file= \ + -drive if=pflash,format=raw,file= \ + -drive file=,if=virtio,format=raw \ + -device virtio-net-pci,netdev=n0 \ + -netdev user,id=n0,hostfwd=tcp::-:22 +``` +OVMF_CODE probe order: `/usr/share/OVMF/OVMF_CODE_4M.fd`, `/usr/share/OVMF/OVMF_CODE.fd`, `/usr/share/edk2-ovmf/x64/OVMF_CODE.fd`. OVMF_VARS: copy a `*_VARS*.fd` template per run (writable). The wic passed to `-drive` must be **decompressed**. The guest reaches the host dummy server at `http://10.0.2.2:` (SLIRP alias). Each run: unique dummy port + sshport + tmp dir. + +--- + +## File Structure + +``` +tests/ota-integration/ + __init__.py + README.md # how to run locally + in CI + requirements.txt # pytest, pexpect, pyyaml, cryptography + pytest.ini # markers: qemu (needs qemu+root), unit + dummy_server.py # DummyOtaServer: endpoints + recorded state + target.py # Target ABC + QemuTarget (+ Cm4Target stub) + image_ops.py # extract_rootfs_bz2(), sha256_file(), decompress_wic() + seed.py # render_config_yaml(), gen_ed25519_key(), seed_data_partition() + conftest.py # fixtures: dummy_server, qemu_target, artifacts + test_dummy_server.py # UNIT: drive the dummy directly, assert recorded state + test_image_ops.py # UNIT: rootfs extraction / sha256 / config render + test_boot.py # T1 (marker: qemu) + test_ota_cycle.py # T2 (marker: qemu) + test_rollback.py # T3 (marker: qemu) +scripts/ + l0a-fstab-uuid-lint.sh # static wks/recipe UUID-pattern lint (every PR) + compute-release-version.sh # next YYYY.MM.DD-HH[a-z]; used by the composite action +.github/actions/compute-version/action.yml # composite action wrapping the script +.github/workflows/ + ci.yml # MODIFY: add L0a job/step + boot-ota-test.yml # NEW reusable (workflow_call): runs pytest -m qemu + boot-ota-pr.yml # NEW: path-filtered PR trigger → build + boot-ota-test + release.yml # MODIFY: workflow_dispatch + compute-version + gate + boot-ota-spike.yml # NEW: workflow_dispatch feasibility spike (TCG boot timing) +meta-oe5xrx-remotestation/recipes-core/images/oe5xrx-remotestation-image.bb # MODIFY: L0b assertion +scripts/release.sh # MODIFY: reduce to a `gh workflow run` dispatcher +``` + +--- + +## Task 1: Test package scaffold + +**Files:** +- Create: `tests/ota-integration/__init__.py` (empty), `tests/ota-integration/requirements.txt`, `tests/ota-integration/pytest.ini`, `tests/ota-integration/README.md` + +**Interfaces:** +- Produces: the package dir + `pytest -m unit` runnable (no tests yet → exit 5 is fine). + +- [ ] **Step 1: Create requirements.txt** +``` +pytest>=8 +pexpect>=4.9 +pyyaml>=6 +cryptography>=43 +``` +- [ ] **Step 2: Create pytest.ini** +```ini +[pytest] +markers = + unit: pure unit tests, no QEMU/root needed + qemu: needs qemu-system-x86_64 + root (loop mount); run in CI or on a capable host +addopts = -ra +``` +- [ ] **Step 3: Create `__init__.py` (empty) and a short README.md** documenting: `pip install -r requirements.txt`; `pytest -m unit` (fast, local); `sudo pytest -m qemu` (needs qemu+OVMF+root); env knobs (`OTA_IT_WIC`, `OTA_IT_LAST_RELEASE_WIC`). +- [ ] **Step 4: Commit** +```bash +git add tests/ota-integration/ +git commit -m "test(ota-it): scaffold ota-integration package" +``` + +--- + +## Task 2: Dummy OTA server (TDD, unit-tested) + +**Files:** +- Create: `tests/ota-integration/dummy_server.py` +- Test: `tests/ota-integration/test_dummy_server.py` + +**Interfaces:** +- Produces: + - `class DummyOtaServer(payload_path: str|None, checksum: str|None, size: int|None, target_tag: str)` with: + - `.start() -> None` / `.stop() -> None` (background `http.server.ThreadingHTTPServer`) + - `.url -> str` (e.g. `http://127.0.0.1:`; bind `0.0.0.0`, report chosen port) + - `.port -> int` + - `.offer_update: bool` (default False; when True `/check/` returns 200) + - `.result_status: str` (what `/check/` reports; default `"pending"`, set to `"rebooting"` to drive post-reboot branch) + - recorded state: `.heartbeats: list[dict]`, `.status_updates: list[dict]`, `.commits: list[dict]`, `.downloads: int` + - `.last_reported_version() -> str|None` (from the most recent commit) + +- [ ] **Step 1: Write failing unit tests** `test_dummy_server.py` (marker `unit`): +```python +import json, urllib.request +import pytest +from tests.ota_integration.dummy_server import DummyOtaServer + +pytestmark = pytest.mark.unit + +def _post(url, obj): + data = json.dumps(obj).encode() + req = urllib.request.Request(url, data=data, method="POST", + headers={"Content-Type": "application/json"}) + with urllib.request.urlopen(req) as r: + return r.status, r.read() + +def test_heartbeat_records_and_returns_ok(): + s = DummyOtaServer(payload_path=None, checksum=None, size=None, target_tag="T"); s.start() + try: + st, body = _post(s.url + "/api/v1/heartbeat/", {"os_version": "OE5XRX Remote Station T"}) + assert st == 200 and json.loads(body) == {"status": "ok"} + assert s.heartbeats[-1]["os_version"] == "OE5XRX Remote Station T" + finally: + s.stop() + +def test_check_204_when_no_update(): + s = DummyOtaServer(payload_path=None, checksum=None, size=None, target_tag="T"); s.start() + try: + req = urllib.request.Request(s.url + "/api/v1/deployments/check/", + data=b'{"current_version":"X"}', method="POST", + headers={"Content-Type": "application/json"}) + try: + urllib.request.urlopen(req) + assert False, "expected 204" + except urllib.error.HTTPError as e: + assert e.code == 204 + finally: + s.stop() + +def test_check_200_offers_update_with_expected_fields(tmp_path): + payload = tmp_path / "rootfs.bz2"; payload.write_bytes(b"hello-bz2") + s = DummyOtaServer(payload_path=str(payload), checksum="deadbeef", + size=9, target_tag="2026.07.11-15") + s.offer_update = True; s.start() + try: + st, body = _post(s.url + "/api/v1/deployments/check/", {"current_version": "old"}) + d = json.loads(body) + assert st == 200 + assert d["target_tag"] == "2026.07.11-15" + assert d["checksum_sha256"] == "deadbeef" + assert d["size_bytes"] == 9 + assert d["download_url"].endswith("/download/") + assert "deployment_result_id" in d and "deployment_id" in d + finally: + s.stop() + +def test_download_serves_payload_bytes(tmp_path): + payload = tmp_path / "rootfs.bz2"; payload.write_bytes(b"PAYLOAD") + s = DummyOtaServer(payload_path=str(payload), checksum="x", size=7, target_tag="T") + s.offer_update = True; s.start() + try: + st, body = _post(s.url + "/api/v1/deployments/check/", {"current_version": "old"}) + url = json.loads(body)["download_url"] + with urllib.request.urlopen(s.url + url) as r: + assert r.status == 200 and r.read() == b"PAYLOAD" + assert s.downloads == 1 + finally: + s.stop() + +def test_status_and_commit_recorded(): + s = DummyOtaServer(payload_path=None, checksum=None, size=None, target_tag="2026.07.11-15") + s.start() + try: + _post(s.url + "/api/v1/deployments/1/status/", {"status": "downloading", "error_message": ""}) + st, _ = _post(s.url + "/api/v1/deployments/commit/", {"version": "2026.07.11-15"}) + assert st == 200 + assert s.status_updates[-1]["status"] == "downloading" + assert s.last_reported_version() == "2026.07.11-15" + finally: + s.stop() +``` +- [ ] **Step 2: Run to confirm failure** — `cd /home/pbuchegger/OE5XRX/linux-image && python -m pytest tests/ota-integration/test_dummy_server.py -q` → FAIL (ImportError). (Add a `conftest.py`/`sys.path` shim or run with `PYTHONPATH=.`; use package import `tests.ota_integration...` — ensure `tests/__init__.py` and `tests/ota_integration` importability, or switch tests to `from dummy_server import ...` with `rootdir` on path. Pick one and be consistent.) +- [ ] **Step 3: Implement `dummy_server.py`** — a `ThreadingHTTPServer` + `BaseHTTPRequestHandler`. Route on `self.path` and method. `/api/v1/heartbeat/` → record JSON body, 200 `{"status":"ok"}`. `/api/v1/deployments/check/` → if `offer_update`: 200 with the dict (using `result_status`, `download_url = f"/api/v1/deployments/{deployment_id}/download/"`), else 204. `GET .../download/` → read `payload_path`, send 200 with `Content-Length` + `Content-Type: application/x-bzip2`, increment `downloads`. `/api/v1/deployments//status/` → append to `status_updates`, 200. `/api/v1/deployments/commit/` → append to `commits`, 200. Ignore all auth headers. Bind to `("0.0.0.0", 0)`, read back `server_address[1]` for `.port`; `.url` uses `127.0.0.1` for local tests (the guest uses 10.0.2.2 separately — the URL the guest sees is set via config, not `.url`). Silence base logging. +- [ ] **Step 4: Run tests to pass** — `python -m pytest tests/ota-integration/test_dummy_server.py -q` → PASS. +- [ ] **Step 5: Commit** — `git add tests/ota-integration/{dummy_server.py,test_dummy_server.py} && git commit -m "test(ota-it): dummy OTA server speaking the agent protocol"` + +--- + +## Task 3: Image ops — rootfs extraction, sha256, config render (TDD, unit where possible) + +**Files:** +- Create: `tests/ota-integration/image_ops.py`, `tests/ota-integration/seed.py` +- Test: `tests/ota-integration/test_image_ops.py` + +**Interfaces:** +- Produces: + - `image_ops.sha256_file(path) -> str` + - `image_ops.bz2_compress(src, dst) -> None` + - `image_ops.extract_rootfs_bz2(wic_path, out_bz2) -> tuple[str, int]` — extracts partition 2 (`root_a`) from the wic, bz2-compresses it to `out_bz2`, returns `(sha256_hex, size_bytes)` of the **compressed** file. Uses `losetup --find --show --partscan` + `dd`/stream through `bz2`; requires root. Guarded so unit runs can skip. + - `seed.render_config_yaml(server_url, station_id=1, key_path="/etc/stationagent/device_key.pem") -> str` (includes `heartbeat_interval: 10`, `ota_check_interval: 1`, `bootloader: grub`) + - `seed.gen_ed25519_key(dest_pem_path) -> None` (PEM private key via `cryptography`) + - `seed.seed_data_partition(wic_path, config_yaml_text, key_pem_path) -> None` — loop-mount partition 4 (`data`), `mkdir -p etc-overlay/stationagent`, write `config.yml` (0600) + `device_key.pem`; root-only. + - `image_ops.decompress_wic(bz2_or_wic_path, out_wic) -> str` — if `.bz2`, stream-decompress; else copy. Returns out path. + +- [ ] **Step 1: Write failing unit tests** (`unit` marker) for the pure functions: +```python +import bz2, hashlib, pytest, yaml +from tests.ota_integration import image_ops, seed +pytestmark = pytest.mark.unit + +def test_sha256_file(tmp_path): + p = tmp_path/"f"; p.write_bytes(b"abc") + assert image_ops.sha256_file(str(p)) == hashlib.sha256(b"abc").hexdigest() + +def test_bz2_roundtrip(tmp_path): + src = tmp_path/"s"; src.write_bytes(b"x"*1000); dst = tmp_path/"s.bz2" + image_ops.bz2_compress(str(src), str(dst)) + assert bz2.decompress(dst.read_bytes()) == b"x"*1000 + +def test_render_config_yaml_has_tuned_timers(): + txt = seed.render_config_yaml("http://10.0.2.2:8080") + d = yaml.safe_load(txt) + assert d["server_url"] == "http://10.0.2.2:8080" + assert d["heartbeat_interval"] == 10 and d["ota_check_interval"] == 1 + assert d["station_id"] == 1 and d["bootloader"] == "grub" + +def test_gen_ed25519_key_writes_pem(tmp_path): + dst = tmp_path/"k.pem"; seed.gen_ed25519_key(str(dst)) + assert dst.read_bytes().startswith(b"-----BEGIN PRIVATE KEY-----") + +def test_decompress_wic_passthrough_and_bz2(tmp_path): + raw = tmp_path/"a.wic"; raw.write_bytes(b"WIC") + assert open(image_ops.decompress_wic(str(raw), str(tmp_path/"o1.wic")),"rb").read() == b"WIC" + comp = tmp_path/"b.wic.bz2"; comp.write_bytes(bz2.compress(b"WIC2")) + assert open(image_ops.decompress_wic(str(comp), str(tmp_path/"o2.wic")),"rb").read() == b"WIC2" +``` +- [ ] **Step 2: Run → FAIL** (ImportError / not implemented). +- [ ] **Step 3: Implement `image_ops.py` + `seed.py`.** Pure funcs as above. `extract_rootfs_bz2` and `seed_data_partition` use `subprocess` (`losetup`, `mount`, `umount`) with a context manager that always detaches the loop device and unmounts (finally). `render_config_yaml` builds the dict and `yaml.safe_dump`s it. +- [ ] **Step 4: Run → PASS** (unit subset; the root-requiring funcs are covered later by the qemu-marked flow). +- [ ] **Step 5: Commit** — `git commit -m "test(ota-it): image ops (rootfs extract, sha256, config seed helpers)"` + +--- + +## Task 4: `Target` interface + `QemuTarget` + Cm4Target stub + +**Files:** +- Create: `tests/ota-integration/target.py` + +**Interfaces:** +- Consumes: `image_ops`, `seed`. +- Produces: + - `class Target(ABC)` with abstract methods: `flash(wic_path)`, `seed_config(config_yaml, key_pem)`, `power_on()`, `power_off()`, `reset()`, `console() -> pexpect.spawn-like`, `dut_server_url(dummy_port) -> str`, `boot_markers() -> dict` (keys: `banner_re`, `login_re`), and `reset_ab_state()` (set grubenv committed slot-A). + - `class QemuTarget(Target)`: + - ctor `(work_dir, mem=1024, cpus=2)`; discovers OVMF; allocates a free ssh port. + - `flash(wic)` = copy the (decompressed) wic into `work_dir` as the per-run disk; `power_on` launches qemu with `-serial pty`, parses the `char device redirected to /dev/pts/N` line, opens a `pexpect.fdpexpect`/`pexpect.spawn` on it. + - `dut_server_url(port)` = `f"http://10.0.2.2:{port}"`. + - `boot_markers()` = `{"banner_re": r"OE5XRX Remote Station (\S+)", "login_re": r"login:"}`. + - `reset_ab_state()` / `seed_config` call `seed.seed_data_partition` / a grubenv reset on the per-run disk (loop-mount, root). + - `class Cm4Target(Target)` — **stub**: every method raises `NotImplementedError("CM4 bench: future work")`, with a module docstring pointing at the spec's CM4 section. Exists so the interface is real and imports don't break. + +- [ ] **Step 1:** Write a `unit`-marked test asserting the interface shape only: `QemuTarget.boot_markers()` returns the two regexes; `dut_server_url(8080) == "http://10.0.2.2:8080"`; `Cm4Target().flash("x")` raises `NotImplementedError`. (No QEMU launched.) +- [ ] **Step 2:** Run → FAIL. +- [ ] **Step 3:** Implement `target.py`. Keep QEMU-launch code in `power_on` (not exercised by unit tests). Use the exact qemu args from the Protocol Reference. Detect `/dev/kvm` and add `-enable-kvm` only if present (TCG otherwise). +- [ ] **Step 4:** Run unit test → PASS. +- [ ] **Step 5:** Commit — `git commit -m "test(ota-it): Target interface + QemuTarget + Cm4Target stub"` + +--- + +## Task 5: Fixtures + T1 (boot smoke) — qemu-marked + +**Files:** +- Create: `tests/ota-integration/conftest.py`, `tests/ota-integration/test_boot.py` + +**Interfaces:** +- Consumes: `DummyOtaServer`, `QemuTarget`, `image_ops`, `seed`. +- Produces: fixtures `built_wic` (env `OTA_IT_WIC` → decompressed path), `qemu_target`, `dummy` (started server). T1 test. + +- [ ] **Step 1:** Write `test_boot.py` (marker `qemu`): +```python +import os, re, pytest +pytestmark = pytest.mark.qemu + +def test_t1_boots_and_agent_checks_in(qemu_target, dummy, built_wic): + # dummy offers NO update; agent must just boot + heartbeat. + tag = os.environ["OTA_IT_EXPECTED_TAG"] # dev-stamp or release tag + qemu_target.flash(built_wic) + qemu_target.reset_ab_state() + url = qemu_target.dut_server_url(dummy.port) + from tests.ota_integration import seed + cfg = seed.render_config_yaml(url) + keyp = qemu_target.work_dir + "/device_key.pem"; seed.gen_ed25519_key(keyp) + qemu_target.seed_config(cfg, keyp) + con = qemu_target.power_on() + m = qemu_target.boot_markers() + con.expect(m["banner_re"], timeout=900) # banner carries the version + banner_ver = con.match.group(1).decode() if isinstance(con.match.group(1), bytes) else con.match.group(1) + con.expect(m["login_re"], timeout=120) + # agent must check in at the expected version + _wait_until(lambda: any(tag in (h.get("os_version") or "") for h in dummy.heartbeats), 300) + assert any(tag in (h.get("os_version") or "") for h in dummy.heartbeats) + assert tag in banner_ver or banner_ver in tag +``` +(`_wait_until` = poll helper in conftest; condition-based, generous timeout.) +- [ ] **Step 2:** Run → FAIL/SKIP without a wic. Document: T1 requires `OTA_IT_WIC` + `OTA_IT_EXPECTED_TAG` + root + qemu. +- [ ] **Step 3:** Implement `conftest.py` fixtures + `_wait_until`. `built_wic` decompresses `OTA_IT_WIC` via `image_ops.decompress_wic`. `dummy` yields a started `DummyOtaServer` (offer_update False for T1). `qemu_target` yields a `QemuTarget(tmp work dir)`, teardown kills qemu + detaches loops. +- [ ] **Step 4:** (CI-only) verify green in the `boot-ota-test.yml` run (Task 9). Locally, `pytest -m unit` still passes; `-m qemu` skipped without env. +- [ ] **Step 5:** Commit — `git commit -m "test(ota-it): T1 boot smoke + fixtures"` + +--- + +## Task 6: T2 (cross-build OTA cycle) + T3 (rollback) — qemu-marked + +**Files:** +- Create: `tests/ota-integration/test_ota_cycle.py`, `tests/ota-integration/test_rollback.py` + +**Interfaces:** +- Consumes: everything above. Env: `OTA_IT_LAST_RELEASE_WIC` (slot A = previous release), `OTA_IT_WIC` (slot B = build under test), `OTA_IT_EXPECTED_TAG` (new tag). + +- [ ] **Step 1: T2** (`test_ota_cycle.py`, marker `qemu`): flash the **last-release** wic; extract `root_a` from the **new build** wic → bz2 + sha256 (`image_ops.extract_rootfs_bz2`); configure `dummy` with that payload, `offer_update=True`, `target_tag=`, `result_status="pending"`. Boot last-release; wait for the agent to check in, download (`dummy.downloads >= 1`), and post `installing`/`rebooting` (`dummy.status_updates`). When the guest reboots, flip `dummy.result_status="rebooting"` so the post-reboot check drives verify+commit. Assert: after reboot the banner shows the **new** tag + login; `dummy.last_reported_version() == ` and a `commit` was recorded. Add the code comment: *"Slot A MUST be a DIFFERENT build than slot B (last release vs build-under-test) — same-build hides the ESP-UUID bug (#37)."* +- [ ] **Step 2: T3** (`test_rollback.py`, marker `qemu`): same setup but serve a **deliberately broken** rootfs (e.g. truncate/zero the payload so the slot is unbootable) OR withhold the commit and let bootcount exceed bootlimit; assert the guest reverts to slot A (banner shows the **old** tag) and the agent reports `rolled_back` (in `dummy.status_updates`). +- [ ] **Step 3:** Run locally `-m unit` (unaffected). `-m qemu` runs in CI. +- [ ] **Step 4:** Commit — `git commit -m "test(ota-it): T2 cross-build OTA + T3 rollback"` + +--- + +## Task 7: L0a static UUID-pattern lint (every PR) + L0b build assertion + +**Files:** +- Create: `scripts/l0a-fstab-uuid-lint.sh`, `tests/ota-integration/test_l0a_lint.py` +- Modify: `meta-oe5xrx-remotestation/recipes-core/images/oe5xrx-remotestation-image.bb` (add L0b) + +**Interfaces:** +- Produces: `l0a-fstab-uuid-lint.sh ` exits non-zero if any wks `part` with a mountpoint uses `--use-uuid` **without** `--no-fstab-update`, or any recipe writes a `UUID=` fstab line. Prints offending file:line. + +- [ ] **Step 1:** Write `test_l0a_lint.py` (`unit`): create a temp wks with `part /boot ... --use-uuid` (no `--no-fstab-update`) → assert the script exits non-zero; a temp wks with `--use-uuid --no-fstab-update` → exits zero; a clean tree → zero. Invoke via `subprocess`. +- [ ] **Step 2:** Run → FAIL (script missing). +- [ ] **Step 3:** Implement `l0a-fstab-uuid-lint.sh` (bash, `set -euo pipefail`): scan `meta-*/wic/*.wks*` for lines matching `^part .*/[^ ]* .*--use-uuid` that do NOT contain `--no-fstab-update`; grep recipes for `UUID=` writes into fstab. Emit `::error file=...::` lines and exit 1 on any hit. **Run it on the current tree — it must exit 0** (the #37 fix already added `--no-fstab-update`). +- [ ] **Step 4:** Run test → PASS; run `scripts/l0a-fstab-uuid-lint.sh .` → exit 0. +- [ ] **Step 5: L0b** — in the image recipe, add a `python assert_no_uuid_fstab()` ROOTFS_POSTPROCESS that reads `${IMAGE_ROOTFS}/etc/fstab` and `bb.fatal`s if any non-comment line's device field matches `UUID=` or `/dev/disk/by-uuid/`. Append `ROOTFS_POSTPROCESS_COMMAND += "assert_no_uuid_fstab;"`. (Mirrors the existing `fix_firmware_fstab`/`add_efi_fstab` style.) +- [ ] **Step 6:** Commit — `git commit -m "test(ota-it): L0a UUID-pattern lint + L0b fstab build assertion"` + +--- + +## Task 8: `compute-release-version.sh` + composite action (TDD) + +**Files:** +- Create: `scripts/compute-release-version.sh`, `.github/actions/compute-version/action.yml`, `tests/ota-integration/test_compute_version.py` + +**Interfaces:** +- Produces: `compute-release-version.sh` prints the next `YYYY.MM.DD-HH[a-z]` to stdout. Given an env/arg list of "existing tags" (for testability) it computes: `base=$(date -u +%Y.%m.%d-%H)`; if no existing tag == base → `base`; else next free `[a-z]` suffix (`base` a? then b? …); error at `z`. Accept existing tags via `--existing ""` (default: `git tag -l "${base}*"`) so tests don't touch git. + +- [ ] **Step 1:** Write `test_compute_version.py` (`unit`): call the script with `--now 2026.07.11-15 --existing ""` → `2026.07.11-15`; `--existing "2026.07.11-15"` → `2026.07.11-15a`; `--existing $'2026.07.11-15\n2026.07.11-15a'` → `2026.07.11-15b`; unrelated tags ignored. (Add `--now` to inject the base for deterministic tests.) +- [ ] **Step 2:** Run → FAIL. +- [ ] **Step 3:** Implement the script (bash). Mirror `scripts/release.sh` letter-progression (`tr 'a-y' 'b-z'`) but computed against the `--existing` list. Validate output against `^[0-9]{4}\.[0-9]{2}\.[0-9]{2}-[0-9]{2}[a-z]?$`. +- [ ] **Step 4:** Run → PASS. +- [ ] **Step 5:** Create `.github/actions/compute-version/action.yml` (composite): `git fetch --tags --quiet`, run the script, set output `version`. +- [ ] **Step 6:** Commit — `git commit -m "ci(release): compute-version script + composite action"` + +--- + +## Task 9: `boot-ota-test.yml` reusable workflow + +**Files:** +- Create: `.github/workflows/boot-ota-test.yml` + +**Interfaces:** +- `workflow_call` inputs: `machine` (default `qemux86-64`), `runner_label` (default `ubuntu-latest`), `new_artifact` (the `yocto-image-` artifact name from the same run), `expected_tag`, `last_release_tag` (optional; empty → skip T2, run T1 only — bootstrap case). + +- [ ] **Step 1:** Write `boot-ota-test.yml`: `runs-on: ${{ inputs.runner_label }}`. Steps: checkout; `sudo apt-get install -y qemu-system-x86 ovmf python3-pip`; `pip install -r tests/ota-integration/requirements.txt`; `download-artifact` the new build wic → set `OTA_IT_WIC`; if `last_release_tag` non-empty, `gh release download --pattern '*qemux86-64*.wic.bz2'` → `OTA_IT_LAST_RELEASE_WIC`; set `OTA_IT_EXPECTED_TAG`; run `sudo -E env "PATH=$PATH" python -m pytest tests/ota-integration -m qemu -v` (T2/T3 auto-skip if `OTA_IT_LAST_RELEASE_WIC` unset via a `skipif`). Add a job-level `timeout-minutes: 60`. +- [ ] **Step 2:** `actionlint`/`yamllint` it (mirror ci.yml's yamllint invocation) → clean. +- [ ] **Step 3:** Commit — `git commit -m "ci: reusable boot-ota-test workflow (runner-parametrized)"` + +--- + +## Task 10: `boot-ota-pr.yml` (path-filtered PR) + `ci.yml` L0a + +**Files:** +- Create: `.github/workflows/boot-ota-pr.yml` +- Modify: `.github/workflows/ci.yml` + +- [ ] **Step 1:** `boot-ota-pr.yml`: `on: pull_request: paths: [meta-oe5xrx-remotestation/wic/**, meta-oe5xrx-remotestation/recipes-bsp/**, meta-oe5xrx-remotestation/recipes-core/ab-layout/**, meta-oe5xrx-remotestation/recipes-core/images/**, meta-oe5xrx-remotestation/recipes-core/station-agent/**, tests/ota-integration/**, .github/workflows/boot-ota-*.yml]`. Job `build-x64` → `uses: ./.github/workflows/build.yml` with `machine: qemux86-64, release_tag: ""` (dev stamp) + `secrets: inherit`. Job `test` → `needs: build-x64`, `uses: ./.github/workflows/boot-ota-test.yml` with `expected_tag: dev`, `last_release_tag: ""` (PR path runs T1 only to keep it bounded; T2 is the release-gate's job), `new_artifact: yocto-image-qemux86-64`. +- [ ] **Step 2:** `ci.yml`: add a step to the `validate` job (or a new `l0a` job) that runs `scripts/l0a-fstab-uuid-lint.sh .`. +- [ ] **Step 3:** yamllint both → clean. +- [ ] **Step 4:** Commit — `git commit -m "ci: L0a on every PR + boot-critical PR boot/OTA test"` + +--- + +## Task 11: Release rework — `workflow_dispatch` + compute-version + gate + +**Files:** +- Modify: `.github/workflows/release.yml`, `scripts/release.sh` + +- [ ] **Step 1:** Rework `release.yml`: + - Trigger → `on: workflow_dispatch: inputs: { dry_run: {type: boolean, default: false} }`. Remove the tag-push trigger. + - `checkout` gains `fetch-depth: 0` where the version is computed. + - New job `version` (`needs: preflight`): uses `./.github/actions/compute-version` → output `version`. + - `build-x64`/`build-rpi`: `needs: [preflight, version]`, `release_tag: ${{ needs.version.outputs.version }}`. + - New job `gate`: `needs: [version, build-x64]`, `uses: ./.github/workflows/boot-ota-test.yml` with `expected_tag: ${{ needs.version.outputs.version }}`, `last_release_tag: ` (a small step: `gh release list -L1 --json tagName -q '.[0].tagName'`), `new_artifact: yocto-image-qemux86-64`. + - `release` (publish): `needs: [build-x64, build-rpi, gate, version]`, `if: ${{ !inputs.dry_run }}`; add to `action-gh-release`: `tag_name: ${{ needs.version.outputs.version }}` + `target_commitish: ${{ github.sha }}`; replace all `${{ github.ref_name }}` with `${{ needs.version.outputs.version }}`. Add a default-branch guard for real (non-dry) runs (mirror FW lines 27-38). + - `dry_run: true` → runs preflight+version+build+gate, skips cosign-sign + create-release. +- [ ] **Step 2:** Reduce `scripts/release.sh` to a thin dispatcher: `gh workflow run release.yml [-f dry_run=true]`, with a note that version is computed server-side. Keep `--dry-run` → `-f dry_run=true`. +- [ ] **Step 3:** yamllint `release.yml` → clean; shellcheck `release.sh`. +- [ ] **Step 4:** Commit — `git commit -m "ci(release): one-button dispatch + auto-version + boot/OTA publish gate"` + +--- + +## Task 12: Feasibility spike workflow + README wiring + +**Files:** +- Create: `.github/workflows/boot-ota-spike.yml` + +- [ ] **Step 1:** `boot-ota-spike.yml`: `on: workflow_dispatch: inputs: { release_tag: {type: string, default: ""} }`. On `ubuntu-latest`: install qemu+ovmf; `gh release download` the given (or latest) release wic; time a headless TCG boot to `login:` (a tiny inline python/pexpect snippet or reuse `test_boot.py` with a no-update dummy); print wall-clock, `free -m`, and `df -h` (disk headroom). Purpose: decide GH-hosted vs Hetzner. Job `timeout-minutes: 45`. +- [ ] **Step 2:** yamllint → clean. +- [ ] **Step 3:** Update `tests/ota-integration/README.md` with the spike + how to flip `runner_label` to a self-hosted Hetzner label if the spike shows GH-hosted is insufficient. +- [ ] **Step 4:** Commit — `git commit -m "ci: TCG boot feasibility spike (GH-hosted vs Hetzner decision)"` + +--- + +## Task 13: Self-verification notes + open final PR + +**Files:** +- Modify: `tests/ota-integration/README.md` + +- [ ] **Step 1:** Add a "Verification of the test itself" section to the README: how to prove the harness catches the known bugs — temporarily revert #37 (`--use-uuid`, drop `--no-fstab-update`) → L0a red, L0b red, T2 red (emergency/rollback); revert #36 (skip relabel) → T2 red. Document as the acceptance check for the harness (run in a throwaway branch, not on main). +- [ ] **Step 2:** Final self-review pass: `yamllint` all workflows, `shellcheck` the new shell scripts, `python -m pytest tests/ota-integration -m unit` green. +- [ ] **Step 3:** Commit — `git commit -m "docs(ota-it): harness self-verification guide"` +- [ ] **Step 4:** Push branch, open PR (do NOT merge — user reviews). PR body: summarize the harness, the two bugs it catches, the cadence (L0a every PR, T1 on boot-critical PRs, T1+T2 as release gate), the runner-feasibility open item, and what is CI-verified vs unit-verified locally. + +--- + +## Self-Review (author checklist — completed at plan time) + +- **Spec coverage:** dummy server (T2/Task2), Target interface incl. CM4 stub (Task4), config injection = seed (Task3), T1/T2/T3 (Tasks 5-6), L0a/L0b (Task7), reusable boot-ota-test + runner param (Task9), boot-critical PR + release gate + dispatch/auto-version (Tasks 10-11), feasibility spike (Task12), self-verification (Task13). ✓ +- **Resolved open items folded in:** data-init preserves seeded config ✓; payload = extracted root_a ext4 bz2 ✓; plain http ok ✓; RPi console ✓ (CM4 stub only). +- **Type consistency:** `DummyOtaServer` fields (`heartbeats/status_updates/commits/downloads/offer_update/result_status/last_reported_version/url/port`), `Target` methods (`flash/seed_config/power_on/power_off/reset/console/dut_server_url/boot_markers/reset_ab_state`), and env vars (`OTA_IT_WIC/OTA_IT_LAST_RELEASE_WIC/OTA_IT_EXPECTED_TAG`) are used identically across tasks. ✓ +- **CM4 = future:** only a `NotImplementedError` stub ships now (Task4); no bench code. ✓ diff --git a/docs/superpowers/specs/2026-07-11-boot-ota-integration-test-design.md b/docs/superpowers/specs/2026-07-11-boot-ota-integration-test-design.md new file mode 100644 index 0000000..198ec14 --- /dev/null +++ b/docs/superpowers/specs/2026-07-11-boot-ota-integration-test-design.md @@ -0,0 +1,198 @@ +# Boot & OTA Integration Test — Design + +**Date:** 2026-07-11 +**Status:** Approved (brainstorming complete, pending implementation plan) +**Repos touched:** `linux-image` (test harness, dummy server, CI wiring, build-time guards, release-workflow rework). Consumes the `station-agent` binary already baked into the image. One small **follow-up in `station-manager`**: a contract test pinning the agent-protocol shapes the dummy mirrors (no code dependency, just drift protection). + +## Problem + +Two OTA-safety bugs shipped in published releases and only bit at runtime, on a real A/B OTA into the *other* slot. Neither was catchable by any existing check (recipe parse, shellcheck, yamllint, sim-harness): + +1. **fs-label mismatch** (fixed in #36): the served rootfs carried label `root_a`; an OTA'd slot B was never relabelled to `root_b`, so x86 GRUB's `search --label root_b` failed and slot B was unbootable → rollback. +2. **ESP mounted by FAT UUID** (fixed in #37): the x86 `/boot` ESP was mounted via a build-time-regenerated FAT UUID. OTA rewrites only the rootfs, never the on-disk ESP, so an OTA'd slot's baked `UUID=` never matched the physical ESP → `/boot` timed out → systemd emergency mode → rollback. Diagnosed live via Proxmox VNC (`Timed out waiting for device /dev/disk/by-uuid/A5E9-662F`). + +**Common shape:** both are runtime boot behaviours of a *cross-build* OTA'd slot. They appear ONLY when a **different** build's rootfs is written into slot B of an existing disk and that slot is actually **booted**. Static/unit tests cannot see them; the current CI never boots the image and never performs an OTA. + +**Critical corollary — the cross-build requirement:** if a test used the *same* build for slot A and slot B, the ESP UUIDs would coincidentally match and bug #2 would hide. The test MUST use two different builds (last published release as slot A, the new build as slot B). This mirrors reality: the field station had `10-21` flashed and OTA'd `09-04`/`15`. + +## Goals + +- **Boot proof:** the freshly built image boots to multi-user (not emergency mode) and the agent reaches the network. +- **OTA proof:** a real cross-build A/B OTA cycle — last release → new build — boots the new slot and commits, driven through the agent's *real* OTA client code path. +- **Cheap first line:** a static lint on every PR + an authoritative build-time guard, both targeting the bug-2 class (any device-`UUID=` mount). +- **Right gate:** never *publish* a release whose x86 image doesn't boot + OTA. +- **One-button release:** the release becomes a manually-dispatched workflow that computes its own next version (mirrors the FW-RemoteStation pattern) — no more tag-push trigger. +- **Reusable workflows + swappable runner:** build, boot-ota-test, and version-compute are reusable; the test runner is a parameter so GH-hosted ↔ Hetzner is a one-line flip. +- **Serial-first architecture** so a CM4 hardware target is a later transport drop-in, not a harness rewrite. + +## Non-Goals (documented follow-ups) + +- **CM4 / real-hardware HIL bench** — deferred, but the **`Target` interface is designed CM4-ready now** (flash/seed/power/console/flash-mode/server-url/boot-markers) so `Cm4Target` + the physical bench are a drop-in, not a rewrite. See Future Work for the bench mechanics. +- **RPi-under-QEMU** (qemu raspi, TCG, slow) — deferred. The release gate therefore tests **x86 only**; a broken RPi image could still publish (known gap). +- **Full `station-manager` E2E** (real server + agent) — the dummy server covers the agent's OTA contract faithfully (see Dummy Server); a real-server E2E is a heavier, separate future layer. +- **Auth verification** — the dummy does **not** verify the agent's Ed25519 request signature (test double). See Auth & Payload Fidelity. +- **Nightly / every-PR full runs** — out. Path-filtered PRs + the release gate cover the boot-critical surface. +- **KVM acceleration** — unavailable on both GH-hosted and Hetzner Cloud (no nested virt); everything runs under TCG. True KVM would need bare-metal, out of scope. + +## Success Criteria (per boot — an AND of two independent layers) + +A boot counts as successful only if **both** hold: + +1. **Serial layer:** the console shows the getty banner `OE5XRX Remote Station ` (from `/etc/issue`, stamped by `stamp_release`) **and then** the `login:` prompt. Emergency mode reaches neither — it drops to a `sulogin`/maintenance prompt — so this cleanly distinguishes bug-2's failure. The banner carries the version, doubling as a version assertion. +2. **Application layer:** the agent checks in with the dummy server (heartbeat), and in the OTA test **commits** the deployment, reporting `new_version == `. + +`` is cross-checked at both layers and against the fetched artifact's tag. + +## Architecture + +Three components in `linux-image` under `tests/ota-integration/` (mirrors the existing `tests/sim-harness/`). + +### 1. Dummy OTA server (standalone, no station-manager dependency) + +A minimal stdlib `http.server` speaking exactly the agent's endpoints (observed in production logs): + +- `POST /api/v1/deployments/check/` → `{no update}` (T1) or `{update available: deployment N, image release , download_url, sha256}` (T2). +- `GET /api/v1/deployments//download/` → streams the new build's **bz2-compressed rootfs** bytes. +- `POST /api/v1/deployments//status/` → records each status transition. +- `POST /api/v1/deployments/commit/` → records the commit (with reported version). +- heartbeat endpoint → records station check-ins (version, slot, health). + +It **records** every agent interaction and exposes it to the test (in-process handle or `GET /_test/state`) so assertions read "did the agent commit with the new version?" rather than scraping logs. + +**Why a standalone dummy, not reused station-manager code — settled by a code fact:** the agent verifies only a **SHA-256** checksum on the OTA download (`ota.py:download_firmware` stream-decompresses the bz2 rootfs into the slot), **not** a cosign signature; and it **signs** its requests with Ed25519 (`signing.py`) which a test double need not verify. So the dummy only has to serve the real `rootfs.bz2` + its `sha256` and accept signed requests — and the agent then runs its **complete real OTA code path** (check → download → sha256 → decompress → install → relabel → trial → reboot → commit). Reusing station-manager server code would add only wire-protocol/auth fidelity, which (a) is station-manager's own test turf — the agent *lives* in that repo and versions with the server — and (b) is not turnkey (no compose stack; would need Postgres+Redis+Django assembled in CI). Not worth the heavy cross-repo dependency for an *image* test. + +**Drift guard (cheap, in place of the dependency):** document the mirrored endpoints inline with a pointer to `station-manager` `apps/deployments/api_views.py`, plus a small **contract test in station-manager** asserting the response shapes the dummy relies on. Makes drift visible without coupling build/test envs. + +### 2. `Target` abstraction + serial-driven harness (pytest + pexpect) + +Test logic is written once against a **`Target` interface** — not merely a "serial transport" (that under-abstracts and forces a CM4 rewrite). Everything device-specific goes through the Target, so the same T1/T2/T3 assertions run on QEMU now and a CM4 bench later: + +| Capability | `QemuTarget` (x86, now) | `Cm4Target` (future, real hardware) | +|---|---|---| +| `flash(wic)` | copy image, point `-drive` at it | **rpiboot/usbboot**: expose eMMC as USB-MSD on the host → `dd`/`bmaptool` | +| `seed_config(files)` | `losetup --partscan` the data partition of the file | mount the exposed-eMMC data partition (via rpiboot) | +| `power_on/off/reset()` | start / kill the QEMU process | controllable power — relay / networked PDU / host GPIO (**open**) | +| `console()` | pexpect pty on `-serial` | UART GPIO14/15 → USB-UART → `/dev/ttyUSB0` (pyserial) | +| `enter_flash_mode()` | n/a | drive the **nRPIBOOT** pin + power sequence | +| `dut_server_url()` | `http://10.0.2.2:` (QEMU SLIRP alias) | `http://:` (real network) | +| `boot_markers` | GRUB → getty banner | u-boot `OE5XRX: slot=? attempt=?/3` → getty banner | + +`QemuTarget` boots with `qemu-system-x86_64` (OVMF/UEFI, the 4-partition A/B disk, headless `-nographic`, user-net), building on `scripts/run-qemu.sh` but attaching serial to a **clean pty** (not `-serial mon:stdio`, whose monitor multiplexing would confuse pexpect). Verified feasible: the x86 cmdline carries `console=tty0 console=ttyS0,115200`, so systemd runs a getty on ttyS0 and the `/etc/issue` banner + `login:` appear there. + +The harness owns a **per-run copy** of the disk image (never mutates the cached artifact), seeds config via `Target.seed_config`, drives/asserts over `Target.console()`, and queries the dummy's recorded state. The dummy binds where `Target.dut_server_url()` points (SLIRP alias for QEMU, bench-host LAN IP for CM4) — "how the DUT reaches the dummy" is never hardcoded to x86. **Condition-based waits only** (pexpect `expect` on banner/login/commit, never fixed sleeps) so TCG slowness never flakes. + +### 3. Config injection — data-partition preseed (image stays unmodified) + +We boot the **identical, signed production artifact**. Only the **data partition** carries test config — exactly how a real station is provisioned. This is `Target.seed_config`; the `QemuTarget` implementation loop-mounts the disk's data partition (`losetup --find --partscan`, root available on the runner), `Cm4Target` mounts the rpiboot-exposed eMMC. Either way it writes into `etc-overlay/stationagent/` (the source of the `etc-stationagent.mount` overlay over `/etc/stationagent`): + +- `config.yml` — `server_url: http://10.0.2.2:`, `station_id: 1`, `ed25519_key_path: /etc/stationagent/device_key.pem`, `bootloader: grub`, and **tuned-down timers** (`heartbeat_interval: 10`, `ota_check_interval: 1`) so the OTA check + post-reboot commit happen in seconds, not the ~4 min seen in production. +- `device_key.pem` — a harness-generated Ed25519 private key (the dummy does not verify it). + +**Open (resolve in the plan):** confirm `data-init.service` / `data-init.sh` does not overwrite a pre-seeded `etc-overlay/stationagent/` on first boot. + +## Auth & Payload Fidelity + +- **Requests:** the agent signs with Ed25519 (`station_id` + `ed25519_key_path`). The dummy accepts signed requests but does **not** verify them (test double). The harness still provides a valid key + `station_id` so the agent's config validates and it sends real signed requests. Signature verification is a future hardening if the dummy should double as an auth test. +- **Payload:** the OTA download is a **bz2-compressed rootfs**, verified by the agent against a **SHA-256** it gets from the deployment/check response — no cosign on the agent side. The dummy serves the real `rootfs.bz2` from the build artifact and the matching `sha256`. (Plan-time: ensure the build artifact includes the rootfs `.bz2` the server would serve, and compute its sha256 for the dummy.) + +## Test Flows + +### T1 — boot smoke (single image) +1. Copy the **newly built** wic, pre-seed test config on its data partition. +2. Dummy up; `/check/` returns "no update". +3. Boot headless. +4. **Assert (serial):** banner `OE5XRX Remote Station ` → `login:`. +5. **Assert (dummy):** agent checked in with `version == `. + +Proves boot-to-multi-user + agent + network. Does NOT catch bug #2 (fresh flash → matching UUID). That's T2. + +### T2 — ★ cross-build OTA cycle (the money test) +1. Fetch the **last published release** wic (via `gh`, as `run-qemu.sh --release`) → disk = slot A + ESP with the *old* build's identifiers. Pre-seed test config. +2. Dummy offers an update pointing at the **newly built** rootfs.bz2 (+ sha256). +3. Boot last-release. Agent checks in → dummy offers update → agent runs its **real** OTA path (download → sha256 → decompress → install slot B → relabel `root_b` → trial → reboot). +4. **Assert (serial):** after reboot, banner shows the **new** tag → `login:` (exactly where bug #2 = emergency mode and bug #1 = unbootable-B → rollback-to-old-tag manifested). +5. **Assert (dummy):** agent committed with `new_version == `. + +A=old build, B=new build ⇒ genuinely cross-build ⇒ catches ESP-UUID **and** fs-label. A code comment on the "fetch last release" step must state *why* it must differ from the build under test, so it's never "optimised" into a same-build shortcut that hides bug #2. + +### T3 — rollback (follow-up, same harness) +Serve a deliberately broken slot-B rootfs (or force `bootcount > bootlimit`) → assert revert to slot A: banner shows the **old** tag, agent reports `rolled_back`. Ships after T1/T2 are green. + +## L0 — Build-time / static guards (cheap) + +Two layers, because the fstab only exists after a build: + +- **L0a — static lint, every PR (GH-hosted, seconds):** grep the wks/recipes for the *dangerous pattern* — a mountpointed `part … --use-uuid` **without** `--no-fstab-update`, or a recipe writing `UUID=` into fstab. Would have caught bug #2 at the source (the wks had exactly that). Heuristic early-warning; runs without a build. +- **L0b — `ROOTFS_POSTPROCESS` assertion, during the build (authoritative):** fails the build on *any* device-`UUID=` mount in the real fstab (also catches wic auto-injection). Runs whenever a build runs (boot-critical PRs + release). + +## Pipeline + +Runner is a **parameter** on `boot-ota-test.yml` (`runs-on` / `runner_label`) — default GH-hosted `ubuntu-latest`, flip to the Hetzner runner in one line if the feasibility spike (below) says GH is insufficient. + +Reusable pieces: `build.yml` (`workflow_call`, existing), **`boot-ota-test.yml`** (`workflow_call`, new — downloads the new x64 artifact from the same run, fetches slot-A release, installs qemu+ovmf, runs pytest T1+T2), and a **`compute-version`** composite action. + +``` +① ci.yml on: pull_request, push→main (GH-hosted, seconds) + validate ─┬─ recipe-parse / shellcheck / yamllint / wks / udev + └─ L0a (static UUID-pattern lint) [NEW] + +② boot-ota-pr.yml on: pull_request, paths: (boot-critical only) + [ wic/** · recipes-bsp/** (grub/u-boot) · ab-layout/** · images/** · station-agent SRCREV pin ] + build-x64 (uses build.yml, machine=qemux86-64, dev stamp) ← L0b runs inside the build + └─► boot-ota-test (uses boot-ota-test.yml) ← T1+T2, fails the PR check on red + +③ release.yml on: workflow_dispatch (ONE-BUTTON + GATE) [REWORKED] + compute-version ─ next YYYY.MM.DD-HH (suffix [a-z] bump on collision; reads existing releases) + → preflight (SRCREV / FM-artifact pins) + → [ build-x64 , build-rpi ] (L0b runs in both) + └─► boot-ota-test (uses boot-ota-test.yml, x64) [NEW gate step] + → publish (softprops/action-gh-release creates the tag) needs:[build-x64, build-rpi, boot-ota-test] + ← publishes ONLY if the test is green +``` + +**Release trigger change:** drop the `on: push: tags` trigger; release is now `workflow_dispatch` (inputs: `dry_run`, optional `version` override). `compute-version` computes the next hour-stamp itself (mirrors `FW-RemoteStation/.github/workflows/release.yml`), and `action-gh-release` creates the tag at publish time. The existing `YYYY.MM.DD-HH[a-z]` scheme is kept (regex, `run-qemu.sh`, all existing tags depend on it) — only the computation moves into the workflow. `scripts/release.sh` becomes a thin `gh workflow run` wrapper or is retired. + +At the gate the new tag isn't published yet, so `gh release list` → latest = the **previous** release = slot A. Slot B = the just-built artifact ⇒ cross-build for free. + +## Runner Feasibility (no KVM → TCG) + +Both GH-hosted and Hetzner Cloud lack nested virt, so QEMU runs under **TCG** either way — the Hetzner fallback is only *faster/more cores* (CCX43 = 8 dedicated vCPU) vs GH-hosted's 4 shared vCPU, not KVM acceleration. + +**Feasibility spike = the first implementation step.** Before building the full harness, boot the current published release under TCG on `ubuntu-latest` and measure: boot-to-`login:` time, guest+host RAM, and especially **disk** (GH-hosted = 4 vCPU / 16 GB RAM / **14 GB disk** — the fetched last-release wic + new rootfs + per-run working copies are the tightest constraint). Decision gate: + +- Boots reliably in acceptable wall-clock and fits disk → **GH-hosted** (free, simple). +- Too slow / OOM / disk-tight → **Hetzner**, preferably by reusing the **ephemeral build server** (build → test-on-same-box → delete; artifact already local, beefier TCG). Flip via the `runs-on` parameter. + +## Risks & Open Items + +- **GH-hosted sufficiency** — resolved by the feasibility spike (above), not assumed. +- **`data-init` overwrite** of the pre-seeded config — verify against `data-init.sh` in the plan. +- **Protocol drift** dummy ↔ station-manager — documented endpoint mirroring + a station-manager contract test. +- **TCG flake** — condition-based waits, generous timeouts, never fixed sleeps. +- **Disk mutation** — always operate on a per-run copy of the fetched/built wic. +- **RPi gate gap** — the release gate tests x86 only; a broken RPi image can still publish. Documented; closable later. +- **Build artifact must include the rootfs `.bz2`** the OTA serves (+ sha256) — confirm in the plan. + +## Verification of the Test Itself + +Before trusting the harness, prove it *fails* on the very bugs it targets: + +- Temporarily revert #37 (re-introduce the ESP `--use-uuid`) → **T2 red** (emergency/rollback), **L0b red** at build time, **L0a red** at lint time. +- Temporarily revert #36 (skip the relabel) → **T2 red** (slot B unbootable → rollback). +- Revert both back out; tests green. A test that can't catch the known bug is worthless. + +## Future Work + +- **CM4 HIL bench** — implement `Cm4Target` against the interface above + build the physical bench: a self-hosted runner on a **bench host** wired to a CM4 carrier. Production stations use **eMMC**; the dev bench uses cheaper/faster-to-flash **CM4 Lite (SD)**. The medium delta is small because the whole boot/OTA path is PARTLABEL-based (SD and eMMC both enumerate as `mmcblk0`); the residual gap (eMMC-specific naming/timing quirks) is documented, not covered — a green Lite run does **not** guarantee eMMC. Mitigations: a `flash()` backend abstraction lets an eMMC-CM4 (rpiboot) be plugged into the same harness later with zero test-logic change, and real eMMC provisioning + first OTA is itself an implicit eMMC boot/OTA check. The carrier exposes rpiboot (USB-OTG + nRPIBOOT — confirmed) and a UART header (confirmed). Mechanics: + - **Flash backend (cost-tiered):** start **mux-less** — keep slot A (last release) pre-flashed, reset A/B state between runs over serial/rescue, re-flash the SD manually only on base change (the OTA writes slot B over the network, so the flasher is out of that loop and rarely needed → €0). Upgrade to a **self-fabbed SDWire** (open hardware, [3mdeb/SDWire](https://github.com/3mdeb/SDWire), BOM ~€10–20, fits the HW-Module-CI fab pipeline) if manual re-flash gets annoying; commercial SDWire / usb-sd-mux are ~€90–120 (functionally equivalent, robustness-for-money). For the eMMC fidelity path, `flash()` uses rpiboot (nRPIBOOT low → eMMC as `/dev/sdX` → `dd`/`bmaptool`); `seed_config` mounts the data partition in the same window. + - **Console:** UART GPIO14/15 → USB-UART → `/dev/ttyUSB0`. The image already enables `serial-getty@ttyAMA0` for RPi; confirm the RPi cmdline `console=` targets it. + - **Power control (OPEN decision):** needed for hard-reset of a hung boot (the emergency case does not reboot itself → no automatic rollback test without it) and any rpiboot power-sequencing. Options: USB relay on the bench host, networked PDU, or host GPIO + MOSFET. To be decided when the bench is built. + - **Value:** exercises the **u-boot path + real ARM hardware + real power-cycle rollback** (things x86-QEMU/GRUB/SLIRP/software-reboot can't) and **closes the RPi release-gate gap**. +- **RPi-under-QEMU** target + gate (nightly, once x86 is stable) — cheaper partial coverage of the RPi image before the hardware bench exists. +- **Full `station-manager` E2E** — real server + agent, to also cover server↔agent protocol and the deployment state machine. +- **Ed25519 verification** in the dummy, if it should double as an auth test. + +## Release-Rework Caveats (process) + +- **`release.yml` is the production release path.** Verify the new `workflow_dispatch` + `compute-version` flow with the `dry_run` input first (must compute the correct next `YYYY.MM.DD-HH[a-z]` and NOT tag/publish) before relying on it. Sequence: land the harness + gate, then flip the trigger. +- **Releases get slower:** build (~10 min) + T2 under TCG (~15–30 min) per release. Acceptable and intended (the gate is the point), but a known wall-clock cost. diff --git a/meta-oe5xrx-remotestation/recipes-core/images/oe5xrx-remotestation-image.bb b/meta-oe5xrx-remotestation/recipes-core/images/oe5xrx-remotestation-image.bb index b3b3cd6..5958b8d 100644 --- a/meta-oe5xrx-remotestation/recipes-core/images/oe5xrx-remotestation-image.bb +++ b/meta-oe5xrx-remotestation/recipes-core/images/oe5xrx-remotestation-image.bb @@ -217,3 +217,29 @@ python add_efi_fstab() { bb.note('add_efi_fstab: ensured PARTLABEL=efi /boot entry') } ROOTFS_POSTPROCESS_COMMAND:append:qemux86-64 = " add_efi_fstab;" + +# L0b — authoritative guard against the #37 bug class: fail the build if the +# baked /etc/fstab mounts any filesystem by a build-unstable device UUID. +# OTA rewrites only the rootfs, so a UUID= (or /dev/disk/by-uuid/) mount never +# matches the on-disk device after a cross-build OTA → boot hang / emergency +# mode → rollback. Only PARTLABEL / LABEL / kernel-cmdline root are OTA-safe. +# The fast static counterpart is scripts/l0a-fstab-uuid-lint.sh (every PR). +python assert_no_uuid_fstab() { + import os + fstab = os.path.join(d.getVar('IMAGE_ROOTFS'), 'etc/fstab') + if not os.path.exists(fstab): + return + with open(fstab) as f: + for lineno, line in enumerate(f, 1): + stripped = line.strip() + if not stripped or stripped.startswith('#'): + continue + device = stripped.split()[0] + if device.startswith('UUID=') or device.startswith('/dev/disk/by-uuid/'): + bb.fatal( + 'assert_no_uuid_fstab: /etc/fstab line %d mounts by device UUID ' + '(%r) — not OTA-safe. Mount by PARTLABEL=