feat: migrate away from Arduino

.changes/esp-idf-6-migration.md

@@ -0,0 +1,14 @@
+---
+kind: changed
+breaking: true
+---
+Rebuild the firmware on the native ESP-IDF platform
+
+## Release Note
+This is one of the biggest updates the firmware has ever had: the whole thing has been rebuilt from the ground up on Espressif's native platform, leaving the old Arduino foundation behind.
+
+You won't see a pile of new buttons from this release — most of the work is under the hood. What you get is a firmware that's faster to start up, lighter on memory, and far more stable, with a much cleaner codebase that makes future features quicker and safer to add. It also brings a real security upgrade: your hub now checks that it's genuinely talking to OpenShock's servers before it trusts them, so the connection between your device and the network is properly protected.
+
+Just about every part of the firmware was touched along the way — WiFi and networking, the way settings are stored, the radio that talks to your shockers, the status LEDs, and the serial console. Everything has been retested and put back together on the new foundation.
+
+Because this rework runs so deep, we're treating it as a major release. It flashes and runs like any other update, and your existing setup carries over.

.gitattributes

@@ -1,2 +1,4 @@
 * text=auto eol=lf
 *.txt text eol=lf
+*.exe filter=lfs diff=lfs merge=lfs -text
+scripts/flatc filter=lfs diff=lfs merge=lfs -text

.github/actions/build-compilationdb/action.yml

@@ -1,5 +1,8 @@
 name: build-compilationdb
-description: Builds the compilation database for the firmware for code analysis
+description: >-
+  Build a representative board with native ESP-IDF to emit
+  build/compile_commands.json (for cpp-linter) and a real compile pass (for
+  CodeQL C/C++). Uses OpenShock-Core-V2 / esp32s3 as the canonical config.
 inputs:
   version:
     required: true
@@ -8,25 +11,20 @@ inputs:
 runs:
   using: composite
   steps:
-    - uses: ./.github/actions/pio-cache
-      with:
-        env: ci-build
+    - uses: ./.github/actions/setup-esp-idf
 
-    - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
-      with:
-        cache: 'pip'
-
-    - name: Install python dependencies
-      shell: bash
-      run: pip install -r requirements.txt
-
-    - name: Build firmware
-      working-directory: .
+    # Build only the `app` target: this configures the project (writing
+    # build/OpenShock-Core-V2/compile_commands.json for cpp-linter) and compiles
+    # the application (so CodeQL traces the C/C++). It skips the static0 image, so
+    # no frontend / littlefs is needed. ccache is left off so CodeQL sees every
+    # compiler invocation.
+    - name: Build firmware (OpenShock-Core-V2 / esp32s3)
       shell: bash
       run: |
+        set -euo pipefail
         export OPENSHOCK_FW_BUILD_DATE="$(git show -s --format=%cI HEAD 2>/dev/null || date -u +%Y-%m-%dT%H:%M:%SZ)"
-        pio run -e ci-build -t compiledb
+        python3 scripts/build.py OpenShock-Core-V2 app
       env:
         OPENSHOCK_FW_VERSION: ${{ inputs.version }}
         OPENSHOCK_FW_GIT_REF: ${{ github.ref }}
-        OPENSHOCK_FW_GIT_COMMIT: ${{ github.sha }}
+        OPENSHOCK_FW_GIT_COMMIT: ${{ github.sha }}

.github/actions/build-firmware/action.yml

@@ -1,46 +1,65 @@
 name: build-firmware
-description: Builds the firmware partitions and uploads them as an artifact
+description: >-
+  Compile one board's C++ firmware with native ESP-IDF and upload its discrete
+  partition images (app, bootloader, partition-table). Builds only the code
+  targets - not the static0 web-assets image - so it runs in parallel with
+  build-staticfs; the merge-partitions job links them together.
 inputs:
   board:
     required: true
-    description: 'Board name to build'
+    description: 'Board name to build (matches boards/<board>.defaults)'
   version:
     required: true
     description: 'Current firmware version'
 
 runs:
   using: composite
   steps:
-    - uses: ./.github/actions/pio-cache
-      with:
-        env: ${{ inputs.board }}
+    - uses: ./.github/actions/setup-esp-idf
 
-    - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
+    # ccache keyed per board's chip (compiler/framework artifacts track the chip).
+    - name: Cache ccache
+      uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae  # v5.0.5
       with:
-        cache: 'pip'
-
-    - name: Install python dependencies
-      shell: bash
-      run: pip install -r requirements.txt
+        path: ~/.cache/ccache
+        key: ccache-${{ runner.os }}-${{ inputs.board }}-${{ github.sha }}
+        restore-keys: |
+          ccache-${{ runner.os }}-${{ inputs.board }}-
 
+    # Build only the code targets (app + bootloader + partition-table). This
+    # skips the static0 image (built separately by build-staticfs), so no
+    # frontend is needed here. scripts/build.py resolves the target from
+    # boards/<board>.defaults, sets OPENSHOCK_BOARD, and layers the sdkconfig.
     - name: Build firmware
-      working-directory: .
       shell: bash
-      # head_commit.timestamp only exists on push events, so derive the build
-      # date from the checked-out commit (falling back to now) - otherwise
-      # schedule/dispatch builds bake in an empty date.
       run: |
+        set -euo pipefail
         export OPENSHOCK_FW_BUILD_DATE="$(git show -s --format=%cI HEAD 2>/dev/null || date -u +%Y-%m-%dT%H:%M:%SZ)"
-        pio run -e ${{ inputs.board }}
+        export IDF_CCACHE_ENABLE=1
+        python3 scripts/build.py "${{ inputs.board }}" app bootloader partition-table
       env:
         OPENSHOCK_FW_VERSION: ${{ inputs.version }}
         OPENSHOCK_FW_GIT_REF: ${{ github.ref }}
         OPENSHOCK_FW_GIT_COMMIT: ${{ github.sha }}
 
-    - name: Upload build artifacts
+    # Flat, canonically-named images. app.bin/staticfs.bin are the names the
+    # device OTA pulls (OtaUpdateManager.cpp); merge_image.py and the CDN upload
+    # consume these too. `project(firmware)` names the app image firmware.bin.
+    - name: Stage partition images
+      shell: bash
+      env:
+        B: build/${{ inputs.board }}
+      run: |
+        set -euo pipefail
+        mkdir -p dist
+        cp "$B/firmware.bin"                        dist/app.bin
+        cp "$B/bootloader/bootloader.bin"           dist/bootloader.bin
+        cp "$B/partition_table/partition-table.bin" dist/partition-table.bin
+
+    - name: Upload partition images
       uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
       with:
         name: firmware_build_${{ inputs.board }}
-        path: .pio/build/${{ inputs.board }}/*.bin
-        retention-days: 1
+        path: dist/*.bin
+        retention-days: 7
         if-no-files-found: error

.github/actions/build-staticfs/action.yml

@@ -1,51 +1,52 @@
 name: build-staticfs
-description: Builds the static filesystem partition and uploads it as an artifact
+description: >-
+  Pack the captive-portal web assets into the static0 littlefs image once
+  (board-independent) via scripts/gen_staticfs.py. Runs in parallel with the
+  per-board C++ build; the merge job combines the two.
 inputs:
   version:
-    required: true
-    description: 'Current firmware version'
+    description: 'Current firmware version (unused in the image, kept for parity)'
+    required: false
+    default: ''
 
 runs:
   using: composite
   steps:
-    - uses: ./.github/actions/pio-cache
-      with:
-        env: fs
-
     - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
       with:
         cache: 'pip'
 
-    - name: Install dependencies
+    - name: Install Python dependencies
       shell: bash
       run: pip install -r requirements.txt
 
-    - name: Download built frontend
+    # The frontend is built once in build-frontend and downloaded here so
+    # gen_staticfs.py packs the prebuilt assets rather than rebuilding them.
+    - name: Download frontend
       uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
       with:
         name: frontend
-        path: frontend/build/
+        path: frontend/build
 
-    - name: Build filesystem partition
+    # static0 is 0x09D000 bytes in the 4MB OTA table (partitions/ota_4mb.csv);
+    # the image is board-independent, so one build serves every board. Geometry
+    # (4 KiB blocks, 128-byte read/prog) matches the runtime mount in
+    # components/fs/src/LfsPartition.cpp and the CMake build.
+    - name: Build static filesystem image
       shell: bash
-      # head_commit.timestamp only exists on push events; derive from the commit
-      # (falling back to now) so schedule/dispatch builds don't get an empty date.
       run: |
-        export OPENSHOCK_FW_BUILD_DATE="$(git show -s --format=%cI HEAD 2>/dev/null || date -u +%Y-%m-%dT%H:%M:%SZ)"
-        pio run --target buildfs -e fs
-      env:
-        OPENSHOCK_FW_VERSION: ${{ inputs.version }}
-        OPENSHOCK_FW_GIT_REF: ${{ github.ref }}
-        OPENSHOCK_FW_GIT_COMMIT: ${{ github.sha }}
-
-    - name: Rename partition binary
-      shell: bash
-      run: mv .pio/build/fs/littlefs.bin staticfs.bin
+        python scripts/gen_staticfs.py \
+          --frontend-dir frontend \
+          --staging-dir staticfs \
+          --output staticfs.bin \
+          --partition-size 0x09D000 \
+          --block-size 4096 --read-size 128 --prog-size 128 \
+          --skip-frontend-build
 
-    - name: Upload internal filesystem artifact
+    - name: Upload static filesystem artifact
       uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
       with:
         name: firmware_staticfs
         path: staticfs.bin
-        retention-days: 1
+        retention-days: 7
         if-no-files-found: error

.github/actions/cdn-upload-firmware/action.yml

@@ -30,9 +30,10 @@ runs:
         pattern: firmware_*
 
     # Assemble the full /<version>/ tree locally, one directory per board, so it
-    # can be pushed in a single mirror. Each board directory keeps the previous
-    # per-board layout: static + build partitions, the merged firmware.bin, and
-    # md5/sha256 checksums over every .bin.
+    # can be pushed in a single mirror. Each board directory keeps the per-board
+    # layout the device OTA expects: the shared staticfs.bin, the board's app.bin
+    # (+ bootloader / partition-table) from firmware_build_<board>, the merged
+    # firmware.bin, and md5/sha256 checksums over every .bin.
     - name: Assemble upload tree
       shell: bash
       env:
@@ -44,7 +45,7 @@ runs:
           [ -z "$board" ] && continue
           dest="upload/$board"
           mkdir -p "$dest"
-          cp artifacts/firmware_staticfs/*.bin "$dest"/
+          cp artifacts/firmware_staticfs/staticfs.bin "$dest"/
           cp "artifacts/firmware_build_$board"/*.bin "$dest"/
           cp "artifacts/firmware_merged_$board"/OpenShock_*.bin "$dest"/firmware.bin
           (

.github/actions/merge-partitions/action.yml

@@ -1,8 +1,11 @@
 name: merge-partitions
-description: Merges multiple partitions into a single flashable binary
+description: >-
+  Link a board's C++ partition images (from build-firmware) with the shared
+  static0 image (from build-staticfs) into a single flashable binary via
+  scripts/merge_image.py. Needs esptool only - no full ESP-IDF install.
 inputs:
   version:
-    description: 'Version of the firmware'
+    description: 'Firmware version'
     required: true
   board:
     description: 'Board name to merge partitions for'
@@ -15,25 +18,31 @@ runs:
       with:
         cache: 'pip'
 
-    - name: Install dependencies
+    - name: Install Python dependencies
       shell: bash
       run: pip install -r requirements.txt
 
-    - name: Download static filesystem partition
+    - name: Download C++ partition images
       uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
       with:
-        name: firmware_staticfs
+        name: firmware_build_${{ inputs.board }}
+        path: merge
 
-    - name: Download firmware partitions
+    - name: Download static filesystem image
       uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
       with:
-        name: firmware_build_${{ inputs.board }}
+        name: firmware_staticfs
+        path: merge
 
     - name: Merge partitions
       shell: bash
       run: |
-        python scripts/merge_image.py ${{ inputs.board }}
-        mv merged.bin OpenShock_${{ inputs.board }}_${{ inputs.version }}.bin
+        set -euo pipefail
+        python scripts/merge_image.py \
+          --board "${{ inputs.board }}" \
+          --bindir merge \
+          --staticfs merge/staticfs.bin \
+          --output "OpenShock_${{ inputs.board }}_${{ inputs.version }}.bin"
 
     - name: Upload merged firmware binary
       uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1