Harden 1.4 against frontend corruption + freeze/lock dependencies

.github/workflows/test-and-deploy.yml

@@ -31,7 +31,7 @@ jobs:
       run: |
         sudo apt-get install --no-install-recommends --assume-yes shellcheck parallel
         python -m pip install --upgrade pip
-        pip install poetry
+        pip install poetry==2.4.1
 
     - name: Run tests
       run: |
@@ -54,10 +54,10 @@ jobs:
 
       - uses: oven-sh/setup-bun@v1
         with:
-          bun-version: 1.0.3
+          bun-version: 1.3.14
 
       - name: Bun install
-        run: bun install --cwd ./core/frontend
+        run: bun install --frozen-lockfile --cwd ./core/frontend
 
       - name: Bun lint
         run: bun --cwd ./core/frontend lint

.gitignore

@@ -12,7 +12,6 @@ node_modules
 env/
 Pipfile
 Pipfile.lock
-poetry.lock
 registry/
 
 # Docker compose binds

.hooks/pre-push

@@ -53,7 +53,7 @@ echo "installing mavlink-router"
 echo "installing mavp2p"
 ./core/tools/mavp2p/bootstrap.sh
 echo "installing ardupilot_tools"
-NOSUDO=1 ./core/tools/ardupilot_tools/bootstrap.sh
+NOSUDO=1 ./core/tools/ardupilot_tools/bootstrap.sh && ./core/tools/ardupilot_tools/setup-python-libs.sh
 
 isort_extra_args="--check-only --diff"
 black_extra_args="--check --diff"
@@ -99,9 +99,14 @@ echo "Running shellcheck..."
 #SC2086: Allow word splitting
 git ls-files '*.sh' | xargs -L 1 shellcheck --exclude=SC2005,SC2015,SC2046,SC2048,SC2086
 
+# Roots of every python project, which isort and mypy check one by one. A project
+# declares itself with either a setup.py or a pyproject.toml.
+python_packages=$(git ls-files '*/setup.py' 'core/libs/*/pyproject.toml' 'core/services/*/pyproject.toml' \
+    | xargs -n 1 dirname | sort -u)
+
 echo "Running isort.."
 # Run isort for each python project
-dirname $(git ls-files "$repository_path/*/setup.py") | xargs -I {} isort --src-path="{}" ${isort_extra_args} "{}"
+echo "$python_packages" | xargs -I {} isort --src-path="{}" ${isort_extra_args} "{}"
 
 # This will only get the python files track by git, not including submodules
 python_files=$(git ls-files '*.py')
@@ -120,7 +125,7 @@ echo "Running pylint.."
 pylint $python_files
 
 echo "Running mypy.."
-git ls-files '*setup.py' | parallel 'mypy $(dirname {}) --cache-dir "$(dirname {})/__mypycache__"'
+echo "$python_packages" | parallel 'mypy {} --cache-dir "{}/__mypycache__"'
 
 echo "Running pytest.."
 pytest -n 10 --durations=0 --cov="$repository_path" --cov-report html

core/.python-version

@@ -0,0 +1 @@
+3.11

core/Dockerfile

@@ -2,7 +2,7 @@
 ARG BASE_IMAGE=bluerobotics/blueos-base:0.2.6
 
 # Build frontend
-FROM --platform=$BUILDPLATFORM oven/bun:1.0.3-slim AS frontend-builder
+FROM --platform=$BUILDPLATFORM oven/bun:1.3.14-slim AS frontend-builder
 
 ARG VITE_APP_GIT_DESCRIBE
 ENV VITE_APP_GIT_DESCRIBE=${VITE_APP_GIT_DESCRIBE:-none/none-0-g00000000}
@@ -21,7 +21,7 @@ COPY frontend /home/pi/frontend
 RUN <<-EOF
 set -e
 
-    bun install --cwd /home/pi/frontend
+    bun install --frozen-lockfile --cwd /home/pi/frontend
     NODE_OPTIONS=--max-old-space-size=8192 bun run --cwd /home/pi/frontend build
 
 EOF
@@ -34,6 +34,39 @@ COPY tools /home/pi/tools
 RUN apt update && apt install -y --no-install-recommends binutils
 RUN /home/pi/tools/install-static-binaries.sh
 
+# Generation of python virtual environment for our libraries and services
+FROM base AS install-services-and-libs
+
+RUN apt update && apt install -y --no-install-recommends g++
+
+# Wifi service / Bind path for wpa
+RUN mkdir -p /var/run/wpa_supplicant/
+
+# Install UV, version pinned so the resolver that reads uv.lock never changes on its own
+ADD https://astral.sh/uv/0.12.0/install.sh /uv-installer.sh
+RUN sh /uv-installer.sh && rm /uv-installer.sh
+ENV PATH="/root/.local/bin/:$PATH"
+
+# Project files
+COPY ./uv.lock /home/pi/uv.lock
+COPY ./pyproject.toml /home/pi/pyproject.toml
+COPY ./.python-version /home/pi/.python-version
+
+COPY ./libs /home/pi/libs
+COPY ./services /home/pi/services
+
+COPY ./tools /home/pi/tools
+
+# Virtual environment creation and packages installation
+WORKDIR /home/pi
+# --locked fails the build when uv.lock is out of sync with the pyprojects, so dependency
+# changes have to be relocked and committed instead of silently resolved at build time
+RUN uv venv && uv sync --locked
+
+# Post install scripts
+ENV VIRTUAL_ENV=/home/pi/.venv
+RUN /home/pi/tools/install-python-libs.sh
+
 # BlueOS base image
 FROM base
 
@@ -70,17 +103,15 @@ ENV TARGETVARIANT=${TARGETVARIANT}
 COPY tools /home/pi/tools
 RUN /home/pi/tools/install-system-tools.sh
 
-# Install custom libraries
+# Copy libs and services code
 COPY libs /home/pi/libs
-RUN /home/pi/libs/install-libs.sh
+COPY services /home/pi/services
 
 # Set tmux configuration file
 COPY configuration/tmux.conf /etc/tmux.conf
 COPY configuration/motd /etc/motd
 
-# Install services
-COPY services /home/pi/services
-RUN /home/pi/services/install-services.sh
+# Copy configuration files
 COPY start-blueos-core /usr/bin/start-blueos-core
 COPY run-service.sh /usr/bin/run-service
 
@@ -99,6 +130,7 @@ COPY --from=download-binaries \
 
 # Copy frontend built on frontend-builder to this stage
 COPY --from=frontend-builder /home/pi/frontend/dist /home/pi/frontend
+COPY --from=install-services-and-libs /home/pi/.venv /usr/blueos/venv
 
 RUN <<-EOF
 set -e
@@ -113,7 +145,7 @@ set -e
     RCFILE_PATH="/etc/blueosrc"
     echo "export GIT_DESCRIBE_TAGS=$GIT_DESCRIBE_TAGS" >> $RCFILE_PATH
     echo "export HISTFILE=/etc/blueos/.bash_history" >> $RCFILE_PATH
-    echo "export PATH=$(python -m site --user-base)/bin:/usr/blueos/bin:$PATH" >> $RCFILE_PATH
+    echo "export PATH=/usr/blueos/venv/bin:/usr/blueos/bin:$PATH" >> $RCFILE_PATH
     echo "export RUST_BACKTRACE=1" >> $RCFILE_PATH
 
     # Setup shortcuts

core/frontend/.gitignore

@@ -4,4 +4,3 @@ dist/
 yarn.lock
 dev-dist/
 components.d.ts
-bun.lockb

core/frontend/vite.config.js

@@ -118,6 +118,51 @@ export default defineConfig(({ command, mode }) => {
         deleteOriginFile: true,
         filter: /\.(js|css|json|svg|txt|xml|wasm|glb)$/i,
       }),
+      // Parse every shipped script, so a corrupt chunk fails the build instead of only breaking
+      // the route that imports it at runtime, like 1.4.4-beta.14 shipped a source map as a chunk.
+      // Runs after compression to check the exact bytes nginx serves.
+      {
+        name: 'validate-emitted-javascript',
+        apply: 'build',
+        enforce: 'post',
+        // closeBundle is a parallel hook, sequential makes it wait for the compression above
+        closeBundle: {
+          sequential: true,
+          handler() {
+            const fs = require('fs')
+            const { gunzipSync } = require('zlib')
+            const { transformSync } = require('esbuild')
+            const distPath = path.resolve(__dirname, 'dist')
+
+            const collectScripts = (dir) => fs.readdirSync(dir, { withFileTypes: true }).flatMap((entry) => {
+              const fullPath = path.join(dir, entry.name)
+              if (entry.isDirectory()) return collectScripts(fullPath)
+              return /\.js(\.gz)?$/.test(entry.name) ? [fullPath] : []
+            })
+
+            const scripts = collectScripts(distPath)
+            if (scripts.length === 0) {
+              throw new Error('No javascript emitted to dist/, the build produced nothing to validate')
+            }
+
+            const failures = scripts.flatMap((script) => {
+              try {
+                const bytes = fs.readFileSync(script)
+                transformSync((script.endsWith('.gz') ? gunzipSync(bytes) : bytes).toString('utf8'), { loader: 'js' })
+                return []
+              } catch (error) {
+                return [`${path.relative(distPath, script)}: ${error.errors?.[0]?.text ?? error.message}`]
+              }
+            })
+
+            if (failures.length > 0) {
+              throw new Error(`Corrupt javascript in dist/:\n  ${failures.join('\n  ')}`)
+            }
+
+            console.log(`validated ${scripts.length} emitted javascript files`)
+          }
+        }
+      },
       // Fix Draco imports in dev server
       {
         name: 'draco-dev-fix',