diff --git a/.github/actions/codegen_drift/action.yml b/.github/actions/codegen_drift/action.yml new file mode 100644 index 00000000000..83fc03e26ff --- /dev/null +++ b/.github/actions/codegen_drift/action.yml @@ -0,0 +1,104 @@ +name: 'Codegen drift check' +description: > + Runs a Justfile codegen recipe and reports a warning - without failing the + build - when the committed output differs from what the recipe produces. + +inputs: + recipe: + description: 'Justfile recipe to run (e.g. gen-fonts)' + required: true + paths: + description: 'Git pathspec of the generated files to check for drift' + required: true + label: + description: 'Human readable name of the generated artefact' + required: true + just-version: + description: 'Version of just to install' + default: '1.57.0' + +runs: + using: "composite" + steps: + + - name: Install just + shell: bash + env: + JUST_VERSION: ${{ inputs.just-version }} + run: | + # The edgetx-dev image ships wget but not curl. + wget -qO- \ + "https://github.com/casey/just/releases/download/${JUST_VERSION}/just-${JUST_VERSION}-x86_64-unknown-linux-musl.tar.gz" \ + | tar xz -C /usr/local/bin just + just --version + + - name: Allow git to read the workspace + shell: bash + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Regenerate ${{ inputs.label }} + shell: bash + env: + RECIPE: ${{ inputs.recipe }} + run: just "${RECIPE}" 2>&1 | tee /tmp/codegen.log + + - name: Check for incomplete generation + shell: bash + env: + RECIPE: ${{ inputs.recipe }} + run: | + # make_fonts.sh warns and carries on when a translation header yields no + # characters, so a clean diff on its own does not prove a complete run. + if grep -q 'No characters found' /tmp/codegen.log; then + echo "::warning::'just ${RECIPE}' skipped one or more font sets ('No characters found' in the log), so the drift result below may be incomplete." + fi + + - name: Check for drift + shell: bash + env: + PATHS: ${{ inputs.paths }} + RECIPE: ${{ inputs.recipe }} + LABEL: ${{ inputs.label }} + run: | + # Deliberately non-failing. Contributors without the local toolchain must + # not be blocked; this only surfaces that the committed files have drifted. + # Word splitting on PATHS is intentional - it may hold several pathspecs. + # shellcheck disable=SC2086 + changed=$(git status --porcelain -- ${PATHS}) + if [ -z "${changed}" ]; then + echo "${LABEL}: up to date" + exit 0 + fi + + echo "::warning::${LABEL} is out of date. Run 'just ${RECIPE}' and commit the result." + + # A full regeneration diff can run to megabytes (the fonts are the worst + # case), so truncate well inside the 1 MiB step summary limit. Truncation + # uses bash substring expansion rather than `head`: piping into `head` + # would SIGPIPE the producer once it closed the pipe, and under `pipefail` + # that exits 141 - failing a step that must never fail. + # shellcheck disable=SC2086 + diff=$(git diff -- ${PATHS} || true) + truncated="${diff:0:60000}" + if [ "${#diff}" -gt "${#truncated}" ]; then + truncated="${truncated}"$'\n''... diff truncated ...' + fi + + { + echo "### :warning: ${LABEL} is out of date" + echo + echo "The committed files differ from what \`just ${RECIPE}\` produces in" + echo "\`ghcr.io/edgetx/edgetx-dev:latest\`. Run \`just ${RECIPE}\` and commit the result." + echo + echo '```' + echo "${changed}" + echo '```' + echo + echo "
Diff" + echo + echo '```diff' + echo "${truncated}" + echo '```' + echo + echo "
" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/codegen_drift.yml b/.github/workflows/codegen_drift.yml new file mode 100644 index 00000000000..9639c383dc7 --- /dev/null +++ b/.github/workflows/codegen_drift.yml @@ -0,0 +1,182 @@ +--- +name: Codegen drift +on: + push: + branches: + - 'main' + - '[0-9]+.[0-9]+' + paths: &trigger-paths + - '.github/workflows/codegen_drift.yml' + - '.github/actions/codegen_drift/action.yml' + - 'Justfile' + # cfn sort order + - 'tools/cfn_sorter.sh' + - 'tools/cfn_sorter.cpp' + - 'tools/copyright-header.txt' + - 'radio/src/dataconstants.h' + - 'radio/src/cfn_sort.cpp' + # LVGL fonts + - 'radio/src/fonts/**' + # shared: the i18n headers feed both the fonts and the cfn sort order + - 'radio/src/translations/i18n/**' + # YAML parsers + - 'tools/generate-yaml.sh' + - 'tools/build-common.sh' + - 'radio/src/myeeprom.h' + - 'radio/src/datastructs.h' + - 'radio/src/storage/yaml/**' + - 'radio/util/generate_yaml.py' + - 'radio/util/yaml_parser.tmpl' + - 'radio/util/hw_defs/**' + - 'radio/src/boards/hw_defs/**' + pull_request: + paths: *trigger-paths + workflow_dispatch: + +# These jobs only ever report - they must never block a contributor who does not +# have the local toolchain, so nothing here fails on drift. +permissions: + contents: read + pull-requests: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + JUST_VERSION: '1.57.0' + +jobs: + changes: + name: Detect changed inputs + runs-on: ubuntu-latest + outputs: + cfn: ${{ steps.filter.outputs.cfn }} + fonts: ${{ steps.filter.outputs.fonts }} + yaml: ${{ steps.filter.outputs.yaml }} + steps: + - name: Check out the repo + uses: actions/checkout@v7 + + - name: Install just + run: | + wget -qO- "https://github.com/casey/just/releases/download/${JUST_VERSION}/just-${JUST_VERSION}-x86_64-unknown-linux-musl.tar.gz" \ + | tar xz -C /usr/local/bin just + + # A Justfile change cannot cause drift, but it can rename a recipe out + # from under the jobs below, so check they still resolve rather than + # regenerating everything. + - name: Check the recipes this workflow calls still resolve + run: | + just --list >/dev/null + for recipe in cfn-sort gen-fonts gen-yaml; do + just --show "${recipe}" >/dev/null + done + + # Skipped on workflow_dispatch, which has no base revision to diff + # against; the jobs below run unconditionally in that case. + - name: Filter changed paths + uses: dorny/paths-filter@v4 + id: filter + if: github.event_name != 'workflow_dispatch' + with: + filters: | + cfn: + - '.github/workflows/codegen_drift.yml' + - '.github/actions/codegen_drift/action.yml' + - 'tools/cfn_sorter.sh' + - 'tools/cfn_sorter.cpp' + - 'tools/copyright-header.txt' + - 'radio/src/dataconstants.h' + - 'radio/src/cfn_sort.cpp' + - 'radio/src/translations/i18n/**' + fonts: + - '.github/workflows/codegen_drift.yml' + - '.github/actions/codegen_drift/action.yml' + - 'radio/src/fonts/**' + - 'radio/src/translations/i18n/**' + yaml: + - '.github/workflows/codegen_drift.yml' + - '.github/actions/codegen_drift/action.yml' + - 'tools/generate-yaml.sh' + - 'tools/build-common.sh' + - 'radio/src/myeeprom.h' + - 'radio/src/datastructs.h' + - 'radio/src/dataconstants.h' + - 'radio/src/storage/yaml/**' + - 'radio/util/generate_yaml.py' + - 'radio/util/yaml_parser.tmpl' + - 'radio/util/hw_defs/**' + - 'radio/src/boards/hw_defs/**' + + cfn-sort: + name: Custom function sort order + needs: changes + if: needs.changes.outputs.cfn == 'true' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 15 + container: + image: ghcr.io/edgetx/edgetx-dev:latest + volumes: + - ${{ github.workspace }}:/src + steps: + - name: Check out the repo + uses: actions/checkout@v7 + with: + submodules: recursive + + - name: Check for drift + uses: ./.github/actions/codegen_drift + with: + just-version: ${{ env.JUST_VERSION }} + recipe: cfn-sort + paths: radio/src/cfn_sort.cpp + label: Custom function sort order + + fonts: + name: LVGL fonts + needs: changes + if: needs.changes.outputs.fonts == 'true' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 30 + container: + image: ghcr.io/edgetx/edgetx-dev:latest + volumes: + - ${{ github.workspace }}:/src + steps: + - name: Check out the repo + uses: actions/checkout@v7 + with: + submodules: recursive + + - name: Check for drift + uses: ./.github/actions/codegen_drift + with: + just-version: ${{ env.JUST_VERSION }} + recipe: gen-fonts + paths: radio/src/fonts/lvgl + label: LVGL fonts + + yaml: + name: YAML parsers + needs: changes + if: needs.changes.outputs.yaml == 'true' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 30 + container: + image: ghcr.io/edgetx/edgetx-dev:latest + volumes: + - ${{ github.workspace }}:/src + steps: + - name: Check out the repo + uses: actions/checkout@v7 + with: + submodules: recursive + + - name: Check for drift + uses: ./.github/actions/codegen_drift + with: + just-version: ${{ env.JUST_VERSION }} + recipe: gen-yaml + paths: radio/src/storage/yaml + label: YAML parsers diff --git a/Justfile b/Justfile new file mode 100644 index 00000000000..cce6e3c3581 --- /dev/null +++ b/Justfile @@ -0,0 +1,65 @@ +# EdgeTX code generation helpers. +# +# These recipes wrap existing scripts that regenerate files COMMITTED to git. +# After running one, review and commit the result. +# +# The codegen recipes run on the host and need the tools installed locally. The +# docker- variants run the same scripts in the dev container instead, which +# already has them. CI uses the host recipes, as its jobs run in that image. + +IMAGE := "ghcr.io/edgetx/edgetx-dev:latest" + +# --user keeps generated files owned by the invoking user rather than root +_docker := 'docker run --rm --user "$(id -u):$(id -g)" -v "$PWD":/src -w /src ' + IMAGE + +# Show available recipes +default: + @just --list + +# Needs: lv_font_conv (npm), python3, gcc, and the lvgl submodule checked out. +[doc('Regenerate the LVGL fonts (radio/src/fonts/lvgl/{std,sml,lrg}/lv_font_*.c)')] +[group('codegen')] +gen-fonts: + radio/src/fonts/lvgl/make_fonts.sh + +# Needs: a C++ compiler, plus the 19 system locales listed at the top of +# tools/cfn_sorter.sh (apt install locales && locale-gen ...). +[doc('Regenerate the custom-function sort order (radio/src/cfn_sort.cpp)')] +[group('codegen')] +cfn-sort: + tools/cfn_sorter.sh + +# FLAVOR is a semicolon-separated target list; empty uses the script's default. +# Recreates ./build from scratch, and needs the libclang version from the +# edgetx-dev container. +[doc('Regenerate the YAML parsers (radio/src/storage/yaml/yaml_datastructs_*.cpp)')] +[group('codegen')] +gen-yaml FLAVOR='': + FLAVOR="{{ FLAVOR }}" tools/generate-yaml.sh + +[doc('Regenerate the YAML parsers, LVGL fonts and cfn sort order')] +[group('codegen')] +codegen: gen-fonts cfn-sort gen-yaml + +[doc('Regenerate the LVGL fonts in the dev container')] +[group('codegen (docker)')] +docker-gen-fonts: + {{ _docker }} radio/src/fonts/lvgl/make_fonts.sh + +[doc('Regenerate the custom-function sort order in the dev container')] +[group('codegen (docker)')] +docker-cfn-sort: + {{ _docker }} tools/cfn_sorter.sh + +# Uses its own FetchContent cache: the default one is inside the repo, so it +# would be shared with host builds and their CMake state is not portable here. +[doc('Regenerate the YAML parsers in the dev container')] +[group('codegen (docker)')] +docker-gen-yaml FLAVOR='': + {{ _docker }} env FLAVOR="{{ FLAVOR }}" \ + FETCHCONTENT_BASE_DIR=/src/.cache/fetchcontent-docker \ + tools/generate-yaml.sh + +[doc('Regenerate everything in the dev container')] +[group('codegen (docker)')] +docker-codegen: docker-gen-fonts docker-cfn-sort docker-gen-yaml diff --git a/radio/src/fonts/.gitignore b/radio/src/fonts/.gitignore index 5e7095013d6..4350affd24d 100644 --- a/radio/src/fonts/.gitignore +++ b/radio/src/fonts/.gitignore @@ -1,3 +1,7 @@ /*/*.lbm /font_*.png /font_*.specs + +# make_fonts.sh intermediates - removed when it completes, left behind if it aborts +/lvgl/lv_font.inc +/lvgl/lz4_font diff --git a/tools/.gitignore b/tools/.gitignore index bee8a64b79a..214161498ba 100644 --- a/tools/.gitignore +++ b/tools/.gitignore @@ -1 +1,4 @@ __pycache__ + +# cfn_sorter.sh intermediate - removed when it completes, left behind if it aborts +/a.out