feat(bench): 构建引擎基准套件 —— 把一次性脚本变成跨平台、可扩展的测量设施 (2026.8.12.1) #19
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: bench | |
| # Build-engine benchmark. Runs on changes under `bench/` and on demand. | |
| # | |
| # WHY IT IS PATH-SCOPED RATHER THAN ON EVERY PUSH: | |
| # | |
| # * it is heavy — a full matrix compiles the same fixture six ways per platform | |
| # * it is noisy — cloud runners are shared, and the CPU model changes under you | |
| # * it asserts nothing — no threshold, no pass/fail on timings | |
| # | |
| # So it fires when the SUITE itself changes, where the question "did I break the | |
| # harness / did this shift the numbers" is actually being asked, and stays off | |
| # every unrelated PR. A timing threshold on a shared runner would turn normal | |
| # variance into red crosses people learn to ignore, so there is none: results | |
| # are uploaded as artifacts and comparing them is a human act. | |
| # | |
| # The matrix runs platforms in parallel and `fail-fast: false`, because one | |
| # platform missing an engine must not cancel the data from the others. | |
| # | |
| # See bench/README.md for the measurement contract before quoting any number. | |
| on: | |
| # Changes that can MOVE THE NUMBERS: the harness itself, the build | |
| # descriptions of the projects it measures, and its own tests. | |
| # | |
| # Documentation and past results are excluded on purpose. A README edit cannot | |
| # change a measurement, and running a two-hour matrix to prove that teaches | |
| # everyone to ignore the check — which is how a benchmark stops being read. | |
| # `bench/results/**` is excluded for the same reason AND a sharper one: this | |
| # workflow's own artifacts land there, so including it would let a results | |
| # commit trigger the run that produces the next results commit. | |
| push: | |
| paths: | |
| - 'bench/**' | |
| - '!bench/**/*.md' | |
| - '!bench/results/**' | |
| - '.github/workflows/bench.yml' | |
| pull_request: | |
| paths: | |
| - 'bench/**' | |
| - '!bench/**/*.md' | |
| - '!bench/results/**' | |
| - '.github/workflows/bench.yml' | |
| workflow_dispatch: | |
| inputs: | |
| engines: | |
| description: 'comma-separated: mcpp,mcpp-opt,cmake,xmake,meson,bazel' | |
| required: false | |
| default: 'mcpp,mcpp-opt,cmake,xmake' | |
| variants: | |
| description: 'comma-separated: headers,modules,modules-impl' | |
| required: false | |
| default: 'headers,modules,modules-impl' | |
| scenarios: | |
| description: 'comma-separated: cold,noop,touch-hub,touch-leaf,edit-body,edit-comment' | |
| required: false | |
| # All of them. A scenario left out of the default is a scenario nobody | |
| # ever runs — `touch-leaf` was defined, documented and advertised, and | |
| # had never appeared in a single result file. | |
| default: 'cold,noop,touch-hub,touch-leaf,edit-body,edit-comment' | |
| preset: | |
| description: 'named fixture size: smoke | standard | large (overridden by units/fanin/weight below)' | |
| required: false | |
| default: 'standard' | |
| units: | |
| description: 'fixture translation units (0 = use the preset)' | |
| required: false | |
| default: '0' | |
| fanin: | |
| description: 'dependencies per unit (controls graph depth)' | |
| required: false | |
| default: '3' | |
| runs: | |
| description: 'repetitions per cell (0 = per-scenario default)' | |
| required: false | |
| default: '0' | |
| profile: | |
| description: 'release | debug' | |
| required: false | |
| default: 'release' | |
| platforms: | |
| description: 'comma-separated: linux,macos,windows' | |
| required: false | |
| default: 'linux,macos,windows' | |
| concurrency: | |
| group: bench-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| # The matrix is computed rather than written out, so `platforms: linux` runs | |
| # ONE job instead of three jobs where two are skipped — a skipped job still | |
| # queues a runner and still reports a check. | |
| plan: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.plan.outputs.matrix }} | |
| steps: | |
| - id: plan | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| # `inputs.*` is empty on a push/pull_request trigger, so every input needs | |
| # a fallback here — an empty `platforms` would otherwise plan an empty | |
| # matrix and the job would silently do nothing. | |
| want="${{ inputs.platforms || 'linux,macos,windows' }}" | |
| entries=() | |
| case ",$want," in *,linux,*) entries+=('{"os":"ubuntu-24.04","name":"linux"}');; esac | |
| case ",$want," in *,macos,*) entries+=('{"os":"macos-14","name":"macos"}');; esac | |
| case ",$want," in *,windows,*) entries+=('{"os":"windows-2022","name":"windows"}');; esac | |
| if [ ${#entries[@]} -eq 0 ]; then | |
| echo "no platform selected from '$want'" >&2 | |
| exit 1 | |
| fi | |
| printf 'matrix={"include":[%s]}\n' "$(IFS=,; echo "${entries[*]}")" >> "$GITHUB_OUTPUT" | |
| bench: | |
| needs: plan | |
| strategy: | |
| fail-fast: false # one platform's engine gap must not cancel the rest | |
| matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 120 | |
| name: bench (${{ matrix.name }}) | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ./.github/actions/bootstrap-mcpp | |
| - name: Build the harness | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| cd bench | |
| "$MCPP" build --release | |
| # Resolve the produced binary once; the fingerprint directory name is | |
| # not predictable from here. | |
| BIN=$(find target -type f -name 'bench' -o -type f -name 'bench.exe' | head -1) | |
| [ -n "$BIN" ] || { echo "harness binary not found under bench/target" >&2; exit 1; } | |
| echo "BENCH=$PWD/$BIN" >> "$GITHUB_ENV" | |
| # Engines beyond mcpp are optional by design: a missing one is reported as | |
| # `unavailable` with a reason, never as a slow or broken engine. Installing | |
| # them is therefore best-effort and never fails the job. | |
| - name: Install comparison engines (best effort) | |
| shell: bash | |
| continue-on-error: true | |
| run: | | |
| set -uo pipefail | |
| xlings install bazel -y || echo "bazel unavailable on this runner" | |
| xlings install xmake -y || echo "xmake unavailable on this runner" | |
| python3 -m pip install --quiet meson || echo "meson unavailable on this runner" | |
| cmake --version || true | |
| ninja --version || true | |
| - name: Report engine availability | |
| shell: bash | |
| run: | | |
| "$BENCH" --list | |
| - name: Run benchmark | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| # The preset names the size; units/fanin override it only when set to a | |
| # positive number. Passing raw numbers unconditionally would make every | |
| # run's size an accident of this file rather than a named, comparable | |
| # workload — and --preset must come first so the overrides still win. | |
| # Every `inputs.*` needs a fallback: on a push/pull_request trigger | |
| # they are all EMPTY, and an empty --engines would run nothing while | |
| # still reporting success. | |
| args=( --preset "${{ inputs.preset || 'smoke' }}" ) | |
| [ "${{ inputs.units || 0 }}" -gt 0 ] 2>/dev/null && args+=( --units "${{ inputs.units }}" ) | |
| [ "${{ inputs.fanin || 0 }}" -gt 0 ] 2>/dev/null && args+=( --fanin "${{ inputs.fanin }}" ) | |
| "$BENCH" \ | |
| --engines '${{ inputs.engines || 'mcpp,cmake,xmake,meson,bazel' }}' \ | |
| --variants '${{ inputs.variants || 'headers,modules,modules-impl' }}' \ | |
| --scenarios '${{ inputs.scenarios || 'cold,noop,touch-hub,touch-leaf,edit-body,edit-comment' }}' \ | |
| --profile '${{ inputs.profile || 'release' }}' \ | |
| "${args[@]}" \ | |
| --runs '${{ inputs.runs || 0 }}' \ | |
| --work "$RUNNER_TEMP/bench-work" \ | |
| --out "bench-${{ matrix.name }}.json" | |
| - name: Upload report | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: bench-${{ matrix.name }} | |
| path: bench-${{ matrix.name }}.json | |
| if-no-files-found: error |