From 65791607fc2be5406bb6a5280ff376e237cb33c8 Mon Sep 17 00:00:00 2001 From: danceratopz Date: Mon, 13 Jul 2026 11:07:27 +0200 Subject: [PATCH 1/6] refactor(ci): extract shared fill-release just recipe Move the fill flags shared by all fixture releases (`-n`, `--no-html`, `--durations=100`, `--log-level=DEBUG`) out of the inline fill step in `.github/actions/build-fixtures` into a new `just fill-release` recipe. Callers append the feature's fill params, fork range and output via pass-through args; for repeated flags, the last occurrence wins. The xdist worker count is plumbed via `PYTEST_XDIST_AUTO_NUM_WORKERS` (`xdist: auto` in `evm.yaml` for all EVM types, unchanged). Reimplement `fill-nightly` as a thin wrapper around `fill-release` so the nightly fill shares the release flags. Deliberate changes to the nightly invocation: it gains `--no-html` and `--log-level=DEBUG`, and it drops flags that defaults already provide: `--dist=loadgroup` and the trailing `tests` path (both defaults in `pytest-fill.ini`), and the explicit `-n` (now supplied by `fill-release`, same value). --- .github/actions/build-fixtures/action.yaml | 13 ++++++++----- Justfile | 19 ++++++++++++++----- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/.github/actions/build-fixtures/action.yaml b/.github/actions/build-fixtures/action.yaml index 719c3221a75..2bd8d636480 100644 --- a/.github/actions/build-fixtures/action.yaml +++ b/.github/actions/build-fixtures/action.yaml @@ -50,6 +50,8 @@ runs: run: sudo apt-get install -y pigz - name: Generate fixtures using fill shell: bash + env: + PYTEST_XDIST_AUTO_NUM_WORKERS: ${{ steps.evm-builder.outputs.xdist }} run: | IS_SPLIT="${{ inputs.split_label }}" @@ -61,13 +63,14 @@ runs: FORK_ARGS="" fi + EVM_ARGS="" + if [ "${{ steps.evm-builder.outputs.impl }}" != "eels" ]; then + EVM_ARGS="--evm-bin=${{ steps.evm-builder.outputs.evm-bin }}" + fi + # Allow exit code 5 (NO_TESTS_COLLECTED) for fork ranges with no tests. EXIT_CODE=0 - if [ "${{ steps.evm-builder.outputs.impl }}" = "eels" ]; then - uv run fill -n ${{ steps.evm-builder.outputs.xdist }} ${{ steps.properties.outputs.fill-params }} $FORK_ARGS $OUTPUT_ARG --build-name ${{ inputs.release_name }} --no-html --durations=100 --log-level=DEBUG || EXIT_CODE=$? - else - uv run fill -n ${{ steps.evm-builder.outputs.xdist }} --evm-bin=${{ steps.evm-builder.outputs.evm-bin }} ${{ steps.properties.outputs.fill-params }} $FORK_ARGS $OUTPUT_ARG --build-name ${{ inputs.release_name }} --no-html --durations=100 --log-level=DEBUG || EXIT_CODE=$? - fi + just fill-release $EVM_ARGS ${{ steps.properties.outputs.fill-params }} $FORK_ARGS $OUTPUT_ARG --build-name ${{ inputs.release_name }} || EXIT_CODE=$? if [ "$EXIT_CODE" -ne 0 ] && [ "$EXIT_CODE" -ne 5 ]; then exit "$EXIT_CODE" fi diff --git a/Justfile b/Justfile index 9091cd882c1..4c1f4c1aaf7 100644 --- a/Justfile +++ b/Justfile @@ -124,12 +124,23 @@ fill *args: "$@" \ tests +# Callers append the feature params, fork range and output; last flag wins. +# Fill fixtures with the flags shared by all fixture releases +[group('consensus tests')] +fill-release *args: + uv run fill \ + -n {{ xdist_workers }} \ + --output="{{ output_dir }}/fill-release/fixtures" \ + --no-html \ + --durations=100 \ + --log-level=DEBUG \ + "$@" + # Fill all tests (slow included) in all fixture formats [group('consensus tests')] fill-nightly *args: @mkdir -p "{{ output_dir }}/fill-nightly/tmp" "{{ output_dir }}/fill-nightly/logs" - uv run fill \ - -n {{ xdist_workers }} --dist=loadgroup \ + just fill-release \ --skip-index \ --generate-all-formats \ --output="{{ output_dir }}/fill-nightly/fixtures" \ @@ -137,9 +148,7 @@ fill-nightly *args: --log-to "{{ output_dir }}/fill-nightly/logs" \ --clean \ --until "{{ latest_fork }}" \ - --durations=100 \ - "$@" \ - tests + "$@" # --- Integration Tests --- From bf52dbb2e116a060aed23d33140edee0e229b78c Mon Sep 17 00:00:00 2001 From: danceratopz Date: Mon, 13 Jul 2026 11:14:02 +0200 Subject: [PATCH 2/6] feat(ci): run the fixture release flow nightly Replace the standalone nightly-fill workflow with a schedule trigger on `release_fixtures.yaml`: scheduled runs fill the new `nightly` feature (`--until=Amsterdam --generate-all-formats`, slow tests included) through the exact release pipeline (matrix generation, `build-fixtures`, index merge and tarball) and skip only the `release` job, so no tag or draft release is created. The nightly is thereby a daily rehearsal of the release flow itself. Scheduled runs skip when there are no new commits since the last successful scheduled run (query filtered with `event=schedule` so manual releases never advance the nightly baseline) and list the new commits in the step summary. `fail-fast` is disabled on schedule so every fork range reports. `run-name` distinguishes nightly runs from manual releases in the run history. Artifact retention replaces the FIFO rotation of the standalone workflow: the combined `fixtures_nightly.tar.gz` is kept for 5 days (roughly the last five nightly builds at a daily cadence) and split intermediates for 1 day, via a new `split_retention_days` input on the `build-fixtures` action. Manual releases keep the repo defaults. Remove the `fill-nightly` recipe: the nightly fill now goes through `fill-release` inside `build-fixtures`, gaining the fixture index and release tarball packaging that the standalone workflow lacked. --- .github/actions/build-fixtures/action.yaml | 4 + .github/configs/feature.yaml | 7 + .github/workflows/nightly-fill.yaml | 163 --------------------- .github/workflows/release_fixtures.yaml | 79 +++++++++- Justfile | 14 -- docs/dev/releasing_tests.md | 4 + 6 files changed, 90 insertions(+), 181 deletions(-) delete mode 100644 .github/workflows/nightly-fill.yaml diff --git a/.github/actions/build-fixtures/action.yaml b/.github/actions/build-fixtures/action.yaml index 2bd8d636480..18c6fcce126 100644 --- a/.github/actions/build-fixtures/action.yaml +++ b/.github/actions/build-fixtures/action.yaml @@ -13,6 +13,9 @@ inputs: split_label: description: "Label for this fork-range split. Empty for unsplit builds." default: "" + split_retention_days: + description: "retention-days for the split fixture artifact. Empty for the repo default." + default: "" evm: description: "Override the evm impl. Defaults to the feature's evm-type." default: "" @@ -99,3 +102,4 @@ runs: include-hidden-files: true path: fixtures_${{ inputs.release_name }}/ if-no-files-found: ignore + retention-days: ${{ inputs.split_retention_days }} diff --git a/.github/configs/feature.yaml b/.github/configs/feature.yaml index 39297ec6e84..6aadc6f0097 100644 --- a/.github/configs/feature.yaml +++ b/.github/configs/feature.yaml @@ -9,6 +9,13 @@ tests: evm-type: eels fill-params: --until=BPO4 --generate-all-formats +# Filled by the scheduled nightly run of the `release_fixtures` workflow: +# all tests (slow included), all fixture formats, up to the dev fork. Not +# meant for tagged releases. +nightly: + evm-type: eels + fill-params: --until=Amsterdam --generate-all-formats + benchmark: evm-type: benchmark fill-params: --fork=Osaka --generate-all-formats --gas-benchmark-values 1,5,10,30,60,100,150 ./tests/benchmark/compute --maxprocesses=30 --dist=worksteal diff --git a/.github/workflows/nightly-fill.yaml b/.github/workflows/nightly-fill.yaml deleted file mode 100644 index 1439f9a29c4..00000000000 --- a/.github/workflows/nightly-fill.yaml +++ /dev/null @@ -1,163 +0,0 @@ -name: Nightly Fill - -# Runs at 02:00 UTC: the self-hosted runners are past the EU/US daytime -# peaks and results are ready before the EU morning. -on: - schedule: - - cron: "0 2 * * *" - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }} - cancel-in-progress: false - -permissions: - contents: read - # Needed to look up the last successful nightly run and to download the - # per-range fixture artifacts in the `combine` job. - actions: read - -jobs: - setup: - name: Check commits and build fork matrix - runs-on: ubuntu-latest - outputs: - run: ${{ steps.check.outputs.run }} - matrix: ${{ steps.matrix.outputs.matrix }} - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - submodules: false - - name: List commits since the last successful nightly fill - id: check - env: - GH_TOKEN: ${{ github.token }} - run: | - # Head SHA of the last nightly run that filled successfully. Using - # the last *success* (rather than a fixed time window) means a - # nightly that fails or is skipped keeps re-running until it goes - # green, and no commit slips through unfilled. - last_sha=$(gh api \ - "repos/${GITHUB_REPOSITORY}/actions/workflows/nightly-fill.yaml/runs?status=success&per_page=1" \ - --jq '.workflow_runs[0].head_sha // ""') - - if [ -n "$last_sha" ]; then - commits=$(gh api \ - "repos/${GITHUB_REPOSITORY}/compare/${last_sha}...${GITHUB_SHA}" \ - --jq '.commits[] | "- \(.sha[0:7]) \(.commit.message | split("\n")[0])"') - else - # No prior successful run recorded; fill to establish a baseline. - commits="- (no previous successful nightly fill found)" - fi - - count=$(printf '%s\n' "$commits" | grep -c . || true) - if [ "$count" -gt 0 ] || [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then - echo "run=true" >> "$GITHUB_OUTPUT" - { - echo "### Commits since last successful nightly fill" - printf '%s\n' "$commits" - } >> "$GITHUB_STEP_SUMMARY" - else - echo "run=false" >> "$GITHUB_OUTPUT" - echo "No new commits since the last successful nightly fill; skipping." \ - >> "$GITHUB_STEP_SUMMARY" - fi - - - name: Build fork matrix from the shared config - id: matrix - # Reuse .github/configs/fork-ranges.yaml (the same split the release - # workflow uses) so the fork coverage is defined in one place. - run: | - echo "matrix=$(yq -o=json -I=0 '.' .github/configs/fork-ranges.yaml)" >> "$GITHUB_OUTPUT" - - fill-nightly: - name: fill-nightly (${{ matrix.label }}) - runs-on: [self-hosted-ghr, size-xl-x64] - needs: setup - if: needs.setup.outputs.run == 'true' - timeout-minutes: 720 - strategy: - fail-fast: false - matrix: - include: ${{ fromJson(needs.setup.outputs.matrix) }} - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - submodules: recursive - - uses: ./.github/actions/setup-uv - with: - python-version: "3.14" - - uses: ./.github/actions/setup-env - - name: Run nightly fill (${{ matrix.label }}) - run: just fill-nightly --from ${{ matrix.from }} --until ${{ matrix.until }} - env: - PYTEST_XDIST_AUTO_NUM_WORKERS: auto - - name: Upload per-range fixtures (${{ matrix.label }}) - # Intermediate split, merged into a single artifact by `combine`. - # Uploaded even on failure so a regressed fill can still be inspected. - if: always() - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - with: - name: fixtures__${{ matrix.label }} - path: .just/fill-nightly/fixtures - include-hidden-files: true - if-no-files-found: warn - retention-days: 1 - - combine: - name: Combine fixtures - runs-on: [self-hosted-ghr, size-xl-x64] - needs: [setup, fill-nightly] - # Run even if some ranges failed, so partial fixtures are still published. - if: ${{ !cancelled() && needs.setup.outputs.run == 'true' }} - permissions: - contents: read - # Write is needed to delete superseded fixture artifacts (rotation). - actions: write - steps: - - name: Resolve short commit SHA - id: sha - run: echo "short=${GITHUB_SHA:0:8}" >> "$GITHUB_OUTPUT" - - name: Download per-range fixtures - env: - GH_TOKEN: ${{ github.token }} - run: | - gh run download "${{ github.run_id }}" -p "fixtures__*" --dir split \ - || echo "No per-range fixture artifacts found." - - name: Merge into a single fixtures tree - run: | - mkdir -p "fixtures_${{ steps.sha.outputs.short }}" - for d in split/fixtures__*/; do - [ -d "$d" ] || continue - rsync -a "$d" "fixtures_${{ steps.sha.outputs.short }}/" - done - - name: Upload combined fixtures - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - with: - # Keyed by commit so a downloaded set maps back to what filled it. - name: fixtures_${{ steps.sha.outputs.short }} - path: fixtures_${{ steps.sha.outputs.short }} - include-hidden-files: true - if-no-files-found: warn - # Outer bound only; the real policy is "keep the last 5 builds", - # enforced by the rotation step below. - retention-days: 90 - - name: Rotate — keep only the last 5 nightly fixture sets - env: - GH_TOKEN: ${{ github.token }} - run: | - # Combined nightly artifacts are named `fixtures_<8-hex-sha>`. List - # them newest-first and delete everything past the 5 most recent. - # Release artifacts (`fixtures_`) and per-range splits - # (`fixtures__