feat(ci): fill mainnet fixtures nightly, draft cached tests@ releases

.github/actions/build-fixtures/action.yaml

@@ -13,6 +13,9 @@ inputs:
   split_label:
     description: "Label for this fork-range split. Empty for unsplit builds."
     default: ""
+  split_retention_days:
+    description: "retention-days for the split fixture artifact. Empty for the repo default."
+    default: ""
   evm:
     description: "Override the evm impl. Defaults to the feature's evm-type."
     default: ""
@@ -50,6 +53,8 @@ runs:
       run: sudo apt-get install -y pigz
     - name: Generate fixtures using fill
       shell: bash
+      env:
+        PYTEST_XDIST_AUTO_NUM_WORKERS: ${{ steps.evm-builder.outputs.xdist }}
       run: |
         IS_SPLIT="${{ inputs.split_label }}"
 
@@ -61,13 +66,14 @@ runs:
           FORK_ARGS=""
         fi
 
+        EVM_ARGS=""
+        if [ "${{ steps.evm-builder.outputs.impl }}" != "eels" ]; then
+          EVM_ARGS="--evm-bin=${{ steps.evm-builder.outputs.evm-bin }}"
+        fi
+
         # Allow exit code 5 (NO_TESTS_COLLECTED) for fork ranges with no tests.
         EXIT_CODE=0
-        if [ "${{ steps.evm-builder.outputs.impl }}" = "eels" ]; then
-          uv run fill -n ${{ steps.evm-builder.outputs.xdist }} ${{ steps.properties.outputs.fill-params }} $FORK_ARGS $OUTPUT_ARG --build-name ${{ inputs.release_name }} --no-html --durations=100 --log-level=DEBUG || EXIT_CODE=$?
-        else
-          uv run fill -n ${{ steps.evm-builder.outputs.xdist }} --evm-bin=${{ steps.evm-builder.outputs.evm-bin }} ${{ steps.properties.outputs.fill-params }} $FORK_ARGS $OUTPUT_ARG --build-name ${{ inputs.release_name }} --no-html --durations=100 --log-level=DEBUG || EXIT_CODE=$?
-        fi
+        just fill-release $EVM_ARGS ${{ steps.properties.outputs.fill-params }} $FORK_ARGS $OUTPUT_ARG --build-name ${{ inputs.release_name }} || EXIT_CODE=$?
         if [ "$EXIT_CODE" -ne 0 ] && [ "$EXIT_CODE" -ne 5 ]; then
           exit "$EXIT_CODE"
         fi
@@ -96,3 +102,4 @@ runs:
         include-hidden-files: true
         path: fixtures_${{ inputs.release_name }}/
         if-no-files-found: ignore
+        retention-days: ${{ inputs.split_retention_days }}

.github/configs/evm.yaml

@@ -17,12 +17,6 @@ evmone:
   evm-bin: evmone
   xdist: auto
   targets: ["evmone-cli"]
-benchmark:
-  impl: geth
-  repo: ethereum/go-ethereum
-  ref: master
-  evm-bin: evm
-  xdist: auto
 besu:
   impl: besu
   repo: hyperledger/besu

.github/configs/feature.yaml

@@ -5,6 +5,9 @@
 # Any `<feat>-devnet` input resolves to the shared `devnet` entry but keeps
 # its name in the tag; the devnet number lives in the version (X), not the
 # feature name, so this file needs no edits for new devnets.
+# `tests` is also what the scheduled nightly run of the `release_fixtures`
+# workflow fills: mainnet forks only, so the rotating nightly artifact is a
+# release-ready rehearsal of the next tests@ release.
 tests:
   evm-type: eels
   fill-params: --until=BPO4 --generate-all-formats

.github/scripts/check_new_commits.py

@@ -0,0 +1,158 @@
+#!/usr/bin/env -S uv run --script
+#
+# /// script
+# requires-python = ">=3.12"
+# ///
+"""
+Decide whether a scheduled nightly fill has new commits to fill.
+
+Usage: `check_new_commits.py` (all inputs come from the environment).
+
+Compare the current commit against the head SHA of the last scheduled
+run of the release workflow that actually filled: the newest successful
+*scheduled* run that uploaded artifacts. A quiet nightly skips its
+build jobs yet still concludes as a successful run, so plain success is
+no evidence of a fill. Anchoring on real fills means a nightly that
+fails or skips keeps re-running until a fill goes green and no commit
+slips through unfilled; filtering on scheduled runs means manual
+releases never advance the nightly baseline. Manual
+(`workflow_dispatch`) runs always run.
+
+A quiet stretch with no new commits still refreshes: once the last
+fill is `REFRESH_AGE` old -- or its artifact is no longer live -- the
+nightly re-runs anyway, so a live artifact always exists within the
+workflow's five-day retention and the release pipeline keeps getting
+exercised.
+
+Read `GITHUB_EVENT_NAME`, `GITHUB_REPOSITORY` and `GITHUB_SHA` from the
+environment and query the GitHub API via the `gh` CLI (authenticated by
+`GH_TOKEN`). Print `run=true|false` to stdout for `$GITHUB_OUTPUT` and
+append the new-commit list (or a skip notice) to the
+`$GITHUB_STEP_SUMMARY` file.
+"""
+
+import json
+import os
+import subprocess
+import sys
+from datetime import datetime, timedelta, timezone
+
+WORKFLOW_FILE = "release_fixtures.yaml"
+
+# Re-run a quiet nightly once the last successful fill is this old, so
+# a fresh artifact is uploaded before the previous one lapses (the
+# workflow retains scheduled tarballs for five days).
+REFRESH_AGE = timedelta(days=4)
+
+
+def gh_api(path: str) -> str:
+    """Return the stdout of `gh api <path>`, exiting non-zero on error."""
+    result = subprocess.run(
+        ["gh", "api", path], capture_output=True, text=True
+    )
+    if result.returncode != 0:
+        print(f"Error: gh api {path} failed:", file=sys.stderr)
+        print(result.stderr, file=sys.stderr)
+        sys.exit(1)
+    return result.stdout
+
+
+def last_real_nightly(repository: str) -> tuple[str, str, bool]:
+    """
+    Return the head SHA, creation time and artifact liveness of the
+    last scheduled run that actually filled.
+
+    A skipped nightly still concludes as a successful scheduled run,
+    so taking the newest success as the baseline would let skip-runs
+    keep resetting the refresh clock while the last real artifact
+    quietly expires. Walk the recent successful scheduled runs and
+    take the newest one that uploaded artifacts, reporting whether any
+    of them is still live. Return empty strings when none exists yet.
+    """
+    runs = json.loads(
+        gh_api(
+            f"repos/{repository}/actions/workflows/{WORKFLOW_FILE}"
+            "/runs?status=success&event=schedule&per_page=10"
+        )
+    )["workflow_runs"]
+    for run in runs:
+        artifacts = json.loads(
+            gh_api(f"repos/{repository}/actions/runs/{run['id']}/artifacts")
+        )["artifacts"]
+        if artifacts:
+            live = any(not a["expired"] for a in artifacts)
+            return str(run["head_sha"]), str(run["created_at"]), live
+    return "", "", False
+
+
+def is_stale(created_at: str) -> bool:
+    """Return whether a run created at *created_at* is due a refresh."""
+    created = datetime.fromisoformat(created_at)
+    return datetime.now(timezone.utc) - created >= REFRESH_AGE
+
+
+def commits_since(repository: str, last_sha: str, head_sha: str) -> list[str]:
+    """Return `- <sha> <subject>` lines for commits after *last_sha*."""
+    compare = json.loads(
+        gh_api(f"repos/{repository}/compare/{last_sha}...{head_sha}")
+    )
+    return [
+        f"- {c['sha'][:7]} {(c['commit']['message'].splitlines() or [''])[0]}"
+        for c in compare["commits"]
+    ]
+
+
+def append_summary(text: str) -> None:
+    """Append *text* to the GitHub step summary, or stderr if unset."""
+    summary_path = os.environ.get("GITHUB_STEP_SUMMARY")
+    if summary_path:
+        with open(summary_path, "a") as f:
+            f.write(text + "\n")
+    else:
+        print(text, file=sys.stderr)
+
+
+def main() -> None:
+    """Print `run=true|false` and write the step summary."""
+    if os.environ["GITHUB_EVENT_NAME"] != "schedule":
+        # Manual releases always run.
+        print("run=true")
+        return
+
+    repository = os.environ["GITHUB_REPOSITORY"]
+    head_sha = os.environ["GITHUB_SHA"]
+
+    last_sha, last_created, artifact_live = last_real_nightly(repository)
+    if last_sha:
+        commits = commits_since(repository, last_sha, head_sha)
+    else:
+        # No prior successful nightly recorded; fill to get a baseline.
+        commits = ["- (no previous successful nightly fill found)"]
+
+    if commits:
+        print("run=true")
+        append_summary(
+            "### Commits since last successful nightly fill\n"
+            + "\n".join(commits)
+        )
+    elif not artifact_live:
+        print("run=true")
+        append_summary(
+            "No new commits, but no live fixture artifact exists; refilling."
+        )
+    elif is_stale(last_created):
+        print("run=true")
+        append_summary(
+            "No new commits, but the last nightly fill is older than "
+            f"{REFRESH_AGE.days} days; refreshing before its artifact "
+            "retention lapses."
+        )
+    else:
+        print("run=false")
+        append_summary(
+            "No new commits since the last successful nightly fill; skipping."
+        )
+
+
+if __name__ == "__main__":
+    main()

.github/scripts/generate_build_matrix.py

@@ -10,7 +10,7 @@
 Validate release inputs and generate the build matrix for release
 fixture workflows.
 
-Usage: `generate_build_matrix.py <feature> <version> [branch]`.
+Usage: `generate_build_matrix.py <feature> <version> [branch] [evm]`.
 
 First validate the dispatch inputs (see `validate_inputs`), then read
 `.github/configs/feature.yaml` and emit a flat JSON build matrix suitable
@@ -31,6 +31,7 @@
 
 FEATURE_CONFIG = Path(".github/configs/feature.yaml")
 FORK_RANGES_CONFIG = Path(".github/configs/fork-ranges.yaml")
+EVM_CONFIG = Path(".github/configs/evm.yaml")
 
 VERSION_RE = re.compile(r"^v[0-9]+\.[0-9]+\.[0-9]+$")
 
@@ -81,12 +82,13 @@ def fail(message: str) -> NoReturn:
     sys.exit(1)
 
 
-def validate_inputs(feature: str, version: str, branch: str) -> None:
+def validate_inputs(feature: str, version: str, branch: str, evm: str) -> None:
     """
     Validate the release dispatch inputs before building a matrix.
 
-    Centralize the feature/version checks here so they are unit-testable
-    rather than living as inline bash in the release workflow.
+    Centralize the feature/version/evm checks here so they are
+    unit-testable rather than living as inline bash in the release
+    workflow.
 
     For `<feat>-devnet` releases the major version (`X` of `vX.Y.Z`)
     must equal the devnet number encoded in the release branch, so a
@@ -97,6 +99,10 @@ def validate_inputs(feature: str, version: str, branch: str) -> None:
     if not VERSION_RE.match(version):
         fail(f"version '{version}' must match vX.Y.Z (e.g. v20.0.0)")
 
+    # An `evm` override must name a key in evm.yaml.
+    if evm and evm not in load_config(EVM_CONFIG):
+        fail(f"evm '{evm}' is not a key in {EVM_CONFIG}")
+
     # A bare `devnet` has no friendly `<feat>-` prefix to tag with.
     if feature in ("devnet", "-devnet"):
         fail("devnet releases require a <feat>- prefix, e.g. bal-devnet")
@@ -207,16 +213,18 @@ def main() -> None:
     args = sys.argv[1:]
     if len(args) < 2:
         print(
-            "Usage: generate_build_matrix.py <feature> <version> [branch]",
+            "Usage: generate_build_matrix.py "
+            "<feature> <version> [branch] [evm]",
             file=sys.stderr,
         )
         sys.exit(1)
 
     name = args[0]
     version = args[1]
     branch = args[2] if len(args) > 2 else ""
+    evm = args[3] if len(args) > 3 else ""
 
-    validate_inputs(name, version, branch)
+    validate_inputs(name, version, branch, evm)
 
     config = load_config(FEATURE_CONFIG)
     fork_ranges = load_config(FORK_RANGES_CONFIG) or []