diff --git a/.github/actions/build-fixtures/action.yaml b/.github/actions/build-fixtures/action.yaml index 719c3221a75..18c6fcce126 100644 --- a/.github/actions/build-fixtures/action.yaml +++ b/.github/actions/build-fixtures/action.yaml @@ -13,6 +13,9 @@ inputs: split_label: description: "Label for this fork-range split. Empty for unsplit builds." default: "" + split_retention_days: + description: "retention-days for the split fixture artifact. Empty for the repo default." + default: "" evm: description: "Override the evm impl. Defaults to the feature's evm-type." default: "" @@ -50,6 +53,8 @@ runs: run: sudo apt-get install -y pigz - name: Generate fixtures using fill shell: bash + env: + PYTEST_XDIST_AUTO_NUM_WORKERS: ${{ steps.evm-builder.outputs.xdist }} run: | IS_SPLIT="${{ inputs.split_label }}" @@ -61,13 +66,14 @@ runs: FORK_ARGS="" fi + EVM_ARGS="" + if [ "${{ steps.evm-builder.outputs.impl }}" != "eels" ]; then + EVM_ARGS="--evm-bin=${{ steps.evm-builder.outputs.evm-bin }}" + fi + # Allow exit code 5 (NO_TESTS_COLLECTED) for fork ranges with no tests. EXIT_CODE=0 - if [ "${{ steps.evm-builder.outputs.impl }}" = "eels" ]; then - uv run fill -n ${{ steps.evm-builder.outputs.xdist }} ${{ steps.properties.outputs.fill-params }} $FORK_ARGS $OUTPUT_ARG --build-name ${{ inputs.release_name }} --no-html --durations=100 --log-level=DEBUG || EXIT_CODE=$? - else - uv run fill -n ${{ steps.evm-builder.outputs.xdist }} --evm-bin=${{ steps.evm-builder.outputs.evm-bin }} ${{ steps.properties.outputs.fill-params }} $FORK_ARGS $OUTPUT_ARG --build-name ${{ inputs.release_name }} --no-html --durations=100 --log-level=DEBUG || EXIT_CODE=$? - fi + just fill-release $EVM_ARGS ${{ steps.properties.outputs.fill-params }} $FORK_ARGS $OUTPUT_ARG --build-name ${{ inputs.release_name }} || EXIT_CODE=$? if [ "$EXIT_CODE" -ne 0 ] && [ "$EXIT_CODE" -ne 5 ]; then exit "$EXIT_CODE" fi @@ -96,3 +102,4 @@ runs: include-hidden-files: true path: fixtures_${{ inputs.release_name }}/ if-no-files-found: ignore + retention-days: ${{ inputs.split_retention_days }} diff --git a/.github/configs/evm.yaml b/.github/configs/evm.yaml index 621ad39623d..b36b347d981 100644 --- a/.github/configs/evm.yaml +++ b/.github/configs/evm.yaml @@ -17,12 +17,6 @@ evmone: evm-bin: evmone xdist: auto targets: ["evmone-cli"] -benchmark: - impl: geth - repo: ethereum/go-ethereum - ref: master - evm-bin: evm - xdist: auto besu: impl: besu repo: hyperledger/besu diff --git a/.github/configs/feature.yaml b/.github/configs/feature.yaml index 39297ec6e84..398541b7e65 100644 --- a/.github/configs/feature.yaml +++ b/.github/configs/feature.yaml @@ -5,6 +5,9 @@ # Any `-devnet` input resolves to the shared `devnet` entry but keeps # its name in the tag; the devnet number lives in the version (X), not the # feature name, so this file needs no edits for new devnets. +# `tests` is also what the scheduled nightly run of the `release_fixtures` +# workflow fills: mainnet forks only, so the rotating nightly artifact is a +# release-ready rehearsal of the next tests@ release. tests: evm-type: eels fill-params: --until=BPO4 --generate-all-formats diff --git a/.github/scripts/check_new_commits.py b/.github/scripts/check_new_commits.py new file mode 100644 index 00000000000..da778daf459 --- /dev/null +++ b/.github/scripts/check_new_commits.py @@ -0,0 +1,158 @@ +#!/usr/bin/env -S uv run --script +# +# /// script +# requires-python = ">=3.12" +# /// +""" +Decide whether a scheduled nightly fill has new commits to fill. + +Usage: `check_new_commits.py` (all inputs come from the environment). + +Compare the current commit against the head SHA of the last scheduled +run of the release workflow that actually filled: the newest successful +*scheduled* run that uploaded artifacts. A quiet nightly skips its +build jobs yet still concludes as a successful run, so plain success is +no evidence of a fill. Anchoring on real fills means a nightly that +fails or skips keeps re-running until a fill goes green and no commit +slips through unfilled; filtering on scheduled runs means manual +releases never advance the nightly baseline. Manual +(`workflow_dispatch`) runs always run. + +A quiet stretch with no new commits still refreshes: once the last +fill is `REFRESH_AGE` old -- or its artifact is no longer live -- the +nightly re-runs anyway, so a live artifact always exists within the +workflow's five-day retention and the release pipeline keeps getting +exercised. + +Read `GITHUB_EVENT_NAME`, `GITHUB_REPOSITORY` and `GITHUB_SHA` from the +environment and query the GitHub API via the `gh` CLI (authenticated by +`GH_TOKEN`). Print `run=true|false` to stdout for `$GITHUB_OUTPUT` and +append the new-commit list (or a skip notice) to the +`$GITHUB_STEP_SUMMARY` file. +""" + +import json +import os +import subprocess +import sys +from datetime import datetime, timedelta, timezone + +WORKFLOW_FILE = "release_fixtures.yaml" + +# Re-run a quiet nightly once the last successful fill is this old, so +# a fresh artifact is uploaded before the previous one lapses (the +# workflow retains scheduled tarballs for five days). +REFRESH_AGE = timedelta(days=4) + + +def gh_api(path: str) -> str: + """Return the stdout of `gh api `, exiting non-zero on error.""" + result = subprocess.run( + ["gh", "api", path], capture_output=True, text=True + ) + if result.returncode != 0: + print(f"Error: gh api {path} failed:", file=sys.stderr) + print(result.stderr, file=sys.stderr) + sys.exit(1) + return result.stdout + + +def last_real_nightly(repository: str) -> tuple[str, str, bool]: + """ + Return the head SHA, creation time and artifact liveness of the + last scheduled run that actually filled. + + A skipped nightly still concludes as a successful scheduled run, + so taking the newest success as the baseline would let skip-runs + keep resetting the refresh clock while the last real artifact + quietly expires. Walk the recent successful scheduled runs and + take the newest one that uploaded artifacts, reporting whether any + of them is still live. Return empty strings when none exists yet. + """ + runs = json.loads( + gh_api( + f"repos/{repository}/actions/workflows/{WORKFLOW_FILE}" + "/runs?status=success&event=schedule&per_page=10" + ) + )["workflow_runs"] + for run in runs: + artifacts = json.loads( + gh_api(f"repos/{repository}/actions/runs/{run['id']}/artifacts") + )["artifacts"] + if artifacts: + live = any(not a["expired"] for a in artifacts) + return str(run["head_sha"]), str(run["created_at"]), live + return "", "", False + + +def is_stale(created_at: str) -> bool: + """Return whether a run created at *created_at* is due a refresh.""" + created = datetime.fromisoformat(created_at) + return datetime.now(timezone.utc) - created >= REFRESH_AGE + + +def commits_since(repository: str, last_sha: str, head_sha: str) -> list[str]: + """Return `- ` lines for commits after *last_sha*.""" + compare = json.loads( + gh_api(f"repos/{repository}/compare/{last_sha}...{head_sha}") + ) + return [ + f"- {c['sha'][:7]} {(c['commit']['message'].splitlines() or [''])[0]}" + for c in compare["commits"] + ] + + +def append_summary(text: str) -> None: + """Append *text* to the GitHub step summary, or stderr if unset.""" + summary_path = os.environ.get("GITHUB_STEP_SUMMARY") + if summary_path: + with open(summary_path, "a") as f: + f.write(text + "\n") + else: + print(text, file=sys.stderr) + + +def main() -> None: + """Print `run=true|false` and write the step summary.""" + if os.environ["GITHUB_EVENT_NAME"] != "schedule": + # Manual releases always run. + print("run=true") + return + + repository = os.environ["GITHUB_REPOSITORY"] + head_sha = os.environ["GITHUB_SHA"] + + last_sha, last_created, artifact_live = last_real_nightly(repository) + if last_sha: + commits = commits_since(repository, last_sha, head_sha) + else: + # No prior successful nightly recorded; fill to get a baseline. + commits = ["- (no previous successful nightly fill found)"] + + if commits: + print("run=true") + append_summary( + "### Commits since last successful nightly fill\n" + + "\n".join(commits) + ) + elif not artifact_live: + print("run=true") + append_summary( + "No new commits, but no live fixture artifact exists; refilling." + ) + elif is_stale(last_created): + print("run=true") + append_summary( + "No new commits, but the last nightly fill is older than " + f"{REFRESH_AGE.days} days; refreshing before its artifact " + "retention lapses." + ) + else: + print("run=false") + append_summary( + "No new commits since the last successful nightly fill; skipping." + ) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/generate_build_matrix.py b/.github/scripts/generate_build_matrix.py index 02fde3087dc..200bdb788ad 100644 --- a/.github/scripts/generate_build_matrix.py +++ b/.github/scripts/generate_build_matrix.py @@ -10,7 +10,7 @@ Validate release inputs and generate the build matrix for release fixture workflows. -Usage: `generate_build_matrix.py [branch]`. +Usage: `generate_build_matrix.py [branch] [evm]`. First validate the dispatch inputs (see `validate_inputs`), then read `.github/configs/feature.yaml` and emit a flat JSON build matrix suitable @@ -31,6 +31,7 @@ FEATURE_CONFIG = Path(".github/configs/feature.yaml") FORK_RANGES_CONFIG = Path(".github/configs/fork-ranges.yaml") +EVM_CONFIG = Path(".github/configs/evm.yaml") VERSION_RE = re.compile(r"^v[0-9]+\.[0-9]+\.[0-9]+$") @@ -81,12 +82,13 @@ def fail(message: str) -> NoReturn: sys.exit(1) -def validate_inputs(feature: str, version: str, branch: str) -> None: +def validate_inputs(feature: str, version: str, branch: str, evm: str) -> None: """ Validate the release dispatch inputs before building a matrix. - Centralize the feature/version checks here so they are unit-testable - rather than living as inline bash in the release workflow. + Centralize the feature/version/evm checks here so they are + unit-testable rather than living as inline bash in the release + workflow. For `-devnet` releases the major version (`X` of `vX.Y.Z`) must equal the devnet number encoded in the release branch, so a @@ -97,6 +99,10 @@ def validate_inputs(feature: str, version: str, branch: str) -> None: if not VERSION_RE.match(version): fail(f"version '{version}' must match vX.Y.Z (e.g. v20.0.0)") + # An `evm` override must name a key in evm.yaml. + if evm and evm not in load_config(EVM_CONFIG): + fail(f"evm '{evm}' is not a key in {EVM_CONFIG}") + # A bare `devnet` has no friendly `-` prefix to tag with. if feature in ("devnet", "-devnet"): fail("devnet releases require a - prefix, e.g. bal-devnet") @@ -207,7 +213,8 @@ def main() -> None: args = sys.argv[1:] if len(args) < 2: print( - "Usage: generate_build_matrix.py [branch]", + "Usage: generate_build_matrix.py " + " [branch] [evm]", file=sys.stderr, ) sys.exit(1) @@ -215,8 +222,9 @@ def main() -> None: name = args[0] version = args[1] branch = args[2] if len(args) > 2 else "" + evm = args[3] if len(args) > 3 else "" - validate_inputs(name, version, branch) + validate_inputs(name, version, branch, evm) config = load_config(FEATURE_CONFIG) fork_ranges = load_config(FORK_RANGES_CONFIG) or [] diff --git a/.github/scripts/resolve_cached_release.py b/.github/scripts/resolve_cached_release.py new file mode 100644 index 00000000000..6c8f9450133 --- /dev/null +++ b/.github/scripts/resolve_cached_release.py @@ -0,0 +1,296 @@ +#!/usr/bin/env -S uv run --script +# +# /// script +# requires-python = ">=3.12" +# /// +""" +Resolve the nightly fill whose artifact a cached release reuses. + +Usage: `resolve_cached_release.py` (all inputs come from the +environment). + +Dispatching `release_fixtures.yaml` with the `cached` flag drafts a +`tests@` release from the newest nightly artifact instead of +refilling: the scheduled nightly runs already build the mainnet +`tests` feature into a release-shaped `fixtures_` artifact. +The `commit` input picks the nightly built at that commit instead of +the newest one. This script validates the request, picks the nightly +run whose artifact the release job downloads, and pins the exact +commit that run built so the release tag lands on it. + +Checks performed, failing fast on the first violation: + +- The release is for the `tests` feature on the default branch (no + `branch` input): that is what the nightly fills. +- `INPUT_VERSION` matches `vX.Y.Z` and is greater than the newest + existing `tests@` tag (releases always move forward; anything + unusual belongs in a fresh fill). +- The resolved run is a successful *scheduled* run of + `release_fixtures.yaml` with a live (unexpired) tarball artifact + named for the run's own commit: the newest one, or with + `INPUT_COMMIT` (7+ hex characters) the one built at that commit. + Skip-runs upload no artifacts and expired fills cannot be + downloaded, so both are passed over. +- The resolved commit contains the newest existing `tests@` release, + so a cached release never regresses content (re-releasing the same + commit stays allowed). +- The resolved commit is an ancestor of the current branch head. + Commits after it are listed in the step summary so the releaser can + see what the release will NOT contain. + +Read `GITHUB_REPOSITORY`, `GITHUB_SHA`, `INPUT_FEATURE`, +`INPUT_BRANCH`, `INPUT_VERSION` and `INPUT_COMMIT` from the +environment and query the +GitHub API via the `gh` CLI (authenticated by `GH_TOKEN`). Print +`run_id`, `target_sha` and `artifact_name` as `key=value` lines for +`$GITHUB_OUTPUT`. +""" + +import json +import os +import re +import subprocess +import sys +from typing import NoReturn + +WORKFLOW_FILE = "release_fixtures.yaml" + +# The combined-tarball artifact a nightly `tests` fill uploads is +# named for the short hash of the built commit; only that artifact is +# ever reused by a cached release. +ARTIFACT_PREFIX = "fixtures" + +VERSION_RE = re.compile(r"^v([0-9]+)\.([0-9]+)\.([0-9]+)$") +COMMIT_RE = re.compile(r"^[0-9a-f]{7,40}$") + + +def artifact_name(head_sha: str) -> str: + """Return the tarball artifact name of a nightly built at *head_sha*.""" + return f"{ARTIFACT_PREFIX}_{head_sha[:7]}" + + +def fail(message: str) -> NoReturn: + """Print an error to stderr and exit non-zero.""" + print(f"Error: {message}", file=sys.stderr) + sys.exit(1) + + +def gh_api(path: str, paginate: bool = False) -> str: + """ + Return the stdout of `gh api `, exiting non-zero on error. + + With *paginate*, follow the Link header through every page and + return a JSON array of per-page responses (`--slurp`). + """ + flags = ["--paginate", "--slurp"] if paginate else [] + result = subprocess.run( + ["gh", "api", *flags, path], capture_output=True, text=True + ) + if result.returncode != 0: + print(f"Error: gh api {path} failed:", file=sys.stderr) + print(result.stderr, file=sys.stderr) + sys.exit(1) + return result.stdout + + +def append_summary(text: str) -> None: + """Append *text* to the GitHub step summary, or stderr if unset.""" + summary_path = os.environ.get("GITHUB_STEP_SUMMARY") + if summary_path: + with open(summary_path, "a") as f: + f.write(text + "\n") + else: + print(text, file=sys.stderr) + + +def parse_version(version: str) -> tuple[int, int, int]: + """Return the (major, minor, patch) tuple of a `vX.Y.Z` version.""" + m = VERSION_RE.match(version) + if not m: + fail(f"version '{version}' must match vX.Y.Z (e.g. v5.0.0)") + major, minor, patch = (int(g) for g in m.groups()) + return major, minor, patch + + +def newest_tests_tag(repository: str) -> str: + """ + Return the newest existing `tests@vX.Y.Z` tag, or "" when none. + + The `tests@` ref prefix cannot match any other feature's tags + (those are namespaced `tests-@`), so every match is a + mainnet tests release. + + The listing is paginated in ref-name order, not version order + (`tests@v9...` sorts after `tests@v20...`), so every page must be + fetched before taking the maximum. + """ + pages = json.loads( + gh_api( + f"repos/{repository}/git/matching-refs/tags/tests@", + paginate=True, + ) + ) + refs = [ref for page in pages for ref in page] + tags = [ref["ref"].removeprefix("refs/tags/") for ref in refs] + versioned = [ + (parse_version(tag.removeprefix("tests@")), tag) + for tag in tags + if VERSION_RE.match(tag.removeprefix("tests@")) + ] + if not versioned: + return "" + return max(versioned)[1] + + +def has_live_tests_artifact( + repository: str, run_id: str, head_sha: str +) -> bool: + """ + Return whether *run_id* has a live tarball artifact. + + The artifact name is derived from the run's own head SHA, so a + name that does not match the commit it claims to be built from is + passed over. + """ + artifacts = json.loads( + gh_api(f"repos/{repository}/actions/runs/{run_id}/artifacts") + )["artifacts"] + name = artifact_name(head_sha) + return any(a["name"] == name and not a["expired"] for a in artifacts) + + +def cached_run(repository: str, commit: str) -> tuple[str, str]: + """ + Return the (run id, head SHA) of the nightly run to reuse. + + Take the newest successful scheduled run with a live artifact, or + with *commit* the run built at that commit (skip-runs upload no + artifacts and expired fills cannot be downloaded, so both are + passed over). On a commit miss, list the reusable nightlies. + """ + runs = json.loads( + gh_api( + f"repos/{repository}/actions/workflows/{WORKFLOW_FILE}" + "/runs?status=success&event=schedule&per_page=10" + ) + )["workflow_runs"] + live: list[str] = [] + for run in runs: + run_id, head_sha = str(run["id"]), str(run["head_sha"]) + if not has_live_tests_artifact(repository, run_id, head_sha): + continue + if not commit or head_sha.startswith(commit): + return run_id, head_sha + live.append(head_sha[:7]) + if commit: + available = ", ".join(live) if live else "none" + fail( + f"no nightly with a live artifact was built at {commit} " + f"(reusable nightlies: {available}); dispatch a fresh fill " + "instead" + ) + fail( + f"no scheduled run of {WORKFLOW_FILE} with a live " + f"`{ARTIFACT_PREFIX}_` artifact found; dispatch a fresh " + "fill instead" + ) + + +def ensure_not_behind(repository: str, prev_tag: str, target_sha: str) -> None: + """ + Fail when *target_sha* does not contain the *prev_tag* release. + + A cached release must never regress content: the resolved nightly + has to be at or after the newest `tests@` tag. Re-releasing the + identical commit stays allowed. + """ + compare = json.loads( + gh_api(f"repos/{repository}/compare/{prev_tag}...{target_sha}") + ) + if compare["status"] not in ("identical", "ahead"): + fail( + f"the resolved nightly ({target_sha}) does not contain the " + f"newest tests release ({prev_tag}); a cached release must " + "not regress content" + ) + + +def commits_after( + repository: str, target_sha: str, head_sha: str +) -> list[str]: + """ + Return `- ` lines for commits after *target_sha*. + + Fail when *target_sha* is not an ancestor of *head_sha*: a nightly + built from a rewritten or foreign branch must not be released. + """ + compare = json.loads( + gh_api(f"repos/{repository}/compare/{target_sha}...{head_sha}") + ) + if compare["status"] not in ("identical", "ahead"): + fail( + f"nightly commit {target_sha} is not an ancestor of " + f"{head_sha} (compare status: {compare['status']})" + ) + return [ + f"- {c['sha'][:7]} {(c['commit']['message'].splitlines() or [''])[0]}" + for c in compare["commits"] + ] + + +def main() -> None: + """Print the resolved run for `$GITHUB_OUTPUT` and the summary.""" + repository = os.environ["GITHUB_REPOSITORY"] + head_sha = os.environ["GITHUB_SHA"] + version = os.environ["INPUT_VERSION"] + + if os.environ.get("INPUT_FEATURE") != "tests": + fail("cached releases are only available for feature=tests") + if os.environ.get("INPUT_BRANCH"): + fail( + "cached releases reuse a default-branch nightly; drop the " + "`branch` input or dispatch a fresh fill" + ) + + commit = os.environ.get("INPUT_COMMIT", "") + if commit and not COMMIT_RE.match(commit): + fail(f"commit '{commit}' must be 7 to 40 lowercase hex characters") + + requested = parse_version(version) + prev_tag = newest_tests_tag(repository) + if prev_tag and requested <= parse_version( + prev_tag.removeprefix("tests@") + ): + fail( + f"version '{version}' must be greater than the newest " + f"tests release ({prev_tag})" + ) + + run_id, target_sha = cached_run(repository, commit) + if prev_tag: + ensure_not_behind(repository, prev_tag, target_sha) + missing = commits_after(repository, target_sha, head_sha) + + print(f"run_id={run_id}") + print(f"target_sha={target_sha}") + print(f"artifact_name={artifact_name(target_sha)}") + + run_url = f"https://github.com/{repository}/actions/runs/{run_id}" + append_summary( + f"Reusing nightly fill run [{run_id}]({run_url}) " + f"(built at `{target_sha}`) for the `tests@{version}` draft." + ) + if missing: + append_summary( + "### Commits NOT included in this release\n" + + "\n".join(missing) + + "\n\nDispatch a fresh fill to include them." + ) + else: + append_summary( + "The nightly is up to date with the current branch head." + ) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/tests/test_release_scripts.py b/.github/scripts/tests/test_release_scripts.py index a74c4c67c08..44e6dbcab77 100644 --- a/.github/scripts/tests/test_release_scripts.py +++ b/.github/scripts/tests/test_release_scripts.py @@ -6,8 +6,10 @@ """ import json +import os import subprocess import tarfile +from datetime import datetime, timedelta, timezone from pathlib import Path SCRIPTS_DIR = Path(__file__).parent.parent @@ -16,6 +18,8 @@ BUILD_MATRIX_SCRIPT = SCRIPTS_DIR / "generate_build_matrix.py" TARBALL_SCRIPT = SCRIPTS_DIR / "create_release_tarball.py" MERGE_INDEX_SCRIPT = SCRIPTS_DIR / "merge_index_files.py" +CHECK_COMMITS_SCRIPT = SCRIPTS_DIR / "check_new_commits.py" +RESOLVE_CACHED_SCRIPT = SCRIPTS_DIR / "resolve_cached_release.py" def run_script(script: Path, *args: str) -> subprocess.CompletedProcess: @@ -170,6 +174,520 @@ def test_devnet_matching_major_passes(self): out = parse_matrix_output(result.stdout) assert out["feature_name"] == "glamsterdam-devnet" + def test_unknown_evm_fails(self): + """Verify an evm override missing from evm.yaml is rejected.""" + result = run_script( + BUILD_MATRIX_SCRIPT, "tests", "v24.0.0", "", "nonexistent" + ) + assert result.returncode == 1 + assert "not a key" in result.stderr + + def test_known_evm_passes(self): + """Verify an evm override that is a key in evm.yaml passes.""" + result = run_script( + BUILD_MATRIX_SCRIPT, "tests", "v24.0.0", "", "evmone" + ) + assert result.returncode == 0 + + +# Fake `gh` served from PATH: answers the API calls the commit-check +# and cached-release scripts make with canned JSON from env vars, and +# fails loudly on any other (or unconfigured) call. The API path is +# the last argument (flags such as `--paginate --slurp` may precede +# it). Per-run artifact responses come from +# `FAKE_GH_ARTIFACTS_`, falling back to `FAKE_GH_ARTIFACTS`. +FAKE_GH = """#!/usr/bin/env bash +path="${@: -1}" +case "$path" in + *actions/workflows*) response="$FAKE_GH_RUNS" ;; + */artifacts) + run_id="${path##*/runs/}" + run_id="${run_id%%/*}" + var="FAKE_GH_ARTIFACTS_${run_id}" + response="${!var:-$FAKE_GH_ARTIFACTS}" + ;; + *matching-refs*) response="$FAKE_GH_TAGS" ;; + *compare/tests@*) response="$FAKE_GH_COMPARE_TAG" ;; + *compare*) response="$FAKE_GH_COMPARE" ;; + *) response="" ;; +esac +if [ -z "$response" ]; then + echo "unexpected gh call: $*" >&2 + exit 1 +fi +printf '%s' "$response" +""" + +# Canned artifact-list responses for the fake `gh`. +LIVE_ARTIFACTS = '{"artifacts": [{"expired": false}]}' +EXPIRED_ARTIFACTS = '{"artifacts": [{"expired": true}]}' +NO_ARTIFACTS = '{"artifacts": []}' + + +class TestCheckNewCommits: + """Test check_new_commits.py.""" + + def run_check( + self, + tmp_path: Path, + event_name: str, + runs: str = "", + compare: str = "", + artifacts: str = "", + per_run_artifacts: dict[int, str] | None = None, + ) -> tuple[subprocess.CompletedProcess, Path]: + """Run the script with a fake `gh` on PATH; return it + summary.""" + bin_dir = tmp_path / "bin" + bin_dir.mkdir(exist_ok=True) + fake_gh = bin_dir / "gh" + fake_gh.write_text(FAKE_GH) + fake_gh.chmod(0o755) + + summary = tmp_path / "summary.md" + env = os.environ.copy() + env["PATH"] = f"{bin_dir}:{env['PATH']}" + env["GITHUB_EVENT_NAME"] = event_name + env["GITHUB_REPOSITORY"] = "ethereum/execution-specs" + env["GITHUB_SHA"] = "b" * 40 + env["GITHUB_STEP_SUMMARY"] = str(summary) + env["FAKE_GH_RUNS"] = runs + env["FAKE_GH_COMPARE"] = compare + env["FAKE_GH_ARTIFACTS"] = artifacts + for run_id, response in (per_run_artifacts or {}).items(): + env[f"FAKE_GH_ARTIFACTS_{run_id}"] = response + + result = subprocess.run( + ["uv", "run", "-q", str(CHECK_COMMITS_SCRIPT)], + capture_output=True, + text=True, + cwd=REPO_ROOT, + env=env, + ) + return result, summary + + def test_dispatch_always_runs_without_api_calls(self, tmp_path): + """Verify a manual dispatch runs and never calls the API.""" + # The fake `gh` fails every call (no canned responses), so a + # zero exit proves the script made no API call. + result, summary = self.run_check(tmp_path, "workflow_dispatch") + assert result.returncode == 0 + assert result.stdout.strip() == "run=true" + assert not summary.exists() + + def test_schedule_without_prior_run_fills_baseline(self, tmp_path): + """Verify the first scheduled run fills to get a baseline.""" + result, summary = self.run_check( + tmp_path, "schedule", runs='{"workflow_runs": []}' + ) + assert result.returncode == 0 + assert result.stdout.strip() == "run=true" + assert "no previous successful" in summary.read_text() + + @staticmethod + def run_json( + age: timedelta, head_sha: str = "b" * 40, run_id: int = 1 + ) -> dict: + """Return one workflow-run object created *age* ago.""" + created = datetime.now(timezone.utc) - age + return { + "id": run_id, + "head_sha": head_sha, + "created_at": created.isoformat(), + } + + @classmethod + def runs_json( + cls, age: timedelta, head_sha: str = "b" * 40, run_id: int = 1 + ) -> str: + """Return a last-successful-run response created *age* ago.""" + return json.dumps( + {"workflow_runs": [cls.run_json(age, head_sha, run_id)]} + ) + + def test_schedule_with_new_commits_runs(self, tmp_path): + """Verify new commits since the baseline trigger a run.""" + commit = { + "sha": "abcdef1" + "0" * 33, + "commit": {"message": "feat(x): subject\n\nbody"}, + } + result, summary = self.run_check( + tmp_path, + "schedule", + runs=self.runs_json(timedelta(hours=25), head_sha="a" * 40), + compare=json.dumps({"commits": [commit]}), + artifacts=LIVE_ARTIFACTS, + ) + assert result.returncode == 0 + assert result.stdout.strip() == "run=true" + text = summary.read_text() + assert "### Commits since last successful nightly fill" in text + # Short SHA plus the commit subject, without the body. + assert "- abcdef1 feat(x): subject" in text + assert "body" not in text + + def test_schedule_without_new_commits_skips(self, tmp_path): + """Verify no commits since a recent baseline skips the run.""" + result, summary = self.run_check( + tmp_path, + "schedule", + # Just inside the refresh age: pin the four-day boundary. + runs=self.runs_json(timedelta(days=3, hours=23)), + compare=json.dumps({"commits": []}), + artifacts=LIVE_ARTIFACTS, + ) + assert result.returncode == 0 + assert result.stdout.strip() == "run=false" + assert "skipping" in summary.read_text() + + def test_schedule_stale_quiet_baseline_refreshes(self, tmp_path): + """Verify a quiet nightly re-runs once its artifact nears expiry.""" + result, summary = self.run_check( + tmp_path, + "schedule", + runs=self.runs_json(timedelta(days=4, hours=1)), + compare=json.dumps({"commits": []}), + artifacts=LIVE_ARTIFACTS, + ) + assert result.returncode == 0 + assert result.stdout.strip() == "run=true" + assert "refreshing" in summary.read_text() + + def test_schedule_skip_runs_do_not_reset_refresh(self, tmp_path): + """ + Verify skip-runs neither advance the baseline nor its clock. + + A quiet nightly that skips its build still concludes as a + successful scheduled run; if it counted as the baseline, a + stretch of skip-runs would keep resetting the refresh clock + while the last real artifact quietly expired. + """ + runs = json.dumps( + { + "workflow_runs": [ + # Newest success skipped its build: no artifacts. + self.run_json(timedelta(hours=1), run_id=2), + # The last real fill is past the refresh age. + self.run_json(timedelta(days=4, hours=1), run_id=1), + ] + } + ) + result, summary = self.run_check( + tmp_path, + "schedule", + runs=runs, + compare=json.dumps({"commits": []}), + per_run_artifacts={2: NO_ARTIFACTS, 1: LIVE_ARTIFACTS}, + ) + assert result.returncode == 0 + assert result.stdout.strip() == "run=true" + assert "refreshing" in summary.read_text() + + def test_schedule_dead_artifact_refills(self, tmp_path): + """Verify a fill whose artifact is gone refills immediately.""" + result, summary = self.run_check( + tmp_path, + "schedule", + runs=self.runs_json(timedelta(days=1)), + compare=json.dumps({"commits": []}), + artifacts=EXPIRED_ARTIFACTS, + ) + assert result.returncode == 0 + assert result.stdout.strip() == "run=true" + assert "no live fixture artifact" in summary.read_text() + + def test_gh_failure_fails_the_check(self, tmp_path): + """Verify a failing `gh` call fails the script.""" + result, _ = self.run_check(tmp_path, "schedule") + assert result.returncode == 1 + assert "gh api" in result.stderr + + +# Canned responses for the cached-release script. Unlike the commit +# check, it matches artifacts by the commit-derived +# `fixtures_` name, so the canned listings are built +# per head SHA. The tag listing is fetched with `--paginate --slurp` +# (a JSON array of pages); spreading the refs over two pages makes +# every test exercise the page flattening. +TESTS_TAGS = json.dumps( + [ + [{"ref": "refs/tags/tests@v3.1.2"}], + [{"ref": "refs/tags/tests@v4.0.0"}], + ] +) +NO_TAGS = "[[]]" +UP_TO_DATE = json.dumps({"status": "identical", "commits": []}) + + +def artifact_listing(head_sha: str, expired: bool = False) -> str: + """Return an artifact listing with a tarball named for *head_sha*.""" + return json.dumps( + { + "artifacts": [ + { + "name": f"fixtures_{head_sha[:7]}", + "expired": expired, + } + ] + } + ) + + +class TestResolveCachedRelease: + """Test resolve_cached_release.py.""" + + def run_resolve( + self, + tmp_path: Path, + version: str, + feature: str = "tests", + branch: str = "", + commit: str = "", + runs: str = "", + artifacts: str = "", + per_run_artifacts: dict[int, str] | None = None, + tags: str = "", + compare: str = "", + tag_compare: str = UP_TO_DATE, + ) -> tuple[subprocess.CompletedProcess, Path]: + """Run the script with a fake `gh` on PATH; return it + summary.""" + bin_dir = tmp_path / "bin" + bin_dir.mkdir(exist_ok=True) + fake_gh = bin_dir / "gh" + fake_gh.write_text(FAKE_GH) + fake_gh.chmod(0o755) + + summary = tmp_path / "summary.md" + env = os.environ.copy() + env["PATH"] = f"{bin_dir}:{env['PATH']}" + env["GITHUB_REPOSITORY"] = "ethereum/execution-specs" + env["GITHUB_SHA"] = "b" * 40 + env["GITHUB_STEP_SUMMARY"] = str(summary) + env["INPUT_VERSION"] = version + env["INPUT_FEATURE"] = feature + env["INPUT_BRANCH"] = branch + env["INPUT_COMMIT"] = commit + env["FAKE_GH_RUNS"] = runs + env["FAKE_GH_ARTIFACTS"] = artifacts + env["FAKE_GH_TAGS"] = tags + env["FAKE_GH_COMPARE"] = compare + env["FAKE_GH_COMPARE_TAG"] = tag_compare + for run_id, response in (per_run_artifacts or {}).items(): + env[f"FAKE_GH_ARTIFACTS_{run_id}"] = response + + result = subprocess.run( + ["uv", "run", "-q", str(RESOLVE_CACHED_SCRIPT)], + capture_output=True, + text=True, + cwd=REPO_ROOT, + env=env, + ) + return result, summary + + @staticmethod + def parse_outputs(stdout: str) -> dict[str, str]: + """Parse the key=value lines written for `$GITHUB_OUTPUT`.""" + return dict(line.split("=", 1) for line in stdout.strip().splitlines()) + + @staticmethod + def runs_json(*runs: dict) -> str: + """Return a workflow-runs listing response.""" + return json.dumps({"workflow_runs": list(runs)}) + + def test_reuses_newest_run_with_live_artifact(self, tmp_path): + """Verify skip-runs and expired fills are passed over.""" + commit = { + "sha": "abcdef1" + "0" * 33, + "commit": {"message": "feat(x): subject\n\nbody"}, + } + result, summary = self.run_resolve( + tmp_path, + "v4.0.1", + runs=self.runs_json( + # Newest success skipped its build: no artifacts. + {"id": 3, "head_sha": "c" * 40}, + {"id": 2, "head_sha": "a" * 40}, + {"id": 1, "head_sha": "d" * 40}, + ), + per_run_artifacts={ + 3: NO_ARTIFACTS, + 2: artifact_listing("a" * 40), + 1: artifact_listing("d" * 40, expired=True), + }, + tags=TESTS_TAGS, + compare=json.dumps({"status": "ahead", "commits": [commit]}), + ) + assert result.returncode == 0 + out = self.parse_outputs(result.stdout) + assert out["run_id"] == "2" + assert out["target_sha"] == "a" * 40 + assert out["artifact_name"] == "fixtures_aaaaaaa" + text = summary.read_text() + assert "### Commits NOT included in this release" in text + # Short SHA plus the commit subject, without the body. + assert "- abcdef1 feat(x): subject" in text + assert "body" not in text + + def test_up_to_date_nightly_resolves_cleanly(self, tmp_path): + """Verify no missing-commit section when nothing landed since.""" + result, summary = self.run_resolve( + tmp_path, + "v4.0.1", + runs=self.runs_json({"id": 2, "head_sha": "b" * 40}), + artifacts=artifact_listing("b" * 40), + tags=TESTS_TAGS, + compare=UP_TO_DATE, + ) + assert result.returncode == 0 + text = summary.read_text() + assert "up to date" in text + assert "NOT included" not in text + + def test_first_release_without_tags_resolves(self, tmp_path): + """Verify a cached release works before any tests@ tag exists.""" + result, _ = self.run_resolve( + tmp_path, + "v1.0.0", + runs=self.runs_json({"id": 2, "head_sha": "b" * 40}), + artifacts=artifact_listing("b" * 40), + tags=NO_TAGS, + compare=UP_TO_DATE, + ) + assert result.returncode == 0 + assert self.parse_outputs(result.stdout)["run_id"] == "2" + + def test_non_tests_feature_fails(self, tmp_path): + """Verify only the tests feature can release cached.""" + # The fake `gh` fails every call (no canned responses), so a + # clean feature error proves the guard fires before the API. + result, _ = self.run_resolve(tmp_path, "v4.0.1", feature="bal-devnet") + assert result.returncode == 1 + assert "only available for feature=tests" in result.stderr + + def test_branch_input_fails(self, tmp_path): + """Verify a cached release rejects a branch input.""" + result, _ = self.run_resolve( + tmp_path, "v4.0.1", branch="devnets/bal/7" + ) + assert result.returncode == 1 + assert "drop the `branch` input" in result.stderr + + def test_bad_version_format_fails(self, tmp_path): + """Verify a non vX.Y.Z version is rejected before any API call.""" + result, _ = self.run_resolve(tmp_path, "4.0.1") + assert result.returncode == 1 + assert "must match vX.Y.Z" in result.stderr + + def test_version_not_greater_than_newest_tag_fails(self, tmp_path): + """Verify the version must move past the newest tests@ tag.""" + result, _ = self.run_resolve(tmp_path, "v4.0.0", tags=TESTS_TAGS) + assert result.returncode == 1 + assert "must be greater" in result.stderr + assert "tests@v4.0.0" in result.stderr + + def test_no_reusable_run_fails(self, tmp_path): + """Verify a helpful error when every artifact has expired.""" + result, _ = self.run_resolve( + tmp_path, + "v4.0.1", + runs=self.runs_json({"id": 2, "head_sha": "a" * 40}), + artifacts=artifact_listing("a" * 40, expired=True), + tags=TESTS_TAGS, + ) + assert result.returncode == 1 + assert "dispatch a fresh fill instead" in result.stderr + + def test_mismatched_artifact_name_is_skipped(self, tmp_path): + """Verify an artifact named for another commit is not reused.""" + result, _ = self.run_resolve( + tmp_path, + "v4.0.1", + runs=self.runs_json({"id": 2, "head_sha": "a" * 40}), + # Live, but named for a different commit than the run built. + artifacts=artifact_listing("f" * 40), + tags=TESTS_TAGS, + ) + assert result.returncode == 1 + assert "dispatch a fresh fill instead" in result.stderr + + def test_commit_input_selects_that_nightly(self, tmp_path): + """Verify `commit` picks an older nightly over the newest.""" + result, _ = self.run_resolve( + tmp_path, + "v4.0.1", + commit="d" * 7, + runs=self.runs_json( + {"id": 3, "head_sha": "c" * 40}, + {"id": 2, "head_sha": "a" * 40}, + {"id": 1, "head_sha": "d" * 40}, + ), + per_run_artifacts={ + 3: NO_ARTIFACTS, + 2: artifact_listing("a" * 40), + 1: artifact_listing("d" * 40), + }, + tags=TESTS_TAGS, + compare=UP_TO_DATE, + ) + assert result.returncode == 0 + out = self.parse_outputs(result.stdout) + assert out["run_id"] == "1" + assert out["target_sha"] == "d" * 40 + assert out["artifact_name"] == "fixtures_ddddddd" + + def test_commit_input_without_match_fails(self, tmp_path): + """Verify a commit with no live nightly lists the candidates.""" + result, _ = self.run_resolve( + tmp_path, + "v4.0.1", + commit="beef111", + runs=self.runs_json({"id": 2, "head_sha": "a" * 40}), + artifacts=artifact_listing("a" * 40), + tags=TESTS_TAGS, + ) + assert result.returncode == 1 + assert "was built at beef111" in result.stderr + assert "aaaaaaa" in result.stderr + + def test_bad_commit_format_fails(self, tmp_path): + """Verify a malformed commit is rejected before any lookup.""" + result, _ = self.run_resolve( + tmp_path, "v4.0.1", commit="xyz", tags=TESTS_TAGS + ) + assert result.returncode == 1 + assert "hex characters" in result.stderr + + def test_release_behind_previous_fails(self, tmp_path): + """Verify a nightly older than the newest release is rejected.""" + result, _ = self.run_resolve( + tmp_path, + "v4.0.1", + runs=self.runs_json({"id": 2, "head_sha": "a" * 40}), + artifacts=artifact_listing("a" * 40), + tags=TESTS_TAGS, + tag_compare=json.dumps({"status": "behind", "commits": []}), + ) + assert result.returncode == 1 + assert "must not regress content" in result.stderr + + def test_diverged_nightly_fails(self, tmp_path): + """Verify a nightly off the branch history is not reused.""" + result, _ = self.run_resolve( + tmp_path, + "v4.0.1", + runs=self.runs_json({"id": 2, "head_sha": "a" * 40}), + artifacts=artifact_listing("a" * 40), + tags=TESTS_TAGS, + compare=json.dumps({"status": "diverged", "commits": []}), + ) + assert result.returncode == 1 + assert "not an ancestor" in result.stderr + + def test_gh_failure_fails_the_resolution(self, tmp_path): + """Verify a failing `gh` call fails the script.""" + result, _ = self.run_resolve(tmp_path, "v4.0.1") + assert result.returncode == 1 + assert "gh api" in result.stderr + class TestCreateReleaseTarball: """Test create_release_tarball.py.""" diff --git a/.github/workflows/release_fixtures.yaml b/.github/workflows/release_fixtures.yaml index 7ac44ac4f66..63e950cf7f5 100644 --- a/.github/workflows/release_fixtures.yaml +++ b/.github/workflows/release_fixtures.yaml @@ -1,6 +1,21 @@ name: Create Fixture Release +run-name: ${{ github.event_name == 'schedule' && 'Nightly Fill' || format('Create Fixture Release {0}@{1}{2}', inputs.feature, inputs.version, (inputs.cached || inputs.commit != '') && ' (cached)' || '') }} + +# Scheduled runs fill the mainnet `tests` feature (all tests, all fixture +# formats, up to the latest mainnet fork -- no dev forks) through the exact +# release pipeline, but skip the `release` job, so no tag or draft release +# is created: a rotating, always-available artifact of the mainnet +# fixtures. The cron fires at 02:00 UTC: the self-hosted runners are past +# the EU/US daytime peaks and results are ready before the EU morning. +# +# A manual `tests` release can reuse the newest of those artifacts +# instead of refilling via the `cached` checkbox: `build` and `combine` +# are skipped and the `release` job drafts from the nightly's tarball, +# tagged at the commit the nightly built. Runs in minutes. on: + schedule: + - cron: "0 2 * * *" workflow_dispatch: inputs: feature: @@ -27,15 +42,39 @@ on: description: "Override the t8n tool branch / tag / commit" required: false type: string + cached: + description: "Draft from the newest nightly artifact instead of refilling (tests only)" + required: false + type: boolean + default: false + commit: + description: "Release the nightly built at this commit (7+ hex chars); implies cached. Empty = newest." + required: false + type: string + +concurrency: + # Scheduled runs queue behind an in-flight nightly (never cancel a + # fill). Cached releases serialize too: drafts do not reserve their + # tag name, so parallel dispatches of the same version would silently + # coexist. Fresh releases are unconstrained (unique group per run). + group: ${{ github.event_name == 'schedule' && 'nightly-fill' || (inputs.cached || inputs.commit != '') && 'cached-release' || github.run_id }} + cancel-in-progress: false jobs: setup: runs-on: ubuntu-latest outputs: + # A cached release skips the fill: `build` (and with it `combine`) + # keys off `run`, and the release job downloads the resolved + # nightly's artifact and tags the commit it was built from. + run: ${{ (inputs.cached || inputs.commit != '') && 'false' || steps.check.outputs.run }} build_matrix: ${{ steps.matrix.outputs.build_matrix }} feature_name: ${{ steps.matrix.outputs.feature_name }} combine_labels: ${{ steps.matrix.outputs.combine_labels }} - target_sha: ${{ steps.target_sha.outputs.sha }} + target_sha: ${{ steps.cached.outputs.target_sha || steps.target_sha.outputs.sha }} + short_sha: ${{ steps.target_sha.outputs.short_sha }} + artifact_run_id: ${{ steps.cached.outputs.run_id }} + artifact_name: ${{ steps.cached.outputs.artifact_name }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -44,36 +83,67 @@ jobs: - name: Resolve target SHA id: target_sha shell: bash - run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + # The short form is a fixed 7-character slice (not `git + # rev-parse --short`, whose length can drift): the cached-release + # resolver derives artifact names from the API's full SHA and the + # two must always agree. + run: | + sha="$(git rev-parse HEAD)" + echo "sha=$sha" >> "$GITHUB_OUTPUT" + echo "short_sha=${sha:0:7}" >> "$GITHUB_OUTPUT" - uses: ./.github/actions/setup-uv + - name: Check for new commits (scheduled runs) + id: check + env: + GH_TOKEN: ${{ github.token }} + run: | + uv run -q .github/scripts/check_new_commits.py >> "$GITHUB_OUTPUT" + + - name: Resolve the nightly artifact to reuse (cached releases) + id: cached + if: inputs.cached || inputs.commit != '' + env: + GH_TOKEN: ${{ github.token }} + INPUT_FEATURE: ${{ inputs.feature }} + INPUT_VERSION: ${{ inputs.version }} + INPUT_BRANCH: ${{ inputs.branch }} + INPUT_COMMIT: ${{ inputs.commit }} + run: | + # The feature/branch/version guards, run resolution and + # ancestry check live in (and are unit-tested via) + # resolve_cached_release.py. + uv run -q .github/scripts/resolve_cached_release.py >> "$GITHUB_OUTPUT" + - name: Validate input and generate build matrix id: matrix shell: bash env: - INPUT_FEATURE: ${{ inputs.feature }} - INPUT_VERSION: ${{ inputs.version }} + # Scheduled runs have no inputs: fill the mainnet `tests` + # feature; the placeholder version passes validation and is + # otherwise unused because the `release` job is skipped for + # scheduled runs. + INPUT_FEATURE: ${{ inputs.feature || 'tests' }} + INPUT_VERSION: ${{ inputs.version || 'v0.0.0' }} INPUT_BRANCH: ${{ inputs.branch }} INPUT_EVM: ${{ inputs.evm }} run: | - # An `evm` override must name a key in evm.yaml; the feature, - # version and devnet-branch validation lives in (and is unit-tested - # via) generate_build_matrix.py. - if [ -n "$INPUT_EVM" ] && ! grep -qE "^${INPUT_EVM}:" .github/configs/evm.yaml; then - echo "::error::evm '$INPUT_EVM' is not a key in .github/configs/evm.yaml" - exit 1 - fi - + # The feature, version, devnet-branch and evm-override validation + # lives in (and is unit-tested via) generate_build_matrix.py. uv run -q .github/scripts/generate_build_matrix.py \ - "$INPUT_FEATURE" "$INPUT_VERSION" "$INPUT_BRANCH" >> "$GITHUB_OUTPUT" + "$INPUT_FEATURE" "$INPUT_VERSION" "$INPUT_BRANCH" "$INPUT_EVM" \ + >> "$GITHUB_OUTPUT" build: name: fill (${{ matrix.label || matrix.feature }}) needs: setup + if: needs.setup.outputs.run == 'true' runs-on: [self-hosted-ghr, size-gigachungus-x64] timeout-minutes: 1440 strategy: - fail-fast: true + # A release must be complete, so abort on the first failed range; a + # nightly wants every range's result for debugging. + fail-fast: ${{ github.event_name != 'schedule' }} matrix: include: ${{ fromJson(needs.setup.outputs.build_matrix) }} steps: @@ -88,6 +158,9 @@ jobs: from_fork: ${{ matrix.from_fork }} until_fork: ${{ matrix.until_fork }} split_label: ${{ matrix.label }} + # Nightly splits are intermediates consumed by `combine` right + # away; don't retain them for the repo-default period. + split_retention_days: ${{ github.event_name == 'schedule' && '1' || '' }} evm: ${{ inputs.evm }} evm_repo: ${{ inputs.evm_repo }} evm_ref: ${{ inputs.evm_ref }} @@ -152,15 +225,32 @@ jobs: - name: Upload combined fixture tarball uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: - name: fixtures_${{ needs.setup.outputs.feature_name }} + # Name the artifact for the built commit so the rolling + # nightly artifacts are tellable apart at a glance; the + # cached-release resolver derives this exact name from each + # run's head SHA. The tarball inside carries the feature name. + name: fixtures_${{ needs.setup.outputs.short_sha }} path: ${{ steps.tarball.outputs.path }} + # Keep nightly tarballs for five days; a quiet nightly re-runs + # after four (see check_new_commits.py), so a live artifact + # always exists. Release tarballs keep the repo default since + # the release job attaches them to a draft release anyway. + retention-days: ${{ github.event_name == 'schedule' && '5' || '' }} release: runs-on: ubuntu-latest needs: [setup, build, combine] - if: always() && needs.build.result == 'success' && (needs.combine.result == 'success' || needs.combine.result == 'skipped') + # Scheduled runs stop after `combine`: no tag, no draft release. + # Cached releases skip the fill, so a skipped `build` is expected; + # `setup` must have succeeded explicitly, because a failed `setup` + # also leaves `build` skipped and would otherwise start this job + # with empty outputs. + if: always() && github.event_name == 'workflow_dispatch' && needs.setup.result == 'success' && (needs.build.result == 'success' || ((inputs.cached || inputs.commit != '') && needs.build.result == 'skipped')) && (needs.combine.result == 'success' || needs.combine.result == 'skipped') permissions: contents: write + # Cached releases download the artifact from the resolved + # nightly run rather than this one. + actions: read steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -171,11 +261,21 @@ jobs: - name: Download release artifacts shell: bash run: | - gh run download ${{ github.run_id }} -p "fixtures_*" --dir ./artifacts - rm -rf ./artifacts/fixtures__*/ - gh run download ${{ github.run_id }} -p "benchmark_genesis_*" --dir ./artifacts || true + if [ -n "$ARTIFACT_RUN_ID" ]; then + # Cached release: download the resolved nightly's tarball by + # exact name -- a young nightly's live split directories + # would also match the fixtures_* pattern. + gh run download "$ARTIFACT_RUN_ID" -n "$ARTIFACT_NAME" \ + --dir "./artifacts/$ARTIFACT_NAME" + else + gh run download ${{ github.run_id }} -p "fixtures_*" --dir ./artifacts + rm -rf ./artifacts/fixtures__*/ + gh run download ${{ github.run_id }} -p "benchmark_genesis_*" --dir ./artifacts || true + fi env: GH_TOKEN: ${{ github.token }} + ARTIFACT_RUN_ID: ${{ needs.setup.outputs.artifact_run_id }} + ARTIFACT_NAME: ${{ needs.setup.outputs.artifact_name }} - name: Draft release shell: bash diff --git a/Justfile b/Justfile index c3b7de95a5c..4e6a97508e5 100644 --- a/Justfile +++ b/Justfile @@ -10,6 +10,11 @@ list: root := justfile_directory() output_dir := root / ".just" xdist_workers := env("PYTEST_XDIST_AUTO_NUM_WORKERS", "6") + +# The env var's job ends with the `-n` value above; export it empty so +# pytest-xdist, which reads it as a numeric worker-count override in +# `-n auto` mode, does not warn on non-numeric values such as "auto". +export PYTEST_XDIST_AUTO_NUM_WORKERS := "" evm_bin := env("EVM_BIN", "evm") latest_fork := "Amsterdam" @@ -124,6 +129,18 @@ fill *args: "$@" \ tests +# Callers append the feature params, fork range and output; last flag wins. +# Fill fixtures with the flags shared by all fixture releases +[group('consensus tests')] +fill-release *args: + uv run fill \ + -n {{ xdist_workers }} \ + --output="{{ output_dir }}/fill-release/fixtures" \ + --no-html \ + --durations=100 \ + --log-level=DEBUG \ + "$@" + # --- Integration Tests --- # Fill the base coverage consensus tests using EELS with PyPy diff --git a/docs/dev/releasing_tests.md b/docs/dev/releasing_tests.md index 0259159c0f8..ffbfe1cf504 100644 --- a/docs/dev/releasing_tests.md +++ b/docs/dev/releasing_tests.md @@ -1,13 +1,8 @@ # Releasing Test Fixtures -This page covers the mechanics of cutting a test fixture release. For the release types, -their versioning, and consumption guidance, see -[EELS Fixture Releases](../running_tests/releases.md). +This page covers the mechanics of cutting a test fixture release. For the release types, their versioning, and consumption guidance, see [EELS Fixture Releases](../running_tests/releases.md). -Fixture releases are produced by manually dispatching the -[`release_fixtures.yaml`](https://github.com/ethereum/execution-specs/blob/master/.github/workflows/release_fixtures.yaml) -workflow. There is no tag to push by hand. The workflow builds the fixtures and, only on -success, creates the tag and the (draft) GitHub release. +Fixture releases are produced by manually dispatching the [`release_fixtures.yaml`](https://github.com/ethereum/execution-specs/blob/master/.github/workflows/release_fixtures.yaml) workflow. There is no tag to push by hand. The workflow builds the fixtures and, only on success, drafts the GitHub release; publishing the draft creates the tag. ```bash gh workflow run release_fixtures.yaml -f feature= -f version=vX.Y.Z [-f branch=] @@ -23,31 +18,22 @@ gh workflow run release_fixtures.yaml -f feature= -f version=vX.Y.Z [-f | `evm` | no | Override the evm impl (e.g. `geth`, `evmone`). Defaults to the feature's `evm-type` in `feature.yaml`. | | `evm_repo` | no | Override the t8n tool repo (e.g. `ethereum/go-ethereum`). | | `evm_ref` | no | Override the t8n tool branch / tag / commit. | +| `cached` | no | Draft from the newest nightly artifact instead of refilling (`tests` only): `build` and `combine` are skipped and the tag targets the nightly's commit. See [Cached releases](#cached-releases). | +| `commit` | no | Release the nightly built at this commit (7+ hex chars) instead of the newest one; implies `cached`. | -`` must be a key in -[`.github/configs/feature.yaml`](https://github.com/ethereum/execution-specs/blob/master/.github/configs/feature.yaml) -(e.g. `tests`, `benchmark`), or a `-devnet` name that resolves to the shared `devnet` -feature. +`` must be a key in [`.github/configs/feature.yaml`](https://github.com/ethereum/execution-specs/blob/master/.github/configs/feature.yaml) (e.g. `tests`, `benchmark`), or a `-devnet` name that resolves to the shared `devnet` feature. -Input validation runs in -[`generate_build_matrix.py`](https://github.com/ethereum/execution-specs/blob/master/.github/scripts/generate_build_matrix.py) -(unit-tested) before any fixtures are built, and fails fast on: +Input validation runs in [`generate_build_matrix.py`](https://github.com/ethereum/execution-specs/blob/master/.github/scripts/generate_build_matrix.py) (unit-tested) before any fixtures are built, and fails fast on: - an empty `feature` or a `version` that is not `vX.Y.Z`; +- an `evm` override that is not a key in `.github/configs/evm.yaml`; - a bare `devnet` feature name (must carry a `-` prefix, e.g. `bal-devnet`); -- a `-devnet-` feature name — the devnet index belongs in the `version` major, not - the feature name (so `feature=bal-devnet-7` is rejected in favour of - `feature=bal-devnet version=v7.0.0`); -- a `*-devnet` release missing a `branch`, a `branch` outside the `devnets//` shape - (e.g. `devnets/bal/7`), or a `version` major that does not equal the devnet number `` in - the branch (so `feature=bal-devnet branch=devnets/bal/7` must use `version=v7.*.*`). +- a `-devnet-` feature name — the devnet index belongs in the `version` major, not the feature name (so `feature=bal-devnet-7` is rejected in favour of `feature=bal-devnet version=v7.0.0`); +- a `*-devnet` release missing a `branch`, a `branch` outside the `devnets//` shape (e.g. `devnets/bal/7`), or a `version` major that does not equal the devnet number `` in the branch (so `feature=bal-devnet branch=devnets/bal/7` must use `version=v7.*.*`). ## Devnet releases -Devnet releases must use a `-devnet` feature name (e.g. `feature=bal-devnet`) and must -specify the branch to release from. Devnet branches follow the `devnets//` scheme -(e.g. `devnets/bal/7`), and the `version` major must match the devnet number `` in the -branch: +Devnet releases must use a `-devnet` feature name (e.g. `feature=bal-devnet`) and must specify the branch to release from. Devnet branches follow the `devnets//` scheme (e.g. `devnets/bal/7`), and the `version` major must match the devnet number `` in the branch: ```bash gh workflow run release_fixtures.yaml -f feature=bal-devnet -f version=v7.0.0 -f branch=devnets/bal/7 @@ -57,14 +43,9 @@ gh workflow run release_fixtures.yaml -f feature=bal-devnet -f version=v7.0.0 -f On success the workflow: -1. Builds `fixtures_.tar.gz` (the `tests` feature builds `fixtures.tar.gz`) for the - resolved feature (per its `evm-type` and `fill-params` in `feature.yaml`). -2. Creates the git tag `tests-@vX.Y.Z` (the `tests` feature tags as `tests@vX.Y.Z`, - no doubled prefix) on the released commit (the SHA resolved once from the `branch` HEAD when - given, otherwise the dispatch commit). -3. Publishes a **draft pre-release** to - [`ethereum/execution-specs`](https://github.com/ethereum/execution-specs/releases), titled - the same as the git tag, with the fixture tarball(s) attached. +1. Builds `fixtures_.tar.gz` (the `tests` feature builds `fixtures.tar.gz`) for the resolved feature (per its `evm-type` and `fill-params` in `feature.yaml`). +2. Drafts a **pre-release** to [`ethereum/execution-specs`](https://github.com/ethereum/execution-specs/releases) with the fixture tarball(s) attached, titled and tagged `tests-@vX.Y.Z` (the `tests` feature tags as `tests@vX.Y.Z`, no doubled prefix). +3. Targets the tag at the released commit (the SHA resolved once from the `branch` HEAD when given, otherwise the dispatch commit). The tag name and target are stored as draft metadata; the git tag itself is only created when the draft is published, so an unpublished draft can be edited or deleted without leaving a tag behind. | Example dispatch | Git tag | Release title | Artifact | | ---------------- | ------- | ------------- | -------- | @@ -75,13 +56,8 @@ The release is created as a draft; review and publish it from the GitHub release ## Cutting a release -1. **Pick the next version** per the - [Versioning Scheme](../running_tests/releases.md#versioning-scheme) for the feature you're - releasing (e.g. the next `tests` release after `tests@v24.1.0` is `tests@v24.1.1` for a - non-breaking/new-tests bump, or `tests@v24.2.0` for a consensus-breaking spec change). -2. **Dispatch the workflow** from the - [Actions tab](https://github.com/ethereum/execution-specs/actions/workflows/release_fixtures.yaml) - or via the CLI: +1. **Pick the next version** per the [Versioning Scheme](../running_tests/releases.md#versioning-scheme) for the feature you're releasing (e.g. the next `tests` release after `tests@v24.1.0` is `tests@v24.1.1` for a non-breaking/new-tests bump, or `tests@v24.2.0` for a consensus-breaking spec change). +2. **Dispatch the workflow** from the [Actions tab](https://github.com/ethereum/execution-specs/actions/workflows/release_fixtures.yaml) or via the CLI: ```bash gh workflow run release_fixtures.yaml -f feature=tests -f version=v24.1.1 @@ -89,22 +65,39 @@ The release is created as a draft; review and publish it from the GitHub release gh workflow run release_fixtures.yaml -f feature=bal-devnet -f version=v7.0.0 -f branch=devnets/bal/7 ``` -3. **Wait for the build to succeed.** On success the workflow creates the - `tests-@vX.Y.Z` tag on the target commit and drafts the GitHub release with the - fixture tarball attached. If any job fails, no tag or release is created — fix the cause - and re-dispatch. -4. **Review and publish the draft.** Open the draft on the - [releases page](https://github.com/ethereum/execution-specs/releases), check the - auto-generated notes (anchored at the prior release on the same feature via - `--notes-start-tag`), and click *Publish release* when ready. +3. **Wait for the build to succeed.** On success the workflow drafts the GitHub release with the fixture tarball attached. If any job fails, no release is drafted: fix the cause and re-dispatch. +4. **Review and publish the draft.** Open the draft on the [releases page](https://github.com/ethereum/execution-specs/releases), check the auto-generated notes (anchored at the prior release on the same feature via `--notes-start-tag`), and click *Publish release* when ready. Publishing creates the `tests-@vX.Y.Z` tag on the target commit; until then a mispicked version can be fixed by editing the draft, with no stray tag to delete. !!! tip "Release features opt into all fixture formats via `feature.yaml`" - Tarball output (`.tar.gz`) does not by itself include the pre-allocation group formats - (`BlockchainEngineXFixture`, `BlockchainEngineStatefulFixture`). A release feature - requests them by adding `--generate-all-formats` to its `fill-params` in - `.github/configs/feature.yaml`: + Tarball output (`.tar.gz`) does not by itself include the pre-allocation group formats (`BlockchainEngineXFixture`, `BlockchainEngineStatefulFixture`). A release feature requests them by adding `--generate-all-formats` to its `fill-params` in `.github/configs/feature.yaml`: ```console # .tar.gz no longer auto-enables all formats (changed in #2702); request # them explicitly with --generate-all-formats uv run fill --generate-all-formats --output=fixtures.tar.gz tests/ ``` + +## Nightly fill + +The same workflow also runs on a nightly schedule (02:00 UTC) as a release rehearsal: it fills the mainnet `tests` feature (all tests, slow included, all fixture formats, up to the latest mainnet fork — dev forks are not included) through the exact release pipeline, but stops after `combine`, so no tag or release is created. Each run uploads a `fixtures_` workflow artifact (short hash of the built commit, containing `fixtures.tar.gz`) with a 5-day retention: a rotating, always-available build of the mainnet fixtures, effectively a `tests@` release candidate on demand. A scheduled run skips itself when there are no new commits since the last nightly that actually filled — a skipped or failed nightly never advances that baseline, so no commit slips through unfilled. A quiet stretch without commits still re-fills once the last fill is four days old, or its artifact is gone, so a live artifact always exists within the five-day retention. + +## Cached releases + +A `tests@` release can reuse the newest nightly artifact instead of refilling: tick the `cached` checkbox in the dispatch UI, or pass the flag on the CLI: + +```bash +gh workflow run release_fixtures.yaml -f feature=tests -f version=vX.Y.Z -f cached=true +# or release the nightly built at a specific commit (implies cached): +gh workflow run release_fixtures.yaml -f feature=tests -f version=vX.Y.Z -f commit= +``` + +The `build` and `combine` jobs are skipped; the `release` job downloads the `fixtures_` artifact from the newest nightly run that actually filled — or, with the `commit` input, from the nightly built at that commit — and drafts the same release a fresh fill would produce, targeted at the exact commit the nightly built, so publishing it creates the `tests@vX.Y.Z` tag on that commit. The whole run takes minutes on a hosted runner. Review and publish the draft exactly as in [Cutting a release](#cutting-a-release). + +The cached path's validation (unit-tested in [`resolve_cached_release.py`](https://github.com/ethereum/execution-specs/blob/master/.github/scripts/resolve_cached_release.py)) fails fast when: + +- the `feature` is not `tests`, or a `branch` is given (the nightly fills the default branch); +- the `version` is not `vX.Y.Z` or does not exceed the newest existing `tests@` tag; +- no nightly run with a live `fixtures_` artifact exists — or none matching the `commit` input; the error lists the reusable nightlies (artifacts are retained for five days: past that, dispatch a fresh fill); +- the resolved nightly does not contain the newest existing `tests@` release (a cached release must never regress content; re-releasing the identical commit is allowed); +- the nightly's commit is not an ancestor of the current default branch head. + +A cached release contains exactly what the resolved nightly filled: commits that landed after it are **not** included, and the run's step summary lists them so the releaser can decide between the cached artifact and a fresh fill.