diff --git a/.github/actions/build-fixtures/action.yaml b/.github/actions/build-fixtures/action.yaml index 719c3221a75..18c6fcce126 100644 --- a/.github/actions/build-fixtures/action.yaml +++ b/.github/actions/build-fixtures/action.yaml @@ -13,6 +13,9 @@ inputs: split_label: description: "Label for this fork-range split. Empty for unsplit builds." default: "" + split_retention_days: + description: "retention-days for the split fixture artifact. Empty for the repo default." + default: "" evm: description: "Override the evm impl. Defaults to the feature's evm-type." default: "" @@ -50,6 +53,8 @@ runs: run: sudo apt-get install -y pigz - name: Generate fixtures using fill shell: bash + env: + PYTEST_XDIST_AUTO_NUM_WORKERS: ${{ steps.evm-builder.outputs.xdist }} run: | IS_SPLIT="${{ inputs.split_label }}" @@ -61,13 +66,14 @@ runs: FORK_ARGS="" fi + EVM_ARGS="" + if [ "${{ steps.evm-builder.outputs.impl }}" != "eels" ]; then + EVM_ARGS="--evm-bin=${{ steps.evm-builder.outputs.evm-bin }}" + fi + # Allow exit code 5 (NO_TESTS_COLLECTED) for fork ranges with no tests. EXIT_CODE=0 - if [ "${{ steps.evm-builder.outputs.impl }}" = "eels" ]; then - uv run fill -n ${{ steps.evm-builder.outputs.xdist }} ${{ steps.properties.outputs.fill-params }} $FORK_ARGS $OUTPUT_ARG --build-name ${{ inputs.release_name }} --no-html --durations=100 --log-level=DEBUG || EXIT_CODE=$? - else - uv run fill -n ${{ steps.evm-builder.outputs.xdist }} --evm-bin=${{ steps.evm-builder.outputs.evm-bin }} ${{ steps.properties.outputs.fill-params }} $FORK_ARGS $OUTPUT_ARG --build-name ${{ inputs.release_name }} --no-html --durations=100 --log-level=DEBUG || EXIT_CODE=$? - fi + just fill-release $EVM_ARGS ${{ steps.properties.outputs.fill-params }} $FORK_ARGS $OUTPUT_ARG --build-name ${{ inputs.release_name }} || EXIT_CODE=$? if [ "$EXIT_CODE" -ne 0 ] && [ "$EXIT_CODE" -ne 5 ]; then exit "$EXIT_CODE" fi @@ -96,3 +102,4 @@ runs: include-hidden-files: true path: fixtures_${{ inputs.release_name }}/ if-no-files-found: ignore + retention-days: ${{ inputs.split_retention_days }} diff --git a/.github/configs/evm.yaml b/.github/configs/evm.yaml index 621ad39623d..b36b347d981 100644 --- a/.github/configs/evm.yaml +++ b/.github/configs/evm.yaml @@ -17,12 +17,6 @@ evmone: evm-bin: evmone xdist: auto targets: ["evmone-cli"] -benchmark: - impl: geth - repo: ethereum/go-ethereum - ref: master - evm-bin: evm - xdist: auto besu: impl: besu repo: hyperledger/besu diff --git a/.github/configs/feature.yaml b/.github/configs/feature.yaml index 39297ec6e84..6aadc6f0097 100644 --- a/.github/configs/feature.yaml +++ b/.github/configs/feature.yaml @@ -9,6 +9,13 @@ tests: evm-type: eels fill-params: --until=BPO4 --generate-all-formats +# Filled by the scheduled nightly run of the `release_fixtures` workflow: +# all tests (slow included), all fixture formats, up to the dev fork. Not +# meant for tagged releases. +nightly: + evm-type: eels + fill-params: --until=Amsterdam --generate-all-formats + benchmark: evm-type: benchmark fill-params: --fork=Osaka --generate-all-formats --gas-benchmark-values 1,5,10,30,60,100,150 ./tests/benchmark/compute --maxprocesses=30 --dist=worksteal diff --git a/.github/scripts/check_new_commits.py b/.github/scripts/check_new_commits.py new file mode 100644 index 00000000000..1aeefd12979 --- /dev/null +++ b/.github/scripts/check_new_commits.py @@ -0,0 +1,140 @@ +#!/usr/bin/env -S uv run --script +# +# /// script +# requires-python = ">=3.12" +# /// +""" +Decide whether a scheduled nightly fill has new commits to fill. + +Usage: `check_new_commits.py` (all inputs come from the environment). + +Compare the current commit against the head SHA of the last successful +*scheduled* run of the release workflow. Using the last success (rather +than a fixed time window) means a nightly that fails or is skipped +keeps re-running until it goes green, and no commit slips through +unfilled; filtering on scheduled runs means manual releases never +advance the nightly baseline. Manual (`workflow_dispatch`) runs always +run. + +A quiet stretch with no new commits still refreshes: once the last +successful fill is `REFRESH_AGE` old, the nightly re-runs anyway, so a +live artifact always exists within the workflow's five-day retention +and the release pipeline keeps getting exercised. + +Read `GITHUB_EVENT_NAME`, `GITHUB_REPOSITORY` and `GITHUB_SHA` from the +environment and query the GitHub API via the `gh` CLI (authenticated by +`GH_TOKEN`). Print `run=true|false` to stdout for `$GITHUB_OUTPUT` and +append the new-commit list (or a skip notice) to the +`$GITHUB_STEP_SUMMARY` file. +""" + +import json +import os +import subprocess +import sys +from datetime import datetime, timedelta, timezone + +WORKFLOW_FILE = "release_fixtures.yaml" + +# Re-run a quiet nightly once the last successful fill is this old, so +# a fresh artifact is uploaded before the previous one lapses (the +# workflow retains scheduled tarballs for five days). +REFRESH_AGE = timedelta(days=4) + + +def gh_api(path: str) -> str: + """Return the stdout of `gh api `, exiting non-zero on error.""" + result = subprocess.run( + ["gh", "api", path], capture_output=True, text=True + ) + if result.returncode != 0: + print(f"Error: gh api {path} failed:", file=sys.stderr) + print(result.stderr, file=sys.stderr) + sys.exit(1) + return result.stdout + + +def last_successful_nightly(repository: str) -> tuple[str, str]: + """ + Return the head SHA and creation time of the last scheduled run. + + Only successful scheduled runs count; return empty strings when + none exists yet. + """ + runs = json.loads( + gh_api( + f"repos/{repository}/actions/workflows/{WORKFLOW_FILE}" + "/runs?status=success&event=schedule&per_page=1" + ) + )["workflow_runs"] + if not runs: + return "", "" + return str(runs[0]["head_sha"]), str(runs[0]["created_at"]) + + +def is_stale(created_at: str) -> bool: + """Return whether a run created at *created_at* is due a refresh.""" + created = datetime.fromisoformat(created_at) + return datetime.now(timezone.utc) - created >= REFRESH_AGE + + +def commits_since(repository: str, last_sha: str, head_sha: str) -> list[str]: + """Return `- ` lines for commits after *last_sha*.""" + compare = json.loads( + gh_api(f"repos/{repository}/compare/{last_sha}...{head_sha}") + ) + return [ + f"- {c['sha'][:7]} {(c['commit']['message'].splitlines() or [''])[0]}" + for c in compare["commits"] + ] + + +def append_summary(text: str) -> None: + """Append *text* to the GitHub step summary, or stderr if unset.""" + summary_path = os.environ.get("GITHUB_STEP_SUMMARY") + if summary_path: + with open(summary_path, "a") as f: + f.write(text + "\n") + else: + print(text, file=sys.stderr) + + +def main() -> None: + """Print `run=true|false` and write the step summary.""" + if os.environ["GITHUB_EVENT_NAME"] != "schedule": + # Manual releases always run. + print("run=true") + return + + repository = os.environ["GITHUB_REPOSITORY"] + head_sha = os.environ["GITHUB_SHA"] + + last_sha, last_created = last_successful_nightly(repository) + if last_sha: + commits = commits_since(repository, last_sha, head_sha) + else: + # No prior successful nightly recorded; fill to get a baseline. + commits = ["- (no previous successful nightly fill found)"] + + if commits: + print("run=true") + append_summary( + "### Commits since last successful nightly fill\n" + + "\n".join(commits) + ) + elif is_stale(last_created): + print("run=true") + append_summary( + "No new commits, but the last nightly fill is older than " + f"{REFRESH_AGE.days} days; refreshing before its artifact " + "retention lapses." + ) + else: + print("run=false") + append_summary( + "No new commits since the last successful nightly fill; skipping." + ) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/generate_build_matrix.py b/.github/scripts/generate_build_matrix.py index 02fde3087dc..200bdb788ad 100644 --- a/.github/scripts/generate_build_matrix.py +++ b/.github/scripts/generate_build_matrix.py @@ -10,7 +10,7 @@ Validate release inputs and generate the build matrix for release fixture workflows. -Usage: `generate_build_matrix.py [branch]`. +Usage: `generate_build_matrix.py [branch] [evm]`. First validate the dispatch inputs (see `validate_inputs`), then read `.github/configs/feature.yaml` and emit a flat JSON build matrix suitable @@ -31,6 +31,7 @@ FEATURE_CONFIG = Path(".github/configs/feature.yaml") FORK_RANGES_CONFIG = Path(".github/configs/fork-ranges.yaml") +EVM_CONFIG = Path(".github/configs/evm.yaml") VERSION_RE = re.compile(r"^v[0-9]+\.[0-9]+\.[0-9]+$") @@ -81,12 +82,13 @@ def fail(message: str) -> NoReturn: sys.exit(1) -def validate_inputs(feature: str, version: str, branch: str) -> None: +def validate_inputs(feature: str, version: str, branch: str, evm: str) -> None: """ Validate the release dispatch inputs before building a matrix. - Centralize the feature/version checks here so they are unit-testable - rather than living as inline bash in the release workflow. + Centralize the feature/version/evm checks here so they are + unit-testable rather than living as inline bash in the release + workflow. For `-devnet` releases the major version (`X` of `vX.Y.Z`) must equal the devnet number encoded in the release branch, so a @@ -97,6 +99,10 @@ def validate_inputs(feature: str, version: str, branch: str) -> None: if not VERSION_RE.match(version): fail(f"version '{version}' must match vX.Y.Z (e.g. v20.0.0)") + # An `evm` override must name a key in evm.yaml. + if evm and evm not in load_config(EVM_CONFIG): + fail(f"evm '{evm}' is not a key in {EVM_CONFIG}") + # A bare `devnet` has no friendly `-` prefix to tag with. if feature in ("devnet", "-devnet"): fail("devnet releases require a - prefix, e.g. bal-devnet") @@ -207,7 +213,8 @@ def main() -> None: args = sys.argv[1:] if len(args) < 2: print( - "Usage: generate_build_matrix.py [branch]", + "Usage: generate_build_matrix.py " + " [branch] [evm]", file=sys.stderr, ) sys.exit(1) @@ -215,8 +222,9 @@ def main() -> None: name = args[0] version = args[1] branch = args[2] if len(args) > 2 else "" + evm = args[3] if len(args) > 3 else "" - validate_inputs(name, version, branch) + validate_inputs(name, version, branch, evm) config = load_config(FEATURE_CONFIG) fork_ranges = load_config(FORK_RANGES_CONFIG) or [] diff --git a/.github/scripts/tests/test_release_scripts.py b/.github/scripts/tests/test_release_scripts.py index a74c4c67c08..3d2f5c2e50a 100644 --- a/.github/scripts/tests/test_release_scripts.py +++ b/.github/scripts/tests/test_release_scripts.py @@ -6,8 +6,10 @@ """ import json +import os import subprocess import tarfile +from datetime import datetime, timedelta, timezone from pathlib import Path SCRIPTS_DIR = Path(__file__).parent.parent @@ -16,6 +18,7 @@ BUILD_MATRIX_SCRIPT = SCRIPTS_DIR / "generate_build_matrix.py" TARBALL_SCRIPT = SCRIPTS_DIR / "create_release_tarball.py" MERGE_INDEX_SCRIPT = SCRIPTS_DIR / "merge_index_files.py" +CHECK_COMMITS_SCRIPT = SCRIPTS_DIR / "check_new_commits.py" def run_script(script: Path, *args: str) -> subprocess.CompletedProcess: @@ -170,6 +173,159 @@ def test_devnet_matching_major_passes(self): out = parse_matrix_output(result.stdout) assert out["feature_name"] == "glamsterdam-devnet" + def test_unknown_evm_fails(self): + """Verify an evm override missing from evm.yaml is rejected.""" + result = run_script( + BUILD_MATRIX_SCRIPT, "tests", "v24.0.0", "", "nonexistent" + ) + assert result.returncode == 1 + assert "not a key" in result.stderr + + def test_known_evm_passes(self): + """Verify an evm override that is a key in evm.yaml passes.""" + result = run_script( + BUILD_MATRIX_SCRIPT, "tests", "v24.0.0", "", "evmone" + ) + assert result.returncode == 0 + + +# Fake `gh` served from PATH: answers the two API calls the commit-check +# script makes with canned JSON from env vars, and fails loudly on any +# other (or unconfigured) call. +FAKE_GH = """#!/usr/bin/env bash +case "$2" in + *actions/workflows*) response="$FAKE_GH_RUNS" ;; + *compare*) response="$FAKE_GH_COMPARE" ;; + *) response="" ;; +esac +if [ -z "$response" ]; then + echo "unexpected gh call: $*" >&2 + exit 1 +fi +printf '%s' "$response" +""" + + +class TestCheckNewCommits: + """Test check_new_commits.py.""" + + def run_check( + self, + tmp_path: Path, + event_name: str, + runs: str = "", + compare: str = "", + ) -> tuple[subprocess.CompletedProcess, Path]: + """Run the script with a fake `gh` on PATH; return it + summary.""" + bin_dir = tmp_path / "bin" + bin_dir.mkdir(exist_ok=True) + fake_gh = bin_dir / "gh" + fake_gh.write_text(FAKE_GH) + fake_gh.chmod(0o755) + + summary = tmp_path / "summary.md" + env = os.environ.copy() + env["PATH"] = f"{bin_dir}:{env['PATH']}" + env["GITHUB_EVENT_NAME"] = event_name + env["GITHUB_REPOSITORY"] = "ethereum/execution-specs" + env["GITHUB_SHA"] = "b" * 40 + env["GITHUB_STEP_SUMMARY"] = str(summary) + env["FAKE_GH_RUNS"] = runs + env["FAKE_GH_COMPARE"] = compare + + result = subprocess.run( + ["uv", "run", "-q", str(CHECK_COMMITS_SCRIPT)], + capture_output=True, + text=True, + cwd=REPO_ROOT, + env=env, + ) + return result, summary + + def test_dispatch_always_runs_without_api_calls(self, tmp_path): + """Verify a manual dispatch runs and never calls the API.""" + # The fake `gh` fails every call (no canned responses), so a + # zero exit proves the script made no API call. + result, summary = self.run_check(tmp_path, "workflow_dispatch") + assert result.returncode == 0 + assert result.stdout.strip() == "run=true" + assert not summary.exists() + + def test_schedule_without_prior_run_fills_baseline(self, tmp_path): + """Verify the first scheduled run fills to get a baseline.""" + result, summary = self.run_check( + tmp_path, "schedule", runs='{"workflow_runs": []}' + ) + assert result.returncode == 0 + assert result.stdout.strip() == "run=true" + assert "no previous successful" in summary.read_text() + + @staticmethod + def runs_json(age: timedelta, head_sha: str = "b" * 40) -> str: + """Return a last-successful-run response created *age* ago.""" + created = datetime.now(timezone.utc) - age + return json.dumps( + { + "workflow_runs": [ + { + "head_sha": head_sha, + "created_at": created.isoformat(), + } + ] + } + ) + + def test_schedule_with_new_commits_runs(self, tmp_path): + """Verify new commits since the baseline trigger a run.""" + commit = { + "sha": "abcdef1" + "0" * 33, + "commit": {"message": "feat(x): subject\n\nbody"}, + } + result, summary = self.run_check( + tmp_path, + "schedule", + runs=self.runs_json(timedelta(hours=25), head_sha="a" * 40), + compare=json.dumps({"commits": [commit]}), + ) + assert result.returncode == 0 + assert result.stdout.strip() == "run=true" + text = summary.read_text() + assert "### Commits since last successful nightly fill" in text + # Short SHA plus the commit subject, without the body. + assert "- abcdef1 feat(x): subject" in text + assert "body" not in text + + def test_schedule_without_new_commits_skips(self, tmp_path): + """Verify no commits since a recent baseline skips the run.""" + result, summary = self.run_check( + tmp_path, + "schedule", + # Just inside the refresh age: pin the four-day boundary. + runs=self.runs_json(timedelta(days=3, hours=23)), + compare=json.dumps({"commits": []}), + ) + assert result.returncode == 0 + assert result.stdout.strip() == "run=false" + assert "skipping" in summary.read_text() + + def test_schedule_stale_quiet_baseline_refreshes(self, tmp_path): + """Verify a quiet nightly re-runs once its artifact nears expiry.""" + result, summary = self.run_check( + tmp_path, + "schedule", + runs=self.runs_json(timedelta(days=4, hours=1)), + compare=json.dumps({"commits": []}), + ) + assert result.returncode == 0 + assert result.stdout.strip() == "run=true" + assert "refreshing" in summary.read_text() + + def test_gh_failure_fails_the_check(self, tmp_path): + """Verify a failing `gh` call fails the script.""" + result, _ = self.run_check(tmp_path, "schedule") + assert result.returncode == 1 + assert "gh api" in result.stderr + class TestCreateReleaseTarball: """Test create_release_tarball.py.""" diff --git a/.github/workflows/nightly-fill.yaml b/.github/workflows/nightly-fill.yaml deleted file mode 100644 index 1439f9a29c4..00000000000 --- a/.github/workflows/nightly-fill.yaml +++ /dev/null @@ -1,163 +0,0 @@ -name: Nightly Fill - -# Runs at 02:00 UTC: the self-hosted runners are past the EU/US daytime -# peaks and results are ready before the EU morning. -on: - schedule: - - cron: "0 2 * * *" - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }} - cancel-in-progress: false - -permissions: - contents: read - # Needed to look up the last successful nightly run and to download the - # per-range fixture artifacts in the `combine` job. - actions: read - -jobs: - setup: - name: Check commits and build fork matrix - runs-on: ubuntu-latest - outputs: - run: ${{ steps.check.outputs.run }} - matrix: ${{ steps.matrix.outputs.matrix }} - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - submodules: false - - name: List commits since the last successful nightly fill - id: check - env: - GH_TOKEN: ${{ github.token }} - run: | - # Head SHA of the last nightly run that filled successfully. Using - # the last *success* (rather than a fixed time window) means a - # nightly that fails or is skipped keeps re-running until it goes - # green, and no commit slips through unfilled. - last_sha=$(gh api \ - "repos/${GITHUB_REPOSITORY}/actions/workflows/nightly-fill.yaml/runs?status=success&per_page=1" \ - --jq '.workflow_runs[0].head_sha // ""') - - if [ -n "$last_sha" ]; then - commits=$(gh api \ - "repos/${GITHUB_REPOSITORY}/compare/${last_sha}...${GITHUB_SHA}" \ - --jq '.commits[] | "- \(.sha[0:7]) \(.commit.message | split("\n")[0])"') - else - # No prior successful run recorded; fill to establish a baseline. - commits="- (no previous successful nightly fill found)" - fi - - count=$(printf '%s\n' "$commits" | grep -c . || true) - if [ "$count" -gt 0 ] || [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then - echo "run=true" >> "$GITHUB_OUTPUT" - { - echo "### Commits since last successful nightly fill" - printf '%s\n' "$commits" - } >> "$GITHUB_STEP_SUMMARY" - else - echo "run=false" >> "$GITHUB_OUTPUT" - echo "No new commits since the last successful nightly fill; skipping." \ - >> "$GITHUB_STEP_SUMMARY" - fi - - - name: Build fork matrix from the shared config - id: matrix - # Reuse .github/configs/fork-ranges.yaml (the same split the release - # workflow uses) so the fork coverage is defined in one place. - run: | - echo "matrix=$(yq -o=json -I=0 '.' .github/configs/fork-ranges.yaml)" >> "$GITHUB_OUTPUT" - - fill-nightly: - name: fill-nightly (${{ matrix.label }}) - runs-on: [self-hosted-ghr, size-xl-x64] - needs: setup - if: needs.setup.outputs.run == 'true' - timeout-minutes: 720 - strategy: - fail-fast: false - matrix: - include: ${{ fromJson(needs.setup.outputs.matrix) }} - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - submodules: recursive - - uses: ./.github/actions/setup-uv - with: - python-version: "3.14" - - uses: ./.github/actions/setup-env - - name: Run nightly fill (${{ matrix.label }}) - run: just fill-nightly --from ${{ matrix.from }} --until ${{ matrix.until }} - env: - PYTEST_XDIST_AUTO_NUM_WORKERS: auto - - name: Upload per-range fixtures (${{ matrix.label }}) - # Intermediate split, merged into a single artifact by `combine`. - # Uploaded even on failure so a regressed fill can still be inspected. - if: always() - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - with: - name: fixtures__${{ matrix.label }} - path: .just/fill-nightly/fixtures - include-hidden-files: true - if-no-files-found: warn - retention-days: 1 - - combine: - name: Combine fixtures - runs-on: [self-hosted-ghr, size-xl-x64] - needs: [setup, fill-nightly] - # Run even if some ranges failed, so partial fixtures are still published. - if: ${{ !cancelled() && needs.setup.outputs.run == 'true' }} - permissions: - contents: read - # Write is needed to delete superseded fixture artifacts (rotation). - actions: write - steps: - - name: Resolve short commit SHA - id: sha - run: echo "short=${GITHUB_SHA:0:8}" >> "$GITHUB_OUTPUT" - - name: Download per-range fixtures - env: - GH_TOKEN: ${{ github.token }} - run: | - gh run download "${{ github.run_id }}" -p "fixtures__*" --dir split \ - || echo "No per-range fixture artifacts found." - - name: Merge into a single fixtures tree - run: | - mkdir -p "fixtures_${{ steps.sha.outputs.short }}" - for d in split/fixtures__*/; do - [ -d "$d" ] || continue - rsync -a "$d" "fixtures_${{ steps.sha.outputs.short }}/" - done - - name: Upload combined fixtures - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - with: - # Keyed by commit so a downloaded set maps back to what filled it. - name: fixtures_${{ steps.sha.outputs.short }} - path: fixtures_${{ steps.sha.outputs.short }} - include-hidden-files: true - if-no-files-found: warn - # Outer bound only; the real policy is "keep the last 5 builds", - # enforced by the rotation step below. - retention-days: 90 - - name: Rotate — keep only the last 5 nightly fixture sets - env: - GH_TOKEN: ${{ github.token }} - run: | - # Combined nightly artifacts are named `fixtures_<8-hex-sha>`. List - # them newest-first and delete everything past the 5 most recent. - # Release artifacts (`fixtures_`) and per-range splits - # (`fixtures__