From 653c6467c5f2a80de5b4c924a074f6fa97bf114b Mon Sep 17 00:00:00 2001 From: spencer-tb Date: Fri, 3 Jul 2026 16:18:21 +0100 Subject: [PATCH 1/4] feat(docs): parallel PR spec-doc builds, publish-only social cards --- .github/workflows/docs-build.yaml | 7 ++ Justfile | 31 ++++++++ mkdocs.yml | 5 +- pyproject.toml | 4 + src/ethereum_spec_tools/docc.py | 124 ++++++++++++++++++++++++++++++ vulture_whitelist.py | 2 + 6 files changed, 172 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docs-build.yaml b/.github/workflows/docs-build.yaml index b49c2c518a9..4d167b159f1 100644 --- a/.github/workflows/docs-build.yaml +++ b/.github/workflows/docs-build.yaml @@ -227,6 +227,8 @@ jobs: - name: Build MkDocs documentation env: SITE_URL: ${{ needs.check-should-publish.outputs.site_url }} + # Social cards are publish-only; PR builds skip them for speed. + DOCS_SOCIAL_CARDS: ${{ github.event_name != 'pull_request' }} run: | echo "Building MkDocs with SITE_URL=$SITE_URL" just docs @@ -255,7 +257,12 @@ jobs: - uses: ./.github/actions/setup-uv + - name: Build spec documentation (parallel shards) + if: github.event_name == 'pull_request' + run: just docs-spec-parallel + - name: Build spec documentation + if: github.event_name != 'pull_request' run: just docs-spec env: DOCC_SKIP_DIFFS: ${{ case(github.event_name == 'push' && github.ref_name == github.event.repository.default_branch, '', '1') }} diff --git a/Justfile b/Justfile index c3b7de95a5c..9c0d0748678 100644 --- a/Justfile +++ b/Justfile @@ -316,6 +316,37 @@ docs-spec $DOCC_SKIP_DIFFS=env_var_or_default("DOCC_SKIP_DIFFS", ""): [group('docs')] docs-spec-fast: (docs-spec "1") +# Build spec docs in parallel shards of consecutive forks (PR validation). +# Each shard overlaps its predecessor fork by one so every previous-fork +# reference is validated in some shard; outputs are per-shard and not merged. +[group('docs')] +docs-spec-parallel shards="4": + #!/usr/bin/env bash + set -euo pipefail + read -ra forks <<< "$(uv run python -c 'from ethereum_spec_tools.forks import Hardfork; print(" ".join(h.short_name for h in Hardfork.discover()))')" + total=${#forks[@]} + n={{ shards }} + per=$(( (total + n - 1) / n )) + pids=() + for ((i = 0; i < n; i++)); do + start=$(( i * per )) + [ "$start" -ge "$total" ] && break + end=$(( start + per )) + [ "$end" -gt "$total" ] && end=$total + s=$start + [ "$s" -gt 0 ] && s=$(( s - 1 )) + shard=$(IFS=,; echo "${forks[*]:$s:$(( end - s ))}") + echo "shard $i: $shard" + DOCC_SKIP_DIFFS=1 DOCC_ONLY_FORKS="$shard" \ + uv run docc --output "{{ output_dir }}/docs-spec-parallel/shard-$i" & + pids+=($!) + done + fail=0 + for pid in "${pids[@]}"; do + wait "$pid" || fail=1 + done + exit $fail + # Build HTML site documentation with mkdocs [group('docs')] docs *args: diff --git a/mkdocs.yml b/mkdocs.yml index f5f9fa55257..5824e80e677 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -22,7 +22,10 @@ plugins: members_order: source group_by_category: false - search - - social + # Social cards render a PNG per page (~2m for the full site) and need + # native cairo; only publish builds enable them (see docs-build.yaml). + - social: + enabled: !ENV [DOCS_SOCIAL_CARDS, false] - gen-files: scripts: - docs/scripts/copy_repo_docs_to_mkdocs.py diff --git a/pyproject.toml b/pyproject.toml index 99ed678edc7..91116305b6e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -277,6 +277,7 @@ whitelist = "ethereum_spec_tools.whitelist:main" "ethereum_spec_tools.docc.build" = "ethereum_spec_tools.docc:EthereumBuilder" "ethereum_spec_tools.docc.fix-indexes" = "ethereum_spec_tools.docc:FixIndexTransform" "ethereum_spec_tools.docc.minimize-diffs" = "ethereum_spec_tools.docc:MinimizeDiffsTransform" +"ethereum_spec_tools.docc.prune-references" = "ethereum_spec_tools.docc:PruneReferencesTransform" [project.entry-points."docc.plugins.html"] "ethereum_spec_tools.docc:DiffNode" = "ethereum_spec_tools.docc:render_diff" @@ -341,6 +342,9 @@ transform = [ "ethereum_spec_tools.docc.fix-indexes", "ethereum_spec_tools.docc.minimize-diffs", "docc.references.index", + # prune-references must run after the index is populated and before + # any transform that resolves references (search, html). + "ethereum_spec_tools.docc.prune-references", "docc.search.transform", "docc.html.transform", ] diff --git a/src/ethereum_spec_tools/docc.py b/src/ethereum_spec_tools/docc.py index c93a943eeb1..550d97685ab 100644 --- a/src/ethereum_spec_tools/docc.py +++ b/src/ethereum_spec_tools/docc.py @@ -55,6 +55,8 @@ from docc.plugins.python import PythonBuilder, PythonDiscover from docc.plugins.python.cst import PythonSource from docc.plugins.references import Definition, Reference +from docc.plugins.references import Index as ReferenceIndex +from docc.plugins.references import ReferenceError as DoccReferenceError from docc.settings import PluginSettings from docc.source import Source from docc.transform import Transform @@ -144,6 +146,17 @@ def _find_forks(config: PluginSettings) -> List[Hardfork]: return Hardfork.discover([str(forks)]) +def _only_forks() -> Set[str]: + """ + Parse the `DOCC_ONLY_FORKS` fork subset from the environment. + + Return the lower-cased fork short names to render, or an empty set + when the whole fork range should be rendered. + """ + value = os.environ.get("DOCC_ONLY_FORKS", "") + return {f.strip().lower() for f in value.split(",") if f.strip()} + + def _diff_path(before: Hardfork, after: Hardfork) -> PurePath: return PurePath("diffs") / before.short_name / after.short_name @@ -210,6 +223,43 @@ class EthereumPythonDiscover(PythonDiscover): def __init__(self, config: PluginSettings) -> None: super().__init__(config) self._fork_order = _ForkOrder(config) + self._apply_fork_filter(config) + + def _apply_fork_filter(self, config: PluginSettings) -> None: + """ + Exclude fork packages not listed in `DOCC_ONLY_FORKS`. + + The variable holds comma-separated fork short names (for example + `amsterdam,osaka`). When unset or empty, render every fork. When + no listed name matches a known fork, disable the filter so a bad + value cannot produce an empty (but successful) build. + """ + keep = _only_forks() + if not keep: + return + forks = _find_forks(config) + known = {f.short_name.lower() for f in forks} + if not keep & known: + logging.warning( + "DOCC_ONLY_FORKS matches no known fork; rendering all" + ) + return + dropped = [ + config.unresolve_path(PurePath(f.path)) + for f in forks + if f.path is not None and f.short_name.lower() not in keep + ] + logging.info( + "DOCC_ONLY_FORKS: rendering %d of %d fork package(s)", + len(forks) - len(dropped), + len(forks), + ) + # `excluded_paths` is declared `Final` upstream; replace it here, + # before discovery runs, to narrow the rendered sources. + self.excluded_paths = [ # type: ignore[misc] + *self.excluded_paths, + *dropped, + ] @override def _python_source( @@ -231,6 +281,75 @@ def _python_source( ) +class PruneReferencesTransform(Transform): + """ + Drop references into fork packages excluded from the build. + + When `DOCC_ONLY_FORKS` narrows discovery, links into excluded fork + packages have no definition. Replace each such reference with its + plain content so rendering succeeds without a link. References to + anything else are left alone, so genuinely broken identifiers still + fail the build. + """ + + def __init__(self, config: PluginSettings) -> None: + pass + + def transform(self, context: Context) -> None: + """ + Apply the transformation to the given document. + """ + keep = _only_forks() + if not keep: + return + context[Document].root.visit(_PruneReferencesVisitor(context, keep)) + + +class _PruneReferencesVisitor(Visitor): + _context: Context + _keep: Set[str] + _stack: List[Node] + + def __init__(self, context: Context, keep: Set[str]) -> None: + self._context = context + self._keep = keep + self._stack = [] + + def _prunable(self, node: Node) -> bool: + """ + Check whether the node links into an excluded fork package. + """ + if not isinstance(node, Reference): + return False + parts = node.identifier.split(".") + if parts[:2] != ["ethereum", "forks"] or len(parts) < 3: + return False + if parts[2].lower() in self._keep: + return False + try: + self._context[ReferenceIndex].lookup(node.identifier) + except DoccReferenceError: + return True + return False + + @override + def enter(self, node: Node) -> Visit: + if self._stack: + replacement = node + while self._prunable(replacement): + assert isinstance(replacement, Reference) + replacement = replacement.child + if replacement is not node: + self._stack[-1].replace_child(node, replacement) + node = replacement + self._stack.append(node) + return Visit.TraverseChildren + + @override + def exit(self, node: Node) -> None: + self._stack.pop() + + class EthereumDiscover(Discover): """ Creates sources that represent the diff between two other sources, one per @@ -252,6 +371,11 @@ def discover(self, known: FrozenSet[T]) -> Iterator[Source]: logging.info("Skipping diff discovery (DOCC_SKIP_DIFFS)") return + if _only_forks(): + # Fork-subset builds have no complete fork pairs to diff. + logging.info("Skipping diff discovery (DOCC_ONLY_FORKS)") + return + forks = {f.path: f for f in self.forks if f.path is not None} by_fork: Dict[Hardfork, Dict[PurePath, Source]] = defaultdict(dict) diff --git a/vulture_whitelist.py b/vulture_whitelist.py index ffd8e3992bd..fd1f6712690 100644 --- a/vulture_whitelist.py +++ b/vulture_whitelist.py @@ -77,6 +77,8 @@ docc.FixIndexTransform.transform docc.MinimizeDiffsTransform docc.MinimizeDiffsTransform.transform +docc.PruneReferencesTransform +docc.PruneReferencesTransform.transform docc._FixIndexVisitor.enter docc._DoccAdapter.shallow_equals docc._DoccAdapter.shallow_hash From 31b36ba2c5ab211546732d40269d2cf7467a63b9 Mon Sep 17 00:00:00 2001 From: danceratopz Date: Mon, 6 Jul 2026 22:21:28 +0100 Subject: [PATCH 2/4] refactor(docs): replace docs-spec-parallel bash with a Python planner --- Justfile | 31 +------ .../cli/pytest_commands/fill.py | 1 + src/ethereum_spec_tools/docc_shards.py | 83 +++++++++++++++++++ tests/docc/test_docc_shards.py | 50 +++++++++++ 4 files changed, 137 insertions(+), 28 deletions(-) create mode 100644 src/ethereum_spec_tools/docc_shards.py create mode 100644 tests/docc/test_docc_shards.py diff --git a/Justfile b/Justfile index 9c0d0748678..2db39e944ca 100644 --- a/Justfile +++ b/Justfile @@ -316,36 +316,11 @@ docs-spec $DOCC_SKIP_DIFFS=env_var_or_default("DOCC_SKIP_DIFFS", ""): [group('docs')] docs-spec-fast: (docs-spec "1") -# Build spec docs in parallel shards of consecutive forks (PR validation). -# Each shard overlaps its predecessor fork by one so every previous-fork -# reference is validated in some shard; outputs are per-shard and not merged. +# Build spec docs in parallel shards for fast PR validation [group('docs')] docs-spec-parallel shards="4": - #!/usr/bin/env bash - set -euo pipefail - read -ra forks <<< "$(uv run python -c 'from ethereum_spec_tools.forks import Hardfork; print(" ".join(h.short_name for h in Hardfork.discover()))')" - total=${#forks[@]} - n={{ shards }} - per=$(( (total + n - 1) / n )) - pids=() - for ((i = 0; i < n; i++)); do - start=$(( i * per )) - [ "$start" -ge "$total" ] && break - end=$(( start + per )) - [ "$end" -gt "$total" ] && end=$total - s=$start - [ "$s" -gt 0 ] && s=$(( s - 1 )) - shard=$(IFS=,; echo "${forks[*]:$s:$(( end - s ))}") - echo "shard $i: $shard" - DOCC_SKIP_DIFFS=1 DOCC_ONLY_FORKS="$shard" \ - uv run docc --output "{{ output_dir }}/docs-spec-parallel/shard-$i" & - pids+=($!) - done - fail=0 - for pid in "${pids[@]}"; do - wait "$pid" || fail=1 - done - exit $fail + uv run python -m ethereum_spec_tools.docc_shards \ + -n {{ shards }} -o "{{ output_dir }}/docs-spec-parallel" # Build HTML site documentation with mkdocs [group('docs')] diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/fill.py b/packages/testing/src/execution_testing/cli/pytest_commands/fill.py index fbd513b823d..c18114fc904 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/fill.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/fill.py @@ -140,6 +140,7 @@ def _add_default_ignores(self, args: List[str]) -> List[str]: """Add default ignore paths for directories not used by fill.""" # Directories to ignore by default default_ignores = [ + "tests/docc", "tests/evm_tools", "tests/json_loader", "tests/fixtures", diff --git a/src/ethereum_spec_tools/docc_shards.py b/src/ethereum_spec_tools/docc_shards.py new file mode 100644 index 00000000000..c2030a08b9c --- /dev/null +++ b/src/ethereum_spec_tools/docc_shards.py @@ -0,0 +1,83 @@ +""" +Build the docc spec docs as parallel shards of consecutive forks. + +Fast PR-time validation only: the fork range is split into ``n`` +contiguous shards, each overlapping its predecessor by one fork so a +fork's reference to the previous fork resolves within its shard. One +``docc`` process renders each shard concurrently; the per-shard outputs +are not merged. +""" + +import argparse +import os +import subprocess +import sys +from pathlib import Path +from typing import List + +from .forks import Hardfork + + +def compute_shards(forks: List[str], n: int) -> List[List[str]]: + """ + Split ``forks`` into at most ``n`` contiguous shards. + + Every shard after the first is prefixed with its predecessor fork, so + a fork's reference to the previous fork resolves within its shard. + """ + per = (len(forks) + n - 1) // n + shards: List[List[str]] = [] + for i in range(n): + start = i * per + if start >= len(forks): + break + lo = start - 1 if start > 0 else start + shards.append(forks[lo : min(start + per, len(forks))]) + return shards + + +def main() -> int: + """Discover forks, shard them, and render each shard with ``docc``.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "-n", + "--shards", + type=int, + default=4, + help="number of parallel shards (default: 4)", + ) + parser.add_argument( + "-o", + "--output-dir", + type=Path, + required=True, + help="parent directory for each shard's output", + ) + args = parser.parse_args() + if args.shards < 1: + parser.error("--shards must be a positive integer") + + forks = [fork.short_name for fork in Hardfork.discover()] + if not forks: + print("error: no forks discovered", file=sys.stderr) + return 1 + + processes: List[subprocess.Popen[bytes]] = [] + for i, shard in enumerate(compute_shards(forks, args.shards)): + print(f"shard {i}: {','.join(shard)}", flush=True) + env = { + **os.environ, + "DOCC_SKIP_DIFFS": "1", + "DOCC_ONLY_FORKS": ",".join(shard), + } + output = args.output_dir / f"shard-{i}" + processes.append( + subprocess.Popen(["docc", "--output", str(output)], env=env) + ) + + exit_codes = [process.wait() for process in processes] + return 1 if any(exit_codes) else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/docc/test_docc_shards.py b/tests/docc/test_docc_shards.py new file mode 100644 index 00000000000..456a7d7d72e --- /dev/null +++ b/tests/docc/test_docc_shards.py @@ -0,0 +1,50 @@ +""" +Unit tests for the parallel spec-doc shard planner. +""" + +from ethereum_spec_tools.docc_shards import compute_shards +from ethereum_spec_tools.forks import Hardfork + + +def test_shards_cover_every_fork() -> None: + """Every fork lands in at least one shard.""" + forks = [f"f{i}" for i in range(24)] + shards = compute_shards(forks, 4) + covered = {fork for shard in shards for fork in shard} + assert covered == set(forks) + + +def test_shards_are_contiguous_with_one_fork_overlap() -> None: + """Shards tile the range in order, overlapping one fork per seam.""" + forks = [f"f{i}" for i in range(24)] + assert compute_shards(forks, 4) == [ + forks[0:6], + forks[5:12], + forks[11:18], + forks[17:24], + ] + + +def test_each_fork_shares_a_shard_with_its_predecessor() -> None: + """The overlap keeps every fork beside its immediate predecessor.""" + forks = [f"f{i}" for i in range(24)] + shards = compute_shards(forks, 4) + for i in range(1, len(forks)): + assert any( + forks[i] in shard and forks[i - 1] in shard for shard in shards + ) + + +def test_more_shards_than_forks_still_covers_all() -> None: + """Requesting more shards than forks leaves no fork uncovered.""" + shards = compute_shards(["a", "b", "c"], 4) + covered = {fork for shard in shards for fork in shard} + assert covered == {"a", "b", "c"} + + +def test_real_fork_set_is_fully_covered() -> None: + """The discovered fork set shards without dropping any fork.""" + forks = [fork.short_name for fork in Hardfork.discover()] + shards = compute_shards(forks, 4) + covered = {fork for shard in shards for fork in shard} + assert covered == set(forks) From 2eb16699aed557b0ab58e449f650ca7f8ca83010 Mon Sep 17 00:00:00 2001 From: spencer-tb Date: Mon, 6 Jul 2026 22:24:52 +0100 Subject: [PATCH 3/4] docs(spec-tools): note forward refs validated only by the serial build --- src/ethereum_spec_tools/docc_shards.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/ethereum_spec_tools/docc_shards.py b/src/ethereum_spec_tools/docc_shards.py index c2030a08b9c..fbea24eb550 100644 --- a/src/ethereum_spec_tools/docc_shards.py +++ b/src/ethereum_spec_tools/docc_shards.py @@ -6,6 +6,10 @@ fork's reference to the previous fork resolves within its shard. One ``docc`` process renders each shard concurrently; the per-shard outputs are not merged. + +Forward references (a fork referencing a later fork) fall outside their +shard and are pruned, so they are validated only by the serial +default-branch ``docs-spec`` build that gates the docs deploy. """ import argparse From d41d07388b067123edb4165c00f882e8f4a48500 Mon Sep 17 00:00:00 2001 From: danceratopz Date: Tue, 7 Jul 2026 00:15:24 +0200 Subject: [PATCH 4/4] test(test-forks): run docc_shards unit test in the spec-tools job The `docc_shards` unit test never ran in CI: `fill` and the docs build skip `tests/docc` via `_add_default_ignores`, and no other job targets it, so the `compute_shards` coverage and overlap assertions went unchecked. The module itself runs via `docs-spec-parallel`, but its invariants were never asserted. Move the test to `tests/evm_tools/`, which the `spec-tools` job runs, so the assertions now execute in CI. That directory is already in `fill`'s default ignores, so the separate `tests/docc` entry is removed. --- .../testing/src/execution_testing/cli/pytest_commands/fill.py | 1 - tests/{docc => evm_tools}/test_docc_shards.py | 0 2 files changed, 1 deletion(-) rename tests/{docc => evm_tools}/test_docc_shards.py (100%) diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/fill.py b/packages/testing/src/execution_testing/cli/pytest_commands/fill.py index c18114fc904..fbd513b823d 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/fill.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/fill.py @@ -140,7 +140,6 @@ def _add_default_ignores(self, args: List[str]) -> List[str]: """Add default ignore paths for directories not used by fill.""" # Directories to ignore by default default_ignores = [ - "tests/docc", "tests/evm_tools", "tests/json_loader", "tests/fixtures", diff --git a/tests/docc/test_docc_shards.py b/tests/evm_tools/test_docc_shards.py similarity index 100% rename from tests/docc/test_docc_shards.py rename to tests/evm_tools/test_docc_shards.py