diff --git a/.github/workflows/docs-build.yaml b/.github/workflows/docs-build.yaml index b49c2c518a9..4d167b159f1 100644 --- a/.github/workflows/docs-build.yaml +++ b/.github/workflows/docs-build.yaml @@ -227,6 +227,8 @@ jobs: - name: Build MkDocs documentation env: SITE_URL: ${{ needs.check-should-publish.outputs.site_url }} + # Social cards are publish-only; PR builds skip them for speed. + DOCS_SOCIAL_CARDS: ${{ github.event_name != 'pull_request' }} run: | echo "Building MkDocs with SITE_URL=$SITE_URL" just docs @@ -255,7 +257,12 @@ jobs: - uses: ./.github/actions/setup-uv + - name: Build spec documentation (parallel shards) + if: github.event_name == 'pull_request' + run: just docs-spec-parallel + - name: Build spec documentation + if: github.event_name != 'pull_request' run: just docs-spec env: DOCC_SKIP_DIFFS: ${{ case(github.event_name == 'push' && github.ref_name == github.event.repository.default_branch, '', '1') }} diff --git a/Justfile b/Justfile index c3b7de95a5c..2db39e944ca 100644 --- a/Justfile +++ b/Justfile @@ -316,6 +316,12 @@ docs-spec $DOCC_SKIP_DIFFS=env_var_or_default("DOCC_SKIP_DIFFS", ""): [group('docs')] docs-spec-fast: (docs-spec "1") +# Build spec docs in parallel shards for fast PR validation +[group('docs')] +docs-spec-parallel shards="4": + uv run python -m ethereum_spec_tools.docc_shards \ + -n {{ shards }} -o "{{ output_dir }}/docs-spec-parallel" + # Build HTML site documentation with mkdocs [group('docs')] docs *args: diff --git a/mkdocs.yml b/mkdocs.yml index f5f9fa55257..5824e80e677 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -22,7 +22,10 @@ plugins: members_order: source group_by_category: false - search - - social + # Social cards render a PNG per page (~2m for the full site) and need + # native cairo; only publish builds enable them (see docs-build.yaml). + - social: + enabled: !ENV [DOCS_SOCIAL_CARDS, false] - gen-files: scripts: - docs/scripts/copy_repo_docs_to_mkdocs.py diff --git a/pyproject.toml b/pyproject.toml index 99ed678edc7..91116305b6e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -277,6 +277,7 @@ whitelist = "ethereum_spec_tools.whitelist:main" "ethereum_spec_tools.docc.build" = "ethereum_spec_tools.docc:EthereumBuilder" "ethereum_spec_tools.docc.fix-indexes" = "ethereum_spec_tools.docc:FixIndexTransform" "ethereum_spec_tools.docc.minimize-diffs" = "ethereum_spec_tools.docc:MinimizeDiffsTransform" +"ethereum_spec_tools.docc.prune-references" = "ethereum_spec_tools.docc:PruneReferencesTransform" [project.entry-points."docc.plugins.html"] "ethereum_spec_tools.docc:DiffNode" = "ethereum_spec_tools.docc:render_diff" @@ -341,6 +342,9 @@ transform = [ "ethereum_spec_tools.docc.fix-indexes", "ethereum_spec_tools.docc.minimize-diffs", "docc.references.index", + # prune-references must run after the index is populated and before + # any transform that resolves references (search, html). + "ethereum_spec_tools.docc.prune-references", "docc.search.transform", "docc.html.transform", ] diff --git a/src/ethereum_spec_tools/docc.py b/src/ethereum_spec_tools/docc.py index c93a943eeb1..550d97685ab 100644 --- a/src/ethereum_spec_tools/docc.py +++ b/src/ethereum_spec_tools/docc.py @@ -55,6 +55,8 @@ from docc.plugins.python import PythonBuilder, PythonDiscover from docc.plugins.python.cst import PythonSource from docc.plugins.references import Definition, Reference +from docc.plugins.references import Index as ReferenceIndex +from docc.plugins.references import ReferenceError as DoccReferenceError from docc.settings import PluginSettings from docc.source import Source from docc.transform import Transform @@ -144,6 +146,17 @@ def _find_forks(config: PluginSettings) -> List[Hardfork]: return Hardfork.discover([str(forks)]) +def _only_forks() -> Set[str]: + """ + Parse the `DOCC_ONLY_FORKS` fork subset from the environment. + + Return the lower-cased fork short names to render, or an empty set + when the whole fork range should be rendered. + """ + value = os.environ.get("DOCC_ONLY_FORKS", "") + return {f.strip().lower() for f in value.split(",") if f.strip()} + + def _diff_path(before: Hardfork, after: Hardfork) -> PurePath: return PurePath("diffs") / before.short_name / after.short_name @@ -210,6 +223,43 @@ class EthereumPythonDiscover(PythonDiscover): def __init__(self, config: PluginSettings) -> None: super().__init__(config) self._fork_order = _ForkOrder(config) + self._apply_fork_filter(config) + + def _apply_fork_filter(self, config: PluginSettings) -> None: + """ + Exclude fork packages not listed in `DOCC_ONLY_FORKS`. + + The variable holds comma-separated fork short names (for example + `amsterdam,osaka`). When unset or empty, render every fork. When + no listed name matches a known fork, disable the filter so a bad + value cannot produce an empty (but successful) build. + """ + keep = _only_forks() + if not keep: + return + forks = _find_forks(config) + known = {f.short_name.lower() for f in forks} + if not keep & known: + logging.warning( + "DOCC_ONLY_FORKS matches no known fork; rendering all" + ) + return + dropped = [ + config.unresolve_path(PurePath(f.path)) + for f in forks + if f.path is not None and f.short_name.lower() not in keep + ] + logging.info( + "DOCC_ONLY_FORKS: rendering %d of %d fork package(s)", + len(forks) - len(dropped), + len(forks), + ) + # `excluded_paths` is declared `Final` upstream; replace it here, + # before discovery runs, to narrow the rendered sources. + self.excluded_paths = [ # type: ignore[misc] + *self.excluded_paths, + *dropped, + ] @override def _python_source( @@ -231,6 +281,75 @@ def _python_source( ) +class PruneReferencesTransform(Transform): + """ + Drop references into fork packages excluded from the build. + + When `DOCC_ONLY_FORKS` narrows discovery, links into excluded fork + packages have no definition. Replace each such reference with its + plain content so rendering succeeds without a link. References to + anything else are left alone, so genuinely broken identifiers still + fail the build. + """ + + def __init__(self, config: PluginSettings) -> None: + pass + + def transform(self, context: Context) -> None: + """ + Apply the transformation to the given document. + """ + keep = _only_forks() + if not keep: + return + context[Document].root.visit(_PruneReferencesVisitor(context, keep)) + + +class _PruneReferencesVisitor(Visitor): + _context: Context + _keep: Set[str] + _stack: List[Node] + + def __init__(self, context: Context, keep: Set[str]) -> None: + self._context = context + self._keep = keep + self._stack = [] + + def _prunable(self, node: Node) -> bool: + """ + Check whether the node links into an excluded fork package. + """ + if not isinstance(node, Reference): + return False + parts = node.identifier.split(".") + if parts[:2] != ["ethereum", "forks"] or len(parts) < 3: + return False + if parts[2].lower() in self._keep: + return False + try: + self._context[ReferenceIndex].lookup(node.identifier) + except DoccReferenceError: + return True + return False + + @override + def enter(self, node: Node) -> Visit: + if self._stack: + replacement = node + while self._prunable(replacement): + assert isinstance(replacement, Reference) + replacement = replacement.child + if replacement is not node: + self._stack[-1].replace_child(node, replacement) + node = replacement + self._stack.append(node) + return Visit.TraverseChildren + + @override + def exit(self, node: Node) -> None: + self._stack.pop() + + class EthereumDiscover(Discover): """ Creates sources that represent the diff between two other sources, one per @@ -252,6 +371,11 @@ def discover(self, known: FrozenSet[T]) -> Iterator[Source]: logging.info("Skipping diff discovery (DOCC_SKIP_DIFFS)") return + if _only_forks(): + # Fork-subset builds have no complete fork pairs to diff. + logging.info("Skipping diff discovery (DOCC_ONLY_FORKS)") + return + forks = {f.path: f for f in self.forks if f.path is not None} by_fork: Dict[Hardfork, Dict[PurePath, Source]] = defaultdict(dict) diff --git a/src/ethereum_spec_tools/docc_shards.py b/src/ethereum_spec_tools/docc_shards.py new file mode 100644 index 00000000000..fbea24eb550 --- /dev/null +++ b/src/ethereum_spec_tools/docc_shards.py @@ -0,0 +1,87 @@ +""" +Build the docc spec docs as parallel shards of consecutive forks. + +Fast PR-time validation only: the fork range is split into ``n`` +contiguous shards, each overlapping its predecessor by one fork so a +fork's reference to the previous fork resolves within its shard. One +``docc`` process renders each shard concurrently; the per-shard outputs +are not merged. + +Forward references (a fork referencing a later fork) fall outside their +shard and are pruned, so they are validated only by the serial +default-branch ``docs-spec`` build that gates the docs deploy. +""" + +import argparse +import os +import subprocess +import sys +from pathlib import Path +from typing import List + +from .forks import Hardfork + + +def compute_shards(forks: List[str], n: int) -> List[List[str]]: + """ + Split ``forks`` into at most ``n`` contiguous shards. + + Every shard after the first is prefixed with its predecessor fork, so + a fork's reference to the previous fork resolves within its shard. + """ + per = (len(forks) + n - 1) // n + shards: List[List[str]] = [] + for i in range(n): + start = i * per + if start >= len(forks): + break + lo = start - 1 if start > 0 else start + shards.append(forks[lo : min(start + per, len(forks))]) + return shards + + +def main() -> int: + """Discover forks, shard them, and render each shard with ``docc``.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "-n", + "--shards", + type=int, + default=4, + help="number of parallel shards (default: 4)", + ) + parser.add_argument( + "-o", + "--output-dir", + type=Path, + required=True, + help="parent directory for each shard's output", + ) + args = parser.parse_args() + if args.shards < 1: + parser.error("--shards must be a positive integer") + + forks = [fork.short_name for fork in Hardfork.discover()] + if not forks: + print("error: no forks discovered", file=sys.stderr) + return 1 + + processes: List[subprocess.Popen[bytes]] = [] + for i, shard in enumerate(compute_shards(forks, args.shards)): + print(f"shard {i}: {','.join(shard)}", flush=True) + env = { + **os.environ, + "DOCC_SKIP_DIFFS": "1", + "DOCC_ONLY_FORKS": ",".join(shard), + } + output = args.output_dir / f"shard-{i}" + processes.append( + subprocess.Popen(["docc", "--output", str(output)], env=env) + ) + + exit_codes = [process.wait() for process in processes] + return 1 if any(exit_codes) else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/evm_tools/test_docc_shards.py b/tests/evm_tools/test_docc_shards.py new file mode 100644 index 00000000000..456a7d7d72e --- /dev/null +++ b/tests/evm_tools/test_docc_shards.py @@ -0,0 +1,50 @@ +""" +Unit tests for the parallel spec-doc shard planner. +""" + +from ethereum_spec_tools.docc_shards import compute_shards +from ethereum_spec_tools.forks import Hardfork + + +def test_shards_cover_every_fork() -> None: + """Every fork lands in at least one shard.""" + forks = [f"f{i}" for i in range(24)] + shards = compute_shards(forks, 4) + covered = {fork for shard in shards for fork in shard} + assert covered == set(forks) + + +def test_shards_are_contiguous_with_one_fork_overlap() -> None: + """Shards tile the range in order, overlapping one fork per seam.""" + forks = [f"f{i}" for i in range(24)] + assert compute_shards(forks, 4) == [ + forks[0:6], + forks[5:12], + forks[11:18], + forks[17:24], + ] + + +def test_each_fork_shares_a_shard_with_its_predecessor() -> None: + """The overlap keeps every fork beside its immediate predecessor.""" + forks = [f"f{i}" for i in range(24)] + shards = compute_shards(forks, 4) + for i in range(1, len(forks)): + assert any( + forks[i] in shard and forks[i - 1] in shard for shard in shards + ) + + +def test_more_shards_than_forks_still_covers_all() -> None: + """Requesting more shards than forks leaves no fork uncovered.""" + shards = compute_shards(["a", "b", "c"], 4) + covered = {fork for shard in shards for fork in shard} + assert covered == {"a", "b", "c"} + + +def test_real_fork_set_is_fully_covered() -> None: + """The discovered fork set shards without dropping any fork.""" + forks = [fork.short_name for fork in Hardfork.discover()] + shards = compute_shards(forks, 4) + covered = {fork for shard in shards for fork in shard} + assert covered == set(forks) diff --git a/vulture_whitelist.py b/vulture_whitelist.py index ffd8e3992bd..fd1f6712690 100644 --- a/vulture_whitelist.py +++ b/vulture_whitelist.py @@ -77,6 +77,8 @@ docc.FixIndexTransform.transform docc.MinimizeDiffsTransform docc.MinimizeDiffsTransform.transform +docc.PruneReferencesTransform +docc.PruneReferencesTransform.transform docc._FixIndexVisitor.enter docc._DoccAdapter.shallow_equals docc._DoccAdapter.shallow_hash