perf(spec-tools,ci): parallel PR spec-doc builds, publish-only social cards

.github/workflows/docs-build.yaml

@@ -227,6 +227,8 @@ jobs:
       - name: Build MkDocs documentation
         env:
           SITE_URL: ${{ needs.check-should-publish.outputs.site_url }}
+          # Social cards are publish-only; PR builds skip them for speed.
+          DOCS_SOCIAL_CARDS: ${{ github.event_name != 'pull_request' }}
         run: |
           echo "Building MkDocs with SITE_URL=$SITE_URL"
           just docs
@@ -255,7 +257,12 @@ jobs:
 
       - uses: ./.github/actions/setup-uv
 
+      - name: Build spec documentation (parallel shards)
+        if: github.event_name == 'pull_request'
+        run: just docs-spec-parallel
+
       - name: Build spec documentation
+        if: github.event_name != 'pull_request'
         run: just docs-spec
         env:
           DOCC_SKIP_DIFFS: ${{ case(github.event_name == 'push' && github.ref_name == github.event.repository.default_branch, '', '1') }}

Justfile

@@ -316,6 +316,12 @@ docs-spec $DOCC_SKIP_DIFFS=env_var_or_default("DOCC_SKIP_DIFFS", ""):
 [group('docs')]
 docs-spec-fast: (docs-spec "1")
 
+# Build spec docs in parallel shards for fast PR validation
+[group('docs')]
+docs-spec-parallel shards="4":
+    uv run python -m ethereum_spec_tools.docc_shards \
+        -n {{ shards }} -o "{{ output_dir }}/docs-spec-parallel"
+
 # Build HTML site documentation with mkdocs
 [group('docs')]
 docs *args:

mkdocs.yml

@@ -22,7 +22,10 @@ plugins:
             members_order: source
             group_by_category: false
   - search
-  - social
+  # Social cards render a PNG per page (~2m for the full site) and need
+  # native cairo; only publish builds enable them (see docs-build.yaml).
+  - social:
+      enabled: !ENV [DOCS_SOCIAL_CARDS, false]
   - gen-files:
       scripts:
         - docs/scripts/copy_repo_docs_to_mkdocs.py

pyproject.toml

@@ -277,6 +277,7 @@ whitelist = "ethereum_spec_tools.whitelist:main"
 "ethereum_spec_tools.docc.build" = "ethereum_spec_tools.docc:EthereumBuilder"
 "ethereum_spec_tools.docc.fix-indexes" = "ethereum_spec_tools.docc:FixIndexTransform"
 "ethereum_spec_tools.docc.minimize-diffs" = "ethereum_spec_tools.docc:MinimizeDiffsTransform"
+"ethereum_spec_tools.docc.prune-references" = "ethereum_spec_tools.docc:PruneReferencesTransform"
 
 [project.entry-points."docc.plugins.html"]
 "ethereum_spec_tools.docc:DiffNode" = "ethereum_spec_tools.docc:render_diff"
@@ -341,6 +342,9 @@ transform = [
     "ethereum_spec_tools.docc.fix-indexes",
     "ethereum_spec_tools.docc.minimize-diffs",
     "docc.references.index",
+    # prune-references must run after the index is populated and before
+    # any transform that resolves references (search, html).
+    "ethereum_spec_tools.docc.prune-references",
     "docc.search.transform",
     "docc.html.transform",
 ]

src/ethereum_spec_tools/docc.py

@@ -55,6 +55,8 @@
 from docc.plugins.python import PythonBuilder, PythonDiscover
 from docc.plugins.python.cst import PythonSource
 from docc.plugins.references import Definition, Reference
+from docc.plugins.references import Index as ReferenceIndex
+from docc.plugins.references import ReferenceError as DoccReferenceError
 from docc.settings import PluginSettings
 from docc.source import Source
 from docc.transform import Transform
@@ -144,6 +146,17 @@ def _find_forks(config: PluginSettings) -> List[Hardfork]:
     return Hardfork.discover([str(forks)])
 
 
+def _only_forks() -> Set[str]:
+    """
+    Parse the `DOCC_ONLY_FORKS` fork subset from the environment.
+
+    Return the lower-cased fork short names to render, or an empty set
+    when the whole fork range should be rendered.
+    """
+    value = os.environ.get("DOCC_ONLY_FORKS", "")
+    return {f.strip().lower() for f in value.split(",") if f.strip()}
+
+
 def _diff_path(before: Hardfork, after: Hardfork) -> PurePath:
     return PurePath("diffs") / before.short_name / after.short_name
 
@@ -210,6 +223,43 @@ class EthereumPythonDiscover(PythonDiscover):
     def __init__(self, config: PluginSettings) -> None:
         super().__init__(config)
         self._fork_order = _ForkOrder(config)
+        self._apply_fork_filter(config)
+
+    def _apply_fork_filter(self, config: PluginSettings) -> None:
+        """
+        Exclude fork packages not listed in `DOCC_ONLY_FORKS`.
+
+        The variable holds comma-separated fork short names (for example
+        `amsterdam,osaka`). When unset or empty, render every fork. When
+        no listed name matches a known fork, disable the filter so a bad
+        value cannot produce an empty (but successful) build.
+        """
+        keep = _only_forks()
+        if not keep:
+            return
+        forks = _find_forks(config)
+        known = {f.short_name.lower() for f in forks}
+        if not keep & known:
+            logging.warning(
+                "DOCC_ONLY_FORKS matches no known fork; rendering all"
+            )
+            return
+        dropped = [
+            config.unresolve_path(PurePath(f.path))
+            for f in forks
+            if f.path is not None and f.short_name.lower() not in keep
+        ]
+        logging.info(
+            "DOCC_ONLY_FORKS: rendering %d of %d fork package(s)",
+            len(forks) - len(dropped),
+            len(forks),
+        )
+        # `excluded_paths` is declared `Final` upstream; replace it here,
+        # before discovery runs, to narrow the rendered sources.
+        self.excluded_paths = [  # type: ignore[misc]
+            *self.excluded_paths,
+            *dropped,
+        ]
 
     @override
     def _python_source(
@@ -231,6 +281,75 @@ def _python_source(
         )
 
 
+class PruneReferencesTransform(Transform):
+    """
+    Drop references into fork packages excluded from the build.
+
+    When `DOCC_ONLY_FORKS` narrows discovery, links into excluded fork
+    packages have no definition. Replace each such reference with its
+    plain content so rendering succeeds without a link. References to
+    anything else are left alone, so genuinely broken identifiers still
+    fail the build.
+    """
+
+    def __init__(self, config: PluginSettings) -> None:
+        pass
+
+    def transform(self, context: Context) -> None:
+        """
+        Apply the transformation to the given document.
+        """
+        keep = _only_forks()
+        if not keep:
+            return
+        context[Document].root.visit(_PruneReferencesVisitor(context, keep))
+
+
+class _PruneReferencesVisitor(Visitor):
+    _context: Context
+    _keep: Set[str]
+    _stack: List[Node]
+
+    def __init__(self, context: Context, keep: Set[str]) -> None:
+        self._context = context
+        self._keep = keep
+        self._stack = []
+
+    def _prunable(self, node: Node) -> bool:
+        """
+        Check whether the node links into an excluded fork package.
+        """
+        if not isinstance(node, Reference):
+            return False
+        parts = node.identifier.split(".")
+        if parts[:2] != ["ethereum", "forks"] or len(parts) < 3:
+            return False
+        if parts[2].lower() in self._keep:
+            return False
+        try:
+            self._context[ReferenceIndex].lookup(node.identifier)
+        except DoccReferenceError:
+            return True
+        return False
+
+    @override
+    def enter(self, node: Node) -> Visit:
+        if self._stack:
+            replacement = node
+            while self._prunable(replacement):
+                assert isinstance(replacement, Reference)
+                replacement = replacement.child
+            if replacement is not node:
+                self._stack[-1].replace_child(node, replacement)
+                node = replacement
+        self._stack.append(node)
+        return Visit.TraverseChildren
+
+    @override
+    def exit(self, node: Node) -> None:
+        self._stack.pop()
+
+
 class EthereumDiscover(Discover):
     """
     Creates sources that represent the diff between two other sources, one per
@@ -252,6 +371,11 @@ def discover(self, known: FrozenSet[T]) -> Iterator[Source]:
             logging.info("Skipping diff discovery (DOCC_SKIP_DIFFS)")
             return
 
+        if _only_forks():
+            # Fork-subset builds have no complete fork pairs to diff.
+            logging.info("Skipping diff discovery (DOCC_ONLY_FORKS)")
+            return
+
         forks = {f.path: f for f in self.forks if f.path is not None}
 
         by_fork: Dict[Hardfork, Dict[PurePath, Source]] = defaultdict(dict)

src/ethereum_spec_tools/docc_shards.py

@@ -0,0 +1,87 @@
+"""
+Build the docc spec docs as parallel shards of consecutive forks.
+
+Fast PR-time validation only: the fork range is split into ``n``
+contiguous shards, each overlapping its predecessor by one fork so a
+fork's reference to the previous fork resolves within its shard. One
+``docc`` process renders each shard concurrently; the per-shard outputs
+are not merged.
+
+Forward references (a fork referencing a later fork) fall outside their
+shard and are pruned, so they are validated only by the serial
+default-branch ``docs-spec`` build that gates the docs deploy.
+"""
+
+import argparse
+import os
+import subprocess
+import sys
+from pathlib import Path
+from typing import List
+
+from .forks import Hardfork
+
+
+def compute_shards(forks: List[str], n: int) -> List[List[str]]:
+    """
+    Split ``forks`` into at most ``n`` contiguous shards.
+
+    Every shard after the first is prefixed with its predecessor fork, so
+    a fork's reference to the previous fork resolves within its shard.
+    """
+    per = (len(forks) + n - 1) // n
+    shards: List[List[str]] = []
+    for i in range(n):
+        start = i * per
+        if start >= len(forks):
+            break
+        lo = start - 1 if start > 0 else start
+        shards.append(forks[lo : min(start + per, len(forks))])
+    return shards
+
+
+def main() -> int:
+    """Discover forks, shard them, and render each shard with ``docc``."""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "-n",
+        "--shards",
+        type=int,
+        default=4,
+        help="number of parallel shards (default: 4)",
+    )
+    parser.add_argument(
+        "-o",
+        "--output-dir",
+        type=Path,
+        required=True,
+        help="parent directory for each shard's output",
+    )
+    args = parser.parse_args()
+    if args.shards < 1:
+        parser.error("--shards must be a positive integer")
+
+    forks = [fork.short_name for fork in Hardfork.discover()]
+    if not forks:
+        print("error: no forks discovered", file=sys.stderr)
+        return 1
+
+    processes: List[subprocess.Popen[bytes]] = []
+    for i, shard in enumerate(compute_shards(forks, args.shards)):
+        print(f"shard {i}: {','.join(shard)}", flush=True)
+        env = {
+            **os.environ,
+            "DOCC_SKIP_DIFFS": "1",
+            "DOCC_ONLY_FORKS": ",".join(shard),
+        }
+        output = args.output_dir / f"shard-{i}"
+        processes.append(
+            subprocess.Popen(["docc", "--output", str(output)], env=env)
+        )
+
+    exit_codes = [process.wait() for process in processes]
+    return 1 if any(exit_codes) else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/evm_tools/test_docc_shards.py

@@ -0,0 +1,50 @@
+"""
+Unit tests for the parallel spec-doc shard planner.
+"""
+
+from ethereum_spec_tools.docc_shards import compute_shards
+from ethereum_spec_tools.forks import Hardfork
+
+
+def test_shards_cover_every_fork() -> None:
+    """Every fork lands in at least one shard."""
+    forks = [f"f{i}" for i in range(24)]
+    shards = compute_shards(forks, 4)
+    covered = {fork for shard in shards for fork in shard}
+    assert covered == set(forks)
+
+
+def test_shards_are_contiguous_with_one_fork_overlap() -> None:
+    """Shards tile the range in order, overlapping one fork per seam."""
+    forks = [f"f{i}" for i in range(24)]
+    assert compute_shards(forks, 4) == [
+        forks[0:6],
+        forks[5:12],
+        forks[11:18],
+        forks[17:24],
+    ]
+
+
+def test_each_fork_shares_a_shard_with_its_predecessor() -> None:
+    """The overlap keeps every fork beside its immediate predecessor."""
+    forks = [f"f{i}" for i in range(24)]
+    shards = compute_shards(forks, 4)
+    for i in range(1, len(forks)):
+        assert any(
+            forks[i] in shard and forks[i - 1] in shard for shard in shards
+        )
+
+
+def test_more_shards_than_forks_still_covers_all() -> None:
+    """Requesting more shards than forks leaves no fork uncovered."""
+    shards = compute_shards(["a", "b", "c"], 4)
+    covered = {fork for shard in shards for fork in shard}
+    assert covered == {"a", "b", "c"}
+
+
+def test_real_fork_set_is_fully_covered() -> None:
+    """The discovered fork set shards without dropping any fork."""
+    forks = [fork.short_name for fork in Hardfork.discover()]
+    shards = compute_shards(forks, 4)
+    covered = {fork for shard in shards for fork in shard}
+    assert covered == set(forks)

vulture_whitelist.py

@@ -77,6 +77,8 @@
 docc.FixIndexTransform.transform
 docc.MinimizeDiffsTransform
 docc.MinimizeDiffsTransform.transform
+docc.PruneReferencesTransform
+docc.PruneReferencesTransform.transform
 docc._FixIndexVisitor.enter
 docc._DoccAdapter.shallow_equals
 docc._DoccAdapter.shallow_hash