diff --git a/.claude/README.md b/.claude/README.md new file mode 100644 index 000000000000..f9627a68316b --- /dev/null +++ b/.claude/README.md @@ -0,0 +1,44 @@ +# Claude Code Configuration + +This directory contains shared [Claude Code](https://code.claude.com/) configuration for the Lodestar project. See also [`AGENTS.md`](../AGENTS.md) at the repo root for project conventions and architecture guidance. + +## Structure + +``` +.claude/ +├── settings.json # Shared project settings (permissions allowlist) +├── settings.local.json # Personal settings (gitignored) +├── skills/ # Agent skills — Claude auto-discovers these +│ ├── kurtosis-devnet/ # Multi-client devnet testing +│ ├── release-notes/ # Release note drafting +│ └── local-mainnet-debug/ # Debug with real mainnet peers +└── README.md # This file +``` + +## Skills + +Skills are automatically loaded by Claude Code based on task context. You can also invoke them manually: + +| Skill | When to use | +| ----------------------- | -------------------------------------------------------------------------- | +| **kurtosis-devnet** | Spinning up local testnets, cross-client interop testing, fork transitions | +| **release-notes** | Drafting release notes for GitHub and Discord | +| **local-mainnet-debug** | Debugging networking/peer issues against real mainnet peers | + +## Shared Plugins + +For cross-project Ethereum development resources (consensus specs, client cross-reference), install from the shared marketplace: + +```bash +claude plugin marketplace add ChainSafe/lodestar-claude-plugins +claude plugin install ethereum-rnd +claude plugin install consensus-clients +``` + +## Personal Configuration + +Add personal settings to `.claude/settings.local.json` (gitignored). This is useful for: + +- Personal API keys or tool preferences +- Local MCP server configurations +- Additional permission rules for your workflow diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 000000000000..684ea9bb07df --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,67 @@ +{ + "$schema": "https://json.schemastore.org/claude-code-settings.json", + "permissions": { + "allow": [ + "Bash(pnpm build)", + "Bash(pnpm --filter * build)", + "Bash(pnpm lint)", + "Bash(pnpm lint:fix)", + "Bash(pnpm check-types)", + "Bash(pnpm --filter * check-types)", + "Bash(pnpm test:unit)", + "Bash(pnpm vitest *)", + "Bash(pnpm download-spec-tests)", + "Bash(pnpm test:spec)", + "Bash(pnpm docs:lint)", + "Bash(pnpm docs:lint:fix)", + "Bash(git status *)", + "Bash(git diff *)", + "Bash(git log *)", + "Bash(git branch *)", + "Bash(git show *)", + "Bash(git fetch *)", + "Bash(gh pr view *)", + "Bash(gh pr list *)", + "Bash(gh pr diff *)", + "Bash(gh pr checks *)", + "Bash(gh issue view *)", + "Bash(gh issue list *)", + "Bash(gh run view *)", + "Bash(gh run list *)", + "Bash(gh search *)", + "Bash(grep *)", + "Bash(cat *)", + "Bash(head *)", + "Bash(tail *)", + "Bash(wc *)", + "Bash(ls *)", + "Bash(curl -s http://localhost:*)" + ], + "deny": [ + "Bash(rm -rf / *)", + "Bash(sudo *)" + ] + }, + "enabledPlugins": { + "ethereum-rnd@lodestar-claude-plugins": true, + "consensus-clients@lodestar-claude-plugins": true, + "eth-rnd-archive@lodestar-claude-plugins": true, + "claude-md-management@claude-plugins-official": true, + "code-review@claude-plugins-official": true, + "code-simplifier@claude-plugins-official": true, + "commit-commands@claude-plugins-official": true, + "context7@claude-plugins-official": true, + "feature-dev@claude-plugins-official": true, + "pr-review-toolkit@claude-plugins-official": true, + "superpowers@claude-plugins-official": true, + "typescript-lsp@claude-plugins-official": true + }, + "extraKnownMarketplaces": { + "lodestar-claude-plugins": { + "source": { + "source": "github", + "repo": "ChainSafe/lodestar-claude-plugins" + } + } + } +} diff --git a/.claude/skills/kurtosis-devnet/SKILL.md b/.claude/skills/kurtosis-devnet/SKILL.md new file mode 100644 index 000000000000..b19c6ee82a46 --- /dev/null +++ b/.claude/skills/kurtosis-devnet/SKILL.md @@ -0,0 +1,245 @@ +--- +name: kurtosis-devnet +description: Run Ethereum multi-client devnets using Kurtosis and the ethpandaops/ethereum-package. Use for spinning up local testnets, validating cross-client interop, testing fork transitions, running assertoor checks, debugging CL/EL client interactions, or verifying new feature implementations across multiple consensus and execution clients. +--- + +# Kurtosis Devnet + +Run Ethereum consensus/execution client devnets via [Kurtosis](https://github.com/kurtosis-tech/kurtosis) + [ethereum-package](https://github.com/ethpandaops/ethereum-package). + +## Prerequisites + +- [Kurtosis CLI](https://docs.kurtosis.com/install/) installed +- Docker running with sufficient resources (8GB+ RAM recommended for multi-client devnets) + +## Quick Start + +```bash +# Start devnet from config +# Always use --image-download always to ensure external images are up to date +kurtosis run github.com/ethpandaops/ethereum-package \ + --enclave \ + --args-file network_params.yaml \ + --image-download always + +# List enclaves +kurtosis enclave ls + +# Inspect services +kurtosis enclave inspect + +# View logs +kurtosis service logs +kurtosis service logs --follow + +# Cleanup +kurtosis enclave rm -f +# Or clean all enclaves +kurtosis clean -a +``` + +## Config File (`network_params.yaml`) + +See `references/config-reference.md` for the full config structure. + +Key sections: + +- `participants`: list of CL+EL client pairs with images, flags, validator counts +- `network_params`: fork epochs, slot time, network-level settings +- `additional_services`: dora (explorer), assertoor (testing), prometheus, grafana +- `assertoor_params`: automated chain health checks +- `port_publisher`: expose CL/EL ports to host + +## Building Custom Client Images + +When testing local Lodestar branches, build a Docker image first: + +```bash +# Fast build (recommended for iteration) +cd ~/lodestar && docker build -t lodestar:custom -f Dockerfile.dev . + +# Production build (slower, for final validation only) +cd ~/lodestar && docker build -t lodestar:custom . + +# Then reference in config: +# cl_image: lodestar:custom +``` + +**Always use `Dockerfile.dev` for iterative development.** It caches dependency layers and rebuilds in seconds vs minutes for the production Dockerfile. Only use the production `Dockerfile` for final validation or debugging build issues. + +## Service Naming Convention + +Kurtosis names services as: `{role}-{index}-{cl_type}-{el_type}` + +Examples: + +- `cl-1-lodestar-reth` — first CL node (Lodestar with Reth EL) +- `el-1-reth-lodestar` — corresponding EL node +- `vc-1-lodestar-reth` — validator client + +## Accessing Services + +After `kurtosis enclave inspect `, find mapped ports: + +```bash +# CL beacon API (find actual port from inspect output) +curl http://127.0.0.1:/eth/v1/node/syncing + +# Or use port_publisher for predictable ports: +# port_publisher: +# cl: +# enabled: true +# public_port_start: 33000 # cl-1=33000, cl-2=33005, etc. +# el: +# enabled: true +# public_port_start: 32000 +``` + +Port publisher assigns sequential ports (step of 5 per service). + +## Assertoor (Automated Testing) + +Add to config: + +```yaml +additional_services: + - assertoor + +assertoor_params: + run_stability_check: true # chain stability, finality, no reorgs + run_block_proposal_check: true # every client pair proposes a block +``` + +Check results via the assertoor web UI (port shown in `kurtosis enclave inspect`). + +## Common Devnet Patterns + +### Fork Transition Testing + +```yaml +network_params: + electra_fork_epoch: 0 + fulu_fork_epoch: 1 # fork at epoch 1 (slot 32) + seconds_per_slot: 6 # faster for testing +``` + +### Mixed-Client Topology (Cross-Client Interop) + +```yaml +participants: + - cl_type: lodestar + el_type: reth + count: 2 + validator_count: 128 + - cl_type: lighthouse + el_type: geth + count: 2 + validator_count: 128 +``` + +### Observer Nodes (No Validators) + +```yaml +- cl_type: lodestar + cl_image: lodestar:custom + el_type: reth + count: 1 + validator_count: 0 # observer-only +``` + +### Supernode Mode + +Set `supernode: true` to run beacon+validator in a single process (faster startup, simpler topology): + +```yaml +- cl_type: lodestar + el_type: reth + supernode: true + validator_count: 128 +``` + +### Extra CL/VC Params + +```yaml +cl_extra_params: + - --targetPeers=8 + - --logLevel=debug +vc_extra_params: + - --suggestedFeeRecipient=0x... +``` + +## Monitoring & Debugging + +```bash +# Stream logs from a specific service +kurtosis service logs cl-1-lodestar-reth --follow + +# Save all CL logs for analysis +for svc in $(kurtosis enclave inspect 2>/dev/null | grep -oE 'cl-[0-9]+-[^[:space:]]+'); do + kurtosis service logs $svc > "/tmp/${svc}.log" 2>&1 +done + +# Dora explorer (if enabled) — find port via inspect output + +# Check chain finality +curl -s http://127.0.0.1:/eth/v1/beacon/states/head/finality_checkpoints | jq + +# Check peer count +curl -s http://127.0.0.1:/eth/v1/node/peers | jq '.data | length' + +# Check sync status +curl -s http://127.0.0.1:/eth/v1/node/syncing | jq +``` + +### Wait for Finality + +Finality typically takes 2-3 epochs after genesis. With `seconds_per_slot: 6` and 32 slots/epoch: + +- 1 epoch ≈ 192s (3.2 min) +- First finalization ≈ epoch 3-4 boundary (≈10-13 min) + +Monitor: + +```bash +curl -s http:///eth/v1/beacon/states/head/finality_checkpoints | jq '.data.finalized.epoch' +``` + +### Acceptance Criteria Pattern + +For interop validation, define acceptance criteria **before** running the soak test: + +```bash +# Example: check for zero errors across all CL nodes +for svc in cl-1-lodestar-reth cl-2-lodestar-reth; do + echo "=== $svc ===" + kurtosis service logs $svc 2>&1 | grep -c "ERROR" || echo "0 errors" +done +``` + +## Troubleshooting + +| Issue | Fix | +| ------------------------ | ------------------------------------------------------------------------- | +| Peers not connecting | Increase `--targetPeers`, check `directPeers` config | +| No finality | Need ≥2/3 validators attesting; check VC logs | +| "discv5 has no boot enr" | Harmless startup warning, ignore | +| Port conflicts | Change `public_port_start` or stop conflicting enclaves | +| Image not found | Ensure Docker image is built locally or available in registry | +| Slow startup | Use `Dockerfile.dev` for local builds; reduce validator count | +| Docker bind error | Use non-default port ranges; `kurtosis clean -a` between runs | +| Stale enclave state | Always `kurtosis clean -a` before relaunching (not `docker system prune`) | + +## Tips + +- **Start small:** 2+2 nodes for iteration, scale up for final validation +- **Test both roles:** Validator nodes show different bugs than observers — always check both +- **Multiple soak passes:** First clean run may miss edge cases. Run extended soaks (hours). +- **`kurtosis clean -a`** between runs — never use `docker system prune` (it can remove Kurtosis engine images) +- **Debug logs:** Set `global_log_level: "debug"` for verbose output from all nodes +- **Per-participant log levels:** Use `cl_log_level` / `el_log_level` on individual entries for targeted debugging + +--- + +## Self-Maintenance + +If any commands, file paths, URLs, or configurations in this skill are outdated or no longer work, update this SKILL.md with the correct information after completing your current task. Skills should stay accurate and self-healing — fix what you find broken. diff --git a/.claude/skills/kurtosis-devnet/references/config-reference.md b/.claude/skills/kurtosis-devnet/references/config-reference.md new file mode 100644 index 000000000000..ea85f8e6936a --- /dev/null +++ b/.claude/skills/kurtosis-devnet/references/config-reference.md @@ -0,0 +1,198 @@ +# Kurtosis ethereum-package Config Reference + +Full documentation: https://github.com/ethpandaops/ethereum-package + +## Participant Fields + +```yaml +participants: + - el_type: reth|geth|nethermind|besu|erigon + el_image: # e.g., ghcr.io/paradigmxyz/reth:latest + el_extra_params: [] # extra CLI flags for EL + el_log_level: "" # per-participant EL log level + cl_type: lodestar|lighthouse|prysm|teku|nimbus|grandine + cl_image: # e.g., lodestar:custom or chainsafe/lodestar:latest + cl_extra_params: [] # extra CLI flags for CL beacon + cl_log_level: "" # per-participant CL log level + vc_type: # defaults to cl_type + vc_image: # custom VC image (if different from CL) + vc_extra_params: [] # extra CLI flags for VC + supernode: false # combine beacon+vc in single process + count: 1 # number of instances with this config + validator_count: 64 # validators assigned to this participant +``` + +## Network Parameters + +```yaml +network_params: + # Fork epochs (0 = from genesis, null = disabled) + electra_fork_epoch: 0 + fulu_fork_epoch: 1 + + # Timing + seconds_per_slot: 12 # default 12, use 6 for faster devnets + slots_per_epoch: 32 # default 32 + + # Network + network_id: "3151908" + deposit_contract_address: "0x..." + + # Validator + num_validator_keys_per_node: 64 + preregistered_validator_keys_mnemonic: "..." +``` + +## Additional Services + +```yaml +additional_services: + - dora # chain explorer UI + - assertoor # automated testing + - prometheus # metrics collection + - grafana # metrics dashboards + - blob_spammer # blob transaction generator + - el_forkmon # fork monitor + - beacon_metrics_gazer + - blockscout # block explorer with contract verification +``` + +## Assertoor Parameters + +```yaml +assertoor_params: + run_stability_check: true # chain health, finality, reorgs + run_block_proposal_check: true # every client pair proposes a block + run_transaction_test: false # transaction lifecycle + run_blob_transaction_test: false # blob tx test + run_opcodes_transaction_test: false + tests: [] # custom test configs +``` + +## Port Publisher + +```yaml +port_publisher: + el: + enabled: true + public_port_start: 32000 # EL HTTP/WS + cl: + enabled: true + public_port_start: 33000 # CL beacon API + vc: + enabled: false + public_port_start: 34000 +``` + +Ports increment by 5 per service instance: + +- cl-1: 33000, cl-2: 33005, cl-3: 33010, etc. +- el-1: 32000, el-2: 32005, el-3: 32010, etc. + +## Global Settings + +```yaml +global_log_level: info|debug|warn|error # default: info +``` + +## Example Configs + +### Minimal 2-Node Devnet + +```yaml +participants: + - el_type: geth + cl_type: lodestar + count: 1 + validator_count: 128 + - el_type: reth + cl_type: lighthouse + count: 1 + validator_count: 128 + +network_params: + electra_fork_epoch: 0 + fulu_fork_epoch: 1 + seconds_per_slot: 6 + +additional_services: + - dora + - assertoor + +assertoor_params: + run_stability_check: true + run_block_proposal_check: true +``` + +### Lodestar-Only with Custom Image + +```yaml +participants: + - el_type: reth + cl_type: lodestar + cl_image: lodestar:custom + supernode: true + count: 4 + validator_count: 64 + +network_params: + electra_fork_epoch: 0 + fulu_fork_epoch: 0 + seconds_per_slot: 6 + +additional_services: + - dora + - prometheus + - grafana + +port_publisher: + cl: + enabled: true + public_port_start: 33000 +``` + +### Full Multi-Client Interop (6 clients) + +```yaml +participants: + - cl_type: lodestar + el_type: reth + count: 1 + validator_count: 64 + - cl_type: lighthouse + el_type: geth + count: 1 + validator_count: 64 + - cl_type: prysm + el_type: geth + count: 1 + validator_count: 64 + - cl_type: teku + el_type: besu + count: 1 + validator_count: 64 + - cl_type: nimbus + el_type: nethermind + count: 1 + validator_count: 64 + - cl_type: grandine + el_type: reth + count: 1 + validator_count: 64 + +network_params: + electra_fork_epoch: 0 + fulu_fork_epoch: 1 + seconds_per_slot: 6 + +additional_services: + - dora + - assertoor + - prometheus + - grafana + - el_forkmon + +assertoor_params: + run_stability_check: true + run_block_proposal_check: true +``` diff --git a/.claude/skills/local-mainnet-debug/SKILL.md b/.claude/skills/local-mainnet-debug/SKILL.md new file mode 100644 index 000000000000..9b9202923f9a --- /dev/null +++ b/.claude/skills/local-mainnet-debug/SKILL.md @@ -0,0 +1,220 @@ +--- +name: local-mainnet-debug +description: Debug Lodestar beacon node issues by running a local mainnet node with checkpoint sync and engineMock. Use for investigating networking bugs, peer discovery issues, identify failures, metrics anomalies, or any behavior that needs real-world peer interactions without a full execution client. +--- + +# Local Mainnet Debugging + +Run a local Lodestar beacon node against mainnet peers using checkpoint sync and engineMock to reproduce and debug networking, peer discovery, and protocol-level issues in a real-world environment. + +## Quick Start + +```bash +cd ~/lodestar # or worktree directory + +# Basic run — connects to mainnet peers, no EL needed +./lodestar beacon \ + --network mainnet \ + --dataDir /tmp/lodestar-debug \ + --rest false \ + --metrics \ + --execution.engineMock \ + --port 19771 \ + --logLevel debug \ + --checkpointSyncUrl https://beaconstate-mainnet.chainsafe.io \ + --forceCheckpointSync + +# Time-boxed run (e.g., 2 minutes) +timeout 120 ./lodestar beacon \ + --network mainnet \ + --dataDir /tmp/lodestar-debug \ + --rest false \ + --metrics \ + --execution.engineMock \ + --port 19771 \ + --logLevel debug \ + --checkpointSyncUrl https://beaconstate-mainnet.chainsafe.io \ + --forceCheckpointSync +``` + +## Key Parameters + +| Parameter | Purpose | Notes | +| ------------------------ | --------------------------------------- | ------------------------------------------------------ | +| `--network mainnet` | Connect to real mainnet peers | Use `holesky` for testnet | +| `--execution.engineMock` | Skip EL requirement | Node won't validate execution payloads | +| `--rest false` | Disable REST API | Reduces noise, avoids port conflicts | +| `--metrics` | Enable Prometheus metrics | Scrape at `http://localhost:8008/metrics` | +| `--port 19771` | Custom P2P port | Avoid conflicts with other instances | +| `--logLevel debug` | Verbose logging | Use `trace` for maximum detail | +| `--checkpointSyncUrl` | Checkpoint sync endpoint | `https://beaconstate-mainnet.chainsafe.io` for mainnet | +| `--forceCheckpointSync` | Force checkpoint sync even if DB exists | Clean start each run | + +## Checkpoint Sync Endpoints + +```bash +# Mainnet +--checkpointSyncUrl https://beaconstate-mainnet.chainsafe.io +--checkpointSyncUrl https://mainnet-checkpoint-sync.stakely.io +--checkpointSyncUrl https://sync-mainnet.beaconcha.in + +# Holesky +--checkpointSyncUrl https://beaconstate-holesky.chainsafe.io +--checkpointSyncUrl https://holesky.beaconstate.ethstaker.cc +``` + +## Metrics Scraping + +```bash +# One-shot metric grab +curl -s http://localhost:8008/metrics | grep + +# Periodic sampling (every 30s) +while true; do + echo "=== $(date -u +%H:%M:%S) ===" + curl -s http://localhost:8008/metrics | grep -E 'lodestar_peers_by_client|peer_count' + sleep 30 +done + +# Key metrics for peer debugging +curl -s http://localhost:8008/metrics | grep -E \ + 'lodestar_peers_by_client|libp2p_identify|peer_count|connected_peers' +``` + +## Debugging Techniques + +### 1. A/B Testing with Code Changes + +The most powerful technique — test a hypothesis by comparing control vs modified runs. + +```bash +# Control: capture baseline (2 min) +timeout 120 ./lodestar beacon [flags] 2>&1 | tee /tmp/control.log & +# Sample metrics during run +for i in 1 2 3 4; do sleep 30; curl -s http://localhost:8008/metrics > /tmp/control-$i.metrics; done + +# Test: apply change, repeat +timeout 120 ./lodestar beacon [flags] 2>&1 | tee /tmp/test.log & +for i in 1 2 3 4; do sleep 30; curl -s http://localhost:8008/metrics > /tmp/test-$i.metrics; done + +# Compare +diff <(grep pattern /tmp/control-4.metrics) <(grep pattern /tmp/test-4.metrics) +``` + +### 2. Instrument libp2p Internals (Monkeypatching) + +For deep protocol debugging, add temporary instrumentation to `node_modules`: + +```bash +# Find the file to patch +find node_modules -path '*libp2p*identify*' -name '*.js' | head -20 + +# Key files: +# - node_modules/@libp2p/identify/dist/src/identify.js +# - node_modules/@chainsafe/libp2p-yamux/dist/src/stream.js (yamux streams) +# - node_modules/@libp2p/mplex/dist/src/mplex.js (mplex streams) +``` + +**Important:** Always remove monkeypatches before committing. Use `pnpm install` to restore. + +### 3. Log Analysis + +```bash +# Count specific errors +grep -c "Error setting agentVersion" /tmp/run.log + +# Track identify success/failure over time +grep -E "identify (success|error|timeout)" /tmp/run.log | head -50 + +# Extract peer connection events +grep -E "peer:(connect|disconnect|identify)" /tmp/run.log +``` + +### 4. Stream-Level Debugging + +For protocol stream issues (identify, ping, metadata): + +``` +Trace stream lifecycle: +1. Stream opened (protocol, direction, connection ID) +2. MSS negotiation (success/failure) +3. Data read/write (first frame timing) +4. Stream close (who closed, when) +``` + +Key locations for instrumentation: + +- `@libp2p/identify`: identify.js → `_identify()` method +- Stream open/close: yamux or mplex stream.js +- Protocol negotiation: `@libp2p/multistream-select` + +## Common Issues & Diagnostic Approaches + +### Unknown Peers (identify failures) + +**Symptoms:** High ratio of "Unknown" in `lodestar_peers_by_client` metric. + +**Diagnostic approach:** + +1. Check `lodestar_peers_by_client` for Unknown ratio +2. Enable debug logs, grep for "Error setting agentVersion" +3. Instrument identify stream to check stream state before `pb.read()` +4. A/B test: disable suspect components → if Unknown drops to 0, found the cause + +### Slow Peer Discovery + +**Symptoms:** Low peer count after startup. + +**Diagnostic approach:** + +1. Check `libp2p_dialer_pending_dial_count` and `libp2p_connections` +2. Verify port is accessible: `lsof -i :` +3. Check discv5 logs for bootstrap ENR issues +4. Monitor `lodestar_discovery_find_node_query_*` metrics + +### High Disconnect Rate + +**Symptoms:** Peers connecting and quickly disconnecting. + +**Diagnostic approach:** + +1. Check `libp2p_connections` vs `libp2p_protocol_streams` +2. Grep logs for disconnect reasons +3. Check if peers are being pruned by score: `lodestar_gossip_peer_score_*` +4. Monitor `lodestar_peers_by_direction` for inbound/outbound balance + +## Port Conflicts + +```bash +# Check if port is in use +lsof -i :19771 +lsof -i :8008 + +# Use different ports +--port 19772 --metrics.port 8009 +``` + +## Cleanup + +Always use `--dataDir /tmp/lodestar-debug` when running debug sessions to avoid touching the default mainnet database. This isolates debug data from any production node on the same machine. + +```bash +# Remove isolated debug data directory after testing +rm -r /tmp/lodestar-debug +``` + +## Tips + +- **Short runs are fine.** 60-120 seconds is enough to connect to 15-30 peers and observe identify behavior. +- **engineMock means no execution validation.** The node will sync headers but won't validate blocks. Fine for networking/peer debugging. +- **Custom port avoids conflicts** with any production nodes on the same machine. +- **Always use `--forceCheckpointSync`** to ensure a clean start. Stale DB state can mask issues. +- **Metrics lag behind logs.** After stopping the node, the last metrics scrape reflects final state. Periodic sampling during the run gives time-series data. +- **Debug builds not needed.** Standard `pnpm build` output is sufficient; libp2p debug info comes from log level and instrumentation. +- **Use `--network holesky`** when you need a smaller, faster-syncing network for testing. + +--- + +## Self-Maintenance + +If any commands, file paths, URLs, or configurations in this skill are outdated or no longer work, update this SKILL.md with the correct information after completing your current task. Skills should stay accurate and self-healing — fix what you find broken. diff --git a/.claude/skills/release-notes/SKILL.md b/.claude/skills/release-notes/SKILL.md new file mode 100644 index 000000000000..52b4c6adf524 --- /dev/null +++ b/.claude/skills/release-notes/SKILL.md @@ -0,0 +1,74 @@ +--- +name: release-notes +description: > + Write Lodestar release notes and Discord announcements for new versions. + Use when cutting a new release, drafting announcement text, or updating + the GitHub release body. Covers header text, Discord announcement, + and changelog formatting. Requires access to the Lodestar repo and + GitHub CLI for changelog generation. +disable-model-invocation: true +--- + +# Release Notes + +Write release notes for Lodestar versions by analyzing the changelog, identifying +operator-facing changes, and producing both GitHub release header text and a Discord +announcement. + +## Quick Start + +1. Read `references/style-guide.md` for tone, structure, and past examples +2. Read `references/changelog-analysis.md` for how to categorize and prioritize changes +3. Generate the changelog diff between the previous stable and new version +4. Draft the GitHub release header and Discord announcement +5. Output both as a single markdown file for review + +## Workflow + +### 1. Generate Changelog + +```bash +# Get commits between last stable and new version +git log .. --oneline --no-merges + +# Or use the auto-generated changelog from the RC +gh release view --json body -q '.body' +``` + +### 2. Categorize Changes + +Read `references/changelog-analysis.md` for the prioritization framework. +Focus on what matters to **node operators**, not internal refactors. + +### 3. Write Header Text + +The header text goes at the top of the GitHub release body, before the `# Changelog` section. +Follow the structure in `references/style-guide.md`. + +### 4. Write Discord Announcement + +Shorter version of the header text for the `#lodestar-announcements` channel. +See `references/style-guide.md` for Discord-specific formatting rules. + +### 5. Output + +Produce a single markdown file with two sections: + +- `## GitHub Release Header` — full header text +- `## Discord Announcement` — shorter announcement + +## Key Rules + +- **Operator-first**: Every highlight should answer "why should I upgrade?" +- **Breaking changes up front**: Node.js version drops, flag changes, migration steps +- **PR numbers**: Always link PRs as `(#XXXX)` +- **Recommendation level**: Always state mandatory / recommended / optional +- **Don't over-detail**: The full changelog is linked below the header +- **3-6 highlights**: Enough to convey the release but not overwhelming +- **Group related PRs**: Combine related changes into single highlights + +--- + +## Self-Maintenance + +If any commands, file paths, URLs, or configurations in this skill are outdated or no longer work, update this SKILL.md with the correct information after completing your current task. Skills should stay accurate and self-healing — fix what you find broken. diff --git a/.claude/skills/release-notes/references/changelog-analysis.md b/.claude/skills/release-notes/references/changelog-analysis.md new file mode 100644 index 000000000000..46e4d9137377 --- /dev/null +++ b/.claude/skills/release-notes/references/changelog-analysis.md @@ -0,0 +1,79 @@ +# Changelog Analysis Framework + +## How to Prioritize Changes + +When analyzing a changelog for the release header, categorize each commit: + +### Tier 1: Must Mention (Operator-Facing Impact) + +- **Breaking changes**: Dropped runtime versions, removed flags, changed defaults +- **Security fixes**: Vulnerability patches, dependency CVEs +- **Critical bug fixes**: Crash fixes, data corruption, consensus failures +- **New required flags**: Config changes operators need to make +- **Migration steps**: Things operators must do when upgrading + +### Tier 2: Should Mention (Significant Improvements) + +- **Performance improvements**: Memory reduction, faster processing, lower CPU +- **New features operators care about**: APIs, CLI flags, monitoring endpoints +- **Stability fixes**: OOM prevention, crash-loop fixes, sync improvements +- **PeerDAS/networking changes**: Custody, peering, gossip improvements + +### Tier 3: Brief Mention or Skip (Internal) + +- **Refactors**: Code cleanup, type changes (skip unless perf impact) +- **Test improvements**: CI, E2E, flaky test fixes (skip) +- **Documentation**: Only mention if user-facing docs changed +- **Dependency bumps**: Only mention if security-related +- **Maintenance**: Build system, workflows (skip) + +## Grouping Related Changes + +Combine related PRs into single highlights: + +### Example: Memory improvements + +Instead of listing 3 separate PRs: + +- Remove in-memory state caches (#8813) +- Replace bigint-buffer (#8789) +- Signature sets use indices (#8803) + +Write one highlight: + +> **Reduced memory usage** — In-memory state caches removed, bigint-buffer replaced with a +> lighter alternative, and signature verification now uses validator indices. (#8813, #8789, #8803) + +### Example: PeerDAS improvements + +Instead of listing separately: + +- Block import after half columns (#8818) +- Backpressure fix (#8885) + +Write: + +> **PeerDAS supernode stability** — Blocks import after receiving half the columns, and a new +> write queue backpressure mechanism prevents OOM during finalized sync. (#8818, #8885) + +## Generating the Changelog Section + +The full changelog below the header is auto-generated. Use the RC release body as a starting +point — it's generated by the release workflow from conventional commits. + +Ensure the `Full Changelog` comparison link is correct: + +``` +[Full Changelog](https://github.com/ChainSafe/lodestar/compare/...) +``` + +## Checklist Before Finalizing + +- [ ] Recommendation level stated (mandatory/recommended/optional) +- [ ] Breaking changes called out prominently +- [ ] 3-6 highlights covering the most impactful changes +- [ ] PR numbers referenced for each highlight +- [ ] Migration steps included if any (with code blocks) +- [ ] Link to full changelog at the bottom +- [ ] Discord version fits under 2000 chars +- [ ] No internal/CI/test changes in highlights diff --git a/.claude/skills/release-notes/references/style-guide.md b/.claude/skills/release-notes/references/style-guide.md new file mode 100644 index 000000000000..ee9d1ff1819f --- /dev/null +++ b/.claude/skills/release-notes/references/style-guide.md @@ -0,0 +1,82 @@ +# Release Notes Style Guide + +## Tone & Voice + +- Friendly, professional, operator-focused +- Open with a casual greeting: "Good day Lodestar operators!", "Hey Lodestar users!", "Happy [month]!" +- State version and recommendation level immediately: "We've just released vX.Y.Z and **recommend** users update" +- Recommendation levels: **mandatory** (breaking/security), **recommended** (features/perf), maintenance (minor) +- Close with a link to full changelog and optionally a seasonal/fun sign-off + +## GitHub Release Header Structure + +```markdown +[Greeting]. We've just released vX.Y.Z, a **[level]** upgrade for all [mainnet and testnet] users. +[1-2 sentence summary of the theme of this release.] + +[Optional: Breaking change / migration callout in bold] + +### Highlights + +- **[Feature/Fix name]** — 1-2 sentence description of what changed and why operators care. ([#XXXX]) +- **[Feature/Fix name]** — ... ([#XXXX]) +- [3-6 highlights total, ordered by operator impact] + +[Optional: Additional context paragraphs for complex changes like migrations] + +For the full changelog, please see: https://github.com/ChainSafe/lodestar/releases/vX.Y.Z +``` + +### Formatting Rules + +- Use markdown headers, bold, bullet lists +- PR references: `(#8813)` or `(#8813, #8789)` for related PRs +- Keep highlights to 3-6 items — enough to convey the release but not overwhelming +- Each highlight should be one concept (combine related PRs into one bullet) +- Don't list every PR — that's what the changelog is for + +## Discord Announcement Structure + +``` +[Greeting]. We've just released vX.Y.Z and **[level]** users update [reason]. + +[Breaking change if any] + +Key highlights: +[emoji] [highlight 1] +[emoji] [highlight 2] +[emoji] [highlight 3] + +For the full changelog: +``` + +### Discord-Specific Rules + +- No markdown tables — use bullet lists +- Wrap links in `<>` to suppress embeds (except the main release link) +- Use emoji bullets for visual scanning: 🧠 ⚡ 🔧 🌐 📡 🛡️ 🎯 +- Keep under 2000 chars (Discord message limit) +- No headers — use **bold** or emoji for emphasis + +## Past Examples (Distilled Patterns) + +### Major Feature Release (e.g., mainnet fork activation) + +- "**Mandatory** for all node operators" +- Detailed config changes (new flags, default values) +- Compatibility notes for validators +- Signed off with enthusiasm + emoji + +### Performance Release + +- "**Highly recommended** if you've seen performance declines" +- Runtime changes highlighted (e.g., Node.js version bump) +- Build-from-source migration note +- Holiday sign-off + +### Maintenance Release + +- "**Recommend** users update for latest features and best performance" +- Package manager migration with commands +- Breaking: deprecated feature removal +- Affected-user-specific note (e.g., DVT operators) diff --git a/.github/actions/core-dump/action.yml b/.github/actions/core-dump/action.yml index e2e0b1224912..910747888ab0 100644 --- a/.github/actions/core-dump/action.yml +++ b/.github/actions/core-dump/action.yml @@ -10,7 +10,7 @@ runs: shell: sh - name: Backup core dump - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: core-dump path: /cores/* diff --git a/.github/actions/setup-and-build/action.yml b/.github/actions/setup-and-build/action.yml index d19a8154bc2f..8071ca6b9f4b 100644 --- a/.github/actions/setup-and-build/action.yml +++ b/.github/actions/setup-and-build/action.yml @@ -8,10 +8,10 @@ inputs: runs: using: "composite" steps: - - uses: pnpm/action-setup@v4 + - uses: pnpm/action-setup@c5ba7f7862a0f64c1b1a05fbac13e0b8e86ba08c # v4 - name: Setup Node - uses: actions/setup-node@v6 + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version: ${{inputs.node}} check-latest: true @@ -32,7 +32,7 @@ runs: run: echo "key=build-cache-${{ runner.os }}-${{ runner.arch }}-node-${{ inputs.node }}-${{ github.sha }}" >> $GITHUB_OUTPUT - name: Restore build - uses: actions/cache/restore@v4 + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 id: cache-build-restore with: path: | @@ -58,7 +58,7 @@ runs: run: pnpm check-bundle - name: Cache build artifacts - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: | lib/ diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000000..05c25f68de31 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + groups: + actions: + patterns: + - "*" diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 6781922b5304..d01008ebdce5 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -30,7 +30,7 @@ jobs: steps: # - Uses YAML anchors in the future - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: "./.github/actions/setup-and-build" with: node: 24 @@ -38,7 +38,7 @@ jobs: # Restore performance downloaded states - name: Restore performance state cache - uses: actions/cache@v4 + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 with: path: packages/state-transition/test-cache key: perf-states-${{ hashFiles('packages/state-transition/test/perf/params.ts') }} diff --git a/.github/workflows/binaries.yml b/.github/workflows/binaries.yml index 2bbab6ccdc50..481a4523e107 100644 --- a/.github/workflows/binaries.yml +++ b/.github/workflows/binaries.yml @@ -27,7 +27,7 @@ jobs: arch: arm64 runs-on: ${{matrix.os}} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Install arm64 specifics if: matrix.arch == 'arm64' run: |- @@ -45,13 +45,13 @@ jobs: npx caxa -m "Unpacking Lodestar binary, please wait..." -e "dashboards/**" -e "docs/**" -D -p "pnpm clean:nm && pnpm install --frozen-lockfile --prod" --input . --output "lodestar" -- "{{caxa}}/node_modules/.bin/node" "--max-old-space-size=8192" "{{caxa}}/packages/cli/bin/lodestar.js" tar -czf "dist/lodestar-${{ inputs.version }}-${{ matrix.platform }}-${{ matrix.arch }}.tar.gz" "lodestar" - name: Upload binaries - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: binaries-${{ matrix.os }} path: dist/ if-no-files-found: error - name: Sanity check binary - uses: actions/github-script@v7 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 with: script: | exec.exec('./lodestar dev'); diff --git a/.github/workflows/build-debug-node.yml b/.github/workflows/build-debug-node.yml index f9c4c5802a2e..a450d501f60a 100644 --- a/.github/workflows/build-debug-node.yml +++ b/.github/workflows/build-debug-node.yml @@ -18,7 +18,7 @@ jobs: run: apt-get install python3 g++ make python3-pip - name: Download Node.js source - uses: actions/checkout@v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: repository: "nodejs/node" ref: "v${{ github.event.inputs.version }}" @@ -44,7 +44,7 @@ jobs: working-directory: "nodejs" - name: Upload build to artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: nodejs-debug-build-${{ github.event.inputs.version }} path: nodejs-debug-build-${{ github.event.inputs.version }} diff --git a/.github/workflows/check-specrefs.yml b/.github/workflows/check-specrefs.yml index b7dc415253e8..a0576bbd87e6 100644 --- a/.github/workflows/check-specrefs.yml +++ b/.github/workflows/check-specrefs.yml @@ -16,15 +16,15 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Check version consistency run: | - SPEC_TEST_VERSION=$(grep 'specVersion:' packages/beacon-node/test/spec/specTestVersioning.ts | head -1 | sed 's/.*specVersion: "\(.*\)".*/\1/') + SPEC_TEST_VERSION=$(grep '"specVersion":' spec-tests-version.json | head -1 | sed 's/.*"specVersion": "\(.*\)".*/\1/') ETHSPECIFY_VERSION=$(grep '^version:' specrefs/.ethspecify.yml | sed 's/version: //') if [ "$SPEC_TEST_VERSION" != "$ETHSPECIFY_VERSION" ]; then - echo "Version mismatch between specTestVersioning.ts and ethspecify" - echo " packages/beacon-node/test/spec/specTestVersioning.ts: $SPEC_TEST_VERSION" + echo "Version mismatch between spec-teests-version.json and ethspecify" + echo " spec-tests-version.json: $SPEC_TEST_VERSION" echo " specrefs/.ethspecify.yml: $ETHSPECIFY_VERSION" exit 1 else @@ -32,7 +32,7 @@ jobs: fi - name: Install ethspecify - run: python3 -mpip install ethspecify + run: python3 -mpip install ethspecify==0.3.9 - name: Update spec references run: ethspecify process --path=specrefs diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 2d635afe2688..b1603921d1b1 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -47,11 +47,11 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@b1bff81932f5cdfc8695c7752dcee935dcd061c8 # v4.33.0 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -64,7 +64,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@b1bff81932f5cdfc8695c7752dcee935dcd061c8 # v4.33.0 # ℹ️ Command-line programs to run using the OS shell. # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun @@ -77,4 +77,4 @@ jobs: # ./location_of_script_within_repo/buildscript.sh - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@b1bff81932f5cdfc8695c7752dcee935dcd061c8 # v4.33.0 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d8056e7f03e5..fb6861e2c218 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -29,13 +29,13 @@ jobs: - arch: arm64 runner: buildjet-4vcpu-ubuntu-2204-arm steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ inputs.ref || github.sha }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Login to Docker Hub - uses: docker/login-action@v3 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -74,9 +74,9 @@ jobs: needs: docker steps: - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Login to Docker Hub - uses: docker/login-action@v3 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.github/workflows/docs-backfill.yml b/.github/workflows/docs-backfill.yml index a68f8e99507e..19c69af10a65 100644 --- a/.github/workflows/docs-backfill.yml +++ b/.github/workflows/docs-backfill.yml @@ -22,7 +22,7 @@ jobs: name: Backfill versioned docs runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: unstable fetch-depth: 0 @@ -73,9 +73,9 @@ jobs: - name: Install pnpm if: steps.versions.outputs.missing != '' && github.event.inputs.dry_run != 'true' - uses: pnpm/action-setup@v4 + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4 - - uses: actions/setup-node@v6 + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 if: steps.versions.outputs.missing != '' && github.event.inputs.dry_run != 'true' with: node-version: 24 diff --git a/.github/workflows/docs-check.yml b/.github/workflows/docs-check.yml index 7969c198615d..105faa1744d7 100644 --- a/.github/workflows/docs-check.yml +++ b/.github/workflows/docs-check.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: # - Uses YAML anchors in the future - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: "./.github/actions/setup-and-build" with: node: 24 @@ -29,4 +29,4 @@ jobs: # Run spellcheck AFTER building docs, in case the CLI reference has issues - name: Spellcheck - uses: rojopolis/spellcheck-github-actions@0.32.0 + uses: rojopolis/spellcheck-github-actions@e3cd8e9aec4587ec73bc0e60745aafd45c37aa2e # 0.60.0 diff --git a/.github/workflows/docs-version.yml b/.github/workflows/docs-version.yml index 3535fd32975b..2738e73c27a1 100644 --- a/.github/workflows/docs-version.yml +++ b/.github/workflows/docs-version.yml @@ -33,14 +33,14 @@ jobs: fi # Checkout the tagged release to build its docs - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ github.ref }} - name: Install pnpm - uses: pnpm/action-setup@v4 + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4 - - uses: actions/setup-node@v6 + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version: 24 check-latest: true diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 4171b2ee116d..e23ca9be580f 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -30,14 +30,14 @@ jobs: - name: Log Deployment Ref run: echo "Deploying docs from ref $DEPLOY_REF" - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ env.DEPLOY_REF }} - name: Install pnpm before setup node - uses: pnpm/action-setup@v4 + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4 - - uses: actions/setup-node@v6 + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version: 24 check-latest: true @@ -48,7 +48,7 @@ jobs: run: echo "v8CppApiVersion=$(node --print "process.versions.modules")" >> $GITHUB_OUTPUT - name: Restore dependencies - uses: actions/cache@v4 + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 id: cache-deps with: path: | @@ -91,7 +91,7 @@ jobs: run: pnpm install && pnpm build - name: Deploy - uses: peaceiris/actions-gh-pages@v3 + uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4.0.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: ./docs/build diff --git a/.github/workflows/kurtosis.yml b/.github/workflows/kurtosis.yml index bf1ad642a76f..673d0a36b31a 100644 --- a/.github/workflows/kurtosis.yml +++ b/.github/workflows/kurtosis.yml @@ -11,17 +11,17 @@ jobs: name: Build and run Kurtosis runs-on: buildjet-4vcpu-ubuntu-2204 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Build Docker image - local only run: > docker buildx build . --load \ --tag chainsafe/lodestar:kurtosis-ci \ --build-arg COMMIT=$(git rev-parse HEAD) - name: Run test - uses: ethpandaops/kurtosis-assertoor-github-action@v1 + uses: ethpandaops/kurtosis-assertoor-github-action@5932604b244dbd2ddb811516b516a9094f4d2c2f # v1 with: ethereum_package_args: ".github/workflows/assets/kurtosis_sim_test_config.yaml" diff --git a/.github/workflows/lint-pr-title.yml b/.github/workflows/lint-pr-title.yml index 873343c8d799..4157ee34d802 100644 --- a/.github/workflows/lint-pr-title.yml +++ b/.github/workflows/lint-pr-title.yml @@ -14,7 +14,7 @@ jobs: name: Validate PR title runs-on: ubuntu-latest steps: - - uses: amannn/action-semantic-pull-request@v5 + - uses: amannn/action-semantic-pull-request@48f256284bd46cdaab1048c3721360e808335d50 # v6.1.1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: diff --git a/.github/workflows/native-portability.yml b/.github/workflows/native-portability.yml new file mode 100644 index 000000000000..0cfb51047e07 --- /dev/null +++ b/.github/workflows/native-portability.yml @@ -0,0 +1,32 @@ +name: Native portability +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +on: + push: + branches: [unstable, stable] + pull_request: + workflow_dispatch: + +jobs: + check-native-portability: + name: Check native module portability + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4 + + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + with: + node-version: 24 + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Install binutils + run: sudo apt-get install -y binutils + + - name: Check native module CPU portability + run: bash scripts/ci/check_native_portability.sh diff --git a/.github/workflows/publish-dev.yml b/.github/workflows/publish-dev.yml index 0abec636425a..589620b5e2f1 100644 --- a/.github/workflows/publish-dev.yml +++ b/.github/workflows/publish-dev.yml @@ -19,14 +19,14 @@ jobs: runs-on: ubuntu-latest steps: # - Uses YAML anchors in the future - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - name: Install pnpm before setup node - uses: pnpm/action-setup@v4 + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4 - - uses: actions/setup-node@v6 + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version: 24 registry-url: "https://registry.npmjs.org" @@ -36,7 +36,7 @@ jobs: id: node run: echo "v8CppApiVersion=$(node --print "process.versions.modules")" >> $GITHUB_OUTPUT - name: Restore dependencies - uses: actions/cache@v4 + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 id: cache-deps with: path: | diff --git a/.github/workflows/publish-manual.yml b/.github/workflows/publish-manual.yml index ad7bc6a31321..db7954978ae3 100644 --- a/.github/workflows/publish-manual.yml +++ b/.github/workflows/publish-manual.yml @@ -23,7 +23,7 @@ jobs: name: Validate inputs runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ inputs.ref }} fetch-depth: 0 diff --git a/.github/workflows/publish-nextfork.yml b/.github/workflows/publish-nextfork.yml index 84eabc546728..45f74ff069be 100644 --- a/.github/workflows/publish-nextfork.yml +++ b/.github/workflows/publish-nextfork.yml @@ -22,12 +22,12 @@ jobs: runs-on: ubuntu-latest steps: # - Uses YAML anchors in the future - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - name: Install pnpm before setup node - uses: pnpm/action-setup@v4 - - uses: actions/setup-node@v6 + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4 + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version: 24 registry-url: "https://registry.npmjs.org" @@ -37,7 +37,7 @@ jobs: id: node run: echo "v8CppApiVersion=$(node --print "process.versions.modules")" >> $GITHUB_OUTPUT - name: Restore dependencies - uses: actions/cache@master + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # master id: cache-deps with: path: | diff --git a/.github/workflows/publish-rc.yml b/.github/workflows/publish-rc.yml index aabcde7ee3d5..2ea480971a69 100644 --- a/.github/workflows/publish-rc.yml +++ b/.github/workflows/publish-rc.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 @@ -59,7 +59,7 @@ jobs: needs: [tag, binaries] if: needs.tag.outputs.is_rc == 'true' steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 # Needs full depth for changelog generation @@ -71,14 +71,14 @@ jobs: run: node scripts/generate_changelog.mjs ${{ needs.tag.outputs.prev_tag }} ${{ needs.tag.outputs.tag }} CHANGELOG.md - name: Get binaries - uses: actions/download-artifact@v5 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: path: dist/ merge-multiple: true - name: Create Release id: create_release - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@153bb8e04406b158c6c84fc1615b65b24149a1fe # v2.6.1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: @@ -88,6 +88,7 @@ jobs: body_path: "CHANGELOG.md" name: Release ${{ needs.tag.outputs.tag }} prerelease: true + draft: true - name: Change and commit version # Write version before publishing so it's picked up by `lerna publish from-package`. @@ -108,10 +109,15 @@ jobs: - name: Publish to npm registry run: pnpm run release:publish --dist-tag rc + - name: Publish release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: gh release edit ${{ needs.tag.outputs.tag }} --draft=false + # In case of failure - name: Rollback on failure if: failure() - uses: author/action-rollback@1.0.4 + uses: author/action-rollback@f4473931b1155b601092ec00eb1fa4882b80219f # 1.0.4 with: release_id: ${{ steps.create_release.outputs.id }} tag: ${{ needs.tag.outputs.tag }} @@ -125,7 +131,7 @@ jobs: needs: [tag, npm] if: needs.tag.outputs.is_rc == 'true' steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - run: scripts/await-release.sh ${{ needs.tag.outputs.tag }} rc 900 docker: diff --git a/.github/workflows/publish-stable.yml b/.github/workflows/publish-stable.yml index 7bf82a6fe45d..2d13ce1ea3b3 100644 --- a/.github/workflows/publish-stable.yml +++ b/.github/workflows/publish-stable.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 @@ -65,7 +65,7 @@ jobs: needs: [tag, binaries] if: needs.tag.outputs.is_stable == 'true' steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 # Needs full depth for changelog generation @@ -77,14 +77,14 @@ jobs: run: node scripts/generate_changelog.mjs ${{ needs.tag.outputs.prev_tag }} ${{ needs.tag.outputs.tag }} CHANGELOG.md - name: Get binaries - uses: actions/download-artifact@v4 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: path: dist/ merge-multiple: true - name: Create Release id: create_release - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@153bb8e04406b158c6c84fc1615b65b24149a1fe # v2.6.1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: @@ -94,14 +94,20 @@ jobs: body_path: "CHANGELOG.md" name: Release ${{ needs.tag.outputs.tag }} prerelease: false + draft: true - name: Publish to npm registry (release) run: pnpm run release:publish + - name: Publish release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: gh release edit ${{ needs.tag.outputs.tag }} --draft=false + # In case of failure - name: Rollback on failure if: failure() - uses: author/action-rollback@1.0.4 + uses: author/action-rollback@f4473931b1155b601092ec00eb1fa4882b80219f # 1.0.4 with: release_id: ${{ steps.create_release.outputs.id }} tag: ${{ needs.tag.outputs.tag }} @@ -115,7 +121,7 @@ jobs: needs: [tag, npm] if: needs.tag.outputs.is_stable == 'true' steps: - - uses: nflaig/release-comment-on-pr@v1 + - uses: nflaig/release-comment-on-pr@addf0108e7a60a9aaef4eee2900d69b126a75a5d # v1 with: token: ${{ secrets.GH_PAGES_TOKEN }} @@ -125,7 +131,7 @@ jobs: needs: [tag, npm] if: needs.tag.outputs.is_stable == 'true' steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - run: scripts/await-release.sh ${{ needs.tag.outputs.tag }} latest 900 docker: diff --git a/.github/workflows/test-bun.yml b/.github/workflows/test-bun.yml deleted file mode 100644 index bb2158fd744a..000000000000 --- a/.github/workflows/test-bun.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Bun Tests -# only one can run at a time -concurrency: - # If PR, cancel prev commits. head_ref = source branch name on pull_request, null if push - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -on: - push: - # We intentionally don't run push on feature branches. See PR for rational. - branches: [unstable, stable] - pull_request: - workflow_dispatch: - -jobs: - unit-tests-bun: - name: Unit Tests (Bun) - runs-on: buildjet-4vcpu-ubuntu-2204 - steps: - - uses: actions/checkout@v4 - - name: Install pnpm before setup node - uses: pnpm/action-setup@v4 - - name: Setup Node - uses: actions/setup-node@v6 - with: - node-version: 24 - cache: pnpm - - uses: oven-sh/setup-bun@v2 - with: - bun-version: latest - - name: Install pnpm - run: bun install -g npm:pnpm - - name: Install - run: pnpm install --frozen-lockfile - - name: Build - run: pnpm build - - name: Unit Tests - # These packages are not testable yet in Bun because of `@chainsafe/blst` dependency. - run: excluded=(beacon-node prover light-client cli); for pkg in packages/*/; do [[ ! " ${excluded[@]} " =~ " $(basename "$pkg") " ]] && echo "Testing $(basename "$pkg")" && (cd "$pkg" && bun run --bun test:unit); done - shell: bash diff --git a/.github/workflows/test-sim.yml b/.github/workflows/test-sim.yml index 6cfaf1be3c27..cede48a6ede7 100644 --- a/.github/workflows/test-sim.yml +++ b/.github/workflows/test-sim.yml @@ -28,7 +28,7 @@ jobs: runs-on: buildjet-4vcpu-ubuntu-2204 steps: # - Uses YAML anchors in the future - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: "./.github/actions/setup-and-build" with: node: 24 @@ -39,7 +39,7 @@ jobs: runs-on: buildjet-4vcpu-ubuntu-2204 steps: # - Uses YAML anchors in the future - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: "./.github/actions/setup-and-build" with: node: 24 @@ -57,7 +57,7 @@ jobs: GENESIS_DELAY_SLOTS: ${{github.event.inputs.genesisDelaySlots}} - name: Upload debug log test files for "packages/cli" if: ${{ always() }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: sim-test-multifork-logs path: packages/cli/test-logs @@ -68,7 +68,7 @@ jobs: runs-on: buildjet-4vcpu-ubuntu-2204 steps: # - Uses YAML anchors in the future - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: "./.github/actions/setup-and-build" with: node: 24 @@ -86,7 +86,7 @@ jobs: GENESIS_DELAY_SLOTS: ${{github.event.inputs.genesisDelaySlots}} - name: Upload debug log test files for "packages/cli" if: ${{ always() }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: sim-test-endpoints-logs path: packages/cli/test-logs @@ -97,7 +97,7 @@ jobs: runs-on: buildjet-4vcpu-ubuntu-2204 steps: # - Uses YAML anchors in the future - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: "./.github/actions/setup-and-build" with: node: 24 @@ -115,7 +115,7 @@ jobs: GENESIS_DELAY_SLOTS: ${{github.event.inputs.genesisDelaySlots}} - name: Upload debug log test files for "packages/cli" if: ${{ always() }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: sim-test-eth-backup-provider-logs path: packages/cli/test-logs @@ -126,7 +126,7 @@ jobs: runs-on: buildjet-4vcpu-ubuntu-2204 steps: # - Uses YAML anchors in the future - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: "./.github/actions/setup-and-build" with: node: 24 @@ -144,7 +144,7 @@ jobs: GENESIS_DELAY_SLOTS: ${{github.event.inputs.genesisDelaySlots}} - name: Upload debug log test files for "packages/cli" if: ${{ always() }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: sim-test-mixed-clients-logs path: packages/cli/test-logs diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8b0e2d1dbd50..5f5df7441041 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,7 +22,7 @@ jobs: node: [24] steps: # - Uses YAML anchors in the future - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: "./.github/actions/setup-and-build" with: node: ${{ matrix.node }} @@ -44,7 +44,7 @@ jobs: matrix: node: [24] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: "./.github/actions/setup-and-build" with: node: ${{ matrix.node }} @@ -70,7 +70,7 @@ jobs: matrix: node: [24] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: "./.github/actions/setup-and-build" with: @@ -91,14 +91,14 @@ jobs: matrix: node: [24] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: "./.github/actions/setup-and-build" with: node: ${{ matrix.node }} # Cache validator slashing protection data tests - name: Restore spec tests cache - uses: actions/cache@v4 + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 with: path: packages/validator/spec-tests key: spec-test-data-${{ hashFiles('packages/validator/test/spec/params.ts') }} @@ -114,7 +114,7 @@ jobs: # if: ${{ failure() && steps.unit_tests.conclusion == 'failure' }} - name: Upload coverage data - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 with: token: ${{ secrets.CODECOV_TOKEN }} fail_ci_if_error: true @@ -131,7 +131,7 @@ jobs: node: [24] steps: # - Uses YAML anchors in the future - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: "./.github/actions/setup-and-build" with: @@ -153,7 +153,7 @@ jobs: - name: Upload debug log test for test env if: ${{ always() }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: debug-e2e-test-logs-node-${{matrix.node}} path: test-logs/e2e-test-env @@ -168,7 +168,7 @@ jobs: node: [24] steps: # - Uses YAML anchors in the future - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: "./.github/actions/setup-and-build" with: node: ${{ matrix.node }} @@ -189,17 +189,17 @@ jobs: matrix: node: [24] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: "./.github/actions/setup-and-build" with: node: ${{ matrix.node }} # Download spec tests with cache - name: Restore spec tests cache - uses: actions/cache@v4 + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 with: path: packages/beacon-node/spec-tests - key: spec-test-data-${{ hashFiles('packages/beacon-node/test/spec/specTestVersioning.ts') }} + key: spec-test-data-${{ hashFiles('spec-tests-version.json') }} - name: Download spec tests run: pnpm download-spec-tests # Run them in different steps to quickly identifying which command failed diff --git a/.gitignore b/.gitignore index 4edee34dcdb4..6abb0efe9c48 100644 --- a/.gitignore +++ b/.gitignore @@ -29,7 +29,10 @@ validators .vscode/launch.json !.vscode/settings.json .vscode/tasks.json -.claude/ +# Claude Code — ignore local/personal config, keep shared project config +.claude/settings.local.json +.claude/plans/ +.claude/CLAUDE.local.md # Tests artifacts packages/*/spec-tests* diff --git a/.vscode/settings.json b/.vscode/settings.json index 7ae1039bcebc..4892bc4ce12f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,6 +2,7 @@ "window.title": "${activeEditorShort}${separator}${rootName}${separator}${profileName}${separator}[${activeRepositoryBranchName}]", // For `sysoev.vscode-open-in-github` extension "openInGitHub.defaultBranch": "unstable", + "js/ts.experimental.useTsgo": true, "editor.formatOnSave": true, "editor.codeActionsOnSave": { "source.fixAll.biome": "explicit", diff --git a/.wordlist.txt b/.wordlist.txt index c74ea96cdf3d..a49940b25e20 100644 --- a/.wordlist.txt +++ b/.wordlist.txt @@ -84,6 +84,7 @@ PRs Plaintext PoS Prysm +QUIC Quickstart README READMEs @@ -96,6 +97,7 @@ SSD SSZ Somer Stakehouse +TLS TOC TTD Teku diff --git a/AGENTS.md b/AGENTS.md index 19e3420ca11c..8aac83cac823 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,5 +1,16 @@ # AGENTS.md +## Critical rules + +- **Target branch:** `unstable` (never `stable`) +- **Pre-push:** run `pnpm lint`, `pnpm check-types`, `pnpm test:unit` before every push +- **Relative imports:** use `.js` extension in TypeScript ESM imports +- **No `any`:** avoid `any` / `as any`; use proper types or justified `biome-ignore` +- **No `lib/` edits:** never edit `packages/*/lib/` — these are build outputs +- **Follow existing patterns** before introducing new abstractions +- **Structured logging** with specific error codes (not generic `Error`) +- **Incremental commits** after review starts — do not force push unless maintainer requests it + ## Project overview Lodestar is a TypeScript implementation of the Ethereum consensus client @@ -103,8 +114,12 @@ Lodestar uses [Biome](https://biomejs.dev/) for linting and formatting. - **Naming**: `camelCase` for functions/variables, `PascalCase` for classes, `UPPER_SNAKE_CASE` for constants - **Quotes**: Use double quotes (`"`) not single quotes -- **Types**: All functions must have explicit parameter and return types -- **No `any`**: Avoid TypeScript `any` type +- **Types**: Prefer explicit types on public APIs and complex functions +- **No `any` or `as any`**: Do not use `any` type or `as any` assertions to bypass + the type system. In production code, find the proper type or interface. In test code, + use public APIs rather than accessing private fields via `as any`. If genuinely + unavoidable, add a suppression with the full rule ID and justification: + `// biome-ignore lint/suspicious/noExplicitAny: ` - **Private fields**: No underscore prefix (use `private dirty`, not `private _dirty`) - **Named exports only**: No default exports @@ -117,10 +132,17 @@ Imports are auto-sorted by Biome in this order: 3. `@chainsafe/*` and `@lodestar/*` packages 4. Relative paths -Always use `.js` extension for relative imports (even for `.ts` files): +In TypeScript source and test files, use `.js` extension for relative ESM imports +(even though source files are `.ts`). This is required for Node.js ESM resolution. +This rule does **not** apply to non-TS files (e.g., `package.json`, `.mjs` config). ```typescript +// ✅ Correct import {something} from "./utils.js"; +import {IBeaconStateView} from "../stateView/interface.js"; + +// ❌ Wrong — will break at runtime +import {something} from "./utils.ts"; ``` ### Comments @@ -258,25 +280,8 @@ for (const block of blocks) { ### Running specific tests -Use vitest project filters for targeted test runs: - -```bash -# Unit tests only (from repo root) -pnpm vitest run --project unit test/unit/chain/validation/block.test.ts - -# With pattern matching -pnpm vitest run --project unit -t "should reject" - -# From package directory (no project filter needed) -cd packages/beacon-node -pnpm vitest run test/unit/chain/validation/block.test.ts -``` - -For spec tests with minimal preset (faster): - -```bash -LODESTAR_PRESET=minimal pnpm vitest run --config vitest.spec.config.ts -``` +See **Build commands** above for all test invocations. Use `--project unit` +for targeted runs and `LODESTAR_PRESET=minimal` for faster spec tests. ## Pull request guidelines @@ -321,11 +326,20 @@ refactor(reqresp)!: support byte based handlers ### PR etiquette - Keep PRs as drafts until ready for review -- Don't force push after review starts (use incremental commits) -- Close stale PRs rather than letting them sit +- Avoid force push after review starts unless a maintainer requests it (use incremental commits) +- Flag stale PRs to maintainers rather than letting them sit indefinitely - Respond to review feedback promptly — reply to every comment, including bot reviewers - When updating based on feedback, respond in-thread to acknowledge +## Pre-push checklist + +Before pushing any commit, verify: + +1. `pnpm lint` — Biome enforces formatting; CI catches failures but wastes a round-trip +2. `pnpm check-types` — catch type errors before CI +3. `pnpm docs:lint` — if you edited any `.md` files, check Prettier formatting +4. No edits in `packages/*/lib/` — these are build outputs; edit `src/` instead + ## Common tasks ### Adding a new feature @@ -341,7 +355,7 @@ refactor(reqresp)!: support byte based handlers 1. Write a failing test that reproduces the bug 2. Fix the bug 3. Verify the test passes -4. Run full test suite: `pnpm test:unit` +4. Run checks: `pnpm lint`, `pnpm check-types`, `pnpm test:unit` ### Adding a new SSZ type @@ -360,47 +374,10 @@ refactor(reqresp)!: support byte based handlers ## Style learnings from reviews -### Prefer inline logic over helper functions - -For simple validation logic, inline the check rather than creating a helper: - -```typescript -// Preferred -if (error.code === RegenErrorCode.BLOCK_NOT_IN_FORKCHOICE) { - return GossipAction.REJECT; -} - -// Avoid (unless logic is complex and reused) -function shouldReject(error: Error): boolean { - return error.code === RegenErrorCode.BLOCK_NOT_IN_FORKCHOICE; -} -``` - -### Match existing comment style - -When adding comments to containers or functions modified across forks, -follow the existing style in that file. Don't add unnecessary markers. - -### Error handling patterns - -Use specific error codes when available: - -```typescript -// Preferred -throw new BlockError(block, {code: BlockErrorCode.PARENT_UNKNOWN}); - -// Avoid generic errors when specific ones exist -throw new Error("Parent not found"); -``` - -### Config value coercion - -When reading optional config values, handle undefined explicitly: - -```typescript -const peers = config.directPeers ?? []; -const trimmed = value?.trim() ?? ""; -``` +- **Prefer inline logic** over single-use helper functions for simple checks +- **Match existing patterns** in the file you're modifying (comments, structure) +- **Use specific error codes** (`BlockErrorCode.PARENT_UNKNOWN`) over generic `Error` +- **Handle undefined** explicitly: `config.directPeers ?? []`, `value?.trim() ?? ""` ## Implementing consensus specs @@ -422,8 +399,7 @@ When implementing changes from the consensus specs, the mapping is typically: ### Fork organization -Specs and code are organized by fork: `phase0`, `altair`, `bellatrix`, -`capella`, `deneb`, `electra`, `fulu`, `gloas`. +Forks follow the progression defined in **Architecture patterns > Fork-aware code** above. - **@lodestar/types/src/** - Each fork has its own directory with SSZ type definitions - **@lodestar/state-transition/src/block/** - Block processing functions @@ -431,55 +407,5 @@ Specs and code are organized by fork: `phase0`, `altair`, `bellatrix`, - **@lodestar/state-transition/src/epoch/** - Epoch processing functions - **@lodestar/state-transition/src/slot/** - Slot processing functions -## Important notes - -### Default branch is `unstable` - -All PRs should target `unstable`. The `stable` branch is for releases only -(see RELEASE.md for details). - -### Spec tests require download - -Before running `pnpm test:spec`, download test vectors: - -```bash -pnpm download-spec-tests -``` - -### E2E tests require Docker - -Start the e2e environment before running e2e tests: - -```bash -./scripts/run_e2e_env.sh start -pnpm test:e2e -./scripts/run_e2e_env.sh stop -``` - -### Generated files - -Do not edit files in `packages/*/lib/` - these are build outputs. -Edit source files in `packages/*/src/` instead. - -### Consensus spec references - The `specrefs/` directory contains pinned consensus spec versions. When implementing spec changes, reference the exact spec version. - -## Common pitfalls - -- **Forgetting `pnpm lint` before pushing**: Biome enforces formatting. Always - run it before committing. CI will catch it, but it wastes a round-trip. -- **Forgetting `pnpm docs:lint` after editing docs**: Markdown files are - formatted by Prettier. Run `pnpm docs:lint` (or `pnpm docs:lint:fix` to - auto-fix) before pushing changes to `.md` files. -- **Editing `lib/` instead of `src/`**: Files in `packages/*/lib/` are build - outputs. Always edit in `packages/*/src/`. -- **Stale fork choice head**: After modifying proto-array execution status, - the cached head from `getHead()` is stale. Call `recomputeForkChoiceHead()`. -- **Holding state references**: Beacon state objects are large. Don't store - references beyond their immediate use — let them be garbage collected. -- **Missing `.js` extension**: Relative imports must use `.js` even though - source files are `.ts`. This is required for Node.js ESM resolution. -- **Force pushing after review**: Never force push once a reviewer has started. - Use incremental commits — reviewers track changes between reviews. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000000..535f1fb86767 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,41 @@ +# CLAUDE.md + +See [AGENTS.md](./AGENTS.md) for the full project guide — build commands, code style, architecture patterns, testing, and PR guidelines. + +## Quick Reference + +```bash +# Build +pnpm build + +# Lint (biome) +pnpm lint +pnpm lint:fix + +# Type check +pnpm check-types + +# Unit tests +pnpm test:unit + +# Run specific test +pnpm vitest run --project unit test/unit/path/to/test.test.ts + +# Spec tests (download first) +pnpm download-spec-tests +pnpm test:spec + +# Docs lint (markdown) +pnpm docs:lint +pnpm docs:lint:fix +``` + +## Key Conventions + +- **Default branch:** `unstable` (all PRs target this) +- **Import extensions:** Always use `.js` for relative imports (even for `.ts` files) +- **No default exports:** Named exports only +- **Metrics:** Prometheus naming, always suffix with units (`_seconds`, `_bytes`, `_total`) +- **Fork order:** phase0 → altair → bellatrix → capella → deneb → electra → fulu → gloas +- **Commit style:** [Conventional Commits](https://www.conventionalcommits.org/) (`feat:`, `fix:`, `refactor:`, etc.) +- **AI disclosure:** Required in PR descriptions when AI-assisted diff --git a/biome.jsonc b/biome.jsonc index f58cdb488846..72e43151af09 100644 --- a/biome.jsonc +++ b/biome.jsonc @@ -72,7 +72,7 @@ "useImportExtensions": { "level": "error", "options": { - "forceJsExtensions": false + "forceJsExtensions": true } }, "useParseIntRadix": { diff --git a/dashboards/lodestar_beacon_chain.json b/dashboards/lodestar_beacon_chain.json index d65beb99c217..9c4f2b6ba0a5 100644 --- a/dashboards/lodestar_beacon_chain.json +++ b/dashboards/lodestar_beacon_chain.json @@ -330,7 +330,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(lodestar_seen_block_input_cache_duplicate_block_count[$rate_interval]) * 12", + "expr": "rate(lodestar_seen_block_input_cache_duplicate_block_total[$rate_interval]) * 12", "hide": false, "instant": false, "legendFormat": "duplicate_block_{{source}}", @@ -343,7 +343,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(lodestar_seen_block_input_cache_duplicate_blob_count[$rate_interval]) * 12", + "expr": "rate(lodestar_seen_block_input_cache_duplicate_blob_total[$rate_interval]) * 12", "hide": false, "instant": false, "legendFormat": "duplicate_blob_{{source}}", @@ -356,7 +356,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(lodestar_seen_block_input_cache_duplicate_column_count[$rate_interval]) * 12", + "expr": "rate(lodestar_seen_block_input_cache_duplicate_column_total[$rate_interval]) * 12", "hide": false, "instant": false, "legendFormat": "duplicate_columns_{{source}}", @@ -369,7 +369,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(lodestar_seen_block_input_cache_items_created_by_block[$rate_interval]) * 12", + "expr": "rate(lodestar_seen_block_input_cache_items_created_by_block_total[$rate_interval]) * 12", "hide": false, "instant": false, "legendFormat": "created_by_block", @@ -382,12 +382,37 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(lodestar_seen_block_input_cache_items_created_by_blob[$rate_interval]) * 12", + "expr": "rate(lodestar_seen_block_input_cache_items_created_by_blob_total[$rate_interval]) * 12", "hide": false, "instant": false, "legendFormat": "created_by_blob", "range": true, "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_seen_block_input_cache_items_created_by_column_total[$rate_interval]) * 12", + "hide": false, + "instant": false, + "legendFormat": "created_by_column", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "lodestar_seen_block_input_cache_serialized_object_refs", + "instant": false, + "legendFormat": "serialized_objects", + "range": true, + "refId": "H" } ], "title": "Block Input", diff --git a/dashboards/lodestar_block_processor.json b/dashboards/lodestar_block_processor.json index 4107450b4c86..f46257193f5e 100644 --- a/dashboards/lodestar_block_processor.json +++ b/dashboards/lodestar_block_processor.json @@ -3898,1078 +3898,13 @@ "title": "Prepare Next Epoch hash_tree_root", "type": "heatmap" }, - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 173 - }, - "id": 136, - "panels": [], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "refId": "A" - } - ], - "title": "Fork-Choice Stats", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 28, - "gradientMode": "opacity", - "hideFrom": { - "graph": false, - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "s" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "prepare_next_slot" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 174 - }, - "id": 130, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - }, - "tooltipOptions": { - "mode": "single" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "rate(beacon_fork_choice_find_head_seconds_sum[$rate_interval])/rate(beacon_fork_choice_find_head_seconds_count[$rate_interval])", - "interval": "", - "legendFormat": "{{caller}}", - "range": true, - "refId": "A" - } - ], - "title": "updateHead() avg time", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "graph": false, - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [] - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "beacon_fork_choice_errors_total{group=\"beta\", instance=\"contabo-13\", job=~\"$beacon_job|beacon\"}" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 174 - }, - "id": 140, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - }, - "tooltipOptions": { - "mode": "single" - } - }, - "pluginVersion": "8.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "beacon_fork_choice_errors_total", - "interval": "", - "legendFormat": "", - "range": true, - "refId": "A" - } - ], - "title": "updateHead() errors", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "graph": false, - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [] - }, - "overrides": [ - { - "matcher": { - "id": "byFrameRefID", - "options": "C" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "percentunit" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 182 - }, - "id": 132, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - }, - "tooltipOptions": { - "mode": "single" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "12 * rate(beacon_fork_choice_find_head_seconds_count[$rate_interval])", - "interval": "", - "legendFormat": "{{caller}}_updateHead_calls", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "", - "hide": false, - "interval": "", - "legendFormat": "updateHead calls", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "rate(beacon_fork_choice_find_head_seconds_sum[$rate_interval])", - "hide": false, - "interval": "", - "legendFormat": "{{caller}}_usage_rate", - "range": true, - "refId": "C" - } - ], - "title": "updateHead() calls / slot", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "left", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "opacity", - "hideFrom": { - "graph": false, - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "percentunit" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "{job=~\"$beacon_job|beacon\"}" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "purple", - "mode": "palette-classic" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 182 - }, - "id": 138, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - }, - "tooltipOptions": { - "mode": "single" - } - }, - "pluginVersion": "8.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "rate(beacon_fork_choice_changed_head_total[6m])/rate(beacon_fork_choice_requests_total[6m])", - "hide": false, - "interval": "", - "legendFormat": "Changed Heads", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "", - "hide": false, - "interval": "", - "legendFormat": "Total Requests", - "refId": "B" - } - ], - "title": "ratio of updateHead() calls that change head", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [] - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "queued_attestations" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "validated_attestation_datas" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 190 - }, - "id": 531, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "beacon_fork_choice_votes_count", - "legendFormat": "votes", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "beacon_fork_choice_queued_attestations_count", - "hide": false, - "legendFormat": "queued_attestations", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "beacon_fork_choice_validated_attestation_datas_count", - "hide": false, - "legendFormat": "validated_attestation_datas", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "beacon_fork_choice_balances_length", - "hide": false, - "legendFormat": "balances_length", - "range": true, - "refId": "D" - } - ], - "title": "Forkchoice Internal Data", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [] - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 190 - }, - "id": 533, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "beacon_fork_choice_nodes_count", - "legendFormat": "nodes", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "beacon_fork_choice_indices_count", - "hide": false, - "legendFormat": "indices", - "range": true, - "refId": "B" - } - ], - "title": "Proto Array", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 198 - }, - "id": 587, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "rate(beacon_fork_choice_compute_deltas_seconds_sum[$rate_interval])\n/\nrate(beacon_fork_choice_compute_deltas_seconds_count[$rate_interval])", - "instant": false, - "legendFormat": "duration", - "range": true, - "refId": "A" - } - ], - "title": "computeDeltas() avg time", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [] - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "deltas" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 198 - }, - "id": 588, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "beacon_fork_choice_compute_deltas_deltas_count", - "instant": false, - "legendFormat": "deltas", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "beacon_fork_choice_compute_deltas_zero_deltas_count", - "hide": false, - "instant": false, - "legendFormat": "zero_deltas", - "range": true, - "refId": "B" - } - ], - "title": "computeDeltas() - deltas result", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [] - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 206 - }, - "id": 589, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "beacon_fork_choice_compute_deltas_equivocating_validators_count", - "instant": false, - "legendFormat": "equivocating_validators", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "beacon_fork_choice_compute_deltas_old_inactive_validators_count", - "hide": false, - "instant": false, - "legendFormat": "old_inactive_validators", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "beacon_fork_choice_compute_deltas_new_inactive_validators_count", - "hide": false, - "instant": false, - "legendFormat": "new_inactive_validators", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "beacon_fork_choice_compute_deltas_unchanged_vote_validators_count", - "hide": false, - "instant": false, - "legendFormat": "unchanged_vote_validators", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "beacon_fork_choice_compute_deltas_new_vote_validators_count", - "hide": false, - "instant": false, - "legendFormat": "new_vote_validators", - "range": true, - "refId": "E" - } - ], - "title": "computeDeltas() - validators result", - "type": "timeseries" - }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 214 + "y": 173 }, "id": 569, "panels": [], @@ -5027,7 +3962,7 @@ "h": 8, "w": 12, "x": 0, - "y": 215 + "y": 174 }, "id": 580, "options": { @@ -5241,7 +4176,7 @@ "h": 8, "w": 12, "x": 12, - "y": 215 + "y": 174 }, "id": 578, "options": { @@ -5439,7 +4374,7 @@ "h": 8, "w": 12, "x": 0, - "y": 223 + "y": 182 }, "id": 577, "options": { @@ -5572,7 +4507,7 @@ "h": 8, "w": 12, "x": 12, - "y": 223 + "y": 182 }, "id": 586, "options": { @@ -5655,7 +4590,7 @@ "h": 8, "w": 12, "x": 0, - "y": 231 + "y": 190 }, "id": 584, "options": { @@ -5772,7 +4707,7 @@ "h": 8, "w": 12, "x": 12, - "y": 231 + "y": 190 }, "id": 582, "options": { @@ -5873,7 +4808,7 @@ "h": 8, "w": 12, "x": 12, - "y": 239 + "y": 198 }, "id": 585, "options": { @@ -5931,7 +4866,7 @@ "h": 1, "w": 24, "x": 0, - "y": 247 + "y": 206 }, "id": 538, "panels": [], @@ -5963,7 +4898,7 @@ "h": 8, "w": 12, "x": 0, - "y": 248 + "y": 207 }, "id": 539, "options": { @@ -6045,7 +4980,7 @@ "h": 8, "w": 12, "x": 12, - "y": 248 + "y": 207 }, "id": 540, "options": { @@ -6154,7 +5089,7 @@ "h": 8, "w": 12, "x": 0, - "y": 256 + "y": 215 }, "id": 553, "options": { @@ -6241,7 +5176,7 @@ "h": 8, "w": 12, "x": 12, - "y": 256 + "y": 215 }, "id": 554, "options": { @@ -6300,7 +5235,7 @@ "h": 8, "w": 12, "x": 0, - "y": 264 + "y": 223 }, "id": 556, "options": { @@ -6388,7 +5323,7 @@ "h": 8, "w": 12, "x": 12, - "y": 264 + "y": 223 }, "id": 542, "options": { @@ -6497,7 +5432,7 @@ "h": 8, "w": 12, "x": 0, - "y": 272 + "y": 231 }, "id": 541, "options": { @@ -6583,7 +5518,7 @@ "h": 8, "w": 12, "x": 12, - "y": 272 + "y": 231 }, "id": 555, "options": { @@ -6642,7 +5577,7 @@ "h": 8, "w": 12, "x": 0, - "y": 280 + "y": 239 }, "id": 557, "options": { @@ -6731,7 +5666,7 @@ "h": 8, "w": 12, "x": 12, - "y": 280 + "y": 239 }, "id": 544, "options": { @@ -6847,7 +5782,7 @@ "h": 8, "w": 12, "x": 0, - "y": 288 + "y": 247 }, "id": 543, "options": { @@ -6933,7 +5868,7 @@ "h": 8, "w": 12, "x": 12, - "y": 288 + "y": 247 }, "id": 558, "options": { @@ -6992,7 +5927,7 @@ "h": 8, "w": 12, "x": 0, - "y": 296 + "y": 255 }, "id": 559, "options": { @@ -7081,7 +6016,7 @@ "h": 8, "w": 12, "x": 12, - "y": 296 + "y": 255 }, "id": 546, "options": { @@ -7197,7 +6132,7 @@ "h": 8, "w": 12, "x": 0, - "y": 304 + "y": 263 }, "id": 545, "options": { @@ -7283,7 +6218,7 @@ "h": 8, "w": 12, "x": 12, - "y": 304 + "y": 263 }, "id": 560, "options": { @@ -7342,7 +6277,7 @@ "h": 8, "w": 12, "x": 0, - "y": 312 + "y": 271 }, "id": 561, "options": { @@ -7431,7 +6366,7 @@ "h": 8, "w": 12, "x": 12, - "y": 312 + "y": 271 }, "id": 548, "options": { @@ -7547,7 +6482,7 @@ "h": 8, "w": 12, "x": 0, - "y": 320 + "y": 279 }, "id": 547, "options": { @@ -7633,7 +6568,7 @@ "h": 8, "w": 12, "x": 12, - "y": 320 + "y": 279 }, "id": 562, "options": { @@ -7692,7 +6627,7 @@ "h": 8, "w": 12, "x": 0, - "y": 328 + "y": 287 }, "id": 563, "options": { @@ -7781,7 +6716,7 @@ "h": 8, "w": 12, "x": 12, - "y": 328 + "y": 287 }, "id": 550, "options": { @@ -7897,7 +6832,7 @@ "h": 8, "w": 12, "x": 0, - "y": 336 + "y": 295 }, "id": 549, "options": { @@ -7983,7 +6918,7 @@ "h": 8, "w": 12, "x": 12, - "y": 336 + "y": 295 }, "id": 564, "options": { @@ -8042,7 +6977,7 @@ "h": 8, "w": 12, "x": 0, - "y": 344 + "y": 303 }, "id": 565, "options": { @@ -8124,7 +7059,7 @@ "h": 8, "w": 12, "x": 12, - "y": 344 + "y": 303 }, "id": 552, "options": { @@ -8233,7 +7168,7 @@ "h": 8, "w": 12, "x": 0, - "y": 352 + "y": 311 }, "id": 551, "options": { @@ -8319,7 +7254,7 @@ "h": 8, "w": 12, "x": 12, - "y": 352 + "y": 311 }, "id": 566, "options": { @@ -8474,11 +7409,6 @@ "type": "adhoc" }, { - "current": { - "selected": false, - "text": "${VAR_BEACON_JOB}", - "value": "${VAR_BEACON_JOB}" - }, "description": "Job name used in Prometheus config to scrape Beacon node", "hide": 2, "label": "Beacon node job name", @@ -8510,6 +7440,6 @@ "timezone": "utc", "title": "Lodestar - block processor", "uid": "lodestar_block_processor", - "version": 16, + "version": 24, "weekStart": "monday" } diff --git a/dashboards/lodestar_execution_engine.json b/dashboards/lodestar_execution_engine.json index 3ff00be01c2e..15fb4be6f74d 100644 --- a/dashboards/lodestar_execution_engine.json +++ b/dashboards/lodestar_execution_engine.json @@ -1267,12 +1267,23 @@ }, "editorMode": "builder", "expr": "rate(lodestar_execution_engine_notify_new_payload_result_total[1h])", - "legendFormat": "{{result}}", + "legendFormat": "newPayload_{{result}}", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "rate(lodestar_execution_engine_notify_forkchoice_update_result_total[1h])", + "legendFormat": "fcu_{{result}}", + "range": true, + "refId": "B" } ], - "title": "notifyNewPayload result", + "title": "notifyNewPayload / notifyForkchoiceUpdate result", "type": "timeseries" }, { diff --git a/dashboards/lodestar_fork_choice.json b/dashboards/lodestar_fork_choice.json new file mode 100644 index 000000000000..4f8605a01d18 --- /dev/null +++ b/dashboards/lodestar_fork_choice.json @@ -0,0 +1,1282 @@ +{ + "__inputs": [ + { + "description": "", + "label": "Prometheus", + "name": "DS_PROMETHEUS", + "pluginId": "prometheus", + "pluginName": "Prometheus", + "type": "datasource" + }, + { + "description": "", + "label": "Beacon node job name", + "name": "VAR_BEACON_JOB", + "type": "constant", + "value": "beacon" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "lodestar" + ], + "targetBlank": false, + "title": "Lodestar dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 136, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "refId": "A" + } + ], + "title": "Fork-Choice Stats", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 28, + "gradientMode": "opacity", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "s" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "prepare_next_slot" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 130, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "rate(beacon_fork_choice_find_head_seconds_sum[$rate_interval])/rate(beacon_fork_choice_find_head_seconds_count[$rate_interval])", + "interval": "", + "legendFormat": "{{caller}}", + "range": true, + "refId": "A" + } + ], + "title": "updateHead() avg time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "beacon_fork_choice_errors_total{group=\"beta\", instance=\"contabo-13\", job=~\"$beacon_job|beacon\"}" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 140, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "8.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "beacon_fork_choice_errors_total", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "updateHead() errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "percentunit" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 132, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "12 * rate(beacon_fork_choice_find_head_seconds_count[$rate_interval])", + "interval": "", + "legendFormat": "{{caller}}_updateHead_calls", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": false, + "expr": "", + "hide": false, + "interval": "", + "legendFormat": "updateHead calls", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "rate(beacon_fork_choice_find_head_seconds_sum[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "{{caller}}_usage_rate", + "range": true, + "refId": "C" + } + ], + "title": "updateHead() calls / slot", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "left", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "opacity", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "{job=~\"$beacon_job|beacon\"}" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "palette-classic" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 138, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "8.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": false, + "expr": "rate(beacon_fork_choice_changed_head_total[6m])/rate(beacon_fork_choice_requests_total[6m])", + "hide": false, + "interval": "", + "legendFormat": "Changed Heads", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": false, + "expr": "", + "hide": false, + "interval": "", + "legendFormat": "Total Requests", + "refId": "B" + } + ], + "title": "ratio of updateHead() calls that change head", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "queued_attestations" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "validated_attestation_datas" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 531, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "beacon_fork_choice_votes_count", + "legendFormat": "votes", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "beacon_fork_choice_queued_attestations_count", + "hide": false, + "legendFormat": "queued_attestations", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "beacon_fork_choice_validated_attestation_datas_count", + "hide": false, + "legendFormat": "validated_attestation_datas", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "beacon_fork_choice_balances_length", + "hide": false, + "legendFormat": "balances_length", + "range": true, + "refId": "D" + } + ], + "title": "Forkchoice Internal Data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 533, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "beacon_fork_choice_nodes_count", + "legendFormat": "nodes", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "beacon_fork_choice_indices_count", + "hide": false, + "legendFormat": "indices", + "range": true, + "refId": "B" + } + ], + "title": "Proto Array", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 587, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(beacon_fork_choice_compute_deltas_seconds_sum[$rate_interval])\n/\nrate(beacon_fork_choice_compute_deltas_seconds_count[$rate_interval])", + "instant": false, + "legendFormat": "duration", + "range": true, + "refId": "A" + } + ], + "title": "computeDeltas() avg time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [] + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "deltas" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 588, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "beacon_fork_choice_compute_deltas_deltas_count", + "instant": false, + "legendFormat": "deltas", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "beacon_fork_choice_compute_deltas_zero_deltas_count", + "hide": false, + "instant": false, + "legendFormat": "zero_deltas", + "range": true, + "refId": "B" + } + ], + "title": "computeDeltas() - deltas result", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 33 + }, + "id": 589, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "beacon_fork_choice_compute_deltas_equivocating_validators_count", + "instant": false, + "legendFormat": "equivocating_validators", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "beacon_fork_choice_compute_deltas_old_inactive_validators_count", + "hide": false, + "instant": false, + "legendFormat": "old_inactive_validators", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "beacon_fork_choice_compute_deltas_new_inactive_validators_count", + "hide": false, + "instant": false, + "legendFormat": "new_inactive_validators", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "beacon_fork_choice_compute_deltas_unchanged_vote_validators_count", + "hide": false, + "instant": false, + "legendFormat": "unchanged_vote_validators", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "beacon_fork_choice_compute_deltas_new_vote_validators_count", + "hide": false, + "instant": false, + "legendFormat": "new_vote_validators", + "range": true, + "refId": "E" + } + ], + "title": "computeDeltas() - validators result", + "type": "timeseries" + } + ], + "refresh": "10s", + "schemaVersion": 39, + "tags": [ + "lodestar" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "auto": true, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": false, + "text": "1h", + "value": "1h" + }, + "hide": 0, + "label": "rate() interval", + "name": "rate_interval", + "options": [ + { + "selected": false, + "text": "auto", + "value": "$__auto_interval_rate_interval" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": true, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "queryValue": "", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_local" + }, + "filters": [ + { + "condition": "", + "key": "instance", + "operator": "=", + "value": "unstable-super" + } + ], + "hide": 0, + "name": "Filters", + "skipUrlSync": false, + "type": "adhoc" + }, + { + "description": "Job name used in Prometheus config to scrape Beacon node", + "hide": 2, + "label": "Beacon node job name", + "name": "beacon_job", + "query": "${VAR_BEACON_JOB}", + "skipUrlSync": false, + "type": "constant" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "utc", + "title": "Lodestar - fork choice", + "uid": "lodestar_fork_choice", + "version": 2, + "weekStart": "monday" +} diff --git a/dashboards/lodestar_libp2p.json b/dashboards/lodestar_libp2p.json index 3e93c2eb7606..e242d31c7bbe 100644 --- a/dashboards/lodestar_libp2p.json +++ b/dashboards/lodestar_libp2p.json @@ -214,7 +214,7 @@ "x": 12, "y": 1 }, - "id": 28, + "id": 40, "options": { "legend": { "calcs": [], @@ -223,7 +223,7 @@ "showLegend": true }, "tooltip": { - "mode": "single", + "mode": "multi", "sort": "none" } }, @@ -233,14 +233,28 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "exemplar": false, - "expr": "libp2p_connection_manager_protocol_streams_per_connection_90th_percentile", - "interval": "", - "legendFormat": "{{protocol}}", + "editorMode": "code", + "expr": "libp2p_tcp_inbound_connections_total", + "instant": false, + "legendFormat": "tcp", + "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "libp2p_quic_inbound_connections_total", + "hide": false, + "instant": false, + "legendFormat": "quic", + "range": true, + "refId": "B" } ], - "title": "Streams Per Connection (90th percentile)", + "title": "Inbound Connections by Transport", "type": "timeseries" }, { @@ -367,8 +381,7 @@ "mode": "off" } }, - "mappings": [], - "unit": "none" + "mappings": [] }, "overrides": [] }, @@ -378,7 +391,7 @@ "x": 12, "y": 9 }, - "id": 30, + "id": 28, "options": { "legend": { "calcs": [], @@ -397,16 +410,14 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", "exemplar": false, - "expr": "libp2p_protocol_streams_total", + "expr": "libp2p_connection_manager_protocol_streams_per_connection_90th_percentile", "interval": "", "legendFormat": "{{protocol}}", - "range": true, "refId": "A" } ], - "title": "Total Streams", + "title": "Streams Per Connection (90th percentile)", "type": "timeseries" }, { @@ -493,6 +504,90 @@ "title": "Total Bandwidth", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "libp2p_protocol_streams_total", + "interval": "", + "legendFormat": "{{protocol}}", + "range": true, + "refId": "A" + } + ], + "title": "Total Streams", + "type": "timeseries" + }, { "collapsed": false, "gridPos": { diff --git a/dashboards/lodestar_peerdas.json b/dashboards/lodestar_peerdas.json index 32cc741baccd..92f86bc8cfab 100644 --- a/dashboards/lodestar_peerdas.json +++ b/dashboards/lodestar_peerdas.json @@ -21,6 +21,12 @@ "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, "type": "dashboard" } ] @@ -46,6 +52,7 @@ "url": "" } ], + "liveNow": false, "panels": [ { "collapsed": false, @@ -55,9 +62,9 @@ "x": 0, "y": 0 }, - "id": 61, + "id": 100, "panels": [], - "title": "Network", + "title": "Health Overview", "type": "row" }, { @@ -65,14 +72,175 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "Time elapsed between data column sidecar slot time and the time data column received", + "description": "Total number of custody groups", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "thresholds" + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 5, + "x": 0, + "y": 1 + }, + "id": 36, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "8.5.16", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "beacon_custody_groups", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Custody groups", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of custody data columns expected to have but are missing currently.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 5, + "x": 5, + "y": 1 + }, + "id": 101, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.16", + "targets": [ + { + "expr": "lodestar_data_columns_missing_custody_columns_count", + "legendFormat": "missing", + "refId": "A" + } + ], + "title": "Missing Custody Columns", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total number of backfilled custody groups", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "thresholds" + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 5, + "x": 10, + "y": 1 + }, + "id": 53, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "8.5.16", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "beacon_custody_groups_backfilled", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Backfilled custody groups", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Average number of data column sidecars per slot that were already received via another source per slot (ie scaled x12).", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -86,7 +254,6 @@ "tooltip": false, "viz": false }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -108,10 +275,85 @@ "overrides": [] }, "gridPos": { - "h": 6, + "h": 7, + "w": 9, + "x": 15, + "y": 1 + }, + "id": 103, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "expr": "rate(lodestar_data_column_sidecar_already_added[$rate_interval]) * 12", + "legendFormat": "duplicates", + "refId": "A" + } + ], + "title": "Duplicate Columns per slot", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Average time elapsed between data column's slot time and when it is received", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, "w": 12, "x": 0, - "y": 1 + "y": 8 }, "id": 71, "options": { @@ -133,15 +375,15 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(lodestar_data_column_elapsed_time_till_received_seconds_sum{source=\"gossip\"}[$rate_interval])\n/\nrate(lodestar_data_column_elapsed_time_till_received_seconds_count{source=\"gossip\"}[$rate_interval])", + "expr": "sum by (receivedOrder) (rate(lodestar_data_column_elapsed_time_till_received_seconds_sum[$rate_interval]))\n/\nsum by (receivedOrder) (rate(lodestar_data_column_elapsed_time_till_received_seconds_count[$rate_interval]))", "instant": false, "interval": "", - "legendFormat": "{{source}}", + "legendFormat": "{{receivedOrder}}", "range": true, "refId": "A" } ], - "title": "Data columns receiving delay", + "title": "Receiving delay (avg)", "type": "timeseries" }, { @@ -149,7 +391,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "Time elapsed between data column sidecar slot time and the time data column received", + "description": "Heatmap of data column receiving latency, (columns/second) received within each latency bucket.", "fieldConfig": { "defaults": { "custom": { @@ -166,17 +408,17 @@ "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 12, - "y": 1 + "y": 8 }, "id": 62, "options": { "calculate": false, - "cellGap": 2, + "cellGap": 1, "color": { - "exponent": 0.5, + "exponent": 0.15, "fill": "dark-orange", "mode": "scheme", "reverse": false, @@ -198,16 +440,17 @@ }, "tooltip": { "mode": "single", - "showColorScale": false, - "yHistogram": false + "show": true, + "showColorScale": true, + "yHistogram": true }, "yAxis": { "axisPlacement": "left", "reverse": false, - "unit": "short" + "unit": "s" } }, - "pluginVersion": "10.4.1", + "pluginVersion": "8.5.16", "targets": [ { "datasource": { @@ -215,7 +458,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "12*rate(lodestar_data_column_elapsed_time_till_received_seconds_bucket{source=\"gossip\"}[$rate_interval])", + "expr": "sum by (le) (rate(lodestar_data_column_elapsed_time_till_received_seconds_bucket[$rate_interval]))", "format": "heatmap", "instant": false, "interval": "", @@ -224,25 +467,37 @@ "refId": "A" } ], - "title": "Data columns receiving delay", + "title": "Receiving delay (heatmap)", "type": "heatmap" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 61, + "panels": [], + "title": "Gossip", + "type": "row" + }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "Number of received data columns by source", + "description": "Average number of data columns received per slot, by source, ( ie scaled x12).", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "", + "axisLabel": "columns / slot (scaled)", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", @@ -253,7 +508,6 @@ "tooltip": false, "viz": false }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -275,10 +529,10 @@ "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 0, - "y": 7 + "y": 17 }, "id": 63, "options": { @@ -308,37 +562,24 @@ "refId": "A" } ], - "title": "Data columns source", + "title": "Data columns received per slot (by source)", "type": "timeseries" }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 13 - }, - "id": 16, - "panels": [], - "title": "Data columns gossip verification", - "type": "row" - }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "description": "Number of data column sidecars processed per slot, by outcome (ie scaled x12).", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "", + "axisLabel": "columns / slot (scaled)", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", @@ -349,7 +590,6 @@ "tooltip": false, "viz": false }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -360,21 +600,23 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, - "mappings": [] + "decimals": 2, + "mappings": [], + "unit": "none" }, "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 12, - "x": 0, - "y": 14 + "x": 12, + "y": 17 }, "id": 13, "interval": "1m", @@ -401,11 +643,11 @@ }, "editorMode": "code", "exemplar": false, - "expr": "avg_over_time(\n (rate(beacon_data_column_sidecar_processing_requests_total[1m]) * 12 > 0)[$rate_interval:]\n)", + "expr": "rate(beacon_data_column_sidecar_processing_requests_total[$rate_interval]) * 12", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "befor verification", + "legendFormat": "total", "range": true, "refId": "A" }, @@ -415,11 +657,11 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "avg_over_time(\n (rate(beacon_data_column_sidecar_processing_successes_total[1m]) * 12 > 0)[$rate_interval:]\n)", + "expr": "rate(beacon_data_column_sidecar_processing_successes_total[$rate_interval]) * 12", "hide": false, "instant": false, "interval": "", - "legendFormat": "after verification", + "legendFormat": "verified", "range": true, "refId": "B" }, @@ -429,100 +671,30 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "avg_over_time(\n (rate(beacon_data_column_sidecar_processing_requests_total[1m]) * 12 > 0)[$rate_interval:]\n)\n-\navg_over_time(\n (rate(beacon_data_column_sidecar_processing_successes_total[1m]) * 12 > 0)[$rate_interval:]\n) != 0", + "expr": "rate(beacon_data_column_sidecar_processing_skip_total[$rate_interval]) * 12", "hide": false, "instant": false, "interval": "", - "legendFormat": "failures", + "legendFormat": "skipped", "range": true, "refId": "C" - } - ], - "title": "Data column sidecars gossip verification", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [] - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 14 - }, - "id": 70, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(lodestar_gossip_validation_error_total{topic=\"data_column_sidecar\"}[$rate_interval])", + "expr": "(\n rate(beacon_data_column_sidecar_processing_requests_total[$rate_interval])\n -\n rate(beacon_data_column_sidecar_processing_successes_total[$rate_interval])\n -\n rate(beacon_data_column_sidecar_processing_skip_total[$rate_interval])\n) * 12", + "hide": false, "instant": false, "interval": "", - "legendFormat": "{{error}}", + "legendFormat": "failed", "range": true, - "refId": "A" + "refId": "D" } ], - "title": "Gossip DataColumnSidecar Error", + "title": "Data column sidecars processing(verification) (per slot)", "type": "timeseries" }, { @@ -530,14 +702,13 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "Time elapsed between data column received and data column validated", + "description": "p99 time to verify a gossip data column sidecar", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -551,7 +722,6 @@ "tooltip": false, "viz": false }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -568,17 +738,19 @@ "mode": "off" } }, - "mappings": [] + "mappings": [], + "unit": "s" }, "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 0, - "y": 20 + "y": 25 }, - "id": 69, + "id": 35, + "interval": "30s", "options": { "legend": { "calcs": [], @@ -591,6 +763,7 @@ "sort": "none" } }, + "pluginVersion": "12.0.1", "targets": [ { "datasource": { @@ -598,14 +771,15 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(lodestar_gossip_data_column_received_to_gossip_validate_seconds_bucket[$rate_interval])", + "expr": "histogram_quantile(0.99, rate(beacon_data_column_sidecar_gossip_verification_seconds_bucket [$rate_interval]))\n", "instant": false, - "legendFormat": "__auto", + "interval": "", + "legendFormat": "p99", "range": true, "refId": "A" } ], - "title": "Data column recv to gossip validation delay", + "title": "Gossip verification runtime (p99)", "type": "timeseries" }, { @@ -613,16 +787,16 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "description": "Validation errors while processing gossip sidecars (errors / slot) (ie scaled x12)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "", + "axisLabel": "errors / slot (scaled)", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", @@ -633,7 +807,6 @@ "tooltip": false, "viz": false }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -651,32 +824,29 @@ } }, "mappings": [], - "unit": "s" + "unit": "none" }, "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 12, - "y": 20 + "y": 25 }, - "id": 35, - "interval": "30s", + "id": 70, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": false + "showLegend": true }, "tooltip": { - "hideZeros": false, "mode": "single", "sort": "none" } }, - "pluginVersion": "12.0.1", "targets": [ { "datasource": { @@ -684,15 +854,15 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, rate(beacon_data_column_sidecar_gossip_verification_seconds_bucket [$rate_interval]))\n", + "expr": "rate(lodestar_gossip_validation_error_total{topic=\"data_column_sidecar\"}[$rate_interval]) * 12", "instant": false, "interval": "", - "legendFormat": "{{instance}}", + "legendFormat": "{{error}}", "range": true, "refId": "A" } ], - "title": "Runtime of data column sidecars gossip verification", + "title": "Gossip DataColumnSidecar validation errors (per slot)", "type": "timeseries" }, { @@ -700,6 +870,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "description": "Heatmap of gossip data column sidecar verification latency. Shows (sidecars/sec) verified within each latency bucket over time.", "fieldConfig": { "defaults": { "custom": { @@ -716,10 +887,10 @@ "overrides": [] }, "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 26 + "h": 9, + "w": 24, + "x": 0, + "y": 33 }, "id": 72, "interval": "30s", @@ -727,7 +898,7 @@ "calculate": false, "cellGap": 1, "color": { - "exponent": 0.5, + "exponent": 0.7, "fill": "dark-orange", "mode": "scheme", "reverse": false, @@ -742,22 +913,24 @@ "le": 1e-9 }, "legend": { - "show": true + "show": false }, "rowsFrame": { "layout": "auto" }, "tooltip": { "mode": "single", - "showColorScale": false, - "yHistogram": false + "show": true, + "showColorScale": true, + "yHistogram": true }, "yAxis": { "axisPlacement": "left", - "reverse": false + "reverse": false, + "unit": "s" } }, - "pluginVersion": "10.4.1", + "pluginVersion": "8.5.16", "targets": [ { "datasource": { @@ -769,12 +942,12 @@ "format": "heatmap", "instant": false, "interval": "", - "legendFormat": "{{instance}}", + "legendFormat": "{{le}}", "range": true, "refId": "A" } ], - "title": "Runtime of data column sidecars gossip verification", + "title": "Gossip verification runtime (heatmap)", "type": "heatmap" }, { @@ -783,11 +956,11 @@ "h": 1, "w": 24, "x": 0, - "y": 32 + "y": 42 }, "id": 58, "panels": [], - "title": "engine_getBlobsV2", + "title": "Engine", "type": "row" }, { @@ -795,14 +968,13 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "Total number of engine_getBlobsV2 requests sent and responses received", + "description": "Rate of engine_getBlobsV2 requests sent and responses received (req/s)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -816,7 +988,6 @@ "tooltip": false, "viz": false }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -835,15 +1006,15 @@ }, "decimals": 0, "mappings": [], - "unit": "none" + "unit": "ops" }, "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 0, - "y": 33 + "y": 43 }, "id": 50, "interval": "30s", @@ -883,16 +1054,100 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(beacon_engine_getBlobsV2_responses_total[$rate_interval])", - "hide": false, - "instant": false, + "expr": "rate(beacon_engine_getBlobsV2_responses_total[$rate_interval])", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "responses", + "range": true, + "refId": "B" + } + ], + "title": "engine_getBlobsV2 requests & responses", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Duration of engine_getBlobsV2 requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 43 + }, + "id": 49, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, rate(beacon_engine_getBlobsV2_request_duration_seconds_bucket[$rate_interval]))", "interval": "", - "legendFormat": "responses", + "legendFormat": "p99", "range": true, - "refId": "B" + "refId": "A" } ], - "title": "engine_getBlobsV2 requests & responses", + "title": "Duration of engine_getBlobsV2 requests (p99)", "type": "timeseries" }, { @@ -900,14 +1155,13 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "Duration of engine_getBlobsV2 requests", + "description": "Percentage of engine_getBlobsV2 requests that did not receive a successful (not null) response", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -921,7 +1175,6 @@ "tooltip": false, "viz": false }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -938,18 +1191,20 @@ "mode": "off" } }, + "decimals": 1, "mappings": [], - "unit": "s" + "unit": "percent" }, "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 12, - "x": 12, - "y": 33 + "x": 0, + "y": 51 }, - "id": 49, + "id": 57, + "interval": "30s", "options": { "legend": { "calcs": [], @@ -971,14 +1226,17 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, rate(beacon_engine_getBlobsV2_request_duration_seconds_bucket[$rate_interval]))", + "exemplar": false, + "expr": "(rate(beacon_engine_getBlobsV2_requests_total[$rate_interval])-rate(beacon_engine_getBlobsV2_responses_total[$rate_interval])) / clamp_min(rate(beacon_engine_getBlobsV2_requests_total[$rate_interval]), 1)* 100", + "format": "time_series", "interval": "", - "legendFormat": "{{instance}}", + "intervalFactor": 1, "range": true, - "refId": "A" + "refId": "A", + "legendFormat": "unsuccessful %" } ], - "title": "Duration of engine_getBlobsV2 requests", + "title": "Unsuccessful engine_getBlobsV2 requests rate (%)", "type": "timeseries" }, { @@ -986,28 +1244,26 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "Percentage of engine_getBlobsV2 requests that did not receive a successful (not null) response", + "description": "Rate of engine API results per second, broken down by outcome", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "opacity", + "fillOpacity": 0, + "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1018,58 +1274,44 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, - "decimals": 0, "mappings": [], - "unit": "none" + "unit": "short" }, "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 12, - "x": 0, - "y": 39 + "x": 12, + "y": 51 }, - "id": 57, - "interval": "30s", + "id": 102, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": false + "showLegend": true }, "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" + "mode": "single", + "sort": "none" } }, - "pluginVersion": "12.0.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "(rate(beacon_engine_getBlobsV2_requests_total[$rate_interval])\n-\nrate(beacon_engine_getBlobsV2_responses_total[$rate_interval])) / rate(beacon_engine_getBlobsV2_requests_total[$rate_interval]) * 100", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}", - "range": true, + "expr": "rate(lodestar_data_column_engine_result_total[$rate_interval])", + "legendFormat": "{{result}}", "refId": "A" } ], - "title": "Unsuccessful engine_getBlobsV2 requests rate (%)", + "title": "Engine API Results (per second)", "type": "timeseries" }, { @@ -1078,7 +1320,7 @@ "h": 1, "w": 24, "x": 0, - "y": 45 + "y": 59 }, "id": 60, "panels": [], @@ -1097,7 +1339,6 @@ "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -1111,7 +1352,6 @@ "tooltip": false, "viz": false }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1134,10 +1374,10 @@ "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 0, - "y": 46 + "y": 60 }, "id": 21, "options": { @@ -1161,14 +1401,14 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, rate(beacon_data_column_sidecar_computation_seconds_bucket[$rate_interval]))\n", + "expr": "histogram_quantile(0.99, sum by (le) (rate(beacon_data_column_sidecar_computation_seconds_bucket[$rate_interval])))", "interval": "", - "legendFormat": "{{instance}}", + "legendFormat": "p99", "range": true, "refId": "A" } ], - "title": "Time taken to compute data column sidecar", + "title": "Data column sidecar computation time (p99)", "type": "timeseries" }, { @@ -1182,7 +1422,6 @@ "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -1196,7 +1435,6 @@ "tooltip": false, "viz": false }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1219,10 +1457,10 @@ "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 12, - "y": 46 + "y": 60 }, "id": 22, "options": { @@ -1247,12 +1485,12 @@ }, "editorMode": "code", "expr": "histogram_quantile(0.99, rate(beacon_data_column_sidecar_inclusion_proof_verification_seconds_bucket[$rate_interval]))", - "legendFormat": "{{instance}}", + "legendFormat": "p99", "range": true, "refId": "A" } ], - "title": "Time taken to verify data column sidecar inclusion proof", + "title": "Time taken to verify data column sidecar inclusion proof (p99)", "type": "timeseries" }, { @@ -1266,7 +1504,6 @@ "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -1280,7 +1517,6 @@ "tooltip": false, "viz": false }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1303,10 +1539,10 @@ "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 0, - "y": 52 + "y": 68 }, "id": 24, "options": { @@ -1332,135 +1568,21 @@ "editorMode": "code", "expr": "histogram_quantile(0.99, rate(beacon_kzg_verification_data_column_batch_seconds_bucket[$rate_interval]))", "interval": "", - "legendFormat": "{{instance}}", + "legendFormat": "p99", "range": true, "refId": "A" } ], - "title": "Runtime of batched data column kzg verification", + "title": "Runtime of batched data column kzg verification (p99)", "type": "timeseries" }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Total number of custody groups", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "green", - "mode": "thresholds" - }, - "mappings": [] - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 52 - }, - "id": 36, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "beacon_target_custody_group_count", - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Custody groups", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Total number of backfilled custody groups", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "green", - "mode": "thresholds" - }, - "mappings": [] - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 52 - }, - "id": 53, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "beacon_custody_groups_backfilled", - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Backfilled custody groups", - "type": "stat" - }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 58 + "y": 76 }, "id": 59, "panels": [], @@ -1478,10 +1600,9 @@ "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "", + "axisLabel": "columns/s", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", @@ -1492,7 +1613,6 @@ "tooltip": false, "viz": false }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1510,15 +1630,16 @@ } }, "decimals": 0, - "mappings": [] + "mappings": [], + "unit": "none" }, "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 0, - "y": 59 + "y": 77 }, "id": 14, "interval": "30s", @@ -1543,7 +1664,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "avg_over_time(\n (rate(beacon_data_availability_reconstructed_columns_total[1m]) * 12 > 0)[$rate_interval:]\n)", + "expr": "rate(beacon_data_availability_reconstructed_columns_total[$rate_interval])", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -1552,7 +1673,7 @@ "refId": "A" } ], - "title": "Total count of reconstructed columns", + "title": "Rate of column reconstruction", "type": "timeseries" }, { @@ -1566,7 +1687,6 @@ "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -1580,7 +1700,6 @@ "tooltip": false, "viz": false }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1603,10 +1722,10 @@ "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 12, - "y": 59 + "y": 77 }, "id": 25, "interval": "30s", @@ -1631,13 +1750,13 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, rate(beacon_data_availability_reconstruction_time_seconds_bucket[$rate_interval]))", - "legendFormat": "{{job}}", + "expr": "histogram_quantile(0.99,sum by (le) (rate(beacon_data_availability_reconstruction_time_seconds_bucket[$rate_interval])))", + "legendFormat": "p99", "range": true, "refId": "A" } ], - "title": "Time taken to reconstruct columns", + "title": "Time taken to reconstruct columns (p99)", "type": "timeseries" }, { @@ -1652,7 +1771,6 @@ "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -1666,7 +1784,6 @@ "tooltip": false, "viz": false }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1683,15 +1800,16 @@ "mode": "off" } }, - "mappings": [] + "mappings": [], + "unit": "short" }, "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 0, - "y": 65 + "y": 85 }, "id": 64, "options": { @@ -1713,9 +1831,9 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(lodestar_data_columns_in_custody_before_reconstruction[$rate_interval]) * 12", + "expr": "lodestar_data_columns_in_custody_before_reconstruction", "instant": false, - "legendFormat": "__auto", + "legendFormat": "{{job}}", "range": true, "refId": "A" } @@ -1728,14 +1846,13 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "Data column sidecars reconstruction result ", + "description": "Reconstruction outcomes per slot (ie rate scaled x12), grouped by result code.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -1749,7 +1866,6 @@ "tooltip": false, "viz": false }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1760,7 +1876,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" @@ -1771,10 +1887,10 @@ "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 12, - "y": 65 + "y": 85 }, "id": 67, "options": { @@ -1804,90 +1920,7 @@ "refId": "A" } ], - "title": "Reconstruction result", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Time elapsed between block slot time and the time data column sidecar reconstructed", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [] - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 71 - }, - "id": 68, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "rate(lodestar_data_column_sidecar_elapsed_time_till_reconstructed_seconds_sum[$rate_interval])\n/\nrate(lodestar_data_column_sidecar_elapsed_time_till_reconstructed_seconds_count[$rate_interval])", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Data columns reconstruction delay", + "title": "Reconstruction result (per slot)", "type": "timeseries" } ], diff --git a/docs/pages/run/beacon-management/networking.md b/docs/pages/run/beacon-management/networking.md index 7532fb043a7a..57b0ba9e9007 100644 --- a/docs/pages/run/beacon-management/networking.md +++ b/docs/pages/run/beacon-management/networking.md @@ -10,18 +10,23 @@ Some of the important Lodestar flags related to networking are: - [`--listenAddress`](./beacon-cli#--listenaddress) - [`--port`](./beacon-cli#--port) - [`--discoveryPort`](./beacon-cli#--discoveryport) +- [`--quicPort`](./beacon-cli#--quicport) - [`--listenAddress6`](./beacon-cli#--listenaddress6) - [`--port6`](./beacon-cli#--port6) - [`--discoveryPort6`](./beacon-cli#--discoveryport6) +- [`--quicPort6`](./beacon-cli#--quicport6) +- [`--quic`](./beacon-cli#--quic) - [`--bootnodes`](./beacon-cli#--bootnodes) - [`--subscribeAllSubnets`](./beacon-cli#--subscribeallsubnets) - [`--disablePeerScoring`](./beacon-cli#--disablepeerscoring) - [`--enr.ip`](./beacon-cli#--enrip) - [`--enr.tcp`](./beacon-cli#--enrtcp) - [`--enr.udp`](./beacon-cli#--enrudp) +- [`--enr.quic`](./beacon-cli#--enrquic) - [`--enr.ip6`](./beacon-cli#--enrip6) - [`--enr.tcp6`](./beacon-cli#--enrtcp6) - [`--enr.udp6`](./beacon-cli#--enrudp6) +- [`--enr.quic6`](./beacon-cli#--enrquic6) - [`--nat`](./beacon-cli#--nat) - [`--private`](./beacon-cli#--private) @@ -79,6 +84,38 @@ Libp2p is a modular and extensible network stack that serves as the data transpo Libp2p operates at the lower levels of the OSI model, particularly at the Transport and Network layers. Libp2p supports both TCP and UDP protocols for establishing connections and data transmission. Combined with libp2p's modular design it can integrate with various networking technologies to facilitating both routing and addressing. +### QUIC Transport + +Lodestar supports [QUIC](https://datatracker.ietf.org/doc/html/rfc9000) as a transport alongside TCP. QUIC is a UDP-based transport that provides built-in encryption (TLS 1.3), multiplexed streams, and faster connection establishment compared to TCP. QUIC is disabled by default and can be enabled with the `--quic` flag. When enabled, Lodestar will prefer QUIC when dialing peers that advertise QUIC support. + +#### Enabling QUIC + +To enable QUIC transport: + +```bash +lodestar beacon --quic +``` + +When QUIC is enabled, the node will listen on an additional UDP port and advertise QUIC support in its ENR. + +#### Port Configuration + +With QUIC enabled, Lodestar uses three ports for P2P networking: + +| Port | Protocol | Default | Purpose | +| ----------------- | -------- | ----------------------- | ------------------------- | +| `--port` | TCP | 9000 | TCP transport for libp2p | +| `--discoveryPort` | UDP | 9000 (same as `--port`) | discv5 peer discovery | +| `--quicPort` | UDP | 9001 (`--port` + 1) | QUIC transport for libp2p | + +Note that `--discoveryPort` and `--quicPort` are both UDP but must use different ports. Lodestar will error on startup if they collide. + +For IPv6 dual-stack, equivalent flags are available: `--port6`, `--discoveryPort6`, and `--quicPort6`. + +#### ENR Advertisement + +When QUIC is enabled, the node's ENR automatically includes QUIC port information so that other nodes can discover and connect via QUIC. The ENR fields can be overridden with `--enr.quic` and `--enr.quic6`. + ## Firewall Management If your setup is behind a firewall there are a few ports that will need to be opened to allow for P2P discovery and communication. There are also some ports that need to be protected to prevent unwanted access or DDOS attacks on your node. @@ -86,7 +123,8 @@ If your setup is behind a firewall there are a few ports that will need to be op Ports that must be opened: - 30303/TCP+UDP - Execution layer P2P communication port -- 9000/TCP+UDP - Beacon node IPv4 and IPv6 P2P communication port +- 9000/TCP+UDP - Beacon node P2P communication (TCP transport + discv5 discovery) +- 9001/UDP - Beacon node QUIC transport (only if `--quic` is enabled) Ports that must be protected: diff --git a/docs/pages/run/beacon-management/syncing.md b/docs/pages/run/beacon-management/syncing.md index 7dd05c4b4338..024a5a30c332 100644 --- a/docs/pages/run/beacon-management/syncing.md +++ b/docs/pages/run/beacon-management/syncing.md @@ -32,7 +32,7 @@ This is another version of checkpoint sync that allows a node that has not been ## Syncing Lodestar -The implementation of the different syncing styles in Lodestar are actually one of two types under the hood, range sync and unknown-parent sync. Range sync is used when the start point of syncing is known. In the case of historical and checkpoint sync the starting points are well defined, genesis and the last finalized epoch boundary. Snapshot sync is not supported by Lodestar. If the starting point for sync is not known Lodestar must first determine where the starting point is. While the discussion about how that happens is out of scope for this document, the gist is that the beacon node will listen to gossipsub for blocks being broadcast on the network. It will also request [`MetaData`](https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/p2p-interface.md#getmetadata) from its peers and use that to start requesting the correct blocks from the network. +The implementation of the different syncing styles in Lodestar are actually one of two types under the hood, range sync and unknown-parent sync. Range sync is used when the start point of syncing is known. In the case of historical and checkpoint sync the starting points are well defined, genesis and the last finalized epoch boundary. Snapshot sync is not supported by Lodestar. If the starting point for sync is not known Lodestar must first determine where the starting point is. While the discussion about how that happens is out of scope for this document, the gist is that the beacon node will listen to gossipsub for blocks being broadcast on the network. It will also request [`MetaData`](https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/p2p-interface.md#getmetadata-v1) from its peers and use that to start requesting the correct blocks from the network. There are several flags that can be used to configure the sync process. diff --git a/docs/static/images/apple-touch-icon.png b/docs/static/images/apple-touch-icon.png new file mode 100644 index 000000000000..07e5a1e0b994 Binary files /dev/null and b/docs/static/images/apple-touch-icon.png differ diff --git a/docs/static/images/favicon-192x192.png b/docs/static/images/favicon-192x192.png new file mode 100644 index 000000000000..116522e4ae43 Binary files /dev/null and b/docs/static/images/favicon-192x192.png differ diff --git a/docs/static/images/favicon-512x512.png b/docs/static/images/favicon-512x512.png new file mode 100644 index 000000000000..da5a6698ec8b Binary files /dev/null and b/docs/static/images/favicon-512x512.png differ diff --git a/docs/static/images/favicon.ico b/docs/static/images/favicon.ico index 4e34d94c437f..ab0809ad3eaf 100644 Binary files a/docs/static/images/favicon.ico and b/docs/static/images/favicon.ico differ diff --git a/lerna.json b/lerna.json index e6666ade7b69..55881c183c53 100644 --- a/lerna.json +++ b/lerna.json @@ -2,7 +2,7 @@ "packages": [ "packages/*" ], - "version": "1.40.0", + "version": "1.41.0", "stream": true, "command": { "version": { diff --git a/package.json b/package.json index ef1622ec947c..6d560f3cd853 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "engines": { "node": "^24.13.0" }, - "packageManager": "pnpm@10.24.0", + "packageManager": "pnpm@10.24.0+sha512.01ff8ae71b4419903b65c60fb2dc9d34cf8bb6e06d03bde112ef38f7a34d6904c424ba66bea5cdcf12890230bf39f9580473140ed9c946fef328b6e5238a345a", "workspaces": [ "packages/*" ], @@ -53,6 +53,7 @@ "@lodestar/params": "workspace:", "@types/node": "^24.10.1", "@types/react": "^19.1.12", + "@typescript/native-preview": "7.0.0-dev.20260303.1", "@vitest/browser": "catalog:", "@vitest/browser-playwright": "catalog:", "@vitest/coverage-v8": "catalog:", @@ -64,7 +65,7 @@ "https-browserify": "^1.0.0", "inquirer": "^9.1.5", "jsdom": "^23.0.1", - "libp2p": "2.9.0", + "libp2p": "3.1.6", "node-gyp": "^9.4.0", "npm-run-all": "^4.1.5", "path-browserify": "^1.0.1", @@ -77,8 +78,8 @@ "stream-browserify": "^3.0.0", "stream-http": "^3.2.0", "supertest": "^6.3.3", - "ts-node": "^10.9.2", - "typescript": "^5.9.2", + "ts-node": "^11.0.0-beta.1", + "typescript": "^6.0.1-rc", "vite": "^6.0.11", "vite-plugin-dts": "^4.5.4", "vite-plugin-node-polyfills": "^0.24.0", diff --git a/packages/api/package.json b/packages/api/package.json index 2b6b616151ac..465ddb8c6b5d 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.40.0", + "version": "1.41.0", "type": "module", "exports": { ".": { @@ -62,11 +62,11 @@ ], "scripts": { "clean": "rm -rf lib && rm -f *.tsbuildinfo", - "build": "tsc -p tsconfig.build.json", + "build": "tsgo -p tsconfig.build.json", "build:watch": "pnpm run build --watch", "build:release": "pnpm clean && pnpm run build", "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\"", - "check-types": "tsc", + "check-types": "tsgo", "lint": "biome check src/ test/", "lint:fix": "pnpm run lint --write", "test": "pnpm test:unit", @@ -87,7 +87,7 @@ "@types/eventsource": "^1.1.11", "@types/qs": "^6.9.7", "ajv": "^8.12.0", - "fastify": "^5.7.4" + "fastify": "^5.8.1" }, "keywords": [ "ethereum", diff --git a/packages/api/src/beacon/routes/events.ts b/packages/api/src/beacon/routes/events.ts index d1ea5763dfcf..5841f3dfd7f7 100644 --- a/packages/api/src/beacon/routes/events.ts +++ b/packages/api/src/beacon/routes/events.ts @@ -89,7 +89,7 @@ export enum EventType { blobSidecar = "blob_sidecar", /** The node has received a valid DataColumnSidecar (from P2P or API) */ dataColumnSidecar = "data_column_sidecar", - /** The node has received a valid execution payload envelope and it has been imported (ePBS) */ + /** The node has verified that the execution payload and blobs for a block are available */ executionPayloadAvailable = "execution_payload_available", /** The node has received a valid execution payload bid (ePBS) */ executionPayloadBid = "execution_payload_bid", diff --git a/packages/api/src/beacon/routes/lodestar.ts b/packages/api/src/beacon/routes/lodestar.ts index 5236d3d40e38..ea894391f8f2 100644 --- a/packages/api/src/beacon/routes/lodestar.ts +++ b/packages/api/src/beacon/routes/lodestar.ts @@ -547,6 +547,8 @@ export function getDefinitions(_config: ChainForkConfig): RouteDefinitions 0) { + const timer = metrics?.peerDas.dataColumnSidecarComputationTime.startTimer(); const cellsAndProofs = cachedResult.cells.map((rowCells, rowIndex) => ({ cells: rowCells, proofs: cachedResult.blobsBundle.proofs.slice( @@ -705,14 +711,42 @@ export function getBeaconBlockApi({ })); dataColumnSidecars = getDataColumnSidecarsForGloas(slot, envelope.beaconBlockRoot, cellsAndProofs); + timer?.(); } } else { // TODO GLOAS: will this api be used by builders or only for self-building? } - // Import envelope locally: verifies signature via state transition, notifies EL, - // updates fork choice payload status (PENDING -> FULL), persists envelope, and emits SSE event. - await chain.importExecutionPayloadEnvelope(signedExecutionPayloadEnvelope); + // If called near a slot boundary (e.g. late in slot N-1), hold briefly so gossip aligns with slot N. + const msToBlockSlot = computeTimeAtSlot(config, slot, chain.genesisTime) * 1000 - Date.now(); + if (msToBlockSlot <= MAX_API_CLOCK_DISPARITY_MS && msToBlockSlot > 0) { + await sleep(msToBlockSlot); + } + + // TODO GLOAS: if block and payload are submitted in parallel, payloadInput may not yet exist. + // A queuing mechanism is needed to handle this case. See https://github.com/ChainSafe/lodestar/issues/8915 + const payloadInput = chain.seenPayloadEnvelopeInputCache.get(blockRootHex); + if (!payloadInput) { + throw new ApiError(404, `PayloadEnvelopeInput not found for block root ${blockRootHex}`); + } + + payloadInput.addPayloadEnvelope({ + envelope: signedExecutionPayloadEnvelope, + source: PayloadEnvelopeInputSource.api, + seenTimestampSec, + peerIdStr: undefined, + }); + + if (dataColumnSidecars.length > 0) { + for (const columnSidecar of dataColumnSidecars) { + payloadInput.addColumn({ + columnSidecar, + source: PayloadEnvelopeInputSource.api, + seenTimestampSec, + peerIdStr: undefined, + }); + } + } const valLogMeta = { slot, @@ -722,23 +756,19 @@ export function getBeaconBlockApi({ dataColumns: dataColumnSidecars.length, }; - // If called near a slot boundary (e.g. late in slot N-1), hold briefly so gossip aligns with slot N. - const msToBlockSlot = computeTimeAtSlot(config, slot, chain.genesisTime) * 1000 - Date.now(); - if (msToBlockSlot <= MAX_API_CLOCK_DISPARITY_MS && msToBlockSlot > 0) { - await sleep(msToBlockSlot); - } - const delaySec = seenTimestampSec - computeTimeAtSlot(config, slot, chain.genesisTime); metrics?.gossipExecutionPayloadEnvelope.elapsedTimeTillReceived.observe({source: OpSource.api}, delaySec); + chain.validatorMonitor?.registerExecutionPayloadEnvelope(OpSource.api, delaySec, signedExecutionPayloadEnvelope); chain.logger.info("Publishing execution payload envelope", valLogMeta); - // Publish envelope and data columns const publishPromises = [ // Gossip the signed execution payload envelope first () => network.publishSignedExecutionPayloadEnvelope(signedExecutionPayloadEnvelope), // For self-builds, publish all data column sidecars ...dataColumnSidecars.map((dataColumnSidecar) => () => network.publishDataColumnSidecar(dataColumnSidecar)), + // Import execution payload. Signature already verified above + () => chain.processExecutionPayload(payloadInput, {validSignature: true}), ]; const sentPeersArr = await promiseAllMaybeAsync(publishPromises); @@ -825,11 +855,18 @@ export function getBeaconBlockApi({ } const indicesToReconstruct = indices ?? Array.from({length: blobCount}, (_, i) => i); + + const timer = metrics?.recoverBlobSidecars.reconstructionTime.startTimer(); const blobs = await reconstructBlobs(dataColumnSidecars, indicesToReconstruct); + timer?.(); + metrics?.recoverBlobSidecars.blobsReconstructed.inc(indicesToReconstruct.length); + const signedBlockHeader = signedBlockToSignedHeader(config, block); data = await Promise.all( indicesToReconstruct.map(async (index, i) => { + // record per column computation time + const compTimer = metrics?.peerDas.dataColumnSidecarComputationTime.startTimer(); // Reconstruct blob sidecar from blob const kzgCommitment = blobKzgCommitments[index]; const blob = blobs[i]; // Use i since blobs only contains requested indices @@ -839,6 +876,7 @@ export function getBeaconBlockApi({ block.message.body, index ); + compTimer?.(); return {index, blob, kzgCommitment, kzgProof, signedBlockHeader, kzgCommitmentInclusionProof}; }) ); @@ -920,7 +958,10 @@ export function getBeaconBlockApi({ indicesToReconstruct = Array.from({length: blobCount}, (_, i) => i); } + const timer = metrics?.peerDas.dataColumnsReconstructionTime.startTimer(); blobs = await reconstructBlobs(dataColumnSidecars, indicesToReconstruct); + timer?.(); + metrics?.peerDas.reconstructedColumns.inc(indicesToReconstruct.length); } else { blobs = []; } diff --git a/packages/beacon-node/src/api/impl/beacon/state/index.ts b/packages/beacon-node/src/api/impl/beacon/state/index.ts index ddbf9e73907d..e8b3b92a5155 100644 --- a/packages/beacon-node/src/api/impl/beacon/state/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/state/index.ts @@ -95,14 +95,14 @@ export function getBeaconStateApi({ const {state, executionOptimistic, finalized} = await getState(stateId); const currentEpoch = getCurrentEpoch(state); const {validators, balances} = state; // Get the validators sub tree once for all the loop - const {pubkey2index} = chain; + const {pubkeyCache} = chain; const validatorResponses: routes.beacon.ValidatorResponse[] = []; if (validatorIds.length) { assertUniqueItems(validatorIds, "Duplicate validator IDs provided"); for (const id of validatorIds) { - const resp = getStateValidatorIndex(id, state, pubkey2index); + const resp = getStateValidatorIndex(id, state, pubkeyCache); if (resp.valid) { const validatorIndex = resp.validatorIndex; const validator = validators.getReadonly(validatorIndex); @@ -127,7 +127,7 @@ export function getBeaconStateApi({ if (statuses.length) { assertUniqueItems(statuses, "Duplicate statuses provided"); - const validatorsByStatus = filterStateValidatorsByStatus(statuses, state, pubkey2index, currentEpoch); + const validatorsByStatus = filterStateValidatorsByStatus(statuses, state, pubkeyCache, currentEpoch); return { data: validatorsByStatus, meta: {executionOptimistic, finalized}, @@ -154,7 +154,7 @@ export function getBeaconStateApi({ async postStateValidatorIdentities({stateId, validatorIds = []}) { const {state, executionOptimistic, finalized} = await getState(stateId); - const {pubkey2index} = chain; + const {pubkeyCache} = chain; let validatorIdentities: routes.beacon.ValidatorIdentities; @@ -163,7 +163,7 @@ export function getBeaconStateApi({ validatorIdentities = []; for (const id of validatorIds) { - const resp = getStateValidatorIndex(id, state, pubkey2index); + const resp = getStateValidatorIndex(id, state, pubkeyCache); if (resp.valid) { const index = resp.validatorIndex; const {pubkey, activationEpoch} = state.validators.getReadonly(index); @@ -187,9 +187,9 @@ export function getBeaconStateApi({ async getStateValidator({stateId, validatorId}) { const {state, executionOptimistic, finalized} = await getState(stateId); - const {pubkey2index} = chain; + const {pubkeyCache} = chain; - const resp = getStateValidatorIndex(validatorId, state, pubkey2index); + const resp = getStateValidatorIndex(validatorId, state, pubkeyCache); if (!resp.valid) { throw new ApiError(resp.code, resp.reason); } @@ -214,7 +214,7 @@ export function getBeaconStateApi({ const balances: routes.beacon.ValidatorBalance[] = []; for (const id of validatorIds) { - const resp = getStateValidatorIndex(id, state, chain.pubkey2index); + const resp = getStateValidatorIndex(id, state, chain.pubkeyCache); if (resp.valid) { balances.push({ diff --git a/packages/beacon-node/src/api/impl/beacon/state/utils.ts b/packages/beacon-node/src/api/impl/beacon/state/utils.ts index bf8f95794ae7..db00897794fb 100644 --- a/packages/beacon-node/src/api/impl/beacon/state/utils.ts +++ b/packages/beacon-node/src/api/impl/beacon/state/utils.ts @@ -1,9 +1,17 @@ -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {routes} from "@lodestar/api"; -import {CheckpointWithPayload, IForkChoice, PayloadStatus} from "@lodestar/fork-choice"; +import {CheckpointWithPayloadStatus, IForkChoice, PayloadStatus} from "@lodestar/fork-choice"; import {ForkSeq, GENESIS_SLOT} from "@lodestar/params"; -import {BeaconStateAllForks, CachedBeaconStateAllForks} from "@lodestar/state-transition"; -import {BLSPubkey, Epoch, RootHex, Slot, ValidatorIndex, getValidatorStatus, phase0} from "@lodestar/types"; +import {BeaconStateAllForks, CachedBeaconStateAllForks, PubkeyCache} from "@lodestar/state-transition"; +import { + BLSPubkey, + Epoch, + RootHex, + Slot, + ValidatorIndex, + getValidatorStatus, + mapToGeneralStatus, + phase0, +} from "@lodestar/types"; import {fromHex} from "@lodestar/utils"; import {IBeaconChain} from "../../../../chain/index.js"; import {ApiError, ValidationError} from "../../errors.js"; @@ -11,7 +19,7 @@ import {ApiError, ValidationError} from "../../errors.js"; export function resolveStateId( forkChoice: IForkChoice, stateId: routes.beacon.StateId -): RootHex | Slot | CheckpointWithPayload { +): RootHex | Slot | CheckpointWithPayloadStatus { if (stateId === "head") { return forkChoice.getHead().stateRoot; } @@ -83,28 +91,6 @@ export async function getStateResponseWithRegen( return res; } -type GeneralValidatorStatus = "active" | "pending" | "exited" | "withdrawal"; - -function mapToGeneralStatus(subStatus: routes.beacon.ValidatorStatus): GeneralValidatorStatus { - switch (subStatus) { - case "active_ongoing": - case "active_exiting": - case "active_slashed": - return "active"; - case "pending_initialized": - case "pending_queued": - return "pending"; - case "exited_slashed": - case "exited_unslashed": - return "exited"; - case "withdrawal_possible": - case "withdrawal_done": - return "withdrawal"; - default: - throw new Error(`Unknown substatus: ${subStatus}`); - } -} - export function toValidatorResponse( index: ValidatorIndex, validator: phase0.Validator, @@ -122,7 +108,7 @@ export function toValidatorResponse( export function filterStateValidatorsByStatus( statuses: string[], state: BeaconStateAllForks, - pubkey2index: PubkeyIndexMap, + pubkeyCache: PubkeyCache, currentEpoch: Epoch ): routes.beacon.ValidatorResponse[] { const responses: routes.beacon.ValidatorResponse[] = []; @@ -133,7 +119,7 @@ export function filterStateValidatorsByStatus( const validatorStatus = getValidatorStatus(validator, currentEpoch); const generalStatus = mapToGeneralStatus(validatorStatus); - const resp = getStateValidatorIndex(validator.pubkey, state, pubkey2index); + const resp = getStateValidatorIndex(validator.pubkey, state, pubkeyCache); if (resp.valid && (statusSet.has(validatorStatus) || statusSet.has(generalStatus))) { responses.push( toValidatorResponse(resp.validatorIndex, validator, state.balances.get(resp.validatorIndex), currentEpoch) @@ -150,7 +136,7 @@ type StateValidatorIndexResponse = export function getStateValidatorIndex( id: routes.beacon.ValidatorId | BLSPubkey, state: BeaconStateAllForks, - pubkey2index: PubkeyIndexMap + pubkeyCache: PubkeyCache ): StateValidatorIndexResponse { if (typeof id === "string") { // mutate `id` and fallthrough to below @@ -178,7 +164,7 @@ export function getStateValidatorIndex( } // typeof id === Uint8Array - const validatorIndex = pubkey2index.get(id); + const validatorIndex = pubkeyCache.getIndex(id); if (validatorIndex === null) { return {valid: false, code: 404, reason: "Validator pubkey not found in state"}; } diff --git a/packages/beacon-node/src/api/impl/debug/index.ts b/packages/beacon-node/src/api/impl/debug/index.ts index c4195c89a44e..48bd0cd447ff 100644 --- a/packages/beacon-node/src/api/impl/debug/index.ts +++ b/packages/beacon-node/src/api/impl/debug/index.ts @@ -43,11 +43,11 @@ export function getDebugApi({ validity: (() => { switch (node.executionStatus) { case ExecutionStatus.Valid: + case ExecutionStatus.PayloadSeparated: return "valid"; case ExecutionStatus.Invalid: return "invalid"; case ExecutionStatus.Syncing: - case ExecutionStatus.PendingEnvelope: case ExecutionStatus.PreMerge: return "optimistic"; } diff --git a/packages/beacon-node/src/api/impl/node/utils.ts b/packages/beacon-node/src/api/impl/node/utils.ts index 36c651879730..1a852e4cad1c 100644 --- a/packages/beacon-node/src/api/impl/node/utils.ts +++ b/packages/beacon-node/src/api/impl/node/utils.ts @@ -1,4 +1,4 @@ -import {Connection, StreamStatus} from "@libp2p/interface"; +import type {Connection, ConnectionStatus} from "@libp2p/interface"; import {routes} from "@lodestar/api"; /** @@ -24,7 +24,7 @@ export function formatNodePeer(peerIdStr: string, connections: Connection[]): ro * - Otherwise, the first closed connection */ export function getRelevantConnection(connections: Connection[]): Connection | null { - const byStatus = new Map(); + const byStatus = new Map(); for (const conn of connections) { if (conn.status === "open") return conn; if (!byStatus.has(conn.status)) byStatus.set(conn.status, conn); @@ -37,7 +37,7 @@ export function getRelevantConnection(connections: Connection[]): Connection | n * Map libp2p connection status to the API's peer state notation * @param status */ -function getPeerState(status: StreamStatus): routes.node.PeerState { +function getPeerState(status: ConnectionStatus): routes.node.PeerState { switch (status) { case "open": return "connected"; diff --git a/packages/beacon-node/src/api/impl/validator/index.ts b/packages/beacon-node/src/api/impl/validator/index.ts index 7e940b08c897..7efaeeed298c 100644 --- a/packages/beacon-node/src/api/impl/validator/index.ts +++ b/packages/beacon-node/src/api/impl/validator/index.ts @@ -1,4 +1,3 @@ -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {routes} from "@lodestar/api"; import {ApplicationMethods} from "@lodestar/api/server"; import {ExecutionStatus, PayloadStatus, ProtoBlock} from "@lodestar/fork-choice"; @@ -27,6 +26,7 @@ import { computeStartSlotAtEpoch, computeTimeAtSlot, createCachedBeaconState, + createPubkeyCache, getBlockRootAtSlot, getCurrentSlot, loadState, @@ -1160,8 +1160,7 @@ export function getValidatorApi( { config: chain.config, // Not required to compute proposers - pubkey2index: new PubkeyIndexMap(), - index2pubkey: [], + pubkeyCache: createPubkeyCache(), }, {skipSyncPubkeys: true, skipSyncCommitteeCache: true} ); @@ -1622,7 +1621,7 @@ export function getValidatorApi( const filteredRegistrations = registrations.filter((registration) => { const {pubkey} = registration.message; - const validatorIndex = chain.pubkey2index.get(pubkey); + const validatorIndex = chain.pubkeyCache.getIndex(pubkey); if (validatorIndex === null) return false; const validator = headState.validators.getReadonly(validatorIndex); diff --git a/packages/beacon-node/src/chain/archiveStore/archiveStore.ts b/packages/beacon-node/src/chain/archiveStore/archiveStore.ts index 2770cc6b3643..b78330d76313 100644 --- a/packages/beacon-node/src/chain/archiveStore/archiveStore.ts +++ b/packages/beacon-node/src/chain/archiveStore/archiveStore.ts @@ -1,6 +1,5 @@ -import {CheckpointWithPayload} from "@lodestar/fork-choice"; +import {CheckpointWithPayloadStatus} from "@lodestar/fork-choice"; import {LoggerNode} from "@lodestar/logger/node"; -import {ForkSeq} from "@lodestar/params"; import {Checkpoint} from "@lodestar/types/phase0"; import {callFnWhenAwait} from "@lodestar/utils"; import {IBeaconDb} from "../../db/index.js"; @@ -14,7 +13,6 @@ import {HistoricalStateRegen} from "./historicalState/historicalStateRegen.js"; import {ArchiveMode, ArchiveStoreOpts, StateArchiveStrategy} from "./interface.js"; import {FrequencyStateArchiveStrategy} from "./strategies/frequencyStateArchiveStrategy.js"; import {archiveBlocks} from "./utils/archiveBlocks.js"; -import {archiveExecutionPayloadEnvelopes} from "./utils/archivePayloads.js"; import {pruneHistory} from "./utils/pruneHistory.js"; import {updateBackfillRange} from "./utils/updateBackfillRange.js"; @@ -29,7 +27,6 @@ type ArchiveStoreInitOpts = ArchiveStoreOpts & {dbName: string; anchorState: {fi export enum ArchiveStoreTask { ArchiveBlocks = "archive_blocks", - ArchivePayloads = "archive_payloads", PruneHistory = "prune_history", OnFinalizedCheckpoint = "on_finalized_checkpoint", MaybeArchiveState = "maybe_archive_state", @@ -44,7 +41,7 @@ export enum ArchiveStoreTask { */ export class ArchiveStore { private archiveMode: ArchiveMode; - private jobQueue: JobItemQueue<[CheckpointWithPayload], void>; + private jobQueue: JobItemQueue<[CheckpointWithPayloadStatus], void>; private archiveDataEpochs?: number; private readonly statesArchiverStrategy: StateArchiveStrategy; @@ -67,7 +64,7 @@ export class ArchiveStore { this.archiveMode = opts.archiveMode; this.archiveDataEpochs = opts.archiveDataEpochs; - this.jobQueue = new JobItemQueue<[CheckpointWithPayload], void>(this.processFinalizedCheckpoint, { + this.jobQueue = new JobItemQueue<[CheckpointWithPayloadStatus], void>(this.processFinalizedCheckpoint, { maxLength: PROCESS_FINALIZED_CHECKPOINT_QUEUE_LENGTH, signal, }); @@ -168,7 +165,7 @@ export class ArchiveStore { //------------------------------------------------------------------------- // Event handlers //------------------------------------------------------------------------- - private onFinalizedCheckpoint = (finalized: CheckpointWithPayload): void => { + private onFinalizedCheckpoint = (finalized: CheckpointWithPayloadStatus): void => { this.jobQueue.push(finalized).catch((e) => { if (!isQueueErrorAborted(e)) { this.logger.error("Error queuing finalized checkpoint", {epoch: finalized.epoch}, e as Error); @@ -189,10 +186,9 @@ export class ArchiveStore { }); }; - private processFinalizedCheckpoint = async (finalized: CheckpointWithPayload): Promise => { + private processFinalizedCheckpoint = async (finalized: CheckpointWithPayloadStatus): Promise => { try { const finalizedEpoch = finalized.epoch; - const finalizedFork = this.chain.config.getForkSeqAtEpoch(finalizedEpoch); this.logger.verbose("Start processing finalized checkpoint", {epoch: finalizedEpoch, rootHex: finalized.rootHex}); let timer = this.metrics?.processFinalizedCheckpoint.durationByTask.startTimer(); @@ -210,12 +206,6 @@ export class ArchiveStore { ); timer?.({source: ArchiveStoreTask.ArchiveBlocks}); - if (finalizedFork >= ForkSeq.gloas) { - timer = this.metrics?.processFinalizedCheckpoint.durationByTask.startTimer(); - await archiveExecutionPayloadEnvelopes(this.chain, finalized); - timer?.({source: ArchiveStoreTask.ArchivePayloads}); - } - if (this.opts.pruneHistory) { timer = this.metrics?.processFinalizedCheckpoint.durationByTask.startTimer(); await pruneHistory( diff --git a/packages/beacon-node/src/chain/archiveStore/historicalState/getHistoricalState.ts b/packages/beacon-node/src/chain/archiveStore/historicalState/getHistoricalState.ts index e6381575e2e0..c110e9d78d5a 100644 --- a/packages/beacon-node/src/chain/archiveStore/historicalState/getHistoricalState.ts +++ b/packages/beacon-node/src/chain/archiveStore/historicalState/getHistoricalState.ts @@ -1,10 +1,10 @@ -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {BeaconConfig} from "@lodestar/config"; import { BeaconStateAllForks, CachedBeaconStateAllForks, DataAvailabilityStatus, ExecutionPayloadStatus, + PubkeyCache, createCachedBeaconState, stateTransition, } from "@lodestar/state-transition"; @@ -15,16 +15,16 @@ import {HistoricalStateRegenMetrics} from "./metrics.js"; import {RegenErrorType} from "./types.js"; /** - * Populate a PubkeyIndexMap with any new entries based on a BeaconState + * Populate a PubkeyCache with any new entries based on a BeaconState */ -export function syncPubkeyCache(state: BeaconStateAllForks, pubkey2index: PubkeyIndexMap): void { +export function syncPubkeyCache(state: BeaconStateAllForks, pubkeyCache: PubkeyCache): void { // Get the validators sub tree once for all the loop const validators = state.validators; const newCount = state.validators.length; - for (let i = pubkey2index.size; i < newCount; i++) { + for (let i = pubkeyCache.size; i < newCount; i++) { const pubkey = validators.getReadonly(i).pubkey; - pubkey2index.set(pubkey, i); + pubkeyCache.set(i, pubkey); } } @@ -35,7 +35,7 @@ export async function getNearestState( slot: number, config: BeaconConfig, db: IBeaconDb, - pubkey2index: PubkeyIndexMap + pubkeyCache: PubkeyCache ): Promise { const stateBytesArr = await db.stateArchive.binaries({limit: 1, lte: slot, reverse: true}); if (!stateBytesArr.length) { @@ -44,14 +44,13 @@ export async function getNearestState( const stateBytes = stateBytesArr[0]; const state = getStateTypeFromBytes(config, stateBytes).deserializeToViewDU(stateBytes); - syncPubkeyCache(state, pubkey2index); + syncPubkeyCache(state, pubkeyCache); return createCachedBeaconState( state, { config, - pubkey2index, - index2pubkey: [], + pubkeyCache, }, { skipSyncPubkeys: true, @@ -66,13 +65,13 @@ export async function getHistoricalState( slot: number, config: BeaconConfig, db: IBeaconDb, - pubkey2index: PubkeyIndexMap, + pubkeyCache: PubkeyCache, metrics?: HistoricalStateRegenMetrics ): Promise { const regenTimer = metrics?.regenTime.startTimer(); const loadStateTimer = metrics?.loadStateTime.startTimer(); - let state = await getNearestState(slot, config, db, pubkey2index).catch((e) => { + let state = await getNearestState(slot, config, db, pubkeyCache).catch((e) => { metrics?.regenErrorCount.inc({reason: RegenErrorType.loadState}); throw e; }); diff --git a/packages/beacon-node/src/chain/archiveStore/historicalState/worker.ts b/packages/beacon-node/src/chain/archiveStore/historicalState/worker.ts index 2b8ede8b8e81..62925444b2d8 100644 --- a/packages/beacon-node/src/chain/archiveStore/historicalState/worker.ts +++ b/packages/beacon-node/src/chain/archiveStore/historicalState/worker.ts @@ -1,9 +1,9 @@ import worker from "node:worker_threads"; -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {Transfer, expose} from "@chainsafe/threads/worker"; import {chainConfigFromJson, createBeaconConfig} from "@lodestar/config"; import {LevelDbController} from "@lodestar/db/controller/level"; import {getNodeLogger} from "@lodestar/logger/node"; +import {createPubkeyCache} from "@lodestar/state-transition"; import {BeaconDb} from "../../../db/index.js"; import {RegistryMetricCreator, collectNodeJSMetrics} from "../../../metrics/index.js"; import {JobFnQueue} from "../../../util/queue/fnQueue.js"; @@ -52,7 +52,7 @@ const queue = new JobFnQueue( queueMetrics ); -const pubkey2index = new PubkeyIndexMap(); +const pubkeyCache = createPubkeyCache(); const api: HistoricalStateWorkerApi = { async close() { @@ -65,7 +65,7 @@ const api: HistoricalStateWorkerApi = { historicalStateRegenMetrics?.regenRequestCount.inc(); const stateBytes = await queue.push(() => - getHistoricalState(slot, config, db, pubkey2index, historicalStateRegenMetrics) + getHistoricalState(slot, config, db, pubkeyCache, historicalStateRegenMetrics) ); const result = Transfer(stateBytes, [stateBytes.buffer]) as unknown as Uint8Array; diff --git a/packages/beacon-node/src/chain/archiveStore/interface.ts b/packages/beacon-node/src/chain/archiveStore/interface.ts index f49e4e448285..25b54c4fa38f 100644 --- a/packages/beacon-node/src/chain/archiveStore/interface.ts +++ b/packages/beacon-node/src/chain/archiveStore/interface.ts @@ -1,4 +1,4 @@ -import {CheckpointWithPayload} from "@lodestar/fork-choice"; +import {CheckpointWithPayloadStatus} from "@lodestar/fork-choice"; import {RootHex} from "@lodestar/types"; import {Metrics} from "../../metrics/metrics.js"; @@ -44,9 +44,9 @@ export type FinalizedStats = { export interface StateArchiveStrategy { onCheckpoint(stateRoot: RootHex, metrics?: Metrics | null): Promise; - onFinalizedCheckpoint(finalized: CheckpointWithPayload, metrics?: Metrics | null): Promise; - maybeArchiveState(finalized: CheckpointWithPayload, metrics?: Metrics | null): Promise; - archiveState(finalized: CheckpointWithPayload, metrics?: Metrics | null): Promise; + onFinalizedCheckpoint(finalized: CheckpointWithPayloadStatus, metrics?: Metrics | null): Promise; + maybeArchiveState(finalized: CheckpointWithPayloadStatus, metrics?: Metrics | null): Promise; + archiveState(finalized: CheckpointWithPayloadStatus, metrics?: Metrics | null): Promise; } export interface IArchiveStore { diff --git a/packages/beacon-node/src/chain/archiveStore/strategies/frequencyStateArchiveStrategy.ts b/packages/beacon-node/src/chain/archiveStore/strategies/frequencyStateArchiveStrategy.ts index 51467d722dcc..09be1942c175 100644 --- a/packages/beacon-node/src/chain/archiveStore/strategies/frequencyStateArchiveStrategy.ts +++ b/packages/beacon-node/src/chain/archiveStore/strategies/frequencyStateArchiveStrategy.ts @@ -1,4 +1,4 @@ -import {CheckpointWithPayload} from "@lodestar/fork-choice"; +import {CheckpointWithPayloadStatus} from "@lodestar/fork-choice"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {Epoch, RootHex, Slot} from "@lodestar/types"; @@ -41,7 +41,7 @@ export class FrequencyStateArchiveStrategy implements StateArchiveStrategy { private readonly bufferPool?: BufferPool | null ) {} - async onFinalizedCheckpoint(_finalized: CheckpointWithPayload, _metrics?: Metrics | null): Promise {} + async onFinalizedCheckpoint(_finalized: CheckpointWithPayloadStatus, _metrics?: Metrics | null): Promise {} async onCheckpoint(_stateRoot: RootHex, _metrics?: Metrics | null): Promise {} /** @@ -56,7 +56,7 @@ export class FrequencyStateArchiveStrategy implements StateArchiveStrategy { * epoch - 1024*2 epoch - 1024 epoch - 32 epoch * ``` */ - async maybeArchiveState(finalized: CheckpointWithPayload, metrics?: Metrics | null): Promise { + async maybeArchiveState(finalized: CheckpointWithPayloadStatus, metrics?: Metrics | null): Promise { let timer = metrics?.processFinalizedCheckpoint.frequencyStateArchive.startTimer(); const lastStoredSlot = await this.db.stateArchive.lastKey(); timer?.({step: FrequencyStateArchiveStep.LoadLastStoredSlot}); @@ -105,7 +105,7 @@ export class FrequencyStateArchiveStrategy implements StateArchiveStrategy { * Archives finalized states from active bucket to archive bucket. * Only the new finalized state is stored to disk */ - async archiveState(finalized: CheckpointWithPayload, metrics?: Metrics | null): Promise { + async archiveState(finalized: CheckpointWithPayloadStatus, metrics?: Metrics | null): Promise { // starting from Mar 2024, the finalized state could be from disk or in memory let timer = metrics?.processFinalizedCheckpoint.frequencyStateArchive.startTimer(); // Convert fork-choice checkpoint to beacon-node checkpoint with payloadPresent diff --git a/packages/beacon-node/src/chain/archiveStore/utils/archiveBlocks.ts b/packages/beacon-node/src/chain/archiveStore/utils/archiveBlocks.ts index c226f06c1e5b..6e9959803a93 100644 --- a/packages/beacon-node/src/chain/archiveStore/utils/archiveBlocks.ts +++ b/packages/beacon-node/src/chain/archiveStore/utils/archiveBlocks.ts @@ -1,7 +1,7 @@ import path from "node:path"; import {ChainForkConfig} from "@lodestar/config"; import {KeyValue} from "@lodestar/db"; -import {CheckpointWithPayload, IForkChoice, PayloadStatus} from "@lodestar/fork-choice"; +import {CheckpointWithPayloadStatus, IForkChoice, PayloadStatus, ProtoBlock} from "@lodestar/fork-choice"; import {ForkSeq, SLOTS_PER_EPOCH} from "@lodestar/params"; import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {Epoch, Slot} from "@lodestar/types"; @@ -52,7 +52,7 @@ export async function archiveBlocks( forkChoice: IForkChoice, lightclientServer: LightClientServer | undefined, logger: Logger, - finalizedCheckpoint: CheckpointWithPayload, + finalizedCheckpoint: CheckpointWithPayloadStatus, currentEpoch: Epoch, archiveDataEpochs?: number, persistOrphanedBlocks?: boolean, @@ -118,10 +118,12 @@ export async function archiveBlocks( logger.verbose("Migrated dataColumnSidecars from hot DB to cold DB", {...logCtx, migratedEntries}); } - if (finalizedPostGloas && finalizedCanonicalEnvelopeBlockRoots.length > 0) { + if (finalizedPostGloas) { const migratedEntries = await migrateExecutionPayloadEnvelopesFromHotToColdDb( + config, db, - finalizedCanonicalEnvelopeBlockRoots + logger, + finalizedCanonicalBlocks ); logger.verbose("Migrated executionPayloadEnvelopes from hot DB to cold DB", {...logCtx, migratedEntries}); } @@ -166,6 +168,11 @@ export async function archiveBlocks( await db.dataColumnSidecar.deleteMany(nonCanonicalBlockRoots); logger.verbose("Deleted non canonical dataColumnSidecars from hot DB", logCtx); } + + if (finalizedPostGloas) { + await db.executionPayloadEnvelope.batchDelete(nonCanonicalBlockRoots); + logger.verbose("Deleted non canonical executionPayloadEnvelopes from hot DB", logCtx); + } } // Delete expired blobs @@ -395,42 +402,42 @@ async function migrateDataColumnSidecarsFromHotToColdDb( } async function migrateExecutionPayloadEnvelopesFromHotToColdDb( + config: ChainForkConfig, db: IBeaconDb, - blocks: BlockRootSlot[] + logger: Logger, + canonicalBlocks: ProtoBlock[] ): Promise { let migratedEnvelopes = 0; - for (let i = 0; i < blocks.length; i += BLOCK_BATCH_SIZE) { - const toIdx = Math.min(i + BLOCK_BATCH_SIZE, blocks.length); - const canonicalBlocks = blocks.slice(i, toIdx); - - if (canonicalBlocks.length === 0) break; - const canonicalEnvelopeEntries = await Promise.all( - canonicalBlocks.map(async (block) => { - const envelopeBytes = await db.executionPayloadEnvelope.getBinary(block.root); - if (!envelopeBytes) { - // Not every canonical Gloas block is guaranteed to have a revealed envelope. - // Skip missing envelopes (orphaned/unrevealed payload path) instead of failing - // finalized archival processing. - return null; - } + const payloadBlocks = canonicalBlocks.filter( + (block) => config.getForkSeq(block.slot) >= ForkSeq.gloas && block.payloadStatus === PayloadStatus.FULL + ); + if (payloadBlocks.length === 0) return 0; + const blocks = payloadBlocks.map((block) => ({slot: block.slot, root: fromHex(block.blockRoot)})); - return {slot: block.slot, root: block.root, value: envelopeBytes}; - }) - ); + const envelopeEntries: KeyValue[] = []; + const migratedRoots: Uint8Array[] = []; - const envelopesToArchive = canonicalEnvelopeEntries.filter((entry) => entry !== null); + const envelopeBytesArray = await Promise.all( + blocks.map((block) => db.executionPayloadEnvelope.getBinary(block.root)) + ); - if (envelopesToArchive.length > 0) { - await Promise.all([ - db.executionPayloadEnvelopeArchive.batchPutBinary( - envelopesToArchive.map((entry) => ({key: entry.slot, value: entry.value})) - ), - db.executionPayloadEnvelope.batchDelete(envelopesToArchive.map((entry) => entry.root)), - ]); + for (let i = 0; i < blocks.length; i++) { + const bytes = envelopeBytesArray[i]; + if (bytes !== null) { + envelopeEntries.push({key: blocks[i].slot, value: bytes}); + migratedRoots.push(blocks[i].root); + } else { + logger.debug("Payload in forkchoice but missing in db", {slot: blocks[i].slot, root: toRootHex(blocks[i].root)}); } + } - migratedEnvelopes += envelopesToArchive.length; + if (envelopeEntries.length > 0) { + await Promise.all([ + db.executionPayloadEnvelopeArchive.batchPutBinary(envelopeEntries), + db.executionPayloadEnvelope.batchDelete(migratedRoots), + ]); + migratedEnvelopes = envelopeEntries.length; } return migratedEnvelopes; diff --git a/packages/beacon-node/src/chain/archiveStore/utils/archivePayloads.ts b/packages/beacon-node/src/chain/archiveStore/utils/archivePayloads.ts deleted file mode 100644 index 491ae8b74b8d..000000000000 --- a/packages/beacon-node/src/chain/archiveStore/utils/archivePayloads.ts +++ /dev/null @@ -1,15 +0,0 @@ -import {CheckpointWithHex} from "@lodestar/fork-choice"; -import {IBeaconChain} from "../../interface.js"; - -/** - * Archives execution payload envelopes from hot DB to archive DB after finalization. - */ -export async function archiveExecutionPayloadEnvelopes( - chain: IBeaconChain, - _finalized: CheckpointWithHex -): Promise { - const finalizedBlock = chain.forkChoice.getFinalizedBlock(); - if (!finalizedBlock) return; - - // TODO GLOAS: Implement payload envelope archival after epbs fork choice changes are merged -} diff --git a/packages/beacon-node/src/chain/blocks/blockInput/blockInput.ts b/packages/beacon-node/src/chain/blocks/blockInput/blockInput.ts index 9eaeec5809d5..93602fcde980 100644 --- a/packages/beacon-node/src/chain/blocks/blockInput/blockInput.ts +++ b/packages/beacon-node/src/chain/blocks/blockInput/blockInput.ts @@ -1,5 +1,5 @@ import {ForkName, ForkPostFulu, ForkPostGloas, ForkPreDeneb, ForkPreGloas, NUMBER_OF_COLUMNS} from "@lodestar/params"; -import {BeaconBlockBody, BlobIndex, ColumnIndex, SignedBeaconBlock, Slot, deneb, fulu} from "@lodestar/types"; +import {BeaconBlockBody, BlobIndex, ColumnIndex, SignedBeaconBlock, Slot, deneb, fulu, gloas} from "@lodestar/types"; import {byteArrayEquals, fromHex, prettyBytes, toRootHex, withTimeout} from "@lodestar/utils"; import {VersionedHashes} from "../../../execution/index.js"; import {kzgCommitmentToVersionedHash} from "../../../util/blobs.js"; @@ -24,7 +24,12 @@ import { SourceMeta, } from "./types.js"; -export type BlockInput = BlockInputPreData | BlockInputBlobs | BlockInputColumns | BlockInputPayloadBid; +export type BlockInput = + | BlockInputPreData + | BlockInputBlobs + | BlockInputColumns + | BlockInputPayloadBid + | BlockInputNoData; export function isBlockInputPreDeneb(blockInput: IBlockInput): blockInput is BlockInputPreData { return blockInput.type === DAType.PreData; @@ -41,6 +46,10 @@ export function isBlockInputPayloadBid(blockInput: IBlockInput): blockInput is B return blockInput.type === DAType.PayloadBid; } +export function isBlockInputNoData(blockInput: IBlockInput): blockInput is BlockInputNoData { + return blockInput.type === DAType.NoData; +} + function createPromise(): PromiseParts { let resolve!: (value: T) => void; let reject!: (e: Error) => void; @@ -105,6 +114,7 @@ abstract class AbstractBlockInput): void; + abstract getSerializedCacheKeys(): object[]; hasBlock(): boolean { return this.state.hasBlock; @@ -243,6 +253,10 @@ export class BlockInputPreData extends AbstractBlockInput { ); } } + + getSerializedCacheKeys(): object[] { + return [this.state.block]; + } } // Payload Bid (Post-Gloas) @@ -295,6 +309,10 @@ export class BlockInputPayloadBid extends AbstractBlockInput blobSidecar); } + + getSerializedCacheKeys(): object[] { + const objects: object[] = []; + + if (this.state.hasBlock) { + objects.push(this.state.block); + } + + for (const {blobSidecar} of this.blobsCache.values()) { + objects.push(blobSidecar); + } + + return objects; + } } function blockAndBlobArePaired(block: SignedBeaconBlock, blobSidecar: deneb.BlobSidecar): boolean { @@ -969,4 +1001,81 @@ export class BlockInputColumns extends AbstractBlockInput; + source: SourceMeta; + timeCompleteSec: number; +}; + +export class BlockInputNoData extends AbstractBlockInput { + type = DAType.NoData as const; + + state: BlockInputNoDataState; + + private constructor(init: BlockInputInit, state: BlockInputNoDataState) { + super(init); + this.state = state; + this.dataPromise.resolve(null); + this.blockPromise.resolve(state.block); + } + + static createFromBlock(props: AddBlock & CreateBlockInputMeta): BlockInputNoData { + const init: BlockInputInit = { + daOutOfRange: props.daOutOfRange, + timeCreated: props.seenTimestampSec, + forkName: props.forkName, + slot: props.block.message.slot, + blockRootHex: props.blockRootHex, + parentRootHex: toRootHex(props.block.message.parentRoot), + }; + const state: BlockInputNoDataState = { + hasBlock: true, + hasAllData: true, + block: props.block, + source: { + source: props.source, + seenTimestampSec: props.seenTimestampSec, + peerIdStr: props.peerIdStr, + }, + timeCompleteSec: props.seenTimestampSec, + }; + return new BlockInputNoData(init, state); + } + + addBlock(_: AddBlock, opts = {throwOnDuplicateAdd: true}): void { + if (opts.throwOnDuplicateAdd) { + throw new BlockInputError( + { + code: BlockInputErrorCode.INVALID_CONSTRUCTION, + blockRoot: this.blockRootHex, + }, + "Cannot addBlock to BlockInputNoData - block already exists" + ); + } + } + + getBlobKzgCommitments(): deneb.BlobKzgCommitments { + return (this.state.block.message.body as gloas.BeaconBlockBody).signedExecutionPayloadBid.message + .blobKzgCommitments; + } + + getSerializedCacheKeys(): object[] { + return [this.state.block]; + } } diff --git a/packages/beacon-node/src/chain/blocks/blockInput/types.ts b/packages/beacon-node/src/chain/blocks/blockInput/types.ts index 7becab0452d0..700d4f76a901 100644 --- a/packages/beacon-node/src/chain/blocks/blockInput/types.ts +++ b/packages/beacon-node/src/chain/blocks/blockInput/types.ts @@ -1,5 +1,5 @@ import {ForkName} from "@lodestar/params"; -import {ColumnIndex, RootHex, SignedBeaconBlock, Slot, deneb, fulu} from "@lodestar/types"; +import {ColumnIndex, DataColumnSidecars, RootHex, SignedBeaconBlock, Slot, deneb, fulu} from "@lodestar/types"; import {VersionedHashes} from "../../../execution/index.js"; export enum DAType { @@ -8,6 +8,7 @@ export enum DAType { Columns = "columns", /** Post-Gloas: block contains only a bid, no DA data dependency. Payload delivered separately via envelope. */ PayloadBid = "payload-bid", + NoData = "no-data", } export type DAData = null | deneb.BlobSidecars | fulu.DataColumnSidecars; @@ -101,6 +102,18 @@ export type MissingColumnMeta = { versionedHashes: VersionedHashes; }; +/** + * Minimal interface required to write data columns to the DB. + * Used by `writeDataColumnsToDb` and designed to be reusable across forks (e.g. Fulu, Gloas). + */ +export interface IDataColumnsInput { + readonly slot: Slot; + readonly blockRootHex: string; + getCustodyColumns(): DataColumnSidecars; + hasComputedAllData(): boolean; + waitForComputedAllData(timeout: number, signal?: AbortSignal): Promise; +} + /** * This is used to validate that BlockInput implementations follow some minimal subset of operations * and that adding a new implementation won't break consumers that rely on this subset. @@ -140,6 +153,11 @@ export interface IBlockInput>; waitForAllData(timeout: number, signal?: AbortSignal): Promise; diff --git a/packages/beacon-node/src/chain/blocks/importBlock.ts b/packages/beacon-node/src/chain/blocks/importBlock.ts index ce7643b8173c..ab5558e7826d 100644 --- a/packages/beacon-node/src/chain/blocks/importBlock.ts +++ b/packages/beacon-node/src/chain/blocks/importBlock.ts @@ -7,8 +7,16 @@ import { ForkChoiceErrorCode, NotReorgedReason, getSafeExecutionBlockHash, + isGloasBlock, } from "@lodestar/fork-choice"; -import {ForkPostAltair, ForkPostElectra, ForkSeq, MAX_SEED_LOOKAHEAD, SLOTS_PER_EPOCH} from "@lodestar/params"; +import { + ForkPostAltair, + ForkPostElectra, + ForkPostGloas, + ForkSeq, + MAX_SEED_LOOKAHEAD, + SLOTS_PER_EPOCH, +} from "@lodestar/params"; import { CachedBeaconStateAltair, EpochCache, @@ -20,7 +28,17 @@ import { isStartSlotOfEpoch, isStateValidatorsNodesPopulated, } from "@lodestar/state-transition"; -import {Attestation, BeaconBlock, altair, capella, electra, isGloasBeaconBlock, phase0, ssz} from "@lodestar/types"; +import { + Attestation, + BeaconBlock, + SignedBeaconBlock, + altair, + capella, + electra, + isGloasBeaconBlock, + phase0, + ssz, +} from "@lodestar/types"; import {isErrorAborted, toRootHex} from "@lodestar/utils"; import {ZERO_HASH_HEX} from "../../constants/index.js"; import {callInNextEventLoop} from "../../util/eventLoop.js"; @@ -102,7 +120,7 @@ export async function importBlock( // Without this, a supernode syncing from behind can accumulate many blocks worth of column // data in memory (up to 128 columns per block) causing OOM before persistence catches up. await this.unfinalizedBlockWrites.waitForSpace(); - this.unfinalizedBlockWrites.push([blockInput]).catch((e) => { + this.unfinalizedBlockWrites.push(blockInput).catch((e) => { if (!isQueueErrorAborted(e)) { this.logger.error("Error pushing block to unfinalized write queue", {slot: blockSlot}, e as Error); } @@ -123,17 +141,11 @@ export async function importBlock( // This adds the state necessary to process the next block // Some block event handlers require state being in state cache so need to do this before emitting EventType.block - // Pre-Gloas: blockSummary.payloadStatus is always FULL, payloadPresent = true (execution payload embedded in block) - // Post-Gloas: blockSummary.payloadStatus is always PENDING (EMPTY variant also created), payloadPresent = false (block state only, no payload processing yet) - const isGloasBlock = blockSummary.blockHashFromBid !== null; - const payloadPresent = !isGloasBlock; + // Pre-Gloas: blockSummary.payloadStatus is always FULL, payloadPresent = true + // Post-Gloas: blockSummary.payloadStatus is always PENDING, so payloadPresent = false (block state only, no payload processing yet) + const payloadPresent = !isGloasBlock(blockSummary); // processState manages both block state and payload state variants together for memory/disk management - this.regen.processState(blockRootHex, postState); - this.logger.verbose("Added block to forkchoice and block state cache", { - slot: blockSlot, - root: blockRootHex, - stateRoot: toRootHex(postState.hashTreeRoot()), - }); + this.regen.processBlockState(blockRootHex, postState); if (postEnvelopeState !== null) { this.regen.processPayloadState(postEnvelopeState); @@ -151,12 +163,29 @@ export async function importBlock( }); } + // For Gloas blocks, create PayloadEnvelopeInput so it's available for later payload import + if (fork >= ForkSeq.gloas) { + this.seenPayloadEnvelopeInputCache.add({ + blockRootHex, + block: block as SignedBeaconBlock, + sampledColumns: this.custodyConfig.sampledColumns, + custodyColumns: this.custodyConfig.custodyColumns, + timeCreatedSec: fullyVerifiedBlock.seenTimestampSec, + }); + this.logger.debug("Created PayloadEnvelopeInput for block", { + slot: blockSlot, + root: blockRootHex, + source: source.source, + ...(opts.seenTimestampSec !== undefined ? {recvToImport: Date.now() / 1000 - opts.seenTimestampSec} : {}), + }); + } + this.metrics?.importBlock.bySource.inc({source: source.source}); // Post-Gloas: immediately import pending envelope for this block if available. // This makes the block FULL right away, so child blocks won't need to wait // for the envelope in their verifyBlock parent-envelope polling loop. - if (isGloasBlock) { + if (isGloasBlock(blockSummary)) { const pendingEnvelope = this.pendingEnvelopes.get(blockRootHex); if (pendingEnvelope) { try { @@ -281,6 +310,32 @@ export async function importBlock( } } + // 4.5. Import payload attestations to fork choice (Gloas) + // + if (isGloasBeaconBlock(block.message)) { + for (const payloadAttestation of block.message.body.payloadAttestations) { + try { + // Extract PTC indices from aggregation bits + const ptcIndices: number[] = []; + for (let i = 0; i < payloadAttestation.aggregationBits.bitLen; i++) { + if (payloadAttestation.aggregationBits.get(i)) { + ptcIndices.push(i); + } + } + + if (ptcIndices.length > 0) { + this.forkChoice.notifyPtcMessages( + toRootHex(payloadAttestation.data.beaconBlockRoot), + ptcIndices, + payloadAttestation.data.payloadPresent + ); + } + } catch (e) { + this.logger.warn("Error processing PayloadAttestation from block", {slot: blockSlot}, e as Error); + } + } + } + // 5. Compute head. If new head, immediately stateCache.setHeadState() const oldHead = this.forkChoice.getHead(); @@ -481,8 +536,6 @@ export async function importBlock( // Cache state to preserve epoch transition work const checkpointState = postState; const cp = getCheckpointFromState(checkpointState); - // Pre-Gloas: payloadPresent = true (FULL variant, execution payload embedded in block) - // Post-Gloas: payloadPresent = false (PENDING variant with EMPTY also created, block state only) this.regen.addCheckpointState(cp, checkpointState, payloadPresent); // consumers should not mutate state ever this.emitter.emit(ChainEvent.checkpoint, cp, checkpointState); @@ -581,7 +634,7 @@ export async function importBlock( if (isBlockInputColumns(blockInput)) { for (const {source} of blockInput.getSampledColumnsWithSource()) { - this.metrics?.importBlock.columnsBySource.inc({source}); + this.metrics?.dataColumns.bySource.inc({source}); } } else if (isBlockInputBlobs(blockInput)) { for (const {source} of blockInput.getAllBlobsWithSource()) { diff --git a/packages/beacon-node/src/chain/blocks/importExecutionPayload.ts b/packages/beacon-node/src/chain/blocks/importExecutionPayload.ts new file mode 100644 index 000000000000..c7c1234cac24 --- /dev/null +++ b/packages/beacon-node/src/chain/blocks/importExecutionPayload.ts @@ -0,0 +1,241 @@ +import {routes} from "@lodestar/api"; +import {ForkName} from "@lodestar/params"; +import { + BeaconStateView, + CachedBeaconStateGloas, + getExecutionPayloadEnvelopeSignatureSet, +} from "@lodestar/state-transition"; +import {processExecutionPayloadEnvelope} from "@lodestar/state-transition/block"; +import {byteArrayEquals, fromHex, toRootHex} from "@lodestar/utils"; +import {ExecutionPayloadStatus} from "../../execution/index.js"; +import {isQueueErrorAborted} from "../../util/queue/index.js"; +import {BeaconChain} from "../chain.js"; +import {RegenCaller} from "../regen/interface.js"; +import {PayloadEnvelopeInput} from "../seenCache/seenPayloadEnvelopeInput.js"; +import {ImportPayloadOpts} from "./types.js"; + +const EVENTSTREAM_EMIT_RECENT_EXECUTION_PAYLOAD_SLOTS = 64; + +export enum PayloadErrorCode { + EXECUTION_ENGINE_INVALID = "PAYLOAD_ERROR_EXECUTION_ENGINE_INVALID", + EXECUTION_ENGINE_ERROR = "PAYLOAD_ERROR_EXECUTION_ENGINE_ERROR", + BLOCK_NOT_IN_FORK_CHOICE = "PAYLOAD_ERROR_BLOCK_NOT_IN_FORK_CHOICE", + STATE_TRANSITION_ERROR = "PAYLOAD_ERROR_STATE_TRANSITION_ERROR", + INVALID_SIGNATURE = "PAYLOAD_ERROR_INVALID_SIGNATURE", +} + +export type PayloadErrorType = + | { + code: PayloadErrorCode.EXECUTION_ENGINE_INVALID; + execStatus: ExecutionPayloadStatus; + errorMessage: string; + } + | { + code: PayloadErrorCode.EXECUTION_ENGINE_ERROR; + execStatus: ExecutionPayloadStatus; + errorMessage: string; + } + | { + code: PayloadErrorCode.BLOCK_NOT_IN_FORK_CHOICE; + blockRootHex: string; + } + | { + code: PayloadErrorCode.STATE_TRANSITION_ERROR; + message: string; + } + | { + code: PayloadErrorCode.INVALID_SIGNATURE; + }; + +export class PayloadError extends Error { + type: PayloadErrorType; + + constructor(type: PayloadErrorType, message?: string) { + super(message ?? type.code); + this.type = type; + } +} + +/** + * Import an execution payload envelope after all data is available. + * + * This function: + * 1. Gets the ProtoBlock from fork choice + * 2. Applies write-queue backpressure (waitForSpace) early, before verification + * 3. Regenerates the block state + * 4. Runs EL verification (notifyNewPayload) in parallel with signature verification and processExecutionPayloadEnvelope + * 5. Persists verified payload envelope to hot DB + * 6. Updates fork choice + * 7. Caches the post-execution payload state + * 8. Records metrics for column sources + * + */ +export async function importExecutionPayload( + this: BeaconChain, + payloadInput: PayloadEnvelopeInput, + opts: ImportPayloadOpts = {} +): Promise { + const envelope = payloadInput.getPayloadEnvelope(); + const blockRootHex = payloadInput.blockRootHex; + + // 1. Get ProtoBlock for parent root lookup + const protoBlock = this.forkChoice.getBlockHexDefaultStatus(blockRootHex); + if (!protoBlock) { + throw new PayloadError({ + code: PayloadErrorCode.BLOCK_NOT_IN_FORK_CHOICE, + blockRootHex, + }); + } + + // 2. Apply backpressure from the write queue early, before doing verification work. + // The actual DB write is deferred until after verification succeeds. + await this.unfinalizedPayloadEnvelopeWrites.waitForSpace(); + + // 3. Get pre-state for processExecutionPayloadEnvelope + // We need the block state (post-block, pre-payload) to process the envelope + const blockState = (await this.regen.getBlockSlotState( + protoBlock, + protoBlock.slot, + {dontTransferCache: true}, + RegenCaller.processBlock + )) as CachedBeaconStateGloas; + + // 4. Run verification steps in parallel + // Note: No data availability check needed here - importExecutionPayload is only + // called when payloadInput.isComplete() is true, so all data is already available. + const [execResult, signatureValid, postPayloadResult] = await Promise.all([ + this.executionEngine.notifyNewPayload( + ForkName.gloas, + envelope.message.payload, + payloadInput.getVersionedHashes(), + fromHex(protoBlock.parentRoot), + envelope.message.executionRequests + ), + + opts.validSignature === true + ? Promise.resolve(true) + : (async () => { + const signatureSet = getExecutionPayloadEnvelopeSignatureSet( + this.config, + blockState.epochCtx.pubkeyCache, + new BeaconStateView(blockState), + envelope, + payloadInput.proposerIndex + ); + return this.bls.verifySignatureSets([signatureSet]); + })(), + + // Signature verified separately above. + // State root check is done separately below with better error typing (matching block pipeline pattern). + (async () => { + try { + return { + postPayloadState: processExecutionPayloadEnvelope(blockState, envelope, { + verifySignature: false, + verifyStateRoot: false, + }), + }; + } catch (e) { + throw new PayloadError( + { + code: PayloadErrorCode.STATE_TRANSITION_ERROR, + message: (e as Error).message, + }, + `State transition error: ${(e as Error).message}` + ); + } + })(), + ]); + + // 4b. Check signature verification result + if (!signatureValid) { + throw new PayloadError({code: PayloadErrorCode.INVALID_SIGNATURE}); + } + + // 5. Handle EL response + switch (execResult.status) { + case ExecutionPayloadStatus.VALID: + break; + + case ExecutionPayloadStatus.INVALID: + throw new PayloadError({ + code: PayloadErrorCode.EXECUTION_ENGINE_INVALID, + execStatus: execResult.status, + errorMessage: execResult.validationError ?? "", + }); + + case ExecutionPayloadStatus.ACCEPTED: + case ExecutionPayloadStatus.SYNCING: + // TODO GLOAS: Handle optimistic import for payload - for now treat as error + throw new PayloadError({ + code: PayloadErrorCode.EXECUTION_ENGINE_ERROR, + execStatus: execResult.status, + errorMessage: execResult.validationError ?? "EL syncing, payload not yet validated", + }); + + case ExecutionPayloadStatus.INVALID_BLOCK_HASH: + case ExecutionPayloadStatus.ELERROR: + case ExecutionPayloadStatus.UNAVAILABLE: + throw new PayloadError({ + code: PayloadErrorCode.EXECUTION_ENGINE_ERROR, + execStatus: execResult.status, + errorMessage: execResult.validationError ?? "", + }); + } + + // 5b. Verify envelope state root matches post-state + const postPayloadState = postPayloadResult.postPayloadState; + const postPayloadStateRoot = postPayloadState.hashTreeRoot(); + if (!byteArrayEquals(envelope.message.stateRoot, postPayloadStateRoot)) { + throw new PayloadError({ + code: PayloadErrorCode.STATE_TRANSITION_ERROR, + message: `Envelope state root mismatch expected=${toRootHex(envelope.message.stateRoot)} actual=${toRootHex(postPayloadStateRoot)}`, + }); + } + + // 5c. Persist payload envelope to hot DB (performed asynchronously to avoid blocking) + this.unfinalizedPayloadEnvelopeWrites.push(payloadInput).catch((e) => { + if (!isQueueErrorAborted(e)) { + this.logger.error( + "Error pushing payload envelope to unfinalized write queue", + {slot: payloadInput.slot, root: blockRootHex}, + e as Error + ); + } + }); + + // 6. Update fork choice + this.forkChoice.onExecutionPayload( + blockRootHex, + payloadInput.getBlockHashHex(), + envelope.message.payload.blockNumber, + toRootHex(postPayloadStateRoot) + ); + + // 7. Cache payload state + // TODO GLOAS: Enable when PR #8868 merged (adds processPayloadState) + // this.regen.processPayloadState(postPayloadState); + // if epoch boundary also call + // this.regen.addCheckpointState(cp, checkpointState, true); + + // 8. Record metrics for payload envelope and column sources + this.metrics?.importPayload.bySource.inc({source: payloadInput.getPayloadEnvelopeSource().source}); + for (const {source} of payloadInput.getSampledColumnsWithSource()) { + this.metrics?.importPayload.columnsBySource.inc({source}); + } + + this.logger.verbose("Execution payload imported", { + slot: payloadInput.slot, + root: blockRootHex, + blockHash: payloadInput.getBlockHashHex(), + }); + + // 9. Emit event after payload is fully verified and imported to fork choice, only for recent enough payloads + const currentSlot = this.clock.currentSlot; + if (currentSlot - payloadInput.slot < EVENTSTREAM_EMIT_RECENT_EXECUTION_PAYLOAD_SLOTS) { + this.emitter.emit(routes.events.EventType.executionPayloadAvailable, { + slot: payloadInput.slot, + blockRoot: blockRootHex, + }); + } +} diff --git a/packages/beacon-node/src/chain/blocks/payloadEnvelopeInput/index.ts b/packages/beacon-node/src/chain/blocks/payloadEnvelopeInput/index.ts new file mode 100644 index 000000000000..368f98324f22 --- /dev/null +++ b/packages/beacon-node/src/chain/blocks/payloadEnvelopeInput/index.ts @@ -0,0 +1,2 @@ +export * from "./payloadEnvelopeInput.js"; +export * from "./types.js"; diff --git a/packages/beacon-node/src/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.ts b/packages/beacon-node/src/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.ts new file mode 100644 index 000000000000..313b7dce8f74 --- /dev/null +++ b/packages/beacon-node/src/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.ts @@ -0,0 +1,336 @@ +import {NUMBER_OF_COLUMNS} from "@lodestar/params"; +import {ColumnIndex, DataColumnSidecars, RootHex, Slot, ValidatorIndex, deneb, gloas} from "@lodestar/types"; +import {toRootHex, withTimeout} from "@lodestar/utils"; +import {VersionedHashes} from "../../../execution/index.js"; +import {kzgCommitmentToVersionedHash} from "../../../util/blobs.js"; +import {AddPayloadEnvelopeProps, ColumnWithSource, CreateFromBlockProps, SourceMeta} from "./types.js"; + +export type PayloadEnvelopeInputState = + | { + hasPayload: false; + hasAllData: false; + hasComputedAllData: false; + } + | { + hasPayload: false; + hasAllData: true; + hasComputedAllData: boolean; + } + | { + hasPayload: true; + hasAllData: false; + hasComputedAllData: false; + payloadEnvelope: gloas.SignedExecutionPayloadEnvelope; + payloadEnvelopeSource: SourceMeta; + } + | { + hasPayload: true; + hasAllData: true; + hasComputedAllData: boolean; + payloadEnvelope: gloas.SignedExecutionPayloadEnvelope; + payloadEnvelopeSource: SourceMeta; + timeCompleteSec: number; + }; + +type PromiseParts = { + promise: Promise; + resolve: (value: T) => void; + reject: (e: Error) => void; +}; + +function createPromise(): PromiseParts { + let resolve!: (value: T) => void; + let reject!: (e: Error) => void; + const promise = new Promise((_resolve, _reject) => { + resolve = _resolve; + reject = _reject; + }); + return {promise, resolve, reject}; +} + +/** + * Tracks bid + payload envelope + data columns for a Gloas block. + * + * Created during block import from signedExecutionPayloadBid in block body. + * Always has bid (required for creation). + * + * Completion requires: payload envelope + all sampled columns + */ +export class PayloadEnvelopeInput { + readonly blockRootHex: RootHex; + readonly slot: Slot; + readonly proposerIndex: ValidatorIndex; + readonly bid: gloas.ExecutionPayloadBid; + readonly versionedHashes: VersionedHashes; + + private columnsCache = new Map(); + + private readonly sampledColumns: ColumnIndex[]; + private readonly custodyColumns: ColumnIndex[]; + + private timeCreatedSec: number; + + private readonly payloadEnvelopeDataPromise: PromiseParts; + private readonly columnsDataPromise: PromiseParts; + + state: PayloadEnvelopeInputState; + + private constructor(props: { + blockRootHex: RootHex; + slot: Slot; + proposerIndex: ValidatorIndex; + bid: gloas.ExecutionPayloadBid; + sampledColumns: ColumnIndex[]; + custodyColumns: ColumnIndex[]; + timeCreatedSec: number; + }) { + this.blockRootHex = props.blockRootHex; + this.slot = props.slot; + this.proposerIndex = props.proposerIndex; + this.bid = props.bid; + this.versionedHashes = props.bid.blobKzgCommitments.map(kzgCommitmentToVersionedHash); + this.sampledColumns = props.sampledColumns; + this.custodyColumns = props.custodyColumns; + this.timeCreatedSec = props.timeCreatedSec; + this.payloadEnvelopeDataPromise = createPromise(); + this.columnsDataPromise = createPromise(); + + const noBlobs = props.bid.blobKzgCommitments.length === 0; + const noSampledColumns = props.sampledColumns.length === 0; + const hasAllData = noBlobs || noSampledColumns; + + if (hasAllData) { + this.state = {hasPayload: false, hasAllData: true, hasComputedAllData: true}; + this.columnsDataPromise.resolve(this.getSampledColumns()); + } else { + this.state = {hasPayload: false, hasAllData: false, hasComputedAllData: false}; + } + } + + static createFromBlock(props: CreateFromBlockProps): PayloadEnvelopeInput { + const bid = (props.block.message.body as gloas.BeaconBlockBody).signedExecutionPayloadBid.message; + return new PayloadEnvelopeInput({ + blockRootHex: props.blockRootHex, + slot: props.block.message.slot, + proposerIndex: props.block.message.proposerIndex, + bid, + sampledColumns: props.sampledColumns, + custodyColumns: props.custodyColumns, + timeCreatedSec: props.timeCreatedSec, + }); + } + + getBid(): gloas.ExecutionPayloadBid { + return this.bid; + } + + getBuilderIndex(): ValidatorIndex { + return this.bid.builderIndex; + } + + getBlockHashHex(): RootHex { + return toRootHex(this.bid.blockHash); + } + + getBlobKzgCommitments(): deneb.BlobKzgCommitments { + return this.bid.blobKzgCommitments; + } + + addPayloadEnvelope(props: AddPayloadEnvelopeProps): void { + if (this.state.hasPayload) { + throw new Error(`Payload envelope already set for block ${this.blockRootHex}`); + } + if (toRootHex(props.envelope.message.beaconBlockRoot) !== this.blockRootHex) { + throw new Error("Payload envelope beacon_block_root mismatch"); + } + + const source: SourceMeta = { + source: props.source, + seenTimestampSec: props.seenTimestampSec, + peerIdStr: props.peerIdStr, + }; + + if (this.state.hasAllData) { + // Complete state + this.state = { + hasPayload: true, + hasAllData: true, + hasComputedAllData: this.state.hasComputedAllData, + payloadEnvelope: props.envelope, + payloadEnvelopeSource: source, + timeCompleteSec: props.seenTimestampSec, + }; + this.payloadEnvelopeDataPromise.resolve(props.envelope); + } else { + // Has payload, waiting for columns + this.state = { + hasPayload: true, + hasAllData: false, + hasComputedAllData: false, + payloadEnvelope: props.envelope, + payloadEnvelopeSource: source, + }; + } + } + + addColumn(columnWithSource: ColumnWithSource): void { + const {columnSidecar, seenTimestampSec} = columnWithSource; + this.columnsCache.set(columnSidecar.index, columnWithSource); + + const sampledColumns = this.getSampledColumns(); + const hasAllData = + // already hasAllData + this.state.hasAllData || + // has all sampled columns + sampledColumns.length === this.sampledColumns.length || + // has enough columns to reconstruct the rest + this.columnsCache.size >= NUMBER_OF_COLUMNS / 2; + + const hasComputedAllData = + // has all sampled columns + sampledColumns.length === this.sampledColumns.length; + + if (!hasAllData) { + return; + } + + if (hasComputedAllData) { + this.columnsDataPromise.resolve(sampledColumns); + } + + if (this.state.hasPayload) { + // Complete state + this.state = { + hasPayload: true, + hasAllData: true, + hasComputedAllData: hasComputedAllData || this.state.hasComputedAllData, + payloadEnvelope: this.state.payloadEnvelope, + payloadEnvelopeSource: this.state.payloadEnvelopeSource, + timeCompleteSec: seenTimestampSec, + }; + this.payloadEnvelopeDataPromise.resolve(this.state.payloadEnvelope); + } else { + // No payload yet, all data ready + this.state = { + hasPayload: false, + hasAllData: true, + hasComputedAllData: hasComputedAllData || this.state.hasComputedAllData, + }; + } + } + + getVersionedHashes(): VersionedHashes { + return this.versionedHashes; + } + + hasPayloadEnvelope(): boolean { + return this.state.hasPayload; + } + + getPayloadEnvelope(): gloas.SignedExecutionPayloadEnvelope { + if (!this.state.hasPayload) throw new Error("Payload envelope not set"); + return this.state.payloadEnvelope; + } + + getPayloadEnvelopeSource(): SourceMeta { + if (!this.state.hasPayload) throw new Error("Payload envelope source not set"); + return this.state.payloadEnvelopeSource; + } + + getSampledColumns(): gloas.DataColumnSidecars { + const columns: gloas.DataColumnSidecars = []; + for (const index of this.sampledColumns) { + const column = this.columnsCache.get(index); + if (column) { + columns.push(column.columnSidecar); + } + } + return columns; + } + + getSampledColumnsWithSource(): ColumnWithSource[] { + const columns: ColumnWithSource[] = []; + for (const index of this.sampledColumns) { + const column = this.columnsCache.get(index); + if (column) { + columns.push(column); + } + } + return columns; + } + + getCustodyColumns(): gloas.DataColumnSidecars { + const columns: gloas.DataColumnSidecars = []; + for (const index of this.custodyColumns) { + const column = this.columnsCache.get(index); + if (column) { + columns.push(column.columnSidecar); + } + } + return columns; + } + + hasComputedAllData(): boolean { + return this.state.hasComputedAllData; + } + + waitForComputedAllData(timeout: number, signal?: AbortSignal): Promise { + if (this.state.hasComputedAllData) { + return Promise.resolve(this.getSampledColumns()); + } + return withTimeout(() => this.columnsDataPromise.promise, timeout, signal); + } + + getTimeCreated(): number { + return this.timeCreatedSec; + } + + getTimeComplete(): number { + if (!this.state.hasPayload || !this.state.hasAllData) throw new Error("Not yet complete"); + return this.state.timeCompleteSec; + } + + isComplete(): boolean { + return this.state.hasPayload && this.state.hasAllData; + } + + async waitForData(): Promise { + return this.payloadEnvelopeDataPromise.promise; + } + + getSerializedCacheKeys(): object[] { + const objects: object[] = []; + + if (this.state.hasPayload) { + objects.push(this.state.payloadEnvelope); + } + + for (const {columnSidecar} of this.columnsCache.values()) { + objects.push(columnSidecar); + } + + return objects; + } + + getLogMeta(): { + slot: number; + blockRoot: string; + hasPayload: boolean; + hasAllData: boolean; + hasComputedAllData: boolean; + isComplete: boolean; + columnsCount: number; + sampledColumnsCount: number; + } { + return { + slot: this.slot, + blockRoot: this.blockRootHex, + hasPayload: this.state.hasPayload, + hasAllData: this.state.hasAllData, + hasComputedAllData: this.state.hasComputedAllData, + isComplete: this.isComplete(), + columnsCount: this.columnsCache.size, + sampledColumnsCount: this.sampledColumns.length, + }; + } +} diff --git a/packages/beacon-node/src/chain/blocks/payloadEnvelopeInput/types.ts b/packages/beacon-node/src/chain/blocks/payloadEnvelopeInput/types.ts new file mode 100644 index 000000000000..b4683a245b1f --- /dev/null +++ b/packages/beacon-node/src/chain/blocks/payloadEnvelopeInput/types.ts @@ -0,0 +1,33 @@ +import {ForkPostGloas} from "@lodestar/params"; +import {ColumnIndex, RootHex, SignedBeaconBlock, gloas} from "@lodestar/types"; + +export enum PayloadEnvelopeInputSource { + gossip = "gossip", + api = "api", + engine = "engine", + byRange = "req_resp_by_range", + byRoot = "req_resp_by_root", + recovery = "recovery", +} + +export type SourceMeta = { + source: PayloadEnvelopeInputSource; + seenTimestampSec: number; + peerIdStr?: string; +}; + +export type ColumnWithSource = SourceMeta & { + columnSidecar: gloas.DataColumnSidecar; +}; + +export type CreateFromBlockProps = { + blockRootHex: RootHex; + block: SignedBeaconBlock; + sampledColumns: ColumnIndex[]; + custodyColumns: ColumnIndex[]; + timeCreatedSec: number; +}; + +export type AddPayloadEnvelopeProps = SourceMeta & { + envelope: gloas.SignedExecutionPayloadEnvelope; +}; diff --git a/packages/beacon-node/src/chain/blocks/payloadEnvelopeProcessor.ts b/packages/beacon-node/src/chain/blocks/payloadEnvelopeProcessor.ts new file mode 100644 index 000000000000..e50b53bda93e --- /dev/null +++ b/packages/beacon-node/src/chain/blocks/payloadEnvelopeProcessor.ts @@ -0,0 +1,61 @@ +import {Metrics} from "../../metrics/metrics.js"; +import {JobItemQueue} from "../../util/queue/index.js"; +import type {BeaconChain} from "../chain.js"; +import {PayloadEnvelopeInput} from "../seenCache/seenPayloadEnvelopeInput.js"; +import {importExecutionPayload} from "./importExecutionPayload.js"; +import {ImportPayloadOpts} from "./types.js"; + +// TODO GLOAS: Set to be equal to DEFAULT_MAX_PENDING_UNFINALIZED_PAYLOAD_ENVELOPE_WRITES for now +const QUEUE_MAX_LENGTH = 16; + +enum PayloadEnvelopeImportStatus { + queued = "queued", + importing = "importing", + imported = "imported", +} + +/** + * PayloadEnvelopeProcessor processes payload envelope jobs in a queued fashion, one after the other. + */ +export class PayloadEnvelopeProcessor { + readonly jobQueue: JobItemQueue<[PayloadEnvelopeInput, ImportPayloadOpts], void>; + private readonly importStatus = new WeakMap(); + + constructor(chain: BeaconChain, metrics: Metrics | null, signal: AbortSignal) { + this.jobQueue = new JobItemQueue<[PayloadEnvelopeInput, ImportPayloadOpts], void>( + (payloadInput, opts) => { + this.importStatus.set(payloadInput, PayloadEnvelopeImportStatus.importing); + return importExecutionPayload.call(chain, payloadInput, opts); + }, + {maxLength: QUEUE_MAX_LENGTH, noYieldIfOneItem: true, signal}, + metrics?.payloadEnvelopeProcessorQueue ?? undefined + ); + } + + async processPayloadEnvelopeJob(payloadInput: PayloadEnvelopeInput, opts: ImportPayloadOpts = {}): Promise { + if (!payloadInput.isComplete()) { + return; + } + + if (this.importStatus.get(payloadInput) !== undefined) { + return; + } + + await this.jobQueue.waitForSpace(); + + // Re-check after await, as another call may have queued this payload. + if (this.importStatus.get(payloadInput) !== undefined) { + return; + } + + this.importStatus.set(payloadInput, PayloadEnvelopeImportStatus.queued); + + try { + await this.jobQueue.push(payloadInput, opts); + this.importStatus.set(payloadInput, PayloadEnvelopeImportStatus.imported); + } catch (e) { + this.importStatus.delete(payloadInput); + throw e; + } + } +} diff --git a/packages/beacon-node/src/chain/blocks/types.ts b/packages/beacon-node/src/chain/blocks/types.ts index f77edd29c1f4..ac78e5b9c42d 100644 --- a/packages/beacon-node/src/chain/blocks/types.ts +++ b/packages/beacon-node/src/chain/blocks/types.ts @@ -46,6 +46,14 @@ export enum BlobSidecarValidation { Full, } +export type ImportPayloadOpts = { + /** + * Set to true if envelope signature was already verified (e.g., during gossip/API validation). + * When false/undefined, signature will be verified during import. + */ + validSignature?: boolean; +}; + export type ImportBlockOpts = { /** * TEMP: Review if this is safe, Lighthouse always imports attestations even in finalized sync. diff --git a/packages/beacon-node/src/chain/blocks/verifyBlock.ts b/packages/beacon-node/src/chain/blocks/verifyBlock.ts index fe4123ffd1e9..c8d16c388176 100644 --- a/packages/beacon-node/src/chain/blocks/verifyBlock.ts +++ b/packages/beacon-node/src/chain/blocks/verifyBlock.ts @@ -81,8 +81,9 @@ export async function verifyBlocksInEpoch( ) { const parentRootHex = toRootHex(block0.message.parentRoot); const childParentBlockHash = toRootHex(block0.message.body.signedExecutionPayloadBid.message.parentBlockHash); + const parentPayloadInput = this.seenPayloadEnvelopeInputCache.get(parentRootHex); const childWantsFullParent = - parentBlock.blockHashFromBid !== null && childParentBlockHash === parentBlock.blockHashFromBid; + parentPayloadInput !== undefined && childParentBlockHash === parentPayloadInput.getBlockHashHex(); if (childWantsFullParent) { // Try up to 20 times with 100ms delay (total max wait: ~2s) diff --git a/packages/beacon-node/src/chain/blocks/verifyBlocksDataAvailability.ts b/packages/beacon-node/src/chain/blocks/verifyBlocksDataAvailability.ts index 470158b1aef5..fdf6369c0327 100644 --- a/packages/beacon-node/src/chain/blocks/verifyBlocksDataAvailability.ts +++ b/packages/beacon-node/src/chain/blocks/verifyBlocksDataAvailability.ts @@ -29,6 +29,9 @@ export async function verifyBlocksDataAvailability( const availableTime = Math.max(0, Math.max(...blocks.map((blockInput) => blockInput.getTimeComplete()))); const dataAvailabilityStatuses: DataAvailabilityStatus[] = blocks.map((blockInput) => { + if (blockInput.type === DAType.NoData) { + return DataAvailabilityStatus.NotRequired; + } if (blockInput.type === DAType.PreData) { return DataAvailabilityStatus.PreData; } diff --git a/packages/beacon-node/src/chain/blocks/verifyBlocksExecutionPayloads.ts b/packages/beacon-node/src/chain/blocks/verifyBlocksExecutionPayloads.ts index 1509af1b0135..ee25caad04b5 100644 --- a/packages/beacon-node/src/chain/blocks/verifyBlocksExecutionPayloads.ts +++ b/packages/beacon-node/src/chain/blocks/verifyBlocksExecutionPayloads.ts @@ -23,7 +23,7 @@ import {IClock} from "../../util/clock.js"; import {getBlobKzgCommitments} from "../../util/dataColumns.js"; import {BlockError, BlockErrorCode} from "../errors/index.js"; import {BlockProcessOpts} from "../options.js"; -import {isBlockInputBlobs, isBlockInputColumns} from "./blockInput/blockInput.js"; +import {isBlockInputBlobs, isBlockInputColumns, isBlockInputNoData} from "./blockInput/blockInput.js"; import {IBlockInput} from "./blockInput/types.js"; import {ImportBlockOpts} from "./types.js"; @@ -54,7 +54,7 @@ type VerifyBlockExecutionResponse = | {executionStatus: ExecutionStatus.Valid; lvhResponse: LVHValidResponse; execError: null} | {executionStatus: ExecutionStatus.Syncing; lvhResponse?: LVHValidResponse; execError: null} | {executionStatus: ExecutionStatus.PreMerge; lvhResponse: undefined; execError: null} - | {executionStatus: ExecutionStatus.PendingEnvelope; lvhResponse: undefined; execError: null}; + | {executionStatus: ExecutionStatus.PayloadSeparated; lvhResponse: undefined; execError: null}; /** * Verifies 1 or more execution payloads from a linear sequence of blocks. @@ -158,26 +158,26 @@ export async function verifyBlockExecutionPayload( preState0: CachedBeaconStateAllForks ): Promise { const block = blockInput.getBlock(); - // TODO: Handle better notifyNewPayload() returning error is syncing const fork = blockInput.forkName; - const executionEnabled = - ForkSeq[fork] >= ForkSeq.gloas || (isExecutionStateType(preState0) && isExecutionEnabled(preState0, block.message)); + + // Gloas block doesn't have execution payload. Return right away + if (isBlockInputNoData(blockInput)) { + return {executionStatus: ExecutionStatus.PayloadSeparated, lvhResponse: undefined, execError: null}; + } const envelope = signedEnvelope?.message ?? null; /** Not null if execution payload is embedded in the block body (pre-Gloas post-merge blocks) */ const executionPayloadEnabled = - executionEnabled && isExecutionBlockBodyType(block.message.body) ? block.message.body.executionPayload : null; - const executionPayloadFromEnvelope = executionEnabled && envelope ? envelope.payload : null; + isExecutionStateType(preState0) && + isExecutionBlockBodyType(block.message.body) && + isExecutionEnabled(preState0, block.message) + ? block.message.body.executionPayload + : null; + const executionPayloadFromEnvelope = envelope ? envelope.payload : null; const executionPayload = executionPayloadEnabled ?? executionPayloadFromEnvelope; if (!executionPayload) { - if (executionEnabled) { - // Post-Gloas bid-only blocks: execution payload is delivered separately via envelope. - // Block is valid but payload status is pending until envelope arrives. - return {executionStatus: ExecutionStatus.PendingEnvelope, lvhResponse: undefined, execError: null}; - } - // Pre-merge block, no execution payload to verify return {executionStatus: ExecutionStatus.PreMerge, lvhResponse: undefined, execError: null}; } diff --git a/packages/beacon-node/src/chain/blocks/verifyBlocksSignatures.ts b/packages/beacon-node/src/chain/blocks/verifyBlocksSignatures.ts index 7f78be6dc269..1054ef7985ff 100644 --- a/packages/beacon-node/src/chain/blocks/verifyBlocksSignatures.ts +++ b/packages/beacon-node/src/chain/blocks/verifyBlocksSignatures.ts @@ -109,8 +109,11 @@ async function verifyExecutionPayloadEnvelopeSignature( const envelope = signedEnvelope.message; const pubkey = envelope.builderIndex === BUILDER_INDEX_SELF_BUILD - ? state.epochCtx.index2pubkey[block.message.proposerIndex] + ? state.epochCtx.getPubkey(block.message.proposerIndex) : PublicKey.fromBytes(state.builders.getReadonly(envelope.builderIndex).pubkey); + if (pubkey == null) { + return false; + } const publicKey = pubkey instanceof PublicKey ? pubkey : PublicKey.fromBytes(pubkey); const signatureSet = createSingleSignatureSetFromComponents( publicKey, diff --git a/packages/beacon-node/src/chain/blocks/verifyBlocksStateTransitionOnly.ts b/packages/beacon-node/src/chain/blocks/verifyBlocksStateTransitionOnly.ts index 11fd079c6c5d..98e2a384fad1 100644 --- a/packages/beacon-node/src/chain/blocks/verifyBlocksStateTransitionOnly.ts +++ b/packages/beacon-node/src/chain/blocks/verifyBlocksStateTransitionOnly.ts @@ -121,7 +121,10 @@ export async function verifyBlocksStateTransitionOnly( if (signedEnvelope && isGloasBeaconBlock(block.message)) { const postEnvelopeState = postState.clone(true) as CachedBeaconStateGloas; // Envelope signatures are verified in verifyBlocksSignatures(); avoid duplicate checks here. - processExecutionPayloadEnvelope(postEnvelopeState, signedEnvelope, false); + processExecutionPayloadEnvelope(postEnvelopeState, signedEnvelope, { + verifySignature: false, + verifyStateRoot: false, + }); const envelopeHashTreeRootTimer = metrics?.stateHashTreeRootTime.startTimer({ source: StateHashTreeRootSource.envelopeTransition, diff --git a/packages/beacon-node/src/chain/blocks/writeBlockInputToDb.ts b/packages/beacon-node/src/chain/blocks/writeBlockInputToDb.ts index bcc30c1e40c5..173044a2824f 100644 --- a/packages/beacon-node/src/chain/blocks/writeBlockInputToDb.ts +++ b/packages/beacon-node/src/chain/blocks/writeBlockInputToDb.ts @@ -1,7 +1,9 @@ -import {fulu} from "@lodestar/types"; -import {prettyPrintIndices, toRootHex} from "@lodestar/utils"; +import {ForkPostDeneb, isForkPostDeneb} from "@lodestar/params"; +import {SignedBeaconBlock} from "@lodestar/types"; +import {fromHex, toRootHex} from "@lodestar/utils"; +import {getBlobKzgCommitments} from "../../util/dataColumns.js"; import {BeaconChain} from "../chain.js"; -import {IBlockInput, isBlockInputBlobs, isBlockInputColumns} from "./blockInput/index.js"; +import {IBlockInput, IDataColumnsInput, isBlockInputBlobs, isBlockInputColumns} from "./blockInput/index.js"; import {BLOB_AVAILABILITY_TIMEOUT} from "./verifyBlocksDataAvailability.js"; /** @@ -10,129 +12,128 @@ import {BLOB_AVAILABILITY_TIMEOUT} from "./verifyBlocksDataAvailability.js"; * * This operation may be performed before, during or after importing to the fork-choice. As long as errors * are handled properly for eventual consistency. + * + * Block+blobs (pre-fulu) and data columns (fulu+) are written in parallel. */ -export async function writeBlockInputToDb(this: BeaconChain, blocksInputs: IBlockInput[]): Promise { - const fnPromises: Promise[] = []; - // track slots for logging - const slots: number[] = []; - - for (const blockInput of blocksInputs) { - const block = blockInput.getBlock(); - const slot = block.message.slot; - slots.push(slot); - const blockRoot = this.config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message); - const blockRootHex = toRootHex(blockRoot); - const blockBytes = this.serializedCache.get(block); - if (blockBytes) { - // skip serializing data if we already have it - this.metrics?.importBlock.persistBlockWithSerializedDataCount.inc(); - fnPromises.push(this.db.block.putBinary(this.db.block.getId(block), blockBytes)); - } else { - this.metrics?.importBlock.persistBlockNoSerializedDataCount.inc(); - fnPromises.push(this.db.block.add(block)); - } +export async function writeBlockInputToDb(this: BeaconChain, blockInput: IBlockInput): Promise { + const promises: Promise[] = [writeBlockAndBlobsToDb.call(this, blockInput)]; - this.logger.debug("Persist block to hot DB", { - slot: block.message.slot, - root: blockRootHex, - inputType: blockInput.type, - }); + if (isBlockInputColumns(blockInput)) { + promises.push(writeDataColumnsToDb.call(this, blockInput)); + } - if (!blockInput.hasAllData()) { - await blockInput.waitForAllData(BLOB_AVAILABILITY_TIMEOUT); - } + await Promise.all(promises); + this.logger.debug("Persisted blockInput to db", {slot: blockInput.slot, root: blockInput.blockRootHex}); +} - // NOTE: Old data is pruned on archive - if (isBlockInputColumns(blockInput)) { - if (!blockInput.hasComputedAllData()) { - // Supernodes may only have a subset of the data columns by the time the block begins to be imported - // because full data availability can be assumed after NUMBER_OF_COLUMNS / 2 columns are available. - // Here, however, all data columns must be fully available/reconstructed before persisting to the DB. - await blockInput.waitForComputedAllData(BLOB_AVAILABILITY_TIMEOUT).catch(() => { - this.logger.debug("Failed to wait for computed all data", {slot, blockRoot: blockRootHex}); - }); - } +async function writeBlockAndBlobsToDb(this: BeaconChain, blockInput: IBlockInput): Promise { + const block = blockInput.getBlock(); + const slot = block.message.slot; + const blockRoot = this.config.getForkTypes(slot).BeaconBlock.hashTreeRoot(block.message); + const blockRootHex = toRootHex(blockRoot); + const numBlobs = isForkPostDeneb(blockInput.forkName) + ? getBlobKzgCommitments(blockInput.forkName, block as SignedBeaconBlock).length + : undefined; + const fnPromises: Promise[] = []; - const {custodyColumns} = this.custodyConfig; - const blobsLen = (block.message as fulu.BeaconBlock).body.blobKzgCommitments.length; - let dataColumnsLen: number; - if (blobsLen === 0) { - dataColumnsLen = 0; - } else { - dataColumnsLen = custodyColumns.length; - } + const blockBytes = this.serializedCache.get(block); + if (blockBytes) { + // skip serializing data if we already have it + this.metrics?.importBlock.persistBlockWithSerializedDataCount.inc(); + fnPromises.push(this.db.block.putBinary(this.db.block.getId(block), blockBytes)); + } else { + this.metrics?.importBlock.persistBlockNoSerializedDataCount.inc(); + fnPromises.push(this.db.block.add(block)); + } - const dataColumnSidecars = blockInput.getCustodyColumns(); - if (dataColumnSidecars.length !== dataColumnsLen) { - this.logger.debug( - `Invalid dataColumnSidecars=${dataColumnSidecars.length} for custody expected custodyColumnsLen=${dataColumnsLen}` - ); - } + this.logger.debug("Persist block to hot DB", {slot, root: blockRootHex, inputType: blockInput.type, numBlobs}); - const binaryPuts = []; - const nonbinaryPuts = []; - for (const dataColumnSidecar of dataColumnSidecars) { - // skip reserializing column if we already have it - const serialized = this.serializedCache.get(dataColumnSidecar); - if (serialized) { - binaryPuts.push({key: dataColumnSidecar.index, value: serialized}); - } else { - nonbinaryPuts.push(dataColumnSidecar); + if (isBlockInputBlobs(blockInput)) { + fnPromises.push( + (async () => { + if (!blockInput.hasAllData()) { + await blockInput.waitForAllData(BLOB_AVAILABILITY_TIMEOUT); } - } - fnPromises.push(this.db.dataColumnSidecar.putManyBinary(blockRoot, binaryPuts)); - fnPromises.push(this.db.dataColumnSidecar.putMany(blockRoot, nonbinaryPuts)); - this.logger.debug("Persisted dataColumnSidecars to hot DB", { - slot: block.message.slot, - root: blockRootHex, - dataColumnSidecars: dataColumnSidecars.length, - numBlobs: blobsLen, - custodyColumns: custodyColumns.length, - }); - } else if (isBlockInputBlobs(blockInput)) { - const blobSidecars = blockInput.getBlobs(); - fnPromises.push(this.db.blobSidecars.add({blockRoot, slot: block.message.slot, blobSidecars})); - this.logger.debug("Persisted blobSidecars to hot DB", { - blobsLen: blobSidecars.length, - slot: block.message.slot, - root: blockRootHex, - }); - } + const blobSidecars = blockInput.getBlobs(); + await this.db.blobSidecars.add({blockRoot, slot, blobSidecars}); + this.logger.debug("Persisted blobSidecars to hot DB", { + slot, + root: blockRootHex, + numBlobs: blobSidecars.length, + }); + })() + ); + } + + await Promise.all(fnPromises); +} + +/** + * Persists data columns to DB for a given block. Accepts a narrow sub-interface of IBlockInput + * so it can be reused across forks (e.g. Fulu, Gloas). + * + * NOTE: Old data is pruned on archive. + */ +export async function writeDataColumnsToDb(this: BeaconChain, blockInput: IDataColumnsInput): Promise { + const {slot, blockRootHex} = blockInput; + const blockRoot = fromHex(blockRootHex); - await Promise.all(fnPromises); - this.logger.debug("Persisted blocksInput to db", { - blocksInput: blocksInputs.length, - slots: prettyPrintIndices(slots), + if (!blockInput.hasComputedAllData()) { + // Supernodes may only have a subset of the data columns by the time the block begins to be imported + // because full data availability can be assumed after NUMBER_OF_COLUMNS / 2 columns are available. + // Here, however, all data columns must be fully available/reconstructed before persisting to the DB. + await blockInput.waitForComputedAllData(BLOB_AVAILABILITY_TIMEOUT).catch(() => { + this.logger.debug("Failed to wait for computed all data", {slot, blockRoot: blockRootHex}); }); } + + const {custodyColumns} = this.custodyConfig; + const dataColumnSidecars = blockInput.getCustodyColumns(); + + const binaryPuts: {key: number; value: Uint8Array}[] = []; + const nonbinaryPuts = []; + for (const dataColumnSidecar of dataColumnSidecars) { + // skip reserializing column if we already have it + const serialized = this.serializedCache.get(dataColumnSidecar); + if (serialized) { + binaryPuts.push({key: dataColumnSidecar.index, value: serialized}); + } else { + nonbinaryPuts.push(dataColumnSidecar); + } + } + + await Promise.all([ + this.db.dataColumnSidecar.putManyBinary(blockRoot, binaryPuts), + this.db.dataColumnSidecar.putMany(blockRoot, nonbinaryPuts), + ]); + + this.logger.debug("Persisted dataColumnSidecars to hot DB", { + slot, + root: blockRootHex, + dataColumnSidecars: dataColumnSidecars.length, + custodyColumns: custodyColumns.length, + numBlobs: dataColumnSidecars[0]?.column.length, + }); } -export async function persistBlockInputs(this: BeaconChain, blockInputs: IBlockInput[]): Promise { +export async function persistBlockInput(this: BeaconChain, blockInput: IBlockInput): Promise { await writeBlockInputToDb - .call(this, blockInputs) + .call(this, blockInput) .catch((e) => { this.logger.debug( "Error persisting block input in hot db", { - count: blockInputs.length, - slot: blockInputs[0].slot, - root: blockInputs[0].blockRootHex, + slot: blockInput.slot, + root: blockInput.blockRootHex, }, e ); }) .finally(() => { - for (const blockInput of blockInputs) { - this.seenBlockInputCache.prune(blockInput.blockRootHex); - } - // Without forcefully clearing this cache, we would rely on WeakMap to evict memory which is not reliable. - // Clear here (after the DB write) so that writeBlockInputToDb can still use the cached serialized bytes. - this.serializedCache.clear(); - if (blockInputs.length === 1) { - this.logger.debug("Pruned block input", { - slot: blockInputs[0].slot, - root: blockInputs[0].blockRootHex, - }); - } + this.seenBlockInputCache.prune(blockInput.blockRootHex); + this.logger.debug("Pruned block input", { + slot: blockInput.slot, + root: blockInput.blockRootHex, + }); }); } diff --git a/packages/beacon-node/src/chain/blocks/writePayloadEnvelopeInputToDb.ts b/packages/beacon-node/src/chain/blocks/writePayloadEnvelopeInputToDb.ts new file mode 100644 index 000000000000..425bd83adca8 --- /dev/null +++ b/packages/beacon-node/src/chain/blocks/writePayloadEnvelopeInputToDb.ts @@ -0,0 +1,55 @@ +import {BeaconChain} from "../chain.js"; +import {PayloadEnvelopeInput} from "../seenCache/seenPayloadEnvelopeInput.js"; +import {writeDataColumnsToDb} from "./writeBlockInputToDb.js"; + +/** + * Persists payload envelope data to DB. This operation must be eventually completed if a payload is imported. + * + * TODO GLOAS: Persist envelope metadata (stateRoot, executionRequests, builderIndex, etc.) without the full + * execution payload body — only keep the blockHash reference. The EL already stores the payload. + * See https://github.com/ChainSafe/lodestar/issues/5671 + */ +export async function writePayloadEnvelopeInputToDb( + this: BeaconChain, + payloadInput: PayloadEnvelopeInput +): Promise { + const envelope = payloadInput.getPayloadEnvelope(); + const blockRootHex = payloadInput.blockRootHex; + + const envelopeBytes = this.serializedCache.get(envelope); + const envelopePromise = envelopeBytes + ? this.db.executionPayloadEnvelope.putBinary(this.db.executionPayloadEnvelope.getId(envelope), envelopeBytes) + : this.db.executionPayloadEnvelope.add(envelope); + + // Write envelope and data columns in parallel (reuses shared column writing logic) + await Promise.all([envelopePromise, writeDataColumnsToDb.call(this, payloadInput)]); + this.logger.debug("Persisted payload envelope to db", { + slot: payloadInput.slot, + root: blockRootHex, + }); +} + +export async function persistPayloadEnvelopeInput( + this: BeaconChain, + payloadInput: PayloadEnvelopeInput +): Promise { + await writePayloadEnvelopeInputToDb + .call(this, payloadInput) + .catch((e) => { + this.logger.error( + "Error persisting payload envelope in hot db", + { + slot: payloadInput.slot, + root: payloadInput.blockRootHex, + }, + e + ); + }) + .finally(() => { + this.seenPayloadEnvelopeInputCache.prune(payloadInput.blockRootHex); + this.logger.debug("Pruned payload envelope input", { + slot: payloadInput.slot, + root: payloadInput.blockRootHex, + }); + }); +} diff --git a/packages/beacon-node/src/chain/bls/multithread/index.ts b/packages/beacon-node/src/chain/bls/multithread/index.ts index a3ca9ca803b6..d5b95609afe2 100644 --- a/packages/beacon-node/src/chain/bls/multithread/index.ts +++ b/packages/beacon-node/src/chain/bls/multithread/index.ts @@ -7,7 +7,7 @@ import {Worker, spawn} from "@chainsafe/threads"; self = undefined; import {PublicKey} from "@chainsafe/blst"; -import {ISignatureSet, Index2PubkeyCache} from "@lodestar/state-transition"; +import {ISignatureSet, PubkeyCache} from "@lodestar/state-transition"; import {Logger} from "@lodestar/utils"; import {Metrics} from "../../../metrics/index.js"; import {LinkedList} from "../../../util/array.js"; @@ -34,7 +34,7 @@ const workerDir = process.env.NODE_ENV === "test" ? "../../../../lib/chain/bls/m export type BlsMultiThreadWorkerPoolModules = { logger: Logger; metrics: Metrics | null; - index2pubkey: Index2PubkeyCache; + pubkeyCache: PubkeyCache; }; export type BlsMultiThreadWorkerPoolOptions = { @@ -114,7 +114,7 @@ type WorkerDescriptor = { export class BlsMultiThreadWorkerPool implements IBlsVerifier { private readonly logger: Logger; private readonly metrics: Metrics | null; - private readonly index2pubkey: Index2PubkeyCache; + private readonly pubkeyCache: PubkeyCache; private readonly workers: WorkerDescriptor[]; private readonly jobs = new LinkedList(); @@ -130,10 +130,10 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier { private workersBusy = 0; constructor(options: BlsMultiThreadWorkerPoolOptions, modules: BlsMultiThreadWorkerPoolModules) { - const {logger, metrics, index2pubkey} = modules; + const {logger, metrics, pubkeyCache} = modules; this.logger = logger; this.metrics = metrics; - this.index2pubkey = index2pubkey; + this.pubkeyCache = pubkeyCache; this.blsVerifyAllMultiThread = options.blsVerifyAllMultiThread ?? false; // Use compressed for herumi for now. @@ -173,7 +173,7 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier { try { return verifySignatureSetsMaybeBatch( sets.map((set) => ({ - publicKey: getAggregatedPubkey(set, this.index2pubkey), + publicKey: getAggregatedPubkey(set, this.pubkeyCache), message: set.signingRoot.valueOf(), signature: set.signature, })) @@ -398,7 +398,7 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier { try { // Note: This can throw, must be handled per-job. // Pubkey and signature aggregation is defered here - workReq = await jobItemWorkReq(job, this.index2pubkey, this.metrics); + workReq = await jobItemWorkReq(job, this.pubkeyCache, this.metrics); } catch (e) { this.metrics?.blsThreadPool.errorAggregateSignatureSetsCount.inc({type: job.type}); diff --git a/packages/beacon-node/src/chain/bls/multithread/jobItem.ts b/packages/beacon-node/src/chain/bls/multithread/jobItem.ts index a5d28a524490..1f4b6a7e2f84 100644 --- a/packages/beacon-node/src/chain/bls/multithread/jobItem.ts +++ b/packages/beacon-node/src/chain/bls/multithread/jobItem.ts @@ -1,5 +1,5 @@ import {PublicKey, asyncAggregateWithRandomness} from "@chainsafe/blst"; -import {ISignatureSet, Index2PubkeyCache, SignatureSetType} from "@lodestar/state-transition"; +import {ISignatureSet, PubkeyCache, SignatureSetType} from "@lodestar/state-transition"; import {Metrics} from "../../../metrics/metrics.js"; import {LinkedList} from "../../../util/array.js"; import {VerifySignatureOpts} from "../interface.js"; @@ -50,7 +50,7 @@ export function jobItemSigSets(job: JobQueueItem): number { */ export async function jobItemWorkReq( job: JobQueueItem, - index2pubkey: Index2PubkeyCache, + pubkeyCache: PubkeyCache, metrics: Metrics | null ): Promise { switch (job.type) { @@ -59,7 +59,7 @@ export async function jobItemWorkReq( opts: job.opts, sets: job.sets.map((set) => ({ // this can throw, handled in the consumer code - publicKey: getAggregatedPubkey(set, index2pubkey, metrics).toBytes(), + publicKey: getAggregatedPubkey(set, pubkeyCache, metrics).toBytes(), signature: set.signature, message: set.signingRoot, })), diff --git a/packages/beacon-node/src/chain/bls/singleThread.ts b/packages/beacon-node/src/chain/bls/singleThread.ts index 673035e204c7..1e2f29f791ab 100644 --- a/packages/beacon-node/src/chain/bls/singleThread.ts +++ b/packages/beacon-node/src/chain/bls/singleThread.ts @@ -1,5 +1,5 @@ import {PublicKey, Signature, aggregatePublicKeys, aggregateSignatures, verify} from "@chainsafe/blst"; -import {ISignatureSet, Index2PubkeyCache} from "@lodestar/state-transition"; +import {ISignatureSet, PubkeyCache} from "@lodestar/state-transition"; import {Metrics} from "../../metrics/index.js"; import {IBlsVerifier} from "./interface.js"; import {verifySignatureSetsMaybeBatch} from "./maybeBatch.js"; @@ -7,18 +7,18 @@ import {getAggregatedPubkey, getAggregatedPubkeysCount} from "./utils.js"; export class BlsSingleThreadVerifier implements IBlsVerifier { private readonly metrics: Metrics | null; - private readonly index2pubkey: Index2PubkeyCache; + private readonly pubkeyCache: PubkeyCache; - constructor({metrics = null, index2pubkey}: {metrics: Metrics | null; index2pubkey: Index2PubkeyCache}) { + constructor({metrics = null, pubkeyCache}: {metrics: Metrics | null; pubkeyCache: PubkeyCache}) { this.metrics = metrics; - this.index2pubkey = index2pubkey; + this.pubkeyCache = pubkeyCache; } async verifySignatureSets(sets: ISignatureSet[]): Promise { this.metrics?.bls.aggregatedPubkeys.inc(getAggregatedPubkeysCount(sets)); const setsAggregated = sets.map((set) => ({ - publicKey: getAggregatedPubkey(set, this.index2pubkey, this.metrics), + publicKey: getAggregatedPubkey(set, this.pubkeyCache, this.metrics), message: set.signingRoot, signature: set.signature, })); diff --git a/packages/beacon-node/src/chain/bls/utils.ts b/packages/beacon-node/src/chain/bls/utils.ts index e01ab788026f..d73528dad4ef 100644 --- a/packages/beacon-node/src/chain/bls/utils.ts +++ b/packages/beacon-node/src/chain/bls/utils.ts @@ -1,22 +1,25 @@ import {PublicKey, aggregatePublicKeys} from "@chainsafe/blst"; -import {ISignatureSet, Index2PubkeyCache, SignatureSetType} from "@lodestar/state-transition"; +import {ISignatureSet, PubkeyCache, SignatureSetType} from "@lodestar/state-transition"; import {Metrics} from "../../metrics/metrics.js"; export function getAggregatedPubkey( signatureSet: ISignatureSet, - index2pubkey: Index2PubkeyCache, + pubkeyCache: PubkeyCache, metrics: Metrics | null = null ): PublicKey { switch (signatureSet.type) { case SignatureSetType.single: return signatureSet.pubkey; - case SignatureSetType.indexed: - return index2pubkey[signatureSet.index]; + case SignatureSetType.indexed: { + return pubkeyCache.getOrThrow(signatureSet.index); + } case SignatureSetType.aggregate: { const timer = metrics?.blsThreadPool.pubkeysAggregationMainThreadDuration.startTimer(); - const pubkeys = signatureSet.indices.map((i) => index2pubkey[i]); + const pubkeys = signatureSet.indices.map((i) => { + return pubkeyCache.getOrThrow(i); + }); const aggregated = aggregatePublicKeys(pubkeys); timer?.(); return aggregated; diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 97e1ed8993cb..0edba74d52d6 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -1,15 +1,15 @@ import path from "node:path"; import {PrivateKey} from "@libp2p/interface"; -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {CompositeTypeAny, TreeView, Type} from "@chainsafe/ssz"; import {routes} from "@lodestar/api"; import {BeaconConfig} from "@lodestar/config"; import { - CheckpointWithPayload, + CheckpointWithPayloadStatus, IForkChoice, PayloadStatus, ProtoBlock, UpdateHeadOpt, + getCheckpointPayloadStatus, getSafeExecutionBlockHash, } from "@lodestar/fork-choice"; import {LoggerNode} from "@lodestar/logger/node"; @@ -29,7 +29,7 @@ import { CachedBeaconStateGloas, EffectiveBalanceIncrements, EpochShuffling, - Index2PubkeyCache, + PubkeyCache, computeAnchorCheckpoint, computeAttestationsRewards, computeBlockRewards, @@ -39,7 +39,6 @@ import { computeSyncCommitteeRewards, getEffectiveBalanceIncrementsZeroInactive, getEffectiveBalancesFromStateBytes, - isParentBlockFull, processSlots, } from "@lodestar/state-transition"; import {processExecutionPayloadEnvelope} from "@lodestar/state-transition/block"; @@ -69,7 +68,7 @@ import {Logger, fromHex, gweiToWei, isErrorAborted, pruneSetToMax, sleep, toRoot import {ProcessShutdownCallback} from "@lodestar/validator"; import {GENESIS_EPOCH, ZERO_HASH, ZERO_HASH_HEX} from "../constants/index.js"; import {IBeaconDb} from "../db/index.js"; -import {BLOB_SIDECARS_IN_WRAPPER_INDEX} from "../db/repositories/blobSidecars.ts"; +import {BLOB_SIDECARS_IN_WRAPPER_INDEX} from "../db/repositories/blobSidecars.js"; import {BuilderStatus} from "../execution/builder/http.js"; import {ExecutionPayloadStatus} from "../execution/engine/interface.js"; import {IExecutionBuilder, IExecutionEngine} from "../execution/index.js"; @@ -81,15 +80,18 @@ import {CustodyConfig, getValidatorsCustodyRequirement} from "../util/dataColumn import {callInNextEventLoop} from "../util/eventLoop.js"; import {ensureDir, writeIfNotExist} from "../util/file.js"; import {isOptimisticBlock} from "../util/forkChoice.js"; -import {JobItemQueue} from "../util/queue/itemQueue.ts"; +import {JobItemQueue} from "../util/queue/itemQueue.js"; import {SerializedCache} from "../util/serializedCache.js"; -import {getSlotFromSignedBeaconBlockSerialized} from "../util/sszBytes.ts"; +import {getSlotFromSignedBeaconBlockSerialized} from "../util/sszBytes.js"; import {ArchiveStore} from "./archiveStore/archiveStore.js"; import {CheckpointBalancesCache} from "./balancesCache.js"; import {BeaconProposerCache} from "./beaconProposerCache.js"; import {IBlockInput, isBlockInputBlobs, isBlockInputColumns} from "./blocks/blockInput/index.js"; import {BlockProcessor, ImportBlockOpts} from "./blocks/index.js"; -import {persistBlockInputs} from "./blocks/writeBlockInputToDb.ts"; +import {PayloadEnvelopeProcessor} from "./blocks/payloadEnvelopeProcessor.js"; +import {ImportPayloadOpts} from "./blocks/types.js"; +import {persistBlockInput} from "./blocks/writeBlockInputToDb.js"; +import {persistPayloadEnvelopeInput} from "./blocks/writePayloadEnvelopeInputToDb.js"; import {BlsMultiThreadWorkerPool, BlsSingleThreadVerifier, IBlsVerifier} from "./bls/index.js"; import {ColumnReconstructionTracker} from "./ColumnReconstructionTracker.js"; import {ChainEvent, ChainEventEmitter} from "./emitter.js"; @@ -114,14 +116,15 @@ import {BlockAttributes, produceBlockBody, produceCommonBlockBody} from "./produ import {QueuedStateRegenerator, RegenCaller} from "./regen/index.js"; import {ReprocessController} from "./reprocess.js"; import { + PayloadEnvelopeInput, SeenAggregators, SeenAttesters, SeenBlockProposers, SeenContributionAndProof, SeenExecutionPayloadBids, - SeenExecutionPayloadEnvelopes, SeenPayloadAttesters, SeenPayloadEnvelopeCache, + SeenPayloadEnvelopeInput, SeenSyncCommitteeMessages, } from "./seenCache/index.js"; import {SeenAggregatedAttestations} from "./seenCache/seenAggregateAndProof.js"; @@ -154,6 +157,13 @@ const DEFAULT_MAX_CACHED_PRODUCED_RESULTS = 4; */ const DEFAULT_MAX_PENDING_UNFINALIZED_BLOCK_WRITES = 16; +/** + * The maximum number of pending unfinalized payload envelope writes to the database before backpressure is applied. + * Payload envelope write queue entries hold references to payload inputs (including columns), + * keeping them in memory. Keep moderate to avoid OOM during sync. + */ +const DEFAULT_MAX_PENDING_UNFINALIZED_PAYLOAD_ENVELOPE_WRITES = 16; + export class BeaconChain implements IBeaconChain { readonly genesisTime: UintNum64; readonly genesisValidatorsRoot: Root; @@ -177,7 +187,8 @@ export class BeaconChain implements IBeaconChain { readonly lightClientServer?: LightClientServer; readonly reprocessController: ReprocessController; readonly archiveStore: ArchiveStore; - readonly unfinalizedBlockWrites: JobItemQueue<[IBlockInput[]], void>; + readonly unfinalizedBlockWrites: JobItemQueue<[IBlockInput], void>; + readonly unfinalizedPayloadEnvelopeWrites: JobItemQueue<[PayloadEnvelopeInput], void>; // Ops pool readonly attestationPool: AttestationPool; @@ -193,21 +204,20 @@ export class BeaconChain implements IBeaconChain { readonly seenAggregators = new SeenAggregators(); readonly seenPayloadAttesters = new SeenPayloadAttesters(); readonly seenAggregatedAttestations: SeenAggregatedAttestations; - readonly seenExecutionPayloadEnvelopes = new SeenExecutionPayloadEnvelopes(); readonly seenExecutionPayloadBids = new SeenExecutionPayloadBids(); readonly seenBlockProposers = new SeenBlockProposers(); readonly seenSyncCommitteeMessages = new SeenSyncCommitteeMessages(); readonly seenContributionAndProof: SeenContributionAndProof; readonly seenAttestationDatas: SeenAttestationDatas; readonly seenBlockInputCache: SeenBlockInput; + readonly seenPayloadEnvelopeInputCache: SeenPayloadEnvelopeInput; readonly seenPayloadEnvelopeCache: SeenPayloadEnvelopeCache; readonly pendingEnvelopes: Map = new Map(); // Seen cache for liveness checks readonly seenBlockAttesters = new SeenBlockAttesters(); // Global state caches - readonly pubkey2index: PubkeyIndexMap; - readonly index2pubkey: Index2PubkeyCache; + readonly pubkeyCache: PubkeyCache; readonly beaconProposerCache: BeaconProposerCache; readonly checkpointBalancesCache: CheckpointBalancesCache; @@ -230,6 +240,7 @@ export class BeaconChain implements IBeaconChain { readonly opts: IChainOptions; protected readonly blockProcessor: BlockProcessor; + protected readonly payloadEnvelopeProcessor: PayloadEnvelopeProcessor; protected readonly db: IBeaconDb; // this is only available if nHistoricalStates is enabled private readonly cpStateDatastore?: CPStateDatastore; @@ -253,8 +264,7 @@ export class BeaconChain implements IBeaconChain { { privateKey, config, - pubkey2index, - index2pubkey, + pubkeyCache, db, dbName, dataDir, @@ -270,8 +280,7 @@ export class BeaconChain implements IBeaconChain { }: { privateKey: PrivateKey; config: BeaconConfig; - pubkey2index: PubkeyIndexMap; - index2pubkey: Index2PubkeyCache; + pubkeyCache: PubkeyCache; db: IBeaconDb; dbName: string; dataDir: string; @@ -303,8 +312,8 @@ export class BeaconChain implements IBeaconChain { const emitter = new ChainEventEmitter(); // by default, verify signatures on both main threads and worker threads const bls = opts.blsVerifyAllMainThread - ? new BlsSingleThreadVerifier({metrics, index2pubkey}) - : new BlsMultiThreadWorkerPool(opts, {logger, metrics, index2pubkey}); + ? new BlsSingleThreadVerifier({metrics, pubkeyCache}) + : new BlsMultiThreadWorkerPool(opts, {logger, metrics, pubkeyCache}); if (!clock) clock = new Clock({config, genesisTime: this.genesisTime, signal}); @@ -323,7 +332,9 @@ export class BeaconChain implements IBeaconChain { const nodeId = computeNodeIdFromPrivateKey(privateKey); const initialCustodyGroupCount = opts.initialCustodyGroupCount ?? config.CUSTODY_REQUIREMENT; - this.metrics?.peerDas.targetCustodyGroupCount.set(initialCustodyGroupCount); + this.metrics?.peerDas.custodyGroupCount.set(initialCustodyGroupCount); + // TODO: backfill not implemented yet + this.metrics?.peerDas.custodyGroupsBackfilled.set(0); this.custodyConfig = new CustodyConfig({ nodeId, config, @@ -332,12 +343,21 @@ export class BeaconChain implements IBeaconChain { this.beaconProposerCache = new BeaconProposerCache(opts); this.checkpointBalancesCache = new CheckpointBalancesCache(); + this.serializedCache = new SerializedCache(); this.seenBlockInputCache = new SeenBlockInput({ config, custodyConfig: this.custodyConfig, clock, chainEvents: emitter, signal, + serializedCache: this.serializedCache, + metrics, + logger, + }); + this.seenPayloadEnvelopeInputCache = new SeenPayloadEnvelopeInput({ + chainEvents: emitter, + signal, + serializedCache: this.serializedCache, metrics, logger, }); @@ -361,8 +381,7 @@ export class BeaconChain implements IBeaconChain { ]); // Global cache of validators pubkey/index mapping - this.pubkey2index = pubkey2index; - this.index2pubkey = index2pubkey; + this.pubkeyCache = pubkeyCache; const fileDataStore = opts.nHistoricalStatesFileDataStore ?? true; const blockStateCache = new FIFOBlockStateCache(this.opts, {metrics}); @@ -385,13 +404,8 @@ export class BeaconChain implements IBeaconChain { const {checkpoint} = computeAnchorCheckpoint(config, anchorState); blockStateCache.add(anchorState); blockStateCache.setHeadState(anchorState); - // Determine payload status from anchor state for Gloas - // Pre-Gloas: payloadPresent is always true (execution payload embedded in block) - // Post-Gloas: check if envelope was applied using isParentBlockFull() - const anchorPayloadPresent = isForkPostGloas(config.getForkName(anchorState.slot)) - ? isParentBlockFull(anchorState as CachedBeaconStateGloas) - : true; - checkpointStateCache.add(checkpoint, anchorState, anchorPayloadPresent); + const payloadPresent = getCheckpointPayloadStatus(anchorState, checkpoint.epoch) === PayloadStatus.FULL; + checkpointStateCache.add(checkpoint, anchorState, payloadPresent); const forkChoice = initializeForkChoice( config, @@ -425,6 +439,7 @@ export class BeaconChain implements IBeaconChain { this.reprocessController = new ReprocessController(this.metrics); this.blockProcessor = new BlockProcessor(this, metrics, opts, signal); + this.payloadEnvelopeProcessor = new PayloadEnvelopeProcessor(this, metrics, signal); this.forkChoice = forkChoice; this.clock = clock; @@ -432,8 +447,6 @@ export class BeaconChain implements IBeaconChain { this.bls = bls; this.emitter = emitter; - this.serializedCache = new SerializedCache(); - this.getBlobsTracker = new GetBlobsTracker({ logger, executionEngine: this.executionEngine, @@ -455,7 +468,7 @@ export class BeaconChain implements IBeaconChain { ); this.unfinalizedBlockWrites = new JobItemQueue( - persistBlockInputs.bind(this), + persistBlockInput.bind(this), { maxLength: DEFAULT_MAX_PENDING_UNFINALIZED_BLOCK_WRITES, signal, @@ -463,6 +476,15 @@ export class BeaconChain implements IBeaconChain { metrics?.unfinalizedBlockWritesQueue ); + this.unfinalizedPayloadEnvelopeWrites = new JobItemQueue( + persistPayloadEnvelopeInput.bind(this), + { + maxLength: DEFAULT_MAX_PENDING_UNFINALIZED_PAYLOAD_ENVELOPE_WRITES, + signal, + }, + metrics?.unfinalizedPayloadEnvelopeWritesQueue + ); + // always run PrepareNextSlotScheduler except for fork_choice spec tests if (!opts?.disablePrepareNextSlot) { new PrepareNextSlotScheduler(this, this.config, metrics, this.logger, signal); @@ -493,6 +515,7 @@ export class BeaconChain implements IBeaconChain { // we can abort any ongoing unfinalized block writes. // TODO: persist fork choice to disk and allow unfinalized block writes to complete. this.unfinalizedBlockWrites.dropAllJobs(); + this.unfinalizedPayloadEnvelopeWrites.dropAllJobs(); this.abortController.abort(); } @@ -666,12 +689,14 @@ export class BeaconChain implements IBeaconChain { return this.cpStateDatastore.readLatestSafe(); } - const persistedKey = checkpointToDatastoreKey(checkpoint); + // TODO GLOAS: Need to revisit the design of this api. Currently we just retrieve FULL state of the checkpoint for backwards compatibility. + // because pre-gloas we always store FULL checkpoint state. + const persistedKey = checkpointToDatastoreKey(checkpoint, true); return this.cpStateDatastore.read(persistedKey); } getStateByCheckpoint( - checkpoint: CheckpointWithPayload + checkpoint: CheckpointWithPayloadStatus ): {state: BeaconStateAllForks; executionOptimistic: boolean; finalized: boolean} | null { // finalized or justified checkpoint states maynot be available with PersistentCheckpointStateCache, use getCheckpointStateOrBytes() api to get Uint8Array const checkpointHexPayload = fcCheckpointToHexPayload(checkpoint); @@ -690,7 +715,7 @@ export class BeaconChain implements IBeaconChain { } async getStateOrBytesByCheckpoint( - checkpoint: CheckpointWithPayload + checkpoint: CheckpointWithPayloadStatus ): Promise<{state: CachedBeaconStateAllForks | Uint8Array; executionOptimistic: boolean; finalized: boolean} | null> { const checkpointHexPayload = fcCheckpointToHexPayload(checkpoint); const cachedStateCtx = await this.regen.getCheckpointStateOrBytes(checkpointHexPayload); @@ -999,7 +1024,7 @@ export class BeaconChain implements IBeaconChain { RegenCaller.produceBlock ); const proposerIndex = state.epochCtx.getBeaconProposer(slot); - const proposerPubKey = this.index2pubkey[proposerIndex].toBytes(); + const proposerPubKey = this.pubkeyCache.getOrThrow(proposerIndex).toBytes(); const {body, produceResult, executionPayloadValue, shouldOverrideBuilder} = await produceBlockBody.call( this, @@ -1108,7 +1133,7 @@ export class BeaconChain implements IBeaconChain { )) as CachedBeaconStateGloas; const postEnvelopeState = preEnvelopeState.clone(true) as CachedBeaconStateGloas; - processExecutionPayloadEnvelope(postEnvelopeState, signedEnvelope, true); + processExecutionPayloadEnvelope(postEnvelopeState, signedEnvelope, {verifySignature: true, verifyStateRoot: true}); const executionPayload = envelope.payload; const fork = this.config.getForkName(envelope.slot); @@ -1142,8 +1167,6 @@ export class BeaconChain implements IBeaconChain { this.regen.processPayloadState(postEnvelopeState); await this.db.executionPayloadEnvelope.put(envelope.beaconBlockRoot, signedEnvelope); - this.seenExecutionPayloadEnvelopes.add(blockRootHex, envelope.slot); - this.emitter.emit(routes.events.EventType.executionPayloadAvailable, { slot: envelope.slot, blockRoot: blockRootHex, @@ -1179,6 +1202,10 @@ export class BeaconChain implements IBeaconChain { }); } + async processExecutionPayload(payloadInput: PayloadEnvelopeInput, opts?: ImportPayloadOpts): Promise { + return this.payloadEnvelopeProcessor.processPayloadEnvelopeJob(payloadInput, opts); + } + getStatus(): Status { const head = this.forkChoice.getHead(); const finalizedCheckpoint = this.forkChoice.getFinalizedCheckpoint(); @@ -1369,7 +1396,7 @@ export class BeaconChain implements IBeaconChain { * @param blockState state that declares justified checkpoint `checkpoint` */ private justifiedBalancesGetter( - checkpoint: CheckpointWithPayload, + checkpoint: CheckpointWithPayloadStatus, blockState: CachedBeaconStateAllForks ): EffectiveBalanceIncrements { this.metrics?.balancesCache.requests.inc(); @@ -1408,7 +1435,7 @@ export class BeaconChain implements IBeaconChain { * @param blockState state that declares justified checkpoint `checkpoint` */ private closestJustifiedBalancesStateToCheckpoint( - checkpoint: CheckpointWithPayload, + checkpoint: CheckpointWithPayloadStatus, blockState: CachedBeaconStateAllForks ): {state: CachedBeaconStateAllForks; stateId: string; shouldWarn: boolean} { const checkpointHexPayload = fcCheckpointToHexPayload(checkpoint); @@ -1423,7 +1450,10 @@ export class BeaconChain implements IBeaconChain { } // Find a state in the same branch of checkpoint at same epoch. Balances should exactly the same - for (const descendantBlock of this.forkChoice.forwardIterateDescendants(checkpoint.rootHex)) { + for (const descendantBlock of this.forkChoice.forwardIterateDescendants( + checkpoint.rootHex, + checkpoint.payloadStatus + )) { if (computeEpochAtSlot(descendantBlock.slot) === checkpoint.epoch) { const descendantBlockState = this.regen.getStateSync(descendantBlock.stateRoot); if (descendantBlockState) { @@ -1439,7 +1469,10 @@ export class BeaconChain implements IBeaconChain { // Find a state in the same branch of checkpoint at a latter epoch. Balances are not the same, but should be close // Note: must call .forwardIterateDescendants() again since nodes are not sorted - for (const descendantBlock of this.forkChoice.forwardIterateDescendants(checkpoint.rootHex)) { + for (const descendantBlock of this.forkChoice.forwardIterateDescendants( + checkpoint.rootHex, + checkpoint.payloadStatus + )) { if (computeEpochAtSlot(descendantBlock.slot) > checkpoint.epoch) { const descendantBlockState = this.regen.getStateSync(descendantBlock.stateRoot); if (descendantBlockState) { @@ -1533,6 +1566,10 @@ export class BeaconChain implements IBeaconChain { private onClockEpoch(epoch: Epoch): void { this.metrics?.clockEpoch.set(epoch); + if (epoch === this.config.GLOAS_FORK_EPOCH) { + this.regen.upgradeForGloas(epoch); + } + this.seenAttesters.prune(epoch); this.seenAggregators.prune(epoch); this.seenPayloadAttesters.prune(epoch); @@ -1546,7 +1583,7 @@ export class BeaconChain implements IBeaconChain { this.seenContributionAndProof.prune(head.slot); } - private onForkChoiceJustified(this: BeaconChain, cp: CheckpointWithPayload): void { + private onForkChoiceJustified(this: BeaconChain, cp: CheckpointWithPayloadStatus): void { this.logger.verbose("Fork choice justified", {epoch: cp.epoch, root: cp.rootHex}); } @@ -1557,11 +1594,10 @@ export class BeaconChain implements IBeaconChain { }); } - private async onForkChoiceFinalized(this: BeaconChain, cp: CheckpointWithPayload): Promise { + private async onForkChoiceFinalized(this: BeaconChain, cp: CheckpointWithPayloadStatus): Promise { this.logger.verbose("Fork choice finalized", {epoch: cp.epoch, root: cp.rootHex}); const finalizedSlot = computeStartSlotAtEpoch(cp.epoch); this.seenBlockProposers.prune(finalizedSlot); - this.seenExecutionPayloadEnvelopes.prune(finalizedSlot); this.seenPayloadEnvelopeCache.onFinalized(cp); // Update validator custody to account for effective balance changes @@ -1600,7 +1636,7 @@ export class BeaconChain implements IBeaconChain { } } - private async updateValidatorsCustodyRequirement(finalizedCheckpoint: CheckpointWithPayload): Promise { + private async updateValidatorsCustodyRequirement(finalizedCheckpoint: CheckpointWithPayloadStatus): Promise { if (this.custodyConfig.targetCustodyGroupCount === this.config.NUMBER_OF_CUSTODY_GROUPS) { // Custody requirements can only be increased, we can disable dynamic custody updates // if the node already maintains custody of all custody groups in case it is configured @@ -1646,7 +1682,7 @@ export class BeaconChain implements IBeaconChain { // Only update if target is increased if (targetCustodyGroupCount > this.custodyConfig.targetCustodyGroupCount) { this.custodyConfig.updateTargetCustodyGroupCount(targetCustodyGroupCount); - this.metrics?.peerDas.targetCustodyGroupCount.set(targetCustodyGroupCount); + this.metrics?.peerDas.custodyGroupCount.set(targetCustodyGroupCount); this.logger.verbose("Updated target custody group count", { finalizedEpoch: finalizedCheckpoint.epoch, validatorCount: validatorIndices.length, @@ -1716,7 +1752,7 @@ export class BeaconChain implements IBeaconChain { throw Error(`State is not in cache for slot ${slot}`); } - const rewards = await computeAttestationsRewards(this.config, this.pubkey2index, cachedState, validatorIds); + const rewards = await computeAttestationsRewards(this.config, this.pubkeyCache, cachedState, validatorIds); return {rewards, executionOptimistic, finalized}; } @@ -1733,6 +1769,6 @@ export class BeaconChain implements IBeaconChain { preState = processSlots(preState, block.slot); // Dial preState's slot to block.slot - return computeSyncCommitteeRewards(this.config, this.index2pubkey, block, preState, validatorIds); + return computeSyncCommitteeRewards(this.config, this.pubkeyCache, block, preState, validatorIds); } } diff --git a/packages/beacon-node/src/chain/emitter.ts b/packages/beacon-node/src/chain/emitter.ts index 95a12ead032b..17818b122021 100644 --- a/packages/beacon-node/src/chain/emitter.ts +++ b/packages/beacon-node/src/chain/emitter.ts @@ -1,7 +1,7 @@ import {EventEmitter} from "node:events"; import {StrictEventEmitter} from "strict-event-emitter-types"; import {routes} from "@lodestar/api"; -import {CheckpointWithPayload} from "@lodestar/fork-choice"; +import {CheckpointWithPayloadStatus} from "@lodestar/fork-choice"; import {CachedBeaconStateAllForks} from "@lodestar/state-transition"; import {DataColumnSidecars, RootHex, deneb, phase0} from "@lodestar/types"; import {PeerIdStr} from "../util/peerId.js"; @@ -83,8 +83,8 @@ export type ChainEventData = { export type IChainEvents = ApiEvents & { [ChainEvent.checkpoint]: (checkpoint: phase0.Checkpoint, state: CachedBeaconStateAllForks) => void; - [ChainEvent.forkChoiceJustified]: (checkpoint: CheckpointWithPayload) => void; - [ChainEvent.forkChoiceFinalized]: (checkpoint: CheckpointWithPayload) => void; + [ChainEvent.forkChoiceJustified]: (checkpoint: CheckpointWithPayloadStatus) => void; + [ChainEvent.forkChoiceFinalized]: (checkpoint: CheckpointWithPayloadStatus) => void; [ChainEvent.updateTargetCustodyGroupCount]: (targetGroupCount: number) => void; diff --git a/packages/beacon-node/src/chain/errors/executionPayloadBid.ts b/packages/beacon-node/src/chain/errors/executionPayloadBid.ts index 5770d5efc045..6abb90896c60 100644 --- a/packages/beacon-node/src/chain/errors/executionPayloadBid.ts +++ b/packages/beacon-node/src/chain/errors/executionPayloadBid.ts @@ -1,5 +1,5 @@ import {BuilderIndex, RootHex, Slot} from "@lodestar/types"; -import {GossipActionError} from "./gossipValidation.ts"; +import {GossipActionError} from "./gossipValidation.js"; export enum ExecutionPayloadBidErrorCode { BUILDER_NOT_ELIGIBLE = "EXECUTION_PAYLOAD_BID_ERROR_BUILDER_NOT_ELIGIBLE", diff --git a/packages/beacon-node/src/chain/errors/executionPayloadEnvelope.ts b/packages/beacon-node/src/chain/errors/executionPayloadEnvelope.ts index af3a040a8d17..cf7a0fe4a002 100644 --- a/packages/beacon-node/src/chain/errors/executionPayloadEnvelope.ts +++ b/packages/beacon-node/src/chain/errors/executionPayloadEnvelope.ts @@ -4,17 +4,21 @@ import {GossipActionError} from "./gossipValidation.js"; export enum ExecutionPayloadEnvelopeErrorCode { BELONG_TO_FINALIZED_BLOCK = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_BELONG_TO_FINALIZED_BLOCK", BLOCK_ROOT_UNKNOWN = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_BLOCK_ROOT_UNKNOWN", + PARENT_UNKNOWN = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_PARENT_UNKNOWN", + UNKNOWN_BLOCK_STATE = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_UNKNOWN_BLOCK_STATE", ENVELOPE_ALREADY_KNOWN = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_ALREADY_KNOWN", INVALID_BLOCK = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_INVALID_BLOCK", SLOT_MISMATCH = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_SLOT_MISMATCH", BUILDER_INDEX_MISMATCH = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_BUILDER_INDEX_MISMATCH", BLOCK_HASH_MISMATCH = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_BLOCK_HASH_MISMATCH", INVALID_SIGNATURE = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_INVALID_SIGNATURE", - CACHE_FAIL = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_CACHE_FAIL", + PAYLOAD_ENVELOPE_INPUT_MISSING = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_PAYLOAD_ENVELOPE_INPUT_MISSING", } export type ExecutionPayloadEnvelopeErrorType = | {code: ExecutionPayloadEnvelopeErrorCode.BELONG_TO_FINALIZED_BLOCK; envelopeSlot: Slot; finalizedSlot: Slot} | {code: ExecutionPayloadEnvelopeErrorCode.BLOCK_ROOT_UNKNOWN; blockRoot: RootHex} + | {code: ExecutionPayloadEnvelopeErrorCode.PARENT_UNKNOWN; parentRoot: RootHex; slot: Slot} + | {code: ExecutionPayloadEnvelopeErrorCode.UNKNOWN_BLOCK_STATE; blockRoot: RootHex; slot: Slot} | { code: ExecutionPayloadEnvelopeErrorCode.ENVELOPE_ALREADY_KNOWN; blockRoot: RootHex; @@ -33,6 +37,6 @@ export type ExecutionPayloadEnvelopeErrorType = bidBlockHash: RootHex | null; } | {code: ExecutionPayloadEnvelopeErrorCode.INVALID_SIGNATURE} - | {code: ExecutionPayloadEnvelopeErrorCode.CACHE_FAIL; blockRoot: RootHex}; + | {code: ExecutionPayloadEnvelopeErrorCode.PAYLOAD_ENVELOPE_INPUT_MISSING; blockRoot: RootHex}; export class ExecutionPayloadEnvelopeError extends GossipActionError {} diff --git a/packages/beacon-node/src/chain/errors/payloadAttestation.ts b/packages/beacon-node/src/chain/errors/payloadAttestation.ts index 6acbed4a47f7..c9f15a2b657d 100644 --- a/packages/beacon-node/src/chain/errors/payloadAttestation.ts +++ b/packages/beacon-node/src/chain/errors/payloadAttestation.ts @@ -1,5 +1,5 @@ import {RootHex, Slot, ValidatorIndex} from "@lodestar/types"; -import {GossipActionError} from "./gossipValidation.ts"; +import {GossipActionError} from "./gossipValidation.js"; export enum PayloadAttestationErrorCode { NOT_CURRENT_SLOT = "PAYLOAD_ATTESTATION_ERROR_NOT_CURRENT_SLOT", diff --git a/packages/beacon-node/src/chain/forkChoice/index.ts b/packages/beacon-node/src/chain/forkChoice/index.ts index cbdca692fbb8..1ce23bb91e4a 100644 --- a/packages/beacon-node/src/chain/forkChoice/index.ts +++ b/packages/beacon-node/src/chain/forkChoice/index.ts @@ -10,7 +10,7 @@ import { ForkChoiceOpts as RawForkChoiceOpts, getCheckpointPayloadStatus, } from "@lodestar/fork-choice"; -import {ForkSeq, ZERO_HASH_HEX} from "@lodestar/params"; +import {ZERO_HASH_HEX} from "@lodestar/params"; import { CachedBeaconStateAllForks, CachedBeaconStateGloas, @@ -22,7 +22,6 @@ import { getEffectiveBalanceIncrementsZeroInactive, isExecutionStateType, isMergeTransitionComplete, - isParentBlockFull, } from "@lodestar/state-transition"; import {Slot, ssz} from "@lodestar/types"; import {Logger, toRootHex} from "@lodestar/utils"; @@ -107,7 +106,7 @@ export function initializeForkChoiceFromFinalizedState( // production code use ForkChoice constructor directly const forkchoiceConstructor = opts.forkchoiceConstructor ?? ForkChoice; - const isForkPostGloas = config.getForkSeq(state.slot) >= ForkSeq.gloas; + const isForkPostGloas = (state as CachedBeaconStateGloas).latestBlockHash !== undefined; // Determine justified checkpoint payload status const justifiedPayloadStatus = getCheckpointPayloadStatus(state, justifiedCheckpoint.epoch); @@ -164,15 +163,7 @@ export function initializeForkChoiceFromFinalizedState( : {executionPayloadBlockHash: null, executionStatus: ExecutionStatus.PreMerge}), dataAvailabilityStatus: DataAvailabilityStatus.PreData, - payloadStatus: isForkPostGloas - ? isParentBlockFull(state as CachedBeaconStateGloas) - ? PayloadStatus.FULL - : PayloadStatus.EMPTY - : PayloadStatus.FULL, - builderIndex: isForkPostGloas ? (state as CachedBeaconStateGloas).latestExecutionPayloadBid.builderIndex : null, - blockHashFromBid: isForkPostGloas - ? toRootHex((state as CachedBeaconStateGloas).latestExecutionPayloadBid.blockHash) - : null, + payloadStatus: isForkPostGloas ? PayloadStatus.PENDING : PayloadStatus.FULL, // TODO GLOAS: Post-gloas how do we know if the checkpoint payload is FULL or EMPTY? parentBlockHash: isForkPostGloas ? toRootHex((state as CachedBeaconStateGloas).latestBlockHash) : null, }, currentSlot @@ -217,7 +208,7 @@ export function initializeForkChoiceFromUnfinalizedState( // this is not the justified state, but there is no other ways to get justified balances const justifiedBalances = getEffectiveBalanceIncrementsZeroInactive(unfinalizedState); - const isForkPostGloas = config.getForkSeq(unfinalizedState.slot) >= ForkSeq.gloas; + const isForkPostGloas = (unfinalizedState as CachedBeaconStateGloas).latestBlockHash !== undefined; // For unfinalized state, use getCheckpointPayloadStatus to determine the correct status. // It checks state.execution_payload_availability to determine EMPTY vs FULL. @@ -271,17 +262,7 @@ export function initializeForkChoiceFromUnfinalizedState( : {executionPayloadBlockHash: null, executionStatus: ExecutionStatus.PreMerge}), dataAvailabilityStatus: DataAvailabilityStatus.PreData, - payloadStatus: isForkPostGloas - ? isParentBlockFull(unfinalizedState as CachedBeaconStateGloas) - ? PayloadStatus.FULL - : PayloadStatus.EMPTY - : PayloadStatus.FULL, - builderIndex: isForkPostGloas - ? (unfinalizedState as CachedBeaconStateGloas).latestExecutionPayloadBid.builderIndex - : null, - blockHashFromBid: isForkPostGloas - ? toRootHex((unfinalizedState as CachedBeaconStateGloas).latestExecutionPayloadBid.blockHash) - : null, + payloadStatus: isForkPostGloas ? PayloadStatus.PENDING : PayloadStatus.FULL, // TODO GLOAS: Post-gloas how do we know if the checkpoint payload is FULL or EMPTY? parentBlockHash: isForkPostGloas ? toRootHex((unfinalizedState as CachedBeaconStateGloas).latestBlockHash) : null, }; @@ -324,9 +305,9 @@ export function initializeForkChoiceFromUnfinalizedState( }; const protoArray = ProtoArray.initialize(finalizedBlock, currentSlot); - protoArray.onBlock(justifiedBlock, currentSlot); - protoArray.onBlock(parentBlock, currentSlot); - protoArray.onBlock(headBlock, currentSlot); + protoArray.onBlock(justifiedBlock, currentSlot, null); + protoArray.onBlock(parentBlock, currentSlot, null); + protoArray.onBlock(headBlock, currentSlot, null); logger?.verbose("Initialized protoArray successfully", {...logCtx, length: protoArray.length()}); diff --git a/packages/beacon-node/src/chain/interface.ts b/packages/beacon-node/src/chain/interface.ts index 5bc72b379248..8ab1b4cab383 100644 --- a/packages/beacon-node/src/chain/interface.ts +++ b/packages/beacon-node/src/chain/interface.ts @@ -1,13 +1,7 @@ -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {CompositeTypeAny, TreeView, Type} from "@chainsafe/ssz"; import {BeaconConfig} from "@lodestar/config"; -import {CheckpointWithHex, CheckpointWithPayload, IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; -import { - BeaconStateAllForks, - CachedBeaconStateAllForks, - EpochShuffling, - Index2PubkeyCache, -} from "@lodestar/state-transition"; +import {CheckpointWithHex, CheckpointWithPayloadStatus, IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; +import {BeaconStateAllForks, CachedBeaconStateAllForks, EpochShuffling, PubkeyCache} from "@lodestar/state-transition"; import { BeaconBlock, BlindedBeaconBlock, @@ -39,7 +33,7 @@ import {IArchiveStore} from "./archiveStore/interface.js"; import {CheckpointBalancesCache} from "./balancesCache.js"; import {BeaconProposerCache, ProposerPreparationData} from "./beaconProposerCache.js"; import {IBlockInput} from "./blocks/blockInput/index.js"; -import {ImportBlockOpts} from "./blocks/types.js"; +import {ImportBlockOpts, ImportPayloadOpts} from "./blocks/types.js"; import {IBlsVerifier} from "./bls/index.js"; import {ColumnReconstructionTracker} from "./ColumnReconstructionTracker.js"; import {ChainEventEmitter} from "./emitter.js"; @@ -65,7 +59,6 @@ import { SeenBlockProposers, SeenContributionAndProof, SeenExecutionPayloadBids, - SeenExecutionPayloadEnvelopes, SeenPayloadAttesters, SeenPayloadEnvelopeCache, SeenSyncCommitteeMessages, @@ -74,6 +67,7 @@ import {SeenAggregatedAttestations} from "./seenCache/seenAggregateAndProof.js"; import {SeenAttestationDatas} from "./seenCache/seenAttestationData.js"; import {SeenBlockAttesters} from "./seenCache/seenBlockAttesters.js"; import {SeenBlockInput} from "./seenCache/seenGossipBlockInput.js"; +import {PayloadEnvelopeInput, SeenPayloadEnvelopeInput} from "./seenCache/seenPayloadEnvelopeInput.js"; import {ShufflingCache} from "./shufflingCache.js"; import {ValidatorMonitor} from "./validatorMonitor.js"; @@ -119,8 +113,7 @@ export interface IBeaconChain { readonly regen: IStateRegenerator; readonly lightClientServer?: LightClientServer; readonly reprocessController: ReprocessController; - readonly pubkey2index: PubkeyIndexMap; - readonly index2pubkey: Index2PubkeyCache; + readonly pubkeyCache: PubkeyCache; readonly archiveStore: IArchiveStore; // Ops pool @@ -137,13 +130,13 @@ export interface IBeaconChain { readonly seenAggregators: SeenAggregators; readonly seenPayloadAttesters: SeenPayloadAttesters; readonly seenAggregatedAttestations: SeenAggregatedAttestations; - readonly seenExecutionPayloadEnvelopes: SeenExecutionPayloadEnvelopes; readonly seenExecutionPayloadBids: SeenExecutionPayloadBids; readonly seenBlockProposers: SeenBlockProposers; readonly seenSyncCommitteeMessages: SeenSyncCommitteeMessages; readonly seenContributionAndProof: SeenContributionAndProof; readonly seenAttestationDatas: SeenAttestationDatas; readonly seenBlockInputCache: SeenBlockInput; + readonly seenPayloadEnvelopeInputCache: SeenPayloadEnvelopeInput; readonly seenPayloadEnvelopeCache: SeenPayloadEnvelopeCache; /** Deserialized envelopes cached from gossip, keyed by beacon block root hex. * Block import path consumes these to import parent envelopes before state transition. */ @@ -205,7 +198,7 @@ export interface IBeaconChain { ): {state: BeaconStateAllForks; executionOptimistic: boolean; finalized: boolean} | null; /** Return state bytes by checkpoint */ getStateOrBytesByCheckpoint( - checkpoint: CheckpointWithPayload + checkpoint: CheckpointWithPayloadStatus ): Promise<{state: CachedBeaconStateAllForks | Uint8Array; executionOptimistic: boolean; finalized: boolean} | null>; /** @@ -263,6 +256,9 @@ export interface IBeaconChain { /** Process a signed execution payload envelope (post-gloas) */ importExecutionPayloadEnvelope(signedEnvelope: gloas.SignedExecutionPayloadEnvelope): Promise; + /** Process execution payload envelope: verify, import to fork choice, and persist to DB */ + processExecutionPayload(payloadInput: PayloadEnvelopeInput, opts?: ImportPayloadOpts): Promise; + getStatus(): Status; recomputeForkChoiceHead(caller: ForkchoiceCaller): ProtoBlock; diff --git a/packages/beacon-node/src/chain/opPools/utils.ts b/packages/beacon-node/src/chain/opPools/utils.ts index e136bf1d4094..ea4b2f5a71c3 100644 --- a/packages/beacon-node/src/chain/opPools/utils.ts +++ b/packages/beacon-node/src/chain/opPools/utils.ts @@ -41,7 +41,7 @@ export function isValidBlsToExecutionChangeForBlockInclusion( state: CachedBeaconStateAllForks, signedBLSToExecutionChange: capella.SignedBLSToExecutionChange ): boolean { - // For each condition from https://github.com/ethereum/consensus-specs/blob/dev/specs/capella/beacon-chain.md#new-process_bls_to_execution_change + // For each condition from https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/capella/beacon-chain.md#new-process_bls_to_execution_change // // 1. assert address_change.validator_index < len(state.validators): // If valid before will always be valid in the future, no need to check diff --git a/packages/beacon-node/src/chain/produceBlock/computeNewStateRoot.ts b/packages/beacon-node/src/chain/produceBlock/computeNewStateRoot.ts index 55c44501db74..f150c0275f54 100644 --- a/packages/beacon-node/src/chain/produceBlock/computeNewStateRoot.ts +++ b/packages/beacon-node/src/chain/produceBlock/computeNewStateRoot.ts @@ -61,7 +61,6 @@ export function computeNewStateRoot( * Compute the state root after processing an execution payload envelope. * Similar to `computeNewStateRoot` but for payload envelope processing. * - * The `postBlockState` is mutated in place, callers must ensure it is not needed afterward. */ export function computeEnvelopeStateRoot( metrics: Metrics | null, @@ -74,13 +73,20 @@ export function computeEnvelopeStateRoot( }; const processEnvelopeTimer = metrics?.blockPayload.executionPayloadEnvelopeProcessingTime.startTimer(); - processExecutionPayloadEnvelope(postBlockState, signedEnvelope, false); + const postEnvelopeState = processExecutionPayloadEnvelope(postBlockState, signedEnvelope, { + // Signature is zero-ed (G2_POINT_AT_INFINITY), skip verification + verifySignature: false, + // State root is being computed here, the envelope doesn't have it yet + verifyStateRoot: false, + // Preserve cache in source state, since the resulting state is not added to the state cache + dontTransferCache: true, + }); processEnvelopeTimer?.(); const hashTreeRootTimer = metrics?.stateHashTreeRootTime.startTimer({ source: StateHashTreeRootSource.computeEnvelopeStateRoot, }); - const stateRoot = postBlockState.hashTreeRoot(); + const stateRoot = postEnvelopeState.hashTreeRoot(); hashTreeRootTimer?.(); return stateRoot; diff --git a/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts b/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts index 22d50f529ef7..9f4f3a2e5ec7 100644 --- a/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts +++ b/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts @@ -4,6 +4,7 @@ import { BUILDER_INDEX_SELF_BUILD, ForkName, ForkPostBellatrix, + ForkPostCapella, ForkPostDeneb, ForkPostFulu, ForkPostGloas, @@ -457,7 +458,7 @@ export async function produceBlockBody( parentBlockRoot: toRootHex(parentBlockRoot), feeRecipient, }); - // https://github.com/ethereum/consensus-specs/blob/dev/specs/deneb/validator.md#constructing-the-beaconblockbody + // https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/deneb/validator.md#constructing-the-beaconblockbody const prepareRes = await prepareExecutionPayload( this, this.logger, @@ -527,9 +528,16 @@ export async function produceBlockBody( // NOTE: Even though the fulu.BlobsBundle type is superficially the same as deneb.BlobsBundle, it is NOT. // In fulu, proofs are _cell_ proofs, vs in deneb they are _blob_ proofs. + const timer = this?.metrics?.peerDas.dataColumnSidecarComputationTime.startTimer(); const cells = blobsBundle.blobs.map((blob) => kzg.computeCells(blob)); + timer?.(); if (this.opts.sanityCheckExecutionEngineBlobs) { - await validateCellsAndKzgCommitments(blobsBundle.commitments, blobsBundle.proofs, cells); + const validationTimer = this.metrics?.peerDas.kzgVerificationDataColumnBatchTime.startTimer(); + try { + await validateCellsAndKzgCommitments(blobsBundle.commitments, blobsBundle.proofs, cells); + } finally { + validationTimer?.(); + } } (blockBody as deneb.BeaconBlockBody).blobKzgCommitments = blobsBundle.commitments; @@ -584,8 +592,14 @@ export async function produceBlockBody( }); } - if (ForkSeq[fork] >= ForkSeq.capella && ForkSeq[fork] < ForkSeq.gloas) { - const {blsToExecutionChanges, executionPayload} = blockBody as capella.BeaconBlockBody; + if (ForkSeq[fork] >= ForkSeq.gloas) { + const {blsToExecutionChanges, payloadAttestations} = blockBody as BeaconBlockBody; + Object.assign(logMeta, { + blsToExecutionChanges: blsToExecutionChanges.length, + payloadAttestations: payloadAttestations.length, + }); + } else if (ForkSeq[fork] >= ForkSeq.capella) { + const {blsToExecutionChanges, executionPayload} = blockBody as BeaconBlockBody; Object.assign(logMeta, { blsToExecutionChanges: blsToExecutionChanges.length, }); @@ -596,12 +610,6 @@ export async function produceBlockBody( withdrawals: executionPayload.withdrawals.length, }); } - } else if (ForkSeq[fork] >= ForkSeq.gloas) { - const {blsToExecutionChanges, payloadAttestations} = blockBody as gloas.BeaconBlockBody; - Object.assign(logMeta, { - blsToExecutionChanges: blsToExecutionChanges.length, - payloadAttestations: payloadAttestations.length, - }); } Object.assign(logMeta, {executionPayloadValue}); @@ -750,8 +758,7 @@ export function getPayloadAttributesForSSE( let parentBlockNumber: number; if (isForkPostGloas(fork)) { - // TODO GLOAS: revisit this after fork choice changes are merged - const parentBlock = chain.forkChoice.getBlockDefaultStatus(parentBlockRoot); + const parentBlock = chain.forkChoice.getBlockHexAndBlockHash(toRootHex(parentBlockRoot), toRootHex(parentHash)); if (parentBlock?.executionPayloadBlockHash == null) { throw Error(`Parent block not found in fork choice root=${toRootHex(parentBlockRoot)}`); } diff --git a/packages/beacon-node/src/chain/regen/errors.ts b/packages/beacon-node/src/chain/regen/errors.ts index eb41e8321da3..6352d765ea11 100644 --- a/packages/beacon-node/src/chain/regen/errors.ts +++ b/packages/beacon-node/src/chain/regen/errors.ts @@ -1,3 +1,4 @@ +import {PayloadStatus} from "@lodestar/fork-choice"; import {Root, RootHex, Slot} from "@lodestar/types"; export enum RegenErrorCode { @@ -9,6 +10,8 @@ export enum RegenErrorCode { BLOCK_NOT_IN_DB = "REGEN_ERROR_BLOCK_NOT_IN_DB", STATE_TRANSITION_ERROR = "REGEN_ERROR_STATE_TRANSITION_ERROR", INVALID_STATE_ROOT = "REGEN_ERROR_INVALID_STATE_ROOT", + UNEXPECTED_PAYLOAD_STATUS = "REGEN_ERROR_UNEXPECTED_PAYLOAD_STATUS", + INTERNAL_ERROR = "REGEN_ERROR_INTERNAL_ERROR", } export type RegenErrorType = @@ -19,7 +22,9 @@ export type RegenErrorType = | {code: RegenErrorCode.TOO_MANY_BLOCK_PROCESSED; stateRoot: RootHex | Root} | {code: RegenErrorCode.BLOCK_NOT_IN_DB; blockRoot: RootHex | Root} | {code: RegenErrorCode.STATE_TRANSITION_ERROR; error: Error} - | {code: RegenErrorCode.INVALID_STATE_ROOT; slot: Slot; expected: RootHex; actual: RootHex}; + | {code: RegenErrorCode.INVALID_STATE_ROOT; slot: Slot; expected: RootHex; actual: RootHex} + | {code: RegenErrorCode.UNEXPECTED_PAYLOAD_STATUS; blockRoot: RootHex | Root; payloadStatus: PayloadStatus} + | {code: RegenErrorCode.INTERNAL_ERROR; message: string}; export class RegenError extends Error { type: RegenErrorType; diff --git a/packages/beacon-node/src/chain/regen/interface.ts b/packages/beacon-node/src/chain/regen/interface.ts index 13e76acc47b0..1b6d7a15a5c9 100644 --- a/packages/beacon-node/src/chain/regen/interface.ts +++ b/packages/beacon-node/src/chain/regen/interface.ts @@ -9,8 +9,10 @@ export enum RegenCaller { processBlock = "processBlock", produceBlock = "produceBlock", validateGossipBlock = "validateGossipBlock", + validateGossipPayloadEnvelope = "validateGossipPayloadEnvelope", validateGossipBlob = "validateGossipBlob", validateGossipDataColumn = "validateGossipDataColumn", + validateGossipExecutionPayloadEnvelope = "validateGossipExecutionPayloadEnvelope", precomputeEpoch = "precomputeEpoch", predictProposerHead = "predictProposerHead", produceAttestationData = "produceAttestationData", @@ -50,7 +52,7 @@ export interface IStateRegenerator extends IStateRegeneratorInternal { * @param blockRootHex - Block root hex * @param postState - Cached beacon state after block processing */ - processState(blockRootHex: RootHex, postState: CachedBeaconStateAllForks): void; + processBlockState(blockRootHex: RootHex, postState: CachedBeaconStateAllForks): void; /** * Process payload state for caching (after processExecutionPayloadEnvelope). * Only called for Gloas blocks that have payloads revealed. @@ -68,6 +70,7 @@ export interface IStateRegenerator extends IStateRegeneratorInternal { addCheckpointState(cp: phase0.Checkpoint, item: CachedBeaconStateAllForks, payloadPresent: boolean): void; updateHeadState(newHead: ProtoBlock, maybeHeadState: CachedBeaconStateAllForks): void; updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch, payloadPresent: boolean): number | null; + upgradeForGloas(epoch: Epoch): void; } /** diff --git a/packages/beacon-node/src/chain/regen/queued.ts b/packages/beacon-node/src/chain/regen/queued.ts index e555e30b5b10..1590ddbcd552 100644 --- a/packages/beacon-node/src/chain/regen/queued.ts +++ b/packages/beacon-node/src/chain/regen/queued.ts @@ -3,10 +3,10 @@ import {IForkChoice, PayloadStatus, ProtoBlock} from "@lodestar/fork-choice"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {CachedBeaconStateAllForks, computeEpochAtSlot} from "@lodestar/state-transition"; import {BeaconBlock, Epoch, RootHex, Slot, isGloasBeaconBlock, phase0} from "@lodestar/types"; -import {Logger, toRootHex} from "@lodestar/utils"; +import {Logger, fromHex, toRootHex} from "@lodestar/utils"; import {Metrics} from "../../metrics/index.js"; import {JobItemQueue} from "../../util/queue/index.js"; -import {getCheckpointFromState} from "../blocks/utils/checkpoint.ts"; +import {getCheckpointFromState} from "../blocks/utils/checkpoint.js"; import {BlockStateCache, CheckpointHexPayload, CheckpointStateCache} from "../stateCache/types.js"; import {RegenError, RegenErrorCode} from "./errors.js"; import { @@ -106,22 +106,14 @@ export class QueuedStateRegenerator implements IStateRegenerator { const parentEpoch = computeEpochAtSlot(parentBlock.slot); const blockEpoch = computeEpochAtSlot(block.slot); - // For Gloas blocks that extend the FULL parent path, the parent must actually be FULL. - // If it's PENDING/EMPTY, we can't provide the correct pre-state synchronously — - // fall through to the queued regen path which will throw and trigger retry. - if ( - isGloasBeaconBlock(block) && - parentBlock.payloadStatus !== PayloadStatus.FULL && - parentBlock.blockHashFromBid !== null - ) { - const childParentBlockHash = toRootHex(block.body.signedExecutionPayloadBid.message.parentBlockHash); - if (childParentBlockHash === parentBlock.blockHashFromBid) { - return null; - } + // Convert PayloadStatus to payloadPresent boolean + if (parentBlock.payloadStatus === PayloadStatus.PENDING) { + throw new RegenError({ + code: RegenErrorCode.UNEXPECTED_PAYLOAD_STATUS, + blockRoot: block.parentRoot, + payloadStatus: parentBlock.payloadStatus, + }); } - - // Convert PayloadStatus to payloadPresent boolean. - // PENDING blocks are in fork-choice but lack an envelope — treat as non-FULL. const payloadPresent = parentBlock.payloadStatus === PayloadStatus.FULL; // Check the checkpoint cache (if the pre-state is a checkpoint state) @@ -160,17 +152,18 @@ export class QueuedStateRegenerator implements IStateRegenerator { * Get state closest to head */ getClosestHeadState(head: ProtoBlock): CachedBeaconStateAllForks | null { - // Convert PayloadStatus to payloadPresent boolean. - // PENDING blocks are in fork-choice but lack an envelope — treat as non-FULL. - const preferredPayloadPresent = head.payloadStatus === PayloadStatus.FULL; - - // In some restart edge cases, fork-choice may reference a head variant whose payload status - // differs from the variant persisted in checkpoint cache. Fall back to the opposite variant - // to avoid startup failures (`headState does not exist`). + // Convert PayloadStatus to payloadPresent boolean + if (head.payloadStatus === PayloadStatus.PENDING) { + throw new RegenError({ + code: RegenErrorCode.UNEXPECTED_PAYLOAD_STATUS, + blockRoot: fromHex(head.blockRoot), + payloadStatus: head.payloadStatus, + }); + } + const payloadPresent = head.payloadStatus === PayloadStatus.FULL; return ( - this.checkpointStateCache.getLatest(head.blockRoot, Infinity, preferredPayloadPresent) || - this.blockStateCache.get(head.stateRoot) || - this.checkpointStateCache.getLatest(head.blockRoot, Infinity, !preferredPayloadPresent) + this.checkpointStateCache.getLatest(head.blockRoot, Infinity, payloadPresent) || + this.blockStateCache.get(head.stateRoot) ); } @@ -184,7 +177,7 @@ export class QueuedStateRegenerator implements IStateRegenerator { this.blockStateCache.deleteAllBeforeEpoch(finalizedEpoch); } - processState(blockRootHex: RootHex, postState: CachedBeaconStateAllForks): void { + processBlockState(blockRootHex: RootHex, postState: CachedBeaconStateAllForks): void { this.blockStateCache.add(postState); this.checkpointStateCache.processState(blockRootHex, postState).catch((e) => { this.logger.debug("Error processing block state", {blockRootHex, slot: postState.slot}, e); @@ -248,6 +241,11 @@ export class QueuedStateRegenerator implements IStateRegenerator { return this.checkpointStateCache.updatePreComputedCheckpoint(rootHex, epoch, payloadPresent); } + upgradeForGloas(epoch: Epoch): void { + this.logger.verbose("Upgrading block state cache for Gloas fork", {epoch}); + this.blockStateCache.upgradeToGloas(); + } + /** * Get the state to run with `block`. * - State after `block.parentRoot` dialed forward to block.slot diff --git a/packages/beacon-node/src/chain/regen/regen.ts b/packages/beacon-node/src/chain/regen/regen.ts index 742ae67f3904..b76470ede33a 100644 --- a/packages/beacon-node/src/chain/regen/regen.ts +++ b/packages/beacon-node/src/chain/regen/regen.ts @@ -80,10 +80,10 @@ export class StateRegenerator implements IStateRegeneratorInternal { if ( isGloasBeaconBlock(block) && parentBlock.payloadStatus !== PayloadStatus.FULL && - parentBlock.blockHashFromBid !== null + parentBlock.executionPayloadBlockHash !== null ) { const childParentBlockHash = toRootHex(block.body.signedExecutionPayloadBid.message.parentBlockHash); - if (childParentBlockHash === parentBlock.blockHashFromBid) { + if (childParentBlockHash === parentBlock.executionPayloadBlockHash) { throw new RegenError({ code: RegenErrorCode.BLOCK_NOT_IN_FORKCHOICE, blockRoot: block.parentRoot, @@ -131,9 +131,14 @@ export class StateRegenerator implements IStateRegeneratorInternal { const {checkpointStateCache} = this.modules; const epoch = computeEpochAtSlot(slot); - // Convert PayloadStatus to payloadPresent boolean. - // PENDING blocks are in fork-choice but lack an envelope — treat as non-FULL (payloadPresent=false). - // Their checkpoint states are cached with payloadPresent=false during block import. + // Convert PayloadStatus to payloadPresent boolean + if (block.payloadStatus === PayloadStatus.PENDING) { + throw new RegenError({ + code: RegenErrorCode.UNEXPECTED_PAYLOAD_STATUS, + blockRoot: fromHex(blockRoot), + payloadStatus: block.payloadStatus, + }); + } const payloadPresent = block.payloadStatus === PayloadStatus.FULL; const latestCheckpointStateCtx = allowDiskReload @@ -183,7 +188,7 @@ export class StateRegenerator implements IStateRegeneratorInternal { const getSeedStateTimer = this.modules.metrics?.regenGetState.getSeedState.startTimer({caller}); // iterateAncestorBlocks only returns ancestor blocks, not the block itself - for (const b of this.modules.forkChoice.iterateAncestorBlocks(block.blockRoot)) { + for (const b of this.modules.forkChoice.iterateAncestorBlocks(block.blockRoot, block.payloadStatus)) { state = this.modules.blockStateCache.get(b.stateRoot); if (state) { break; @@ -192,8 +197,13 @@ export class StateRegenerator implements IStateRegeneratorInternal { if (!lastBlockToReplay) continue; const epoch = computeEpochAtSlot(lastBlockToReplay.slot - 1); - // Convert PayloadStatus to payloadPresent boolean. - // PENDING blocks are treated as non-FULL (payloadPresent=false). + // Convert PayloadStatus to payloadPresent boolean + if (b.payloadStatus === PayloadStatus.PENDING) { + throw new RegenError({ + code: RegenErrorCode.INTERNAL_ERROR, + message: `Unexpected PENDING payloadStatus for ancestor block ${b.blockRoot} at slot ${b.slot}`, + }); + } const payloadPresent = b.payloadStatus === PayloadStatus.FULL; state = allowDiskReload @@ -409,14 +419,11 @@ export async function processSlotsToNearestCheckpoint( // This may becomes the "official" checkpoint state if the 1st block of epoch is skipped const checkpointState = postState; const cp = getCheckpointFromState(checkpointState); - // processSlots() only does epoch transitions, never processes payloads. - // Post-Gloas: epoch boundary states are always post-CL (block state, no payload applied), - // regardless of the source block's payload status. Always cache as EMPTY (false) - // to ensure finalized/justified API lookups find the correct variant. - // Pre-Gloas: payloadPresent is always true (execution payload embedded in block). - const isGloas = checkpointState.config.getForkSeq(checkpointState.slot) >= ForkSeq.gloas; - const cpPayloadPresent = !isGloas; - checkpointStateCache.add(cp, checkpointState, cpPayloadPresent); + // processSlots() only does epoch transitions, never processes payloads + // Pre-Gloas: payloadPresent is always true (execution payload embedded in block) + // Post-Gloas: result is a block state (payloadPresent=false) + const isPayloadPresent = checkpointState.config.getForkSeq(checkpointState.slot) < ForkSeq.gloas; + checkpointStateCache.add(cp, checkpointState, isPayloadPresent); // consumers should not mutate state ever emitter?.emit(ChainEvent.checkpoint, cp, checkpointState); diff --git a/packages/beacon-node/src/chain/seenCache/index.ts b/packages/beacon-node/src/chain/seenCache/index.ts index 7fdccb366a39..25929025de28 100644 --- a/packages/beacon-node/src/chain/seenCache/index.ts +++ b/packages/beacon-node/src/chain/seenCache/index.ts @@ -3,6 +3,6 @@ export {SeenBlockProposers} from "./seenBlockProposers.js"; export {SeenSyncCommitteeMessages} from "./seenCommittee.js"; export {SeenContributionAndProof} from "./seenCommitteeContribution.js"; export {SeenExecutionPayloadBids} from "./seenExecutionPayloadBids.js"; -export {SeenExecutionPayloadEnvelopes} from "./seenExecutionPayloadEnvelope.js"; export {SeenBlockInput} from "./seenGossipBlockInput.js"; export {SeenPayloadEnvelopeCache} from "./seenPayloadEnvelopeCache.js"; +export {PayloadEnvelopeInput, SeenPayloadEnvelopeInput} from "./seenPayloadEnvelopeInput.js"; diff --git a/packages/beacon-node/src/chain/seenCache/seenExecutionPayloadEnvelope.ts b/packages/beacon-node/src/chain/seenCache/seenExecutionPayloadEnvelope.ts deleted file mode 100644 index cbd389d29449..000000000000 --- a/packages/beacon-node/src/chain/seenCache/seenExecutionPayloadEnvelope.ts +++ /dev/null @@ -1,34 +0,0 @@ -import {RootHex, Slot} from "@lodestar/types"; - -/** - * Cache to prevent processing multiple execution payload envelopes for the same block root. - * Only one builder qualifies to submit an execution payload for a given slot. - * We only keep track of envelopes of unfinalized slots. - * [IGNORE] The node has not seen another valid `SignedExecutionPayloadEnvelope` for this block root. - */ -export class SeenExecutionPayloadEnvelopes { - private readonly slotByBlockRoot = new Map(); - private finalizedSlot: Slot = 0; - - isKnown(blockRoot: RootHex): boolean { - return this.slotByBlockRoot.has(blockRoot); - } - - add(blockRoot: RootHex, slot: Slot): void { - if (slot < this.finalizedSlot) { - throw Error(`slot ${slot} < finalizedSlot ${this.finalizedSlot}`); - } - - this.slotByBlockRoot.set(blockRoot, slot); - } - - prune(finalizedSlot: Slot): void { - this.finalizedSlot = finalizedSlot; - - for (const [blockRoot, slot] of this.slotByBlockRoot.entries()) { - if (slot < finalizedSlot) { - this.slotByBlockRoot.delete(blockRoot); - } - } - } -} diff --git a/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts b/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts index 29acb0ab704a..336256678962 100644 --- a/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts +++ b/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts @@ -17,11 +17,12 @@ import {Metrics} from "../../metrics/metrics.js"; import {MAX_LOOK_AHEAD_EPOCHS} from "../../sync/constants.js"; import {IClock} from "../../util/clock.js"; import {CustodyConfig} from "../../util/dataColumns.js"; +import {SerializedCache} from "../../util/serializedCache.js"; import { BlockInput, BlockInputBlobs, BlockInputColumns, - BlockInputPayloadBid, + BlockInputNoData, BlockInputPreData, BlockWithSource, DAType, @@ -55,6 +56,7 @@ export type SeenBlockInputCacheModules = { chainEvents: ChainEventEmitter; signal: AbortSignal; custodyConfig: CustodyConfig; + serializedCache: SerializedCache; metrics: Metrics | null; logger?: Logger; }; @@ -101,6 +103,7 @@ export class SeenBlockInput { private readonly clock: IClock; private readonly chainEvents: ChainEventEmitter; private readonly signal: AbortSignal; + private readonly serializedCache: SerializedCache; private readonly metrics: Metrics | null; private readonly logger?: Logger; private blockInputs = new Map(); @@ -109,19 +112,35 @@ export class SeenBlockInput { // and the signature to ensure we only skip verification if both match private verifiedProposerSignatures = new Map>(); - constructor({config, custodyConfig, clock, chainEvents, signal, metrics, logger}: SeenBlockInputCacheModules) { + constructor({ + config, + custodyConfig, + clock, + chainEvents, + signal, + serializedCache, + metrics, + logger, + }: SeenBlockInputCacheModules) { this.config = config; this.custodyConfig = custodyConfig; this.clock = clock; this.chainEvents = chainEvents; this.signal = signal; + this.serializedCache = serializedCache; this.metrics = metrics; this.logger = logger; if (metrics) { - metrics.seenCache.blockInput.blockInputCount.addCollect(() => - metrics.seenCache.blockInput.blockInputCount.set(this.blockInputs.size) - ); + metrics.seenCache.blockInput.blockInputCount.addCollect(() => { + metrics.seenCache.blockInput.blockInputCount.set(this.blockInputs.size); + metrics.seenCache.blockInput.serializedObjectRefs.set( + Array.from(this.blockInputs.values()).reduce( + (count, blockInput) => count + blockInput.getSerializedCacheKeys().length, + 0 + ) + ); + }); } this.chainEvents.on(ChainEvent.forkChoiceFinalized, this.onFinalized); @@ -142,7 +161,10 @@ export class SeenBlockInput { * Removes the single BlockInput from the cache */ remove(rootHex: RootHex): void { - this.blockInputs.delete(rootHex); + const blockInput = this.blockInputs.get(rootHex); + if (blockInput) { + this.evictBlockInput(blockInput); + } } /** @@ -154,24 +176,24 @@ export class SeenBlockInput { let deletedCount = 0; while (blockInput) { deletedCount++; - this.blockInputs.delete(blockInput.blockRootHex); + this.evictBlockInput(blockInput); blockInput = this.blockInputs.get(parentRootHex ?? ""); parentRootHex = blockInput?.parentRootHex; } - this.logger?.debug(`BlockInputCache.prune deleted ${deletedCount} cached BlockInputs`); + this.logger?.debug("BlockInputCache.prune deleted cached BlockInputs", {deletedCount}); this.pruneToMaxSize(); } onFinalized = (checkpoint: CheckpointWithHex) => { let deletedCount = 0; const cutoffSlot = computeStartSlotAtEpoch(checkpoint.epoch); - for (const [rootHex, blockInput] of this.blockInputs) { + for (const [, blockInput] of this.blockInputs) { if (blockInput.slot < cutoffSlot) { deletedCount++; - this.blockInputs.delete(rootHex); + this.evictBlockInput(blockInput); } } - this.logger?.debug(`BlockInputCache.onFinalized deleted ${deletedCount} cached BlockInputs`); + this.logger?.debug("BlockInputCache.onFinalized deleted cached BlockInputs", {deletedCount}); this.pruneToMaxSize(); }; @@ -181,14 +203,13 @@ export class SeenBlockInput { if (!blockInput) { const {forkName, daOutOfRange} = this.buildCommonProps(block.message.slot); - // Post-Gloas: beacon blocks contain only a bid. No DA data dependency — the execution - // payload and data columns are delivered via their own separate gossip pipelines. if (isForkPostGloas(forkName)) { - blockInput = BlockInputPayloadBid.createFromBlock({ + // Post-gloas + blockInput = BlockInputNoData.createFromBlock({ block: block as SignedBeaconBlock, blockRootHex, daOutOfRange, - forkName: forkName as ForkPostGloas, + forkName, source, seenTimestampSec, peerIdStr, @@ -204,8 +225,8 @@ export class SeenBlockInput { seenTimestampSec, peerIdStr, }); - // Fulu Only } else if (isForkPostFulu(forkName)) { + // Fulu Only blockInput = BlockInputColumns.createFromBlock({ block: block as SignedBeaconBlock, blockRootHex, @@ -217,8 +238,8 @@ export class SeenBlockInput { seenTimestampSec, peerIdStr, }); - // Deneb and Electra } else { + // Deneb and Electra blockInput = BlockInputBlobs.createFromBlock({ block: block as SignedBeaconBlock, blockRootHex, @@ -229,6 +250,7 @@ export class SeenBlockInput { peerIdStr, }); } + this.metrics?.seenCache.blockInput.createdByBlock.inc(); this.blockInputs.set(blockInput.blockRootHex, blockInput); } @@ -328,7 +350,7 @@ export class SeenBlockInput { custodyColumns: this.custodyConfig.custodyColumns, sampledColumns: this.custodyConfig.sampledColumns, }); - this.metrics?.seenCache.blockInput.createdByBlob.inc(); + this.metrics?.seenCache.blockInput.createdByColumn.inc(); this.blockInputs.set(blockRootHex, blockInput); } @@ -409,14 +431,20 @@ export class SeenBlockInput { if (itemsToDelete > 0) { const sorted = [...this.blockInputs.entries()].sort((a, b) => a[1].slot - b[1].slot); - for (const [rootHex] of sorted) { - this.blockInputs.delete(rootHex); + for (const [, blockInput] of sorted) { + this.evictBlockInput(blockInput); itemsToDelete--; if (itemsToDelete <= 0) return; } } pruneSetToMax(this.verifiedProposerSignatures, MAX_BLOCK_INPUT_CACHE_SIZE); } + + private evictBlockInput(blockInput: IBlockInput): void { + // Without forcefully clearing this cache, we would rely on WeakMap to evict memory which is not reliable + this.serializedCache.delete(blockInput.getSerializedCacheKeys()); + this.blockInputs.delete(blockInput.blockRootHex); + } } enum SeenBlockInputCacheErrorCode { diff --git a/packages/beacon-node/src/chain/seenCache/seenPayloadEnvelopeInput.ts b/packages/beacon-node/src/chain/seenCache/seenPayloadEnvelopeInput.ts new file mode 100644 index 000000000000..1932a4c38f42 --- /dev/null +++ b/packages/beacon-node/src/chain/seenCache/seenPayloadEnvelopeInput.ts @@ -0,0 +1,106 @@ +import {CheckpointWithHex} from "@lodestar/fork-choice"; +import {computeStartSlotAtEpoch} from "@lodestar/state-transition"; +import {RootHex} from "@lodestar/types"; +import {Logger} from "@lodestar/utils"; +import {Metrics} from "../../metrics/metrics.js"; +import {SerializedCache} from "../../util/serializedCache.js"; +import {CreateFromBlockProps, PayloadEnvelopeInput} from "../blocks/payloadEnvelopeInput/index.js"; +import {ChainEvent, ChainEventEmitter} from "../emitter.js"; + +export type {PayloadEnvelopeInputState} from "../blocks/payloadEnvelopeInput/index.js"; +export {PayloadEnvelopeInput} from "../blocks/payloadEnvelopeInput/index.js"; + +export type SeenPayloadEnvelopeInputModules = { + chainEvents: ChainEventEmitter; + signal: AbortSignal; + serializedCache: SerializedCache; + metrics: Metrics | null; + logger?: Logger; +}; + +/** + * Cache for tracking PayloadEnvelopeInput instances, keyed by beacon block root. + * + * Created during block import when a block is processed. + * Pruned on finalization and after payload is written to DB. + */ +export class SeenPayloadEnvelopeInput { + private readonly chainEvents: ChainEventEmitter; + private readonly signal: AbortSignal; + private readonly serializedCache: SerializedCache; + private readonly metrics: Metrics | null; + private readonly logger?: Logger; + private payloadInputs = new Map(); + + constructor({chainEvents, signal, serializedCache, metrics, logger}: SeenPayloadEnvelopeInputModules) { + this.chainEvents = chainEvents; + this.signal = signal; + this.serializedCache = serializedCache; + this.metrics = metrics; + this.logger = logger; + + if (metrics) { + metrics.seenCache.payloadEnvelopeInput.count.addCollect(() => { + metrics.seenCache.payloadEnvelopeInput.count.set(this.payloadInputs.size); + metrics.seenCache.payloadEnvelopeInput.serializedObjectRefs.set( + Array.from(this.payloadInputs.values()).reduce( + (count, payloadInput) => count + payloadInput.getSerializedCacheKeys().length, + 0 + ) + ); + }); + } + + this.chainEvents.on(ChainEvent.forkChoiceFinalized, this.onFinalized); + this.signal.addEventListener("abort", () => { + this.chainEvents.off(ChainEvent.forkChoiceFinalized, this.onFinalized); + }); + } + + private onFinalized = (checkpoint: CheckpointWithHex): void => { + // Prune all entries with slot < finalized slot + const finalizedSlot = computeStartSlotAtEpoch(checkpoint.epoch); + let deletedCount = 0; + for (const [, input] of this.payloadInputs) { + if (input.slot < finalizedSlot) { + this.evictPayloadInput(input); + deletedCount++; + } + } + this.logger?.debug("SeenPayloadEnvelopeInput.onFinalized deleted cached entries", {deletedCount}); + }; + + add(props: CreateFromBlockProps): PayloadEnvelopeInput { + if (this.payloadInputs.has(props.blockRootHex)) { + throw new Error(`PayloadEnvelopeInput already exists for block ${props.blockRootHex}`); + } + const input = PayloadEnvelopeInput.createFromBlock(props); + this.payloadInputs.set(props.blockRootHex, input); + this.metrics?.seenCache.payloadEnvelopeInput.created.inc(); + return input; + } + + get(blockRootHex: RootHex): PayloadEnvelopeInput | undefined { + return this.payloadInputs.get(blockRootHex); + } + + has(blockRootHex: RootHex): boolean { + return this.payloadInputs.has(blockRootHex); + } + + prune(blockRootHex: RootHex): void { + const payloadInput = this.payloadInputs.get(blockRootHex); + if (payloadInput) { + this.evictPayloadInput(payloadInput); + } + } + + size(): number { + return this.payloadInputs.size; + } + + private evictPayloadInput(payloadInput: PayloadEnvelopeInput): void { + this.serializedCache.delete(payloadInput.getSerializedCacheKeys()); + this.payloadInputs.delete(payloadInput.blockRootHex); + } +} diff --git a/packages/beacon-node/src/chain/stateCache/datastore/db.ts b/packages/beacon-node/src/chain/stateCache/datastore/db.ts index 64c893c9bdd7..7d49169960d9 100644 --- a/packages/beacon-node/src/chain/stateCache/datastore/db.ts +++ b/packages/beacon-node/src/chain/stateCache/datastore/db.ts @@ -1,6 +1,6 @@ import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {Epoch, phase0, ssz} from "@lodestar/types"; -import {MapDef} from "@lodestar/utils"; +import {MapDef, byteArrayEquals} from "@lodestar/utils"; import {IBeaconDb} from "../../../db/interface.js"; import { getLastProcessedSlotFromBeaconStateSerialized, @@ -14,8 +14,8 @@ import {CPStateDatastore, DatastoreKey} from "./types.js"; export class DbCPStateDatastore implements CPStateDatastore { constructor(private readonly db: IBeaconDb) {} - async write(cpKey: phase0.Checkpoint, stateBytes: Uint8Array): Promise { - const serializedCheckpoint = checkpointToDatastoreKey(cpKey); + async write(cpKey: phase0.Checkpoint, stateBytes: Uint8Array, payloadPresent: boolean): Promise { + const serializedCheckpoint = checkpointToDatastoreKey(cpKey, payloadPresent); await this.db.checkpointState.putBinary(serializedCheckpoint, stateBytes); return serializedCheckpoint; } @@ -40,18 +40,30 @@ export class DbCPStateDatastore implements CPStateDatastore { } } +function extractCheckpointBytes(key: DatastoreKey): Uint8Array { + const fixedSize = ssz.phase0.Checkpoint.minSize; + return key.subarray(0, fixedSize); +} + export function datastoreKeyToCheckpoint(key: DatastoreKey): phase0.Checkpoint { - return ssz.phase0.Checkpoint.deserialize(key); + return ssz.phase0.Checkpoint.deserialize(extractCheckpointBytes(key)); +} + +export function checkpointToDatastoreKey(cp: phase0.Checkpoint, payloadPresent: boolean): DatastoreKey { + const cpBytes = ssz.phase0.Checkpoint.serialize(cp); + const key = new Uint8Array(cpBytes.length + 1); + key.set(cpBytes); + key[cpBytes.length] = payloadPresent ? 1 : 0; + return key; } -export function checkpointToDatastoreKey(cp: phase0.Checkpoint): DatastoreKey { - return ssz.phase0.Checkpoint.serialize(cp); +function isPayloadCheckpointState(key: DatastoreKey): boolean { + return key.at(-1) === 1; } /** - * Get the latest safe checkpoint state the node can use to boot from - * - it should be the checkpoint state that's unique in its epoch - * - its last processed block slot should be at epoch boundary or last slot of previous epoch + * Get the latest "safe" checkpoint state the node can use to boot from + * - its last processed block slot should be at epoch boundary (CRCS) or last slot of previous epoch (PRCS) * - state slot should be at epoch boundary * - state slot should be equal to epoch * SLOTS_PER_EPOCH * @@ -70,9 +82,20 @@ export async function getLatestSafeDatastoreKey( const dataStoreKeyByEpoch: Map = new Map(); for (const [epoch, keys] of checkpointsByEpoch.entries()) { - // only consider epochs with a single checkpoint to avoid ambiguity from forks if (keys.length === 1) { + // PRCS (skipped slot) or CRCS and no payloadPresent + // Pre-gloas always fall into this case dataStoreKeyByEpoch.set(epoch, keys[0]); + } else if (keys.length === 2) { + // CRCS without payload and CRCS with payload + // ie Two keys for the same checkpoint with different payloadPresent suffix (FULL/EMPTY) + // TODO GLOAS: Here we pick FULL key, there is a chance that payload is orphaned hence we not be able to sync + const cp0 = extractCheckpointBytes(keys[0]); + const cp1 = extractCheckpointBytes(keys[1]); + if (byteArrayEquals(cp0, cp1)) { + const fullKey = isPayloadCheckpointState(keys[0]) ? keys[0] : keys[1]; + dataStoreKeyByEpoch.set(epoch, fullKey); + } } } diff --git a/packages/beacon-node/src/chain/stateCache/datastore/file.ts b/packages/beacon-node/src/chain/stateCache/datastore/file.ts index 15ccbfb81f43..6c0391c3c29a 100644 --- a/packages/beacon-node/src/chain/stateCache/datastore/file.ts +++ b/packages/beacon-node/src/chain/stateCache/datastore/file.ts @@ -1,12 +1,13 @@ import path from "node:path"; -import {phase0, ssz} from "@lodestar/types"; +import {phase0} from "@lodestar/types"; import {fromHex, toHex} from "@lodestar/utils"; import {ensureDir, readFile, readFileNames, removeFile, writeIfNotExist} from "../../../util/file.js"; -import {getLatestSafeDatastoreKey} from "./db.js"; +import {checkpointToDatastoreKey, getLatestSafeDatastoreKey} from "./db.js"; import {CPStateDatastore, DatastoreKey} from "./types.js"; const CHECKPOINT_STATES_FOLDER = "checkpoint_states"; -const CHECKPOINT_FILE_NAME_LENGTH = 82; +/** 41 bytes (40 checkpoint + 1 payloadPresent) = 82 hex chars + "0x" prefix = 84 */ +const CHECKPOINT_FILE_NAME_LENGTH = 84; /** * Implementation of CPStateDatastore using file system, this is beneficial for debugging. @@ -28,8 +29,8 @@ export class FileCPStateDatastore implements CPStateDatastore { } } - async write(cpKey: phase0.Checkpoint, stateBytes: Uint8Array): Promise { - const serializedCheckpoint = ssz.phase0.Checkpoint.serialize(cpKey); + async write(cpKey: phase0.Checkpoint, stateBytes: Uint8Array, payloadPresent: boolean): Promise { + const serializedCheckpoint = checkpointToDatastoreKey(cpKey, payloadPresent); const filePath = path.join(this.folderPath, toHex(serializedCheckpoint)); await writeIfNotExist(filePath, stateBytes); return serializedCheckpoint; diff --git a/packages/beacon-node/src/chain/stateCache/datastore/types.ts b/packages/beacon-node/src/chain/stateCache/datastore/types.ts index c63c54cca1d1..de25274539d9 100644 --- a/packages/beacon-node/src/chain/stateCache/datastore/types.ts +++ b/packages/beacon-node/src/chain/stateCache/datastore/types.ts @@ -1,11 +1,12 @@ import {phase0} from "@lodestar/types"; -// With db implementation, persistedKey is serialized data of a checkpoint +// With db implementation, persistedKey is serialized data of a checkpoint + 1 +// ie a fixed size of `ssz.phase0.Checkpoint.minSize + 1` export type DatastoreKey = Uint8Array; // Make this generic to support testing export interface CPStateDatastore { - write: (cpKey: phase0.Checkpoint, stateBytes: Uint8Array) => Promise; + write: (cpKey: phase0.Checkpoint, stateBytes: Uint8Array, payloadPresent: boolean) => Promise; remove: (key: DatastoreKey) => Promise; read: (key: DatastoreKey) => Promise; readLatestSafe: () => Promise; diff --git a/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts b/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts index 61ae41db31f4..f29dff2d87b8 100644 --- a/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts +++ b/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts @@ -190,6 +190,10 @@ export class FIFOBlockStateCache implements BlockStateCache { } } + upgradeToGloas(): void { + this.maxStates = DEFAULT_MAX_BLOCK_STATES_GLOAS; + } + /** * No need for this implementation * This is only to conform to the old api diff --git a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts index d8c922dd0924..4562bac1a52b 100644 --- a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts +++ b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts @@ -1,6 +1,6 @@ import {routes} from "@lodestar/api"; import {BeaconConfig} from "@lodestar/config"; -import {CheckpointWithPayload} from "@lodestar/fork-choice"; +import {CheckpointWithPayloadStatus} from "@lodestar/fork-choice"; import { CachedBeaconStateAllForks, computeStartSlotAtEpoch, @@ -55,6 +55,22 @@ type CacheItem = InMemoryCacheItem | PersistedCacheItem; type LoadedStateBytesData = {persistedKey: DatastoreKey; stateBytes: Uint8Array}; +/** Bitmask for tracking which payload variants exist per root in the epochIndex */ +enum PayloadAvailability { + NOT_PRESENT = 1, + PRESENT = 2, +} + +const PAYLOAD_AVAILABILITY_ALL = [PayloadAvailability.NOT_PRESENT, PayloadAvailability.PRESENT] as const; + +function toPayloadAvailability(payloadPresent: boolean): PayloadAvailability { + return payloadPresent ? PayloadAvailability.PRESENT : PayloadAvailability.NOT_PRESENT; +} + +function fromPayloadAvailability(flag: PayloadAvailability): boolean { + return flag === PayloadAvailability.PRESENT; +} + /** * Before n-historical states, lodestar keeps all checkpoint states since finalized * Since Sep 2024, lodestar stores 3 most recent checkpoint states in memory and the rest on disk. The finalized state @@ -107,8 +123,8 @@ const PROCESS_CHECKPOINT_STATES_BPS = 6667; */ export class PersistentCheckpointStateCache implements CheckpointStateCache { private readonly cache: MapTracker; - /** Epoch -> Set */ - private readonly epochIndex = new MapDef>(() => new Set()); + /** Epoch -> Map */ + private readonly epochIndex = new MapDef>(() => new Map()); private readonly config: BeaconConfig; private readonly metrics: Metrics | null | undefined; private readonly logger: Logger; @@ -210,7 +226,12 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { return stateOrStateBytesData ?? null; } const {persistedKey, stateBytes} = stateOrStateBytesData; - const logMeta = {persistedKey: toHex(persistedKey)}; + const logMeta = { + epoch: cp.epoch, + rootHex: cp.rootHex, + payloadPresent: cp.payloadPresent, + persistedKey: toHex(persistedKey), + }; this.logger.debug("Reload: read state successful", logMeta); this.metrics?.cpStateCache.stateReloadSecFromSlot.observe( this.clock?.secFromSlot(this.clock?.currentSlot ?? 0) ?? 0 @@ -251,7 +272,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { // only remove persisted state once we reload successfully const cpKey = toCacheKey(cp); this.cache.set(cpKey, {type: CacheItemType.inMemory, state: newCachedState, persistedKey}); - this.epochIndex.getOrDefault(cp.epoch).add(cp.rootHex); + this.addToEpochIndex(cp.epoch, cp.rootHex, cp.payloadPresent); // don't prune from memory here, call it at the last 1/3 of slot 0 of an epoch return newCachedState; } catch (e) { @@ -305,7 +326,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { /** * Similar to get() api without reloading from disk */ - get(cpOrKey: CheckpointHexPayload | string): CachedBeaconStateAllForks | null { + get(cpOrKey: CheckpointHexPayload | CacheKey): CachedBeaconStateAllForks | null { this.metrics?.cpStateCache.lookups.inc(); const cpKey = typeof cpOrKey === "string" ? cpOrKey : toCacheKey(cpOrKey); const cacheItem = this.cache.get(cpKey); @@ -346,13 +367,18 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { this.logger.verbose("Added checkpoint state to memory but a persisted key existed", { epoch: cp.epoch, rootHex: cpHex.rootHex, + payloadPresent, persistedKey: toHex(persistedKey), }); } else { this.cache.set(key, {type: CacheItemType.inMemory, state}); - this.logger.verbose("Added checkpoint state to memory", {epoch: cp.epoch, rootHex: cpHex.rootHex}); + this.logger.verbose("Added checkpoint state to memory", { + epoch: cp.epoch, + rootHex: cpHex.rootHex, + payloadPresent, + }); } - this.epochIndex.getOrDefault(cp.epoch).add(cpHex.rootHex); + this.addToEpochIndex(cp.epoch, cpHex.rootHex, cpHex.payloadPresent); this.prunePersistedStates(); } @@ -365,7 +391,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { .sort((a, b) => b - a) .filter((e) => e <= maxEpoch); for (const epoch of epochs) { - if (this.epochIndex.get(epoch)?.has(rootHex)) { + if (this.hasPayloadVariant(epoch, rootHex, payloadPresent)) { const inMemoryClonedState = this.get({rootHex, epoch, payloadPresent}); if (inMemoryClonedState) { return inMemoryClonedState; @@ -392,7 +418,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { .sort((a, b) => b - a) .filter((e) => e <= maxEpoch); for (const epoch of epochs) { - if (this.epochIndex.get(epoch)?.has(rootHex)) { + if (this.hasPayloadVariant(epoch, rootHex, payloadPresent)) { try { const state = await this.getOrReload({rootHex, epoch, payloadPresent}); if (state) { @@ -574,9 +600,10 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { return firstState; } - for (const rootHex of this.epochIndex.get(epoch) || []) { - // Try both payloadPresent values when searching for seed state - for (const payloadPresent of [true, false]) { + for (const [rootHex, bitmask] of this.epochIndex.get(epoch) || []) { + for (const flag of PAYLOAD_AVAILABILITY_ALL) { + if (!(bitmask & flag)) continue; + const payloadPresent = fromPayloadAvailability(flag); const cpKey = toCacheKey({rootHex, epoch, payloadPresent}); const cacheItem = this.cache.get(cpKey); if (cacheItem === undefined) { @@ -618,6 +645,31 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { this.epochIndex.clear(); } + private addToEpochIndex(epoch: Epoch, rootHex: RootHex, payloadPresent: boolean): void { + const rootMap = this.epochIndex.getOrDefault(epoch); + rootMap.set(rootHex, (rootMap.get(rootHex) ?? 0) | toPayloadAvailability(payloadPresent)); + } + + private removeFromEpochIndex(epoch: Epoch, rootHex: RootHex, payloadPresent: boolean): void { + const rootMap = this.epochIndex.get(epoch); + if (rootMap === undefined) return; + const existing = rootMap.get(rootHex); + if (existing === undefined) return; + const updated = existing & ~toPayloadAvailability(payloadPresent); + if (updated === 0) { + rootMap.delete(rootHex); + if (rootMap.size === 0) { + this.epochIndex.delete(epoch); + } + } else { + rootMap.set(rootHex, updated); + } + } + + private hasPayloadVariant(epoch: Epoch, rootHex: RootHex, payloadPresent: boolean): boolean { + return Boolean((this.epochIndex.get(epoch)?.get(rootHex) ?? 0) & toPayloadAvailability(payloadPresent)); + } + /** ONLY FOR DEBUGGING PURPOSES. For lodestar debug API */ dumpSummary(): routes.lodestar.StateCacheItem[] { return Array.from(this.cache.keys()).map((key) => { @@ -696,7 +748,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { const prevEpochRoot = toRootHex(getBlockRootAtSlot(state, epochBoundarySlot - 1)); // for each epoch, usually there are 2 rootHexes respective to the 2 checkpoint states: Previous Root Checkpoint State and Current Root Checkpoint State - const cpRootHexes = this.epochIndex.get(epoch) ?? []; + const cpRootHexMap = this.epochIndex.get(epoch) ?? new Map(); const persistedRootHexes = new Set(); // 1) if there is no CRCS, persist PRCS (block 0 of epoch is skipped). In this case prevEpochRoot === epochBoundaryHex @@ -705,15 +757,16 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { persistedRootHexes.add(epochBoundaryHex); // 3) persist any states with unknown roots to this state - for (const rootHex of cpRootHexes) { + for (const rootHex of cpRootHexMap.keys()) { if (rootHex !== epochBoundaryHex && rootHex !== prevEpochRoot) { persistedRootHexes.add(rootHex); } } - for (const rootHex of cpRootHexes) { - // Process both payloadPresent variants for each rootHex - for (const payloadPresent of [true, false]) { + for (const [rootHex, bitmask] of cpRootHexMap) { + for (const flag of PAYLOAD_AVAILABILITY_ALL) { + if (!(bitmask & flag)) continue; + const payloadPresent = fromPayloadAvailability(flag); const cpKey = toCacheKey({epoch: epoch, rootHex, payloadPresent}); const cacheItem = this.cache.get(cpKey); @@ -752,7 +805,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { AllocSource.PERSISTENT_CHECKPOINTS_CACHE_STATE, (stateBytes) => { timer?.(); - return this.datastore.write(cpPersist, stateBytes); + return this.datastore.write(cpPersist, stateBytes, payloadPresent); }, this.bufferPool ); @@ -774,7 +827,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { } else { // delete the state from memory this.cache.delete(cpKey); - this.epochIndex.get(epoch)?.delete(rootHex); + this.removeFromEpochIndex(epoch, rootHex, payloadPresent); } this.metrics?.cpStateCache.statePruneFromMemoryCount.inc(); this.logger.verbose("Pruned checkpoint state from memory", logMeta); @@ -791,10 +844,11 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { */ private async deleteAllEpochItems(epoch: Epoch): Promise { let persistCount = 0; - const rootHexes = this.epochIndex.get(epoch) || []; - for (const rootHex of rootHexes) { - // Delete both payloadPresent variants for each rootHex - for (const payloadPresent of [true, false]) { + const rootHexMap = this.epochIndex.get(epoch) || new Map(); + for (const [rootHex, bitmask] of rootHexMap) { + for (const flag of PAYLOAD_AVAILABILITY_ALL) { + if (!(bitmask & flag)) continue; + const payloadPresent = fromPayloadAvailability(flag); const key = toCacheKey({rootHex, epoch, payloadPresent}); const cacheItem = this.cache.get(key); @@ -807,13 +861,23 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { } } this.cache.delete(key); + this.logger.verbose("Pruned checkpoint state", { + epoch, + rootHex, + payloadPresent, + type: cacheItem ? (isPersistedCacheItem(cacheItem) ? "persisted" : "in-memory") : "missing", + }); } } this.epochIndex.delete(epoch); - this.logger.verbose("Pruned checkpoint states for epoch", { + this.logger.verbose("Pruned all checkpoint states for epoch", { epoch, persistCount, - rootHexes: Array.from(rootHexes).join(","), + items: Array.from(rootHexMap.entries()) + .flatMap(([rootHex, bitmask]) => + PAYLOAD_AVAILABILITY_ALL.filter((f) => bitmask & f).map((f) => `${rootHex}:${fromPayloadAvailability(f)}`) + ) + .join(","), }); } @@ -874,11 +938,11 @@ export function toCheckpointHexPayload(checkpoint: phase0.Checkpoint, payloadPre } /** - * Convert fork-choice CheckpointWithPayload to beacon-node CheckpointHexPayload. + * Convert fork-choice CheckpointWithPayloadStatus to beacon-node CheckpointHexPayload. * Maps PayloadStatus enum to boolean payloadPresent. * @throws Error if checkpoint has PENDING payload status (ambiguous which variant to use) */ -export function fcCheckpointToHexPayload(checkpoint: CheckpointWithPayload): CheckpointHexPayload { +export function fcCheckpointToHexPayload(checkpoint: CheckpointWithPayloadStatus): CheckpointHexPayload { const PayloadStatus = {PENDING: 0, EMPTY: 1, FULL: 2} as const; if (checkpoint.payloadStatus === PayloadStatus.PENDING) { @@ -895,7 +959,7 @@ export function fcCheckpointToHexPayload(checkpoint: CheckpointWithPayload): Che } export function toCheckpointKey(cp: CheckpointHexPayload): string { - return `${cp.rootHex}:${cp.epoch}`; + return `${cp.rootHex}:${cp.epoch}:${cp.payloadPresent}`; } /** diff --git a/packages/beacon-node/src/chain/stateCache/types.ts b/packages/beacon-node/src/chain/stateCache/types.ts index 62c3d04c03c8..3185685eb736 100644 --- a/packages/beacon-node/src/chain/stateCache/types.ts +++ b/packages/beacon-node/src/chain/stateCache/types.ts @@ -38,6 +38,8 @@ export interface BlockStateCache { size: number; prune(headStateRootHex: RootHex): void; deleteAllBeforeEpoch(finalizedEpoch: Epoch): void; + /** Upgrade cache capacity for Gloas fork (2x states for block + payload states) */ + upgradeToGloas(): void; dumpSummary(): routes.lodestar.StateCacheItem[]; /** Expose beacon states stored in cache. Use with caution */ getStates(): IterableIterator; diff --git a/packages/beacon-node/src/chain/validation/attestation.ts b/packages/beacon-node/src/chain/validation/attestation.ts index 45950b259f0b..d410469ed1c6 100644 --- a/packages/beacon-node/src/chain/validation/attestation.ts +++ b/packages/beacon-node/src/chain/validation/attestation.ts @@ -143,7 +143,10 @@ export async function validateGossipAttestationsSameAttData( if (batchableBls) { // all signature sets should have same signing root since we filtered in network processor signatureValids = await chain.bls.verifySignatureSetsSameMessage( - signatureSets.map((set) => ({publicKey: chain.index2pubkey[set.index], signature: set.signature})), + signatureSets.map((set) => { + const publicKey = chain.pubkeyCache.getOrThrow(set.index); + return {publicKey, signature: set.signature}; + }), signatureSets[0].signingRoot ); } else { @@ -183,7 +186,7 @@ export async function validateGossipAttestationsSameAttData( chain.seenAttesters.add(targetEpoch, validatorIndex); } else { step0ResultOrErrors[oldIndex] = { - err: new AttestationError(GossipAction.IGNORE, { + err: new AttestationError(GossipAction.REJECT, { code: AttestationErrorCode.INVALID_SIGNATURE, }), }; @@ -601,10 +604,14 @@ export function verifyPropagationSlotRange(fork: ForkName, chain: IBeaconChain, // // see: https://github.com/ethereum/consensus-specs/pull/3360 if (ForkSeq[fork] < ForkSeq.deneb) { + const currentSlot = chain.clock.currentSlot; + const withinPastDisparity = currentSlot > 0 && chain.clock.isCurrentSlotGivenGossipDisparity(currentSlot - 1); const earliestPermissibleSlot = Math.max( - // slot with past tolerance of MAXIMUM_GOSSIP_CLOCK_DISPARITY - chain.clock.slotWithPastTolerance(chain.config.MAXIMUM_GOSSIP_CLOCK_DISPARITY / 1000) - - chain.config.ATTESTATION_PROPAGATION_SLOT_RANGE, + // Pre-Deneb propagation is time-bounded: an attestation remains valid at the exact old + // boundary `compute_time_at_slot(slot + range + 1) + MAXIMUM_GOSSIP_CLOCK_DISPARITY`. + // Model that boundary by extending the lower slot bound by one additional slot only while + // the clock still considers the previous slot current given gossip disparity. + currentSlot - chain.config.ATTESTATION_PROPAGATION_SLOT_RANGE - (withinPastDisparity ? 1 : 0), 0 ); diff --git a/packages/beacon-node/src/chain/validation/attesterSlashing.ts b/packages/beacon-node/src/chain/validation/attesterSlashing.ts index df6774f26b22..4fd27a1cdff3 100644 --- a/packages/beacon-node/src/chain/validation/attesterSlashing.ts +++ b/packages/beacon-node/src/chain/validation/attesterSlashing.ts @@ -2,6 +2,7 @@ import { assertValidAttesterSlashing, getAttesterSlashableIndices, getAttesterSlashingSignatureSets, + isSlashableValidator, } from "@lodestar/state-transition"; import {AttesterSlashing} from "@lodestar/types"; import {AttesterSlashingError, AttesterSlashingErrorCode, GossipAction} from "../errors/index.js"; @@ -45,7 +46,7 @@ export async function validateAttesterSlashing( // verifySignature = false, verified in batch below assertValidAttesterSlashing( chain.config, - chain.index2pubkey, + chain.pubkeyCache, state.slot, state.validators.length, attesterSlashing, @@ -58,6 +59,14 @@ export async function validateAttesterSlashing( }); } + const currentEpoch = state.epochCtx.epoch; + if (!intersectingIndices.some((index) => isSlashableValidator(state.validators.getReadonly(index), currentEpoch))) { + throw new AttesterSlashingError(GossipAction.REJECT, { + code: AttesterSlashingErrorCode.INVALID, + error: Error("AttesterSlashing has no slashable validators"), + }); + } + const signatureSets = getAttesterSlashingSignatureSets(chain.config, state.slot, attesterSlashing); if (!(await chain.bls.verifySignatureSets(signatureSets, {batchable: true, priority: prioritizeBls}))) { throw new AttesterSlashingError(GossipAction.REJECT, { diff --git a/packages/beacon-node/src/chain/validation/block.ts b/packages/beacon-node/src/chain/validation/block.ts index 8d2df2063ec4..225a208fcf9a 100644 --- a/packages/beacon-node/src/chain/validation/block.ts +++ b/packages/beacon-node/src/chain/validation/block.ts @@ -9,7 +9,7 @@ import { isExecutionEnabled, isExecutionStateType, } from "@lodestar/state-transition"; -import {SignedBeaconBlock, deneb, gloas} from "@lodestar/types"; +import {SignedBeaconBlock, deneb, gloas, isGloasBeaconBlock} from "@lodestar/types"; import {byteArrayEquals, sleep, toRootHex} from "@lodestar/utils"; import {BlockErrorCode, BlockGossipError, GossipAction} from "../errors/index.js"; import {IBeaconChain} from "../interface.js"; @@ -72,12 +72,12 @@ export async function validateGossipBlock( // [REJECT] The current finalized_checkpoint is an ancestor of block -- i.e. // get_ancestor(store, block.parent_root, compute_start_slot_at_epoch(store.finalized_checkpoint.epoch)) == store.finalized_checkpoint.root const parentRoot = toRootHex(block.parentRoot); - // For gossip validation, we only need to verify the parent root exists in fork-choice. - // Post-Gloas blocks use getBlockHexDefaultStatus (root-only) because getBlockHexAndBlockHash - // can return null for bid-only parents whose executionPayloadBlockHash hasn't been set yet - // (envelope not imported). The full block hash verification happens during state transition. - let parentBlock = chain.forkChoice.getBlockHexDefaultStatus(parentRoot); - + let parentBlock = isGloasBeaconBlock(block) + ? chain.forkChoice.getBlockHexAndBlockHash( + parentRoot, + toRootHex(block.body.signedExecutionPayloadBid.message.parentBlockHash) + ) + : chain.forkChoice.getBlockHexDefaultStatus(parentRoot); if (parentBlock === null) { // If we have already seen the parent block, it may be in-flight (validated/imported but not // inserted into fork-choice yet). Give fork-choice a brief chance to catch up before treating @@ -126,7 +126,7 @@ export async function validateGossipBlock( // [REJECT] The block is from a higher slot than its parent. if (parentBlock.slot >= blockSlot) { - throw new BlockGossipError(GossipAction.IGNORE, { + throw new BlockGossipError(GossipAction.REJECT, { code: BlockErrorCode.NOT_LATER_THAN_PARENT, parentSlot: parentBlock.slot, slot: blockSlot, diff --git a/packages/beacon-node/src/chain/validation/dataColumnSidecar.ts b/packages/beacon-node/src/chain/validation/dataColumnSidecar.ts index 585681e5eaf5..322d49ecfbc2 100644 --- a/packages/beacon-node/src/chain/validation/dataColumnSidecar.ts +++ b/packages/beacon-node/src/chain/validation/dataColumnSidecar.ts @@ -22,6 +22,7 @@ import { ssz, } from "@lodestar/types"; import {byteArrayEquals, toRootHex, verifyMerkleBranch} from "@lodestar/utils"; +import {BeaconMetrics} from "../../metrics/metrics/beacon.js"; import {Metrics} from "../../metrics/metrics.js"; import {kzg} from "../../util/kzg.js"; import { @@ -187,6 +188,7 @@ export async function validateGossipDataColumnSidecar( }); } + // single data column is being verified here const kzgProofTimer = metrics?.peerDas.dataColumnSidecarKzgProofsVerificationTime.startTimer(); // 11) [REJECT] The sidecar's column data is valid as verified by verify_data_column_sidecar_kzg_proofs try { @@ -308,220 +310,237 @@ export async function validateBlockDataColumnSidecars( blockRoot: Root, blockBlobCount: number, dataColumnSidecars: DataColumnSidecars, - blockKzgCommitments?: deneb.KZGCommitment[] + blockKzgCommitments?: deneb.KZGCommitment[], + metrics?: BeaconMetrics["peerDas"] | null ): Promise { - if (dataColumnSidecars.length === 0) { - return; - } - - if (blockBlobCount === 0) { - throw new DataColumnSidecarValidationError( - { - code: DataColumnSidecarErrorCode.INCORRECT_SIDECAR_COUNT, - slot: blockSlot, - expected: 0, - actual: dataColumnSidecars.length, - }, - "Block has no blob commitments but data column sidecars were provided" - ); - } - const firstSidecar = dataColumnSidecars[0]; - const isGloas = isGloasDataColumnSidecar(firstSidecar); - - if (isGloas) { - if (firstSidecar.slot !== blockSlot || !byteArrayEquals(blockRoot, firstSidecar.beaconBlockRoot)) { - throw new DataColumnSidecarValidationError( - { - code: DataColumnSidecarErrorCode.INCORRECT_BLOCK, - slot: blockSlot, - columnIndex: 0, - expected: toRootHex(blockRoot), - actual: toRootHex(firstSidecar.beaconBlockRoot), - }, - "DataColumnSidecar doesn't match corresponding block" - ); + metrics?.dataColumnSidecarProcessingRequests.inc(dataColumnSidecars.length); + const verificationTimer = metrics?.dataColumnSidecarGossipVerificationTime.startTimer(); + try { + if (dataColumnSidecars.length === 0) { + return; } - } else { - // Hash the first sidecar block header and compare the rest via (cheaper) equality - const firstSidecarSignedBlockHeader = firstSidecar.signedBlockHeader; - const firstSidecarBlockHeader = firstSidecarSignedBlockHeader.message; - const firstBlockRoot = ssz.phase0.BeaconBlockHeader.hashTreeRoot(firstSidecarBlockHeader); - if (!byteArrayEquals(blockRoot, firstBlockRoot)) { + + if (blockBlobCount === 0) { throw new DataColumnSidecarValidationError( { - code: DataColumnSidecarErrorCode.INCORRECT_BLOCK, + code: DataColumnSidecarErrorCode.INCORRECT_SIDECAR_COUNT, slot: blockSlot, - columnIndex: 0, - expected: toRootHex(blockRoot), - actual: toRootHex(firstBlockRoot), + expected: 0, + actual: dataColumnSidecars.length, }, - "DataColumnSidecar doesn't match corresponding block" + "Block has no blob commitments but data column sidecars were provided" ); } - } - if (chain !== null) { + const firstSidecar = dataColumnSidecars[0]; + const isGloas = isGloasDataColumnSidecar(firstSidecar); + + // For pre-gloas (fulu) sidecars, verify block header root matches + let firstSidecarSignedBlockHeader: fulu.DataColumnSidecar["signedBlockHeader"] | undefined; if (isGloas) { - // Post-gloas sidecars do not include signed block headers. - // Signature verification is handled via envelope/bid validation paths. - // Skip proposer signature verification here. - } else { - const firstSidecarSignedBlockHeader = firstSidecar.signedBlockHeader; - const rootHex = toRootHex(blockRoot); - const slot = firstSidecarSignedBlockHeader.message.slot; - const signature = firstSidecarSignedBlockHeader.signature; - if (!chain.seenBlockInputCache.isVerifiedProposerSignature(slot, rootHex, signature)) { - const signatureSet = getBlockHeaderProposerSignatureSetByHeaderSlot( - chain.config, - firstSidecarSignedBlockHeader + if (firstSidecar.slot !== blockSlot || !byteArrayEquals(blockRoot, firstSidecar.beaconBlockRoot)) { + throw new DataColumnSidecarValidationError( + { + code: DataColumnSidecarErrorCode.INCORRECT_BLOCK, + slot: blockSlot, + columnIndex: 0, + expected: toRootHex(blockRoot), + actual: toRootHex(firstSidecar.beaconBlockRoot), + }, + "DataColumnSidecar doesn't match corresponding block" ); - - if ( - !(await chain.bls.verifySignatureSets([signatureSet], { - verifyOnMainThread: true, - })) - ) { - throw new DataColumnSidecarValidationError({ - code: DataColumnSidecarErrorCode.PROPOSAL_SIGNATURE_INVALID, - blockRoot: rootHex, + } + } else { + // Hash the first sidecar block header and compare the rest via (cheaper) equality + firstSidecarSignedBlockHeader = firstSidecar.signedBlockHeader; + const firstSidecarBlockHeader = firstSidecarSignedBlockHeader.message; + const firstBlockRoot = ssz.phase0.BeaconBlockHeader.hashTreeRoot(firstSidecarBlockHeader); + if (!byteArrayEquals(blockRoot, firstBlockRoot)) { + throw new DataColumnSidecarValidationError( + { + code: DataColumnSidecarErrorCode.INCORRECT_BLOCK, slot: blockSlot, - index: dataColumnSidecars[0].index, - }); - } - - chain.seenBlockInputCache.markVerifiedProposerSignature(slot, rootHex, signature); + columnIndex: 0, + expected: toRootHex(blockRoot), + actual: toRootHex(firstBlockRoot), + }, + "DataColumnSidecar doesn't match corresponding block" + ); } } - } - - const commitments: Uint8Array[] = []; - const cellIndices: number[] = []; - const cells: Uint8Array[] = []; - const proofs: Uint8Array[] = []; - for (let i = 0; i < dataColumnSidecars.length; i++) { - const columnSidecar = dataColumnSidecars[i]; - - if (isGloasDataColumnSidecar(columnSidecar) !== isGloas) { - throw new DataColumnSidecarValidationError({ - code: DataColumnSidecarErrorCode.INCORRECT_HEADER_ROOT, - slot: blockSlot, - expected: "uniform sidecar type", - actual: "mixed sidecar type", - }); - } - if (columnSidecar.index >= NUMBER_OF_COLUMNS) { - throw new DataColumnSidecarValidationError( - { - code: DataColumnSidecarErrorCode.INVALID_INDEX, - slot: blockSlot, - columnIndex: columnSidecar.index, - }, - "DataColumnSidecar has invalid index" - ); + if (chain !== null) { + if (isGloas) { + // Post-gloas sidecars do not include signed block headers. + // Signature verification is handled via envelope/bid validation paths. + // Skip proposer signature verification here. + } else if (firstSidecarSignedBlockHeader) { + const rootHex = toRootHex(blockRoot); + const slot = firstSidecarSignedBlockHeader.message.slot; + const signature = firstSidecarSignedBlockHeader.signature; + if (!chain.seenBlockInputCache.isVerifiedProposerSignature(slot, rootHex, signature)) { + const signatureSet = getBlockHeaderProposerSignatureSetByHeaderSlot( + chain.config, + firstSidecarSignedBlockHeader + ); + + if ( + !(await chain.bls.verifySignatureSets([signatureSet], { + verifyOnMainThread: true, + })) + ) { + throw new DataColumnSidecarValidationError({ + code: DataColumnSidecarErrorCode.PROPOSAL_SIGNATURE_INVALID, + blockRoot: rootHex, + slot: blockSlot, + index: dataColumnSidecars[0].index, + }); + } + + chain.seenBlockInputCache.markVerifiedProposerSignature(slot, rootHex, signature); + } + } } - if (columnSidecar.column.length !== blockBlobCount) { - throw new DataColumnSidecarValidationError({ - code: DataColumnSidecarErrorCode.INCORRECT_CELL_COUNT, - slot: blockSlot, - columnIndex: columnSidecar.index, - expected: blockBlobCount, - actual: columnSidecar.column.length, - }); - } + const commitments: Uint8Array[] = []; + const cellIndices: number[] = []; + const cells: Uint8Array[] = []; + const proofs: Uint8Array[] = []; + for (let i = 0; i < dataColumnSidecars.length; i++) { + const columnSidecar = dataColumnSidecars[i]; - if (columnSidecar.column.length !== columnSidecar.kzgProofs.length) { - throw new DataColumnSidecarValidationError({ - code: DataColumnSidecarErrorCode.INCORRECT_KZG_PROOF_COUNT, - slot: blockSlot, - columnIndex: columnSidecar.index, - expected: columnSidecar.column.length, - actual: columnSidecar.kzgProofs.length, - }); - } - - if (isGloasDataColumnSidecar(columnSidecar)) { - if (columnSidecar.slot !== blockSlot || !byteArrayEquals(columnSidecar.beaconBlockRoot, blockRoot)) { + if (isGloasDataColumnSidecar(columnSidecar) !== isGloas) { throw new DataColumnSidecarValidationError({ - code: DataColumnSidecarErrorCode.INCORRECT_BLOCK, + code: DataColumnSidecarErrorCode.INCORRECT_HEADER_ROOT, slot: blockSlot, - columnIndex: columnSidecar.index, - expected: toRootHex(blockRoot), - actual: toRootHex(columnSidecar.beaconBlockRoot), + expected: "uniform sidecar type", + actual: "mixed sidecar type", }); } - if (!blockKzgCommitments || blockKzgCommitments.length !== blockBlobCount) { + + if (columnSidecar.index >= NUMBER_OF_COLUMNS) { + throw new DataColumnSidecarValidationError( + { + code: DataColumnSidecarErrorCode.INVALID_INDEX, + slot: blockSlot, + columnIndex: columnSidecar.index, + }, + "DataColumnSidecar has invalid index" + ); + } + + if (columnSidecar.column.length !== blockBlobCount) { throw new DataColumnSidecarValidationError({ - code: DataColumnSidecarErrorCode.INCORRECT_KZG_COMMITMENTS_COUNT, + code: DataColumnSidecarErrorCode.INCORRECT_CELL_COUNT, slot: blockSlot, columnIndex: columnSidecar.index, expected: blockBlobCount, - actual: blockKzgCommitments?.length ?? 0, - }); - } - commitments.push(...blockKzgCommitments); - } else { - const firstSidecarSignedBlockHeader = (firstSidecar as fulu.DataColumnSidecar).signedBlockHeader; - if ( - i !== 0 && - !ssz.phase0.SignedBeaconBlockHeader.equals(firstSidecarSignedBlockHeader, columnSidecar.signedBlockHeader) - ) { - throw new DataColumnSidecarValidationError({ - code: DataColumnSidecarErrorCode.INCORRECT_HEADER_ROOT, - slot: blockSlot, - expected: toRootHex(blockRoot), - actual: toRootHex(ssz.phase0.BeaconBlockHeader.hashTreeRoot(columnSidecar.signedBlockHeader.message)), + actual: columnSidecar.column.length, }); } - if (columnSidecar.column.length !== columnSidecar.kzgCommitments.length) { + if (columnSidecar.column.length !== columnSidecar.kzgProofs.length) { throw new DataColumnSidecarValidationError({ - code: DataColumnSidecarErrorCode.INCORRECT_KZG_COMMITMENTS_COUNT, + code: DataColumnSidecarErrorCode.INCORRECT_KZG_PROOF_COUNT, slot: blockSlot, columnIndex: columnSidecar.index, expected: columnSidecar.column.length, - actual: columnSidecar.kzgCommitments.length, + actual: columnSidecar.kzgProofs.length, }); } - if (!verifyDataColumnSidecarInclusionProof(columnSidecar)) { - throw new DataColumnSidecarValidationError( - { - code: DataColumnSidecarErrorCode.INCLUSION_PROOF_INVALID, + if (isGloasDataColumnSidecar(columnSidecar)) { + if (columnSidecar.slot !== blockSlot || !byteArrayEquals(columnSidecar.beaconBlockRoot, blockRoot)) { + throw new DataColumnSidecarValidationError({ + code: DataColumnSidecarErrorCode.INCORRECT_BLOCK, slot: blockSlot, columnIndex: columnSidecar.index, - }, - "DataColumnSidecar has invalid inclusion proof" - ); + expected: toRootHex(blockRoot), + actual: toRootHex(columnSidecar.beaconBlockRoot), + }); + } + if (!blockKzgCommitments || blockKzgCommitments.length !== blockBlobCount) { + throw new DataColumnSidecarValidationError({ + code: DataColumnSidecarErrorCode.INCORRECT_KZG_COMMITMENTS_COUNT, + slot: blockSlot, + columnIndex: columnSidecar.index, + expected: blockBlobCount, + actual: blockKzgCommitments?.length ?? 0, + }); + } + commitments.push(...blockKzgCommitments); + } else { + if ( + i !== 0 && + firstSidecarSignedBlockHeader && + !ssz.phase0.SignedBeaconBlockHeader.equals(firstSidecarSignedBlockHeader, columnSidecar.signedBlockHeader) + ) { + throw new DataColumnSidecarValidationError({ + code: DataColumnSidecarErrorCode.INCORRECT_HEADER_ROOT, + slot: blockSlot, + expected: toRootHex(blockRoot), + actual: toRootHex(ssz.phase0.BeaconBlockHeader.hashTreeRoot(columnSidecar.signedBlockHeader.message)), + }); + } + + if (columnSidecar.column.length !== columnSidecar.kzgCommitments.length) { + throw new DataColumnSidecarValidationError({ + code: DataColumnSidecarErrorCode.INCORRECT_KZG_COMMITMENTS_COUNT, + slot: blockSlot, + columnIndex: columnSidecar.index, + expected: columnSidecar.column.length, + actual: columnSidecar.kzgCommitments.length, + }); + } + + const inclusionProofTimer = metrics?.dataColumnSidecarInclusionProofVerificationTime.startTimer(); + const validInclusionProof = verifyDataColumnSidecarInclusionProof(columnSidecar); + inclusionProofTimer?.(); + if (!validInclusionProof) { + throw new DataColumnSidecarValidationError( + { + code: DataColumnSidecarErrorCode.INCLUSION_PROOF_INVALID, + slot: blockSlot, + columnIndex: columnSidecar.index, + }, + "DataColumnSidecar has invalid inclusion proof" + ); + } + + commitments.push(...columnSidecar.kzgCommitments); } - commitments.push(...columnSidecar.kzgCommitments); + cellIndices.push(...Array.from({length: columnSidecar.column.length}, () => columnSidecar.index)); + cells.push(...columnSidecar.column); + proofs.push(...columnSidecar.kzgProofs); } - cellIndices.push(...Array.from({length: columnSidecar.column.length}, () => columnSidecar.index)); - cells.push(...columnSidecar.column); - proofs.push(...columnSidecar.kzgProofs); - } - - let reason: string | undefined; - try { - const valid = await kzg.asyncVerifyCellKzgProofBatch(commitments, cellIndices, cells, proofs); - if (!valid) { - reason = "Invalid KZG proof batch"; + let reason: string | undefined; + // batch verification for the cases: downloadByRange and downloadByRoot + const kzgVerificationTimer = metrics?.kzgVerificationDataColumnBatchTime.startTimer(); + try { + const valid = await kzg.asyncVerifyCellKzgProofBatch(commitments, cellIndices, cells, proofs); + if (!valid) { + reason = "Invalid KZG proof batch"; + } + } catch (e) { + reason = (e as Error).message; + } finally { + kzgVerificationTimer?.(); } - } catch (e) { - reason = (e as Error).message; - } - if (reason !== undefined) { - throw new DataColumnSidecarValidationError( - { - code: DataColumnSidecarErrorCode.INVALID_KZG_PROOF_BATCH, - slot: blockSlot, - reason, - }, - "DataColumnSidecar has invalid KZG proof batch" - ); + if (reason !== undefined) { + throw new DataColumnSidecarValidationError( + { + code: DataColumnSidecarErrorCode.INVALID_KZG_PROOF_BATCH, + slot: blockSlot, + reason, + }, + "DataColumnSidecar has invalid KZG proof batch" + ); + } + metrics?.dataColumnSidecarProcessingSuccesses.inc(); + } finally { + verificationTimer?.(); } } diff --git a/packages/beacon-node/src/chain/validation/executionPayloadEnvelope.ts b/packages/beacon-node/src/chain/validation/executionPayloadEnvelope.ts index cdc8847539cf..6afd7a6d2b05 100644 --- a/packages/beacon-node/src/chain/validation/executionPayloadEnvelope.ts +++ b/packages/beacon-node/src/chain/validation/executionPayloadEnvelope.ts @@ -1,16 +1,15 @@ -import {PublicKey} from "@chainsafe/blst"; -import {BUILDER_INDEX_SELF_BUILD} from "@lodestar/params"; +import {PayloadStatus} from "@lodestar/fork-choice"; import { + BeaconStateView, CachedBeaconStateGloas, computeStartSlotAtEpoch, - createSingleSignatureSetFromComponents, - getExecutionPayloadEnvelopeSigningRoot, + getExecutionPayloadEnvelopeSignatureSet, } from "@lodestar/state-transition"; import {gloas} from "@lodestar/types"; import {toRootHex} from "@lodestar/utils"; -import type {PayloadEnvelopeInput} from "../blocks/payloadEnvelopeInput.js"; import {ExecutionPayloadEnvelopeError, ExecutionPayloadEnvelopeErrorCode, GossipAction} from "../errors/index.js"; import {IBeaconChain} from "../index.js"; +import {RegenCaller} from "../regen/index.js"; export async function validateApiExecutionPayloadEnvelope( chain: IBeaconChain, @@ -19,68 +18,39 @@ export async function validateApiExecutionPayloadEnvelope( return validateExecutionPayloadEnvelope(chain, executionPayloadEnvelope); } -/** - * Validate an execution payload envelope received via gossip. - * - * When `envelopeInput` is provided, bid info (slot, builderIndex, blockHashFromBid) - * is taken from it instead of looking up the block in fork-choice. This is critical - * because the block may have been gossip-validated (creating the cache entry) but not - * yet fully imported into fork-choice when the envelope arrives. - */ export async function validateGossipExecutionPayloadEnvelope( chain: IBeaconChain, - executionPayloadEnvelope: gloas.SignedExecutionPayloadEnvelope, - envelopeInput?: PayloadEnvelopeInput + executionPayloadEnvelope: gloas.SignedExecutionPayloadEnvelope ): Promise { - return validateExecutionPayloadEnvelope(chain, executionPayloadEnvelope, envelopeInput); + return validateExecutionPayloadEnvelope(chain, executionPayloadEnvelope); } async function validateExecutionPayloadEnvelope( chain: IBeaconChain, - executionPayloadEnvelope: gloas.SignedExecutionPayloadEnvelope, - envelopeInput?: PayloadEnvelopeInput + executionPayloadEnvelope: gloas.SignedExecutionPayloadEnvelope ): Promise { const envelope = executionPayloadEnvelope.message; const {payload} = envelope; const blockRootHex = toRootHex(envelope.beaconBlockRoot); - // Use bid info from the envelope input cache if available (gossip path), - // otherwise fall back to fork-choice lookup (API path / early-envelope replay). - let bidSlot: number; - let bidBuilderIndex: number; - let bidBlockHash: string; - - if (envelopeInput) { - // Gossip path: bid info from SeenPayloadEnvelopeCache, populated during - // block gossip validation before the block is imported into fork-choice. - bidSlot = envelopeInput.slot; - bidBuilderIndex = envelopeInput.builderIndex; - bidBlockHash = envelopeInput.blockHashFromBid; - } else { - // API / early-envelope path: look up block in fork-choice. - const block = chain.forkChoice.getBlockDefaultStatus(envelope.beaconBlockRoot); - if (block === null) { - throw new ExecutionPayloadEnvelopeError(GossipAction.IGNORE, { - code: ExecutionPayloadEnvelopeErrorCode.BLOCK_ROOT_UNKNOWN, - blockRoot: blockRootHex, - }); - } - - if (block.builderIndex == null || block.blockHashFromBid == null) { - throw new ExecutionPayloadEnvelopeError(GossipAction.IGNORE, { - code: ExecutionPayloadEnvelopeErrorCode.CACHE_FAIL, - blockRoot: blockRootHex, - }); - } - - bidSlot = block.slot; - bidBuilderIndex = block.builderIndex; - bidBlockHash = block.blockHashFromBid; + // [IGNORE] The envelope's block root `envelope.block_root` has been seen (via + // gossip or non-gossip sources) (a client MAY queue payload for processing once + // the block is retrieved). + // TODO GLOAS: Need to review this, we should queue the envelope for later + // processing if the block is not yet known, otherwise we would ignore it here + const block = chain.forkChoice.getBlockDefaultStatus(envelope.beaconBlockRoot); + if (block === null) { + throw new ExecutionPayloadEnvelopeError(GossipAction.IGNORE, { + code: ExecutionPayloadEnvelopeErrorCode.BLOCK_ROOT_UNKNOWN, + blockRoot: blockRootHex, + }); } // [IGNORE] The node has not seen another valid // `SignedExecutionPayloadEnvelope` for this block root from this builder. - if (chain.seenExecutionPayloadEnvelopes.isKnown(blockRootHex)) { + const envelopeBlock = chain.forkChoice.getBlockHex(blockRootHex, PayloadStatus.FULL); + const payloadInput = chain.seenPayloadEnvelopeInputCache.get(blockRootHex); + if (envelopeBlock || payloadInput?.hasPayloadEnvelope()) { throw new ExecutionPayloadEnvelopeError(GossipAction.IGNORE, { code: ExecutionPayloadEnvelopeErrorCode.ENVELOPE_ALREADY_KNOWN, blockRoot: blockRootHex, @@ -88,7 +58,15 @@ async function validateExecutionPayloadEnvelope( }); } - // [IGNORE] The envelope is from a slot greater than or equal to the latest finalized slot + if (!payloadInput) { + // PayloadEnvelopeInput should have been created during block import + throw new ExecutionPayloadEnvelopeError(GossipAction.IGNORE, { + code: ExecutionPayloadEnvelopeErrorCode.PAYLOAD_ENVELOPE_INPUT_MISSING, + blockRoot: blockRootHex, + }); + } + + // [IGNORE] The envelope is from a slot greater than or equal to the latest finalized slot -- i.e. validate that `envelope.slot >= compute_start_slot_at_epoch(store.finalized_checkpoint.epoch)` const finalizedCheckpoint = chain.forkChoice.getFinalizedCheckpoint(); const finalizedSlot = computeStartSlotAtEpoch(finalizedCheckpoint.epoch); if (envelope.slot < finalizedSlot) { @@ -100,55 +78,55 @@ async function validateExecutionPayloadEnvelope( } // [REJECT] `block.slot` equals `envelope.slot`. - if (bidSlot !== envelope.slot) { + if (block.slot !== envelope.slot) { throw new ExecutionPayloadEnvelopeError(GossipAction.REJECT, { code: ExecutionPayloadEnvelopeErrorCode.SLOT_MISMATCH, envelopeSlot: envelope.slot, - blockSlot: bidSlot, + blockSlot: block.slot, }); } // [REJECT] `envelope.builder_index == bid.builder_index` - if (envelope.builderIndex !== bidBuilderIndex) { + if (envelope.builderIndex !== payloadInput.getBuilderIndex()) { throw new ExecutionPayloadEnvelopeError(GossipAction.REJECT, { code: ExecutionPayloadEnvelopeErrorCode.BUILDER_INDEX_MISMATCH, envelopeBuilderIndex: envelope.builderIndex, - bidBuilderIndex, + bidBuilderIndex: payloadInput.getBuilderIndex(), }); } // [REJECT] `payload.block_hash == bid.block_hash` - if (toRootHex(payload.blockHash) !== bidBlockHash) { + if (toRootHex(payload.blockHash) !== payloadInput.getBlockHashHex()) { throw new ExecutionPayloadEnvelopeError(GossipAction.REJECT, { code: ExecutionPayloadEnvelopeErrorCode.BLOCK_HASH_MISMATCH, envelopeBlockHash: toRootHex(payload.blockHash), - bidBlockHash, + bidBlockHash: payloadInput.getBlockHashHex(), }); } - // [REJECT] `signed_execution_payload_envelope.signature` is valid with respect to the builder's public key. - // Spec: verify_execution_payload_envelope_signature - // For BUILDER_INDEX_SELF_BUILD: spec requires verifying against the block's proposer pubkey, - // which needs the per-block state (state.latest_block_header.proposer_index). At gossip time - // we only have the head state, which may not match the envelope's block during reorgs or late - // delivery. Full signature verification for self-build is deferred to processExecutionPayloadEnvelope - // where the correct per-block state is available. - // For regular builders: verify against state.builders[builder_index].pubkey (head state is valid - // since builder pubkeys don't change per-block). - if (envelope.builderIndex !== BUILDER_INDEX_SELF_BUILD) { - const state = chain.getHeadState() as CachedBeaconStateGloas; - const signatureSet = createSingleSignatureSetFromComponents( - PublicKey.fromBytes(state.builders.getReadonly(envelope.builderIndex).pubkey), - getExecutionPayloadEnvelopeSigningRoot(chain.config, envelope), - executionPayloadEnvelope.signature - ); - - if (!(await chain.bls.verifySignatureSets([signatureSet]))) { - throw new ExecutionPayloadEnvelopeError(GossipAction.REJECT, { - code: ExecutionPayloadEnvelopeErrorCode.INVALID_SIGNATURE, + // Get the post block state which is the pre-payload state to verify the builder's signature. + const blockState = await chain.regen + .getState(block.stateRoot, RegenCaller.validateGossipPayloadEnvelope) + .catch(() => { + throw new ExecutionPayloadEnvelopeError(GossipAction.IGNORE, { + code: ExecutionPayloadEnvelopeErrorCode.UNKNOWN_BLOCK_STATE, + blockRoot: blockRootHex, + slot: envelope.slot, }); - } - } + }); + + const state = blockState as CachedBeaconStateGloas; + const signatureSet = getExecutionPayloadEnvelopeSignatureSet( + chain.config, + chain.pubkeyCache, + new BeaconStateView(state), + executionPayloadEnvelope, + payloadInput.proposerIndex + ); - chain.seenExecutionPayloadEnvelopes.add(blockRootHex, envelope.slot); + if (!(await chain.bls.verifySignatureSets([signatureSet], {verifyOnMainThread: true}))) { + throw new ExecutionPayloadEnvelopeError(GossipAction.REJECT, { + code: ExecutionPayloadEnvelopeErrorCode.INVALID_SIGNATURE, + }); + } } diff --git a/packages/beacon-node/src/chain/validation/lightClientFinalityUpdate.ts b/packages/beacon-node/src/chain/validation/lightClientFinalityUpdate.ts index e06b61768e5e..58d20a78c8d2 100644 --- a/packages/beacon-node/src/chain/validation/lightClientFinalityUpdate.ts +++ b/packages/beacon-node/src/chain/validation/lightClientFinalityUpdate.ts @@ -6,7 +6,7 @@ import {LightClientError, LightClientErrorCode} from "../errors/lightClientError import {IBeaconChain} from "../interface.js"; import {updateReceivedTooEarly} from "./lightClientOptimisticUpdate.js"; -// https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/p2p-interface.md#light_client_finality_update +// https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/altair/light-client/p2p-interface.md#light_client_finality_update export function validateLightClientFinalityUpdate( config: ChainForkConfig, chain: IBeaconChain, diff --git a/packages/beacon-node/src/chain/validation/lightClientOptimisticUpdate.ts b/packages/beacon-node/src/chain/validation/lightClientOptimisticUpdate.ts index ae9a83086218..7e10d19b4e75 100644 --- a/packages/beacon-node/src/chain/validation/lightClientOptimisticUpdate.ts +++ b/packages/beacon-node/src/chain/validation/lightClientOptimisticUpdate.ts @@ -6,7 +6,7 @@ import {GossipAction} from "../errors/index.js"; import {LightClientError, LightClientErrorCode} from "../errors/lightClientError.js"; import {IBeaconChain} from "../interface.js"; -// https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/p2p-interface.md#light_client_optimistic_update +// https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/altair/light-client/p2p-interface.md#light_client_optimistic_update export function validateLightClientOptimisticUpdate( config: ChainForkConfig, chain: IBeaconChain, diff --git a/packages/beacon-node/src/chain/validation/payloadAttestationMessage.ts b/packages/beacon-node/src/chain/validation/payloadAttestationMessage.ts index a8b1626db493..f4a25a4ed7fb 100644 --- a/packages/beacon-node/src/chain/validation/payloadAttestationMessage.ts +++ b/packages/beacon-node/src/chain/validation/payloadAttestationMessage.ts @@ -86,8 +86,16 @@ async function validatePayloadAttestationMessage( } // [REJECT] `payload_attestation_message.signature` is valid with respect to the validator's public key. + const validatorPubkey = chain.pubkeyCache.get(validatorIndex); + if (!validatorPubkey) { + throw new PayloadAttestationError(GossipAction.REJECT, { + code: PayloadAttestationErrorCode.INVALID_ATTESTER, + attesterIndex: validatorIndex, + }); + } + const signatureSet = createSingleSignatureSetFromComponents( - chain.index2pubkey[validatorIndex], + validatorPubkey, getPayloadAttestationDataSigningRoot(chain.config, data), payloadAttestationMessage.signature ); diff --git a/packages/beacon-node/src/chain/validation/proposerSlashing.ts b/packages/beacon-node/src/chain/validation/proposerSlashing.ts index 350f2e701668..bf8a2f02359e 100644 --- a/packages/beacon-node/src/chain/validation/proposerSlashing.ts +++ b/packages/beacon-node/src/chain/validation/proposerSlashing.ts @@ -37,7 +37,7 @@ async function validateProposerSlashing( try { const proposer = state.validators.getReadonly(proposerSlashing.signedHeader1.message.proposerIndex); // verifySignature = false, verified in batch below - assertValidProposerSlashing(chain.config, chain.index2pubkey, state.slot, proposerSlashing, proposer, false); + assertValidProposerSlashing(chain.config, chain.pubkeyCache, state.slot, proposerSlashing, proposer, false); } catch (e) { throw new ProposerSlashingError(GossipAction.REJECT, { code: ProposerSlashingErrorCode.INVALID, diff --git a/packages/beacon-node/src/chain/validation/syncCommittee.ts b/packages/beacon-node/src/chain/validation/syncCommittee.ts index d1b5c0b31e84..e1a7fcd820d9 100644 --- a/packages/beacon-node/src/chain/validation/syncCommittee.ts +++ b/packages/beacon-node/src/chain/validation/syncCommittee.ts @@ -15,12 +15,12 @@ export async function validateGossipSyncCommittee( chain: IBeaconChain, syncCommittee: altair.SyncCommitteeMessage, subnet: SubnetID -): Promise<{indexInSubcommittee: IndexInSubcommittee}> { +): Promise<{indicesInSubcommittee: IndexInSubcommittee[]}> { const {slot, validatorIndex, beaconBlockRoot} = syncCommittee; const messageRoot = toRootHex(beaconBlockRoot); const headState = chain.getHeadState(); - const indexInSubcommittee = validateGossipSyncCommitteeExceptSig(chain, headState, subnet, syncCommittee); + const indicesInSubcommittee = validateGossipSyncCommitteeExceptSig(chain, headState, subnet, syncCommittee); // [IGNORE] The signature's slot is for the current slot, i.e. sync_committee_signature.slot == current_slot. // > Checked in validateGossipSyncCommitteeExceptSig() @@ -68,7 +68,7 @@ export async function validateGossipSyncCommittee( // Register this valid item as seen chain.seenSyncCommitteeMessages.add(slot, subnet, validatorIndex, messageRoot); - return {indexInSubcommittee}; + return {indicesInSubcommittee}; } export async function validateApiSyncCommittee( @@ -105,7 +105,7 @@ export function validateGossipSyncCommitteeExceptSig( headState: CachedBeaconStateAllForks, subnet: SubnetID, data: Pick -): IndexInSubcommittee { +): IndexInSubcommittee[] { const {slot, validatorIndex} = data; // [IGNORE] The signature's slot is for the current slot, i.e. sync_committee_signature.slot == current_slot. // (with a MAXIMUM_GOSSIP_CLOCK_DISPARITY allowance) @@ -127,26 +127,27 @@ export function validateGossipSyncCommitteeExceptSig( // [REJECT] The subnet_id is valid for the given validator, i.e. subnet_id in compute_subnets_for_sync_committee(state, sync_committee_signature.validator_index). // Note this validation implies the validator is part of the broader current sync committee along with the correct subcommittee. - const indexInSubcommittee = getIndexInSubcommittee(headState, subnet, data); - if (indexInSubcommittee === null) { + const indicesInSubcommittee = getIndicesInSubcommittee(headState, subnet, data); + if (indicesInSubcommittee === null) { throw new SyncCommitteeError(GossipAction.REJECT, { code: SyncCommitteeErrorCode.VALIDATOR_NOT_IN_SYNC_COMMITTEE, validatorIndex, }); } - return indexInSubcommittee; + return indicesInSubcommittee; } /** - * Returns the IndexInSubcommittee of the given `subnet`. - * Returns `null` if not part of the sync committee or not part of the given `subnet` + * Returns all IndexInSubcommittee positions of the given `subnet`. + * Returns `null` if not part of the sync committee or not part of the given `subnet`. + * A validator may appear multiple times in the same subcommittee. */ -function getIndexInSubcommittee( +function getIndicesInSubcommittee( headState: CachedBeaconStateAllForks, subnet: SubnetID, data: Pick -): IndexInSubcommittee | null { +): IndexInSubcommittee[] | null { const syncCommittee = headState.epochCtx.getIndexedSyncCommittee(data.slot); const indexesInCommittee = syncCommittee.validatorIndexMap.get(data.validatorIndex); if (indexesInCommittee === undefined) { @@ -154,12 +155,12 @@ function getIndexInSubcommittee( return null; } + const indices: IndexInSubcommittee[] = []; for (const indexInCommittee of indexesInCommittee) { if (Math.floor(indexInCommittee / SYNC_COMMITTEE_SUBNET_SIZE) === subnet) { - return indexInCommittee % SYNC_COMMITTEE_SUBNET_SIZE; + indices.push(indexInCommittee % SYNC_COMMITTEE_SUBNET_SIZE); } } - // Not part of this specific subnet - return null; + return indices.length > 0 ? indices : null; } diff --git a/packages/beacon-node/src/chain/validation/syncCommitteeContributionAndProof.ts b/packages/beacon-node/src/chain/validation/syncCommitteeContributionAndProof.ts index 7d94b6bf8a53..f70351269c41 100644 --- a/packages/beacon-node/src/chain/validation/syncCommitteeContributionAndProof.ts +++ b/packages/beacon-node/src/chain/validation/syncCommitteeContributionAndProof.ts @@ -106,7 +106,7 @@ export async function validateSyncCommitteeGossipContributionAndProof( /** * Retrieve pubkeys in contribution aggregate using epochCtx: * - currSyncCommitteeIndexes cache - * - index2pubkey cache + * - pubkeyCache */ function getContributionIndices( state: CachedBeaconStateAltair, diff --git a/packages/beacon-node/src/chain/validation/voluntaryExit.ts b/packages/beacon-node/src/chain/validation/voluntaryExit.ts index e8433225dda8..00d1fe8b38e3 100644 --- a/packages/beacon-node/src/chain/validation/voluntaryExit.ts +++ b/packages/beacon-node/src/chain/validation/voluntaryExit.ts @@ -1,4 +1,5 @@ import { + BeaconStateView, VoluntaryExitValidity, getVoluntaryExitSignatureSet, getVoluntaryExitValidity, @@ -59,7 +60,7 @@ async function validateVoluntaryExit( }); } - const signatureSet = getVoluntaryExitSignatureSet(chain.config, state, voluntaryExit); + const signatureSet = getVoluntaryExitSignatureSet(chain.config, new BeaconStateView(state), voluntaryExit); if (!(await chain.bls.verifySignatureSets([signatureSet], {batchable: true, priority: prioritizeBls}))) { throw new VoluntaryExitError(GossipAction.REJECT, { code: VoluntaryExitErrorCode.INVALID_SIGNATURE, diff --git a/packages/beacon-node/src/chain/validatorMonitor.ts b/packages/beacon-node/src/chain/validatorMonitor.ts index 0b7a2c6a0381..45c88e42a99b 100644 --- a/packages/beacon-node/src/chain/validatorMonitor.ts +++ b/packages/beacon-node/src/chain/validatorMonitor.ts @@ -23,6 +23,7 @@ import { ValidatorIndex, altair, deneb, + gloas, } from "@lodestar/types"; import {LogData, LogHandler, LogLevel, Logger, MapDef, MapDefMax, prettyPrintIndices, toRootHex} from "@lodestar/utils"; import {GENESIS_SLOT} from "../constants/constants.js"; @@ -61,6 +62,11 @@ export type ValidatorMonitor = { ): void; registerBeaconBlock(src: OpSource, delaySec: Seconds, block: BeaconBlock): void; registerBlobSidecar(src: OpSource, seenTimestampSec: Seconds, blob: deneb.BlobSidecar): void; + registerExecutionPayloadEnvelope( + src: OpSource, + delaySec: Seconds, + envelope: gloas.SignedExecutionPayloadEnvelope + ): void; registerImportedBlock(block: BeaconBlock, data: {proposerBalanceDelta: number}): void; onPoolSubmitUnaggregatedAttestation( seenTimestampSec: number, @@ -450,6 +456,10 @@ export function createValidatorMonitor( //TODO: freetheblobs }, + registerExecutionPayloadEnvelope(_src, _delaySec, _envelope) { + // TODO GLOAS: implement execution payload envelope monitoring + }, + registerImportedBlock(block, {proposerBalanceDelta}) { const validator = validators.get(block.proposerIndex); if (validator) { @@ -889,7 +899,7 @@ function renderAttestationSummary( summary: AttestationSummary | undefined, flags: ParticipationFlags ): string { - // Reference https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/beacon-chain.md#get_attestation_participation_flag_indices + // Reference https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/altair/beacon-chain.md#get_attestation_participation_flag_indices // // is_matching_source = data.source == justified_checkpoint // is_matching_target = is_matching_source and data.target.root == get_block_root(state, data.target.epoch) diff --git a/packages/beacon-node/src/db/index.ts b/packages/beacon-node/src/db/index.ts index df83fb16e08d..ecd83a045154 100644 --- a/packages/beacon-node/src/db/index.ts +++ b/packages/beacon-node/src/db/index.ts @@ -1,2 +1,3 @@ export {BeaconDb} from "./beacon.js"; +export {Bucket} from "./buckets.js"; export type {IBeaconDb} from "./interface.js"; diff --git a/packages/beacon-node/src/db/repositories/blockArchive.ts b/packages/beacon-node/src/db/repositories/blockArchive.ts index 1f087cea7e93..a2f9adfef591 100644 --- a/packages/beacon-node/src/db/repositories/blockArchive.ts +++ b/packages/beacon-node/src/db/repositories/blockArchive.ts @@ -1,4 +1,3 @@ -import all from "it-all"; import {ChainForkConfig} from "@lodestar/config"; import {Db, FilterOptions, KeyValue, Repository} from "@lodestar/db"; import {Root, SignedBeaconBlock, Slot, ssz} from "@lodestar/types"; @@ -121,7 +120,7 @@ export class BlockArchiveRepository extends Repository } async values(opts?: BlockFilterOptions): Promise { - return all(this.valuesStream(opts)); + return await Array.fromAsync(this.valuesStream(opts)); } // INDEX diff --git a/packages/beacon-node/src/execution/engine/http.ts b/packages/beacon-node/src/execution/engine/http.ts index b88433d374e3..996a9025674a 100644 --- a/packages/beacon-node/src/execution/engine/http.ts +++ b/packages/beacon-node/src/execution/engine/http.ts @@ -128,6 +128,7 @@ const getClientVersionOpts: ReqOpts = {routeId: "getClientVersion"}; */ export class ExecutionEngineHttp implements IExecutionEngine { private logger: Logger; + private metrics: Metrics | null; // The default state is ONLINE, it will be updated to SYNCING once we receive the first payload // This assumption is better than the OFFLINE state, since we can't be sure if the EL is offline and being offline may trigger some notifications @@ -167,6 +168,7 @@ export class ExecutionEngineHttp implements IExecutionEngine { metrics?.engineHttpProcessorQueue ); this.logger = logger; + this.metrics = metrics ?? null; this.rpc.emitter.on(JsonRpcHttpClientEvent.ERROR, ({error}) => { this.updateEngineState(getExecutionEngineState({payloadError: error, oldState: this.state})); @@ -372,6 +374,7 @@ export class ExecutionEngineHttp implements IExecutionEngine { } = await request; this.updateEngineState(getExecutionEngineState({payloadStatus: status, oldState: this.state})); + this.metrics?.engineNotifyForkchoiceUpdateResult.inc({result: status}); switch (status) { case ExecutionPayloadStatus.VALID: diff --git a/packages/beacon-node/src/execution/engine/interface.ts b/packages/beacon-node/src/execution/engine/interface.ts index ca115f4c4fb2..4314fb107593 100644 --- a/packages/beacon-node/src/execution/engine/interface.ts +++ b/packages/beacon-node/src/execution/engine/interface.ts @@ -129,7 +129,7 @@ export interface IExecutionEngine { * corresponding state, up to and including finalized_block_hash. * * The call of the notify_forkchoice_updated function maps on the POS_FORKCHOICE_UPDATED event defined in the EIP-3675. - * https://github.com/ethereum/consensus-specs/blob/dev/specs/merge/fork-choice.md#notify_forkchoice_updated + * https://github.com/ethereum/consensus-specs/blob/v1.1.7/specs/merge/fork-choice.md#notify_forkchoice_updated * * Should be called in response to fork-choice head and finalized events */ @@ -146,7 +146,7 @@ export interface IExecutionEngine { * since the corresponding call to prepare_payload method. * * Required for block producing - * https://github.com/ethereum/consensus-specs/blob/dev/specs/merge/validator.md#get_payload + * https://github.com/ethereum/consensus-specs/blob/v1.1.7/specs/merge/validator.md#get_payload */ getPayload( fork: ForkName, diff --git a/packages/beacon-node/src/metrics/metrics/beacon.ts b/packages/beacon-node/src/metrics/metrics/beacon.ts index 09b5c89b7ee4..4698600dd65c 100644 --- a/packages/beacon-node/src/metrics/metrics/beacon.ts +++ b/packages/beacon-node/src/metrics/metrics/beacon.ts @@ -333,11 +333,13 @@ export function createBeaconMetrics(register: RegistryMetricCreator) { help: "Time taken to verify data_column sidecar inclusion proof", buckets: [0.002, 0.004, 0.006, 0.008, 0.01, 0.05, 1, 2], }), + // single verification dataColumnSidecarKzgProofsVerificationTime: register.histogram({ name: "beacon_data_column_sidecar_kzg_proofs_verification_seconds", - help: "Time taken to verify data_column sidecar kzg proofs", + help: "Time taken to verify single data_column sidecar kzg proofs", buckets: [0.01, 0.02, 0.03, 0.04, 0.05, 0.1, 0.2, 0.5, 1], }), + // batch verification kzgVerificationDataColumnBatchTime: register.histogram({ name: "beacon_kzg_verification_data_column_batch_seconds", help: "Runtime of batched data column kzg verification", @@ -361,10 +363,14 @@ export function createBeaconMetrics(register: RegistryMetricCreator) { help: "Duration of engine_getBlobsV2 requests", buckets: [0.01, 0.05, 0.1, 0.5, 1, 2.5, 5, 7.5], }), - targetCustodyGroupCount: register.gauge({ - name: "beacon_target_custody_group_count", + custodyGroupCount: register.gauge({ + name: "beacon_custody_groups", help: "Total number of custody groups within a node", }), + custodyGroupsBackfilled: register.gauge({ + name: "beacon_custody_groups_backfilled", + help: "Total number of custody groups backfilled by a node", + }), reconstructedColumns: register.counter({ name: "beacon_data_availability_reconstructed_columns_total", help: "Total count of reconstructed columns", diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index 54bb57984d8e..f08a46608956 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -3,6 +3,7 @@ import {NotReorgedReason} from "@lodestar/fork-choice"; import {ArchiveStoreTask} from "../../chain/archiveStore/archiveStore.js"; import {FrequencyStateArchiveStep} from "../../chain/archiveStore/strategies/frequencyStateArchiveStrategy.js"; import {BlockInputSource} from "../../chain/blocks/blockInput/index.js"; +import {PayloadEnvelopeInputSource} from "../../chain/blocks/payloadEnvelopeInput/index.js"; import {JobQueueItemType} from "../../chain/bls/index.js"; import {AttestationErrorCode, BlockErrorCode} from "../../chain/errors/index.js"; import { @@ -237,6 +238,56 @@ export function createLodestarMetrics( }), }, + payloadEnvelopeProcessorQueue: { + length: register.gauge({ + name: "lodestar_payload_envelope_processor_queue_length", + help: "Count of total payload envelope processor queue length", + }), + droppedJobs: register.gauge({ + name: "lodestar_payload_envelope_processor_queue_dropped_jobs_total", + help: "Count of total payload envelope processor queue dropped jobs", + }), + jobTime: register.histogram({ + name: "lodestar_payload_envelope_processor_queue_job_time_seconds", + help: "Time to process payload envelope processor queue job in seconds", + buckets: [0.01, 0.1, 1, 4, 12], + }), + jobWaitTime: register.histogram({ + name: "lodestar_payload_envelope_processor_queue_job_wait_time_seconds", + help: "Time from job added to the payload envelope processor queue to starting in seconds", + buckets: [0.01, 0.1, 1, 4, 12], + }), + concurrency: register.gauge({ + name: "lodestar_payload_envelope_processor_queue_concurrency", + help: "Current concurrency of payload envelope processor queue", + }), + }, + + unfinalizedPayloadEnvelopeWritesQueue: { + length: register.gauge({ + name: "lodestar_unfinalized_payload_envelope_writes_queue_length", + help: "Count of total unfinalized payload envelope writes queue length", + }), + droppedJobs: register.gauge({ + name: "lodestar_unfinalized_payload_envelope_writes_queue_dropped_jobs_total", + help: "Count of total unfinalized payload envelope writes queue dropped jobs", + }), + jobTime: register.histogram({ + name: "lodestar_unfinalized_payload_envelope_writes_queue_job_time_seconds", + help: "Time to process unfinalized payload envelope writes queue job in seconds", + buckets: [0.01, 0.1, 1, 4, 12], + }), + jobWaitTime: register.histogram({ + name: "lodestar_unfinalized_payload_envelope_writes_queue_job_wait_time_seconds", + help: "Time from job added to the unfinalized payload envelope writes queue to starting in seconds", + buckets: [0.01, 0.1, 1, 4, 12], + }), + concurrency: register.gauge({ + name: "lodestar_unfinalized_payload_envelope_writes_queue_concurrency", + help: "Current concurrency of unfinalized payload envelope writes queue", + }), + }, + engineHttpProcessorQueue: { length: register.gauge({ name: "lodestar_engine_http_processor_queue_length", @@ -836,20 +887,23 @@ export function createLodestarMetrics( buckets: [0.5, 1, 2, 4, 6, 12], }), }, - recoverDataColumnSidecars: { - recoverTime: register.histogram({ - name: "lodestar_recover_data_column_sidecar_recover_time_seconds", - help: "Time elapsed to recover data column sidecar", - buckets: [0.5, 1.0, 1.5, 2], + // recovery in the case of specific blob rows required + recoverBlobSidecars: { + blobsReconstructed: register.counter({ + name: "lodestar_blobs_reconstructed_total", + help: "Total count of reconstructed blobs", + }), + reconstructionTime: register.histogram({ + name: "lodestar_blob_reconstruction_seconds", + help: "Time taken to reconstruct blobs", + buckets: [0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 2, 5], }), + }, + recoverDataColumnSidecars: { custodyBeforeReconstruction: register.gauge({ name: "lodestar_data_columns_in_custody_before_reconstruction", help: "Number of data columns in custody before reconstruction", }), - numberOfColumnsRecovered: register.gauge({ - name: "lodestar_recover_data_column_sidecar_recovered_columns_total", - help: "Total number of columns that were recovered", - }), reconstructionResult: register.counter<{result: DataColumnReconstructionCode}>({ name: "lodestar_data_column_sidecars_reconstruction_result", help: "Data column sidecars reconstruction result", @@ -857,6 +911,10 @@ export function createLodestarMetrics( }), }, dataColumns: { + alreadyAdded: register.counter({ + name: "lodestar_data_column_sidecar_already_added", + help: "Total number of columns that were already added by other sources while waiting", + }), bySource: register.gauge<{source: BlockInputSource}>({ name: "lodestar_data_columns_by_source", help: "Number of received data columns by source", @@ -912,22 +970,34 @@ export function createLodestarMetrics( help: "Total number of imported blobs by source", labelNames: ["blobsSource"], }), - columnsBySource: register.gauge<{source: BlockInputSource}>({ - name: "lodestar_import_columns_by_source_total", - help: "Total number of imported columns (sampled columns) by source", - labelNames: ["source"], - }), notOverrideFcuReason: register.counter<{reason: NotReorgedReason}>({ name: "lodestar_import_block_not_override_fcu_reason_total", help: "Reason why the fcu call is not suppressed during block import", labelNames: ["reason"], }), }, + importPayload: { + bySource: register.gauge<{source: PayloadEnvelopeInputSource}>({ + name: "lodestar_import_payload_by_source_total", + help: "Total number of imported execution payload envelopes by source", + labelNames: ["source"], + }), + columnsBySource: register.gauge<{source: PayloadEnvelopeInputSource}>({ + name: "lodestar_import_payload_columns_by_source_total", + help: "Total number of payload-attached columns (sampled columns for Gloas) by source", + labelNames: ["source"], + }), + }, engineNotifyNewPayloadResult: register.gauge<{result: ExecutionPayloadStatus}>({ name: "lodestar_execution_engine_notify_new_payload_result_total", help: "The total result of calling notifyNewPayload execution engine api", labelNames: ["result"], }), + engineNotifyForkchoiceUpdateResult: register.gauge<{result: ExecutionPayloadStatus}>({ + name: "lodestar_execution_engine_notify_forkchoice_update_result_total", + help: "The total result of calling notifyForkchoiceUpdate execution engine api", + labelNames: ["result"], + }), backfillSync: { backfilledTillSlot: register.gauge({ name: "lodestar_backfill_till_slot", @@ -1456,29 +1526,51 @@ export function createLodestarMetrics( name: "lodestar_seen_block_input_cache_size", help: "Number of cached BlockInputs", }), - duplicateBlockCount: register.gauge<{source: BlockInputSource}>({ - name: "lodestar_seen_block_input_cache_duplicate_block_count", + serializedObjectRefs: register.gauge({ + name: "lodestar_seen_block_input_cache_serialized_object_refs", + help: "Number of serialized-cache object refs retained by cached BlockInputs", + }), + duplicateBlockCount: register.counter<{source: BlockInputSource}>({ + name: "lodestar_seen_block_input_cache_duplicate_block_total", help: "Total number of duplicate blocks that pass validation and attempt to be cached but are known", labelNames: ["source"], }), - duplicateBlobCount: register.gauge<{source: BlockInputSource}>({ - name: "lodestar_seen_block_input_cache_duplicate_blob_count", + duplicateBlobCount: register.counter<{source: BlockInputSource}>({ + name: "lodestar_seen_block_input_cache_duplicate_blob_total", help: "Total number of duplicate blobs that pass validation and attempt to be cached but are known", labelNames: ["source"], }), - duplicateColumnCount: register.gauge<{source: BlockInputSource}>({ - name: "lodestar_seen_block_input_cache_duplicate_column_count", + duplicateColumnCount: register.counter<{source: BlockInputSource}>({ + name: "lodestar_seen_block_input_cache_duplicate_column_total", help: "Total number of duplicate columns that pass validation and attempt to be cached but are known", labelNames: ["source"], }), - createdByBlock: register.gauge({ - name: "lodestar_seen_block_input_cache_items_created_by_block", + createdByBlock: register.counter({ + name: "lodestar_seen_block_input_cache_items_created_by_block_total", help: "Number of BlockInputs created via a block being seen first", }), - createdByBlob: register.gauge({ - name: "lodestar_seen_block_input_cache_items_created_by_blob", + createdByBlob: register.counter({ + name: "lodestar_seen_block_input_cache_items_created_by_blob_total", help: "Number of BlockInputs created via a blob being seen first", }), + createdByColumn: register.counter({ + name: "lodestar_seen_block_input_cache_items_created_by_column_total", + help: "Number of BlockInputs created via a data column being seen first", + }), + }, + payloadEnvelopeInput: { + count: register.gauge({ + name: "lodestar_seen_payload_envelope_input_cache_size", + help: "Number of cached PayloadEnvelopeInputs", + }), + serializedObjectRefs: register.gauge({ + name: "lodestar_seen_payload_envelope_input_cache_serialized_object_refs", + help: "Number of serialized-cache object refs retained by cached PayloadEnvelopeInputs", + }), + created: register.counter({ + name: "lodestar_seen_payload_envelope_input_cache_items_created_total", + help: "Number of PayloadEnvelopeInputs created", + }), }, }, diff --git a/packages/beacon-node/src/monitoring/service.ts b/packages/beacon-node/src/monitoring/service.ts index 4a2a155b681b..75f5e28c1b66 100644 --- a/packages/beacon-node/src/monitoring/service.ts +++ b/packages/beacon-node/src/monitoring/service.ts @@ -89,9 +89,9 @@ export class MonitoringService { } /** - * Stop sending client stats + * Stop sending client stats and wait for any pending request to complete */ - close(): void { + async close(): Promise { if (this.status === Status.Closed) return; this.status = Status.Closed; @@ -103,6 +103,7 @@ export class MonitoringService { } if (this.pendingRequest) { this.fetchAbortController?.abort(FetchAbortReason.Close); + await this.pendingRequest; } } diff --git a/packages/beacon-node/src/network/core/networkCore.ts b/packages/beacon-node/src/network/core/networkCore.ts index d5ec1f6e9abe..6e78e1acc3dc 100644 --- a/packages/beacon-node/src/network/core/networkCore.ts +++ b/packages/beacon-node/src/network/core/networkCore.ts @@ -1,8 +1,8 @@ -import {Connection, PrivateKey} from "@libp2p/interface"; +import type {PeerScoreStatsDump} from "@libp2p/gossipsub/score"; +import type {PublishOpts} from "@libp2p/gossipsub/types"; +import type {Connection, PrivateKey} from "@libp2p/interface"; import {peerIdFromPrivateKey} from "@libp2p/peer-id"; import {multiaddr} from "@multiformats/multiaddr"; -import {PeerScoreStatsDump} from "@chainsafe/libp2p-gossipsub/score"; -import {PublishOpts} from "@chainsafe/libp2p-gossipsub/types"; import {routes} from "@lodestar/api"; import {BeaconConfig, ForkBoundary} from "@lodestar/config"; import type {LoggerNode} from "@lodestar/logger/node"; diff --git a/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts b/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts index 194a7a9ea73a..d137f565a5c5 100644 --- a/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts +++ b/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts @@ -1,9 +1,9 @@ import path from "node:path"; import workerThreads from "node:worker_threads"; import {privateKeyToProtobuf} from "@libp2p/crypto/keys"; -import {PrivateKey} from "@libp2p/interface"; -import {PeerScoreStatsDump} from "@chainsafe/libp2p-gossipsub/score"; -import {PublishOpts} from "@chainsafe/libp2p-gossipsub/types"; +import type {PeerScoreStatsDump} from "@libp2p/gossipsub/score"; +import type {PublishOpts} from "@libp2p/gossipsub/types"; +import type {PrivateKey} from "@libp2p/interface"; import {ModuleThread, Thread, Worker, spawn} from "@chainsafe/threads"; import {routes} from "@lodestar/api"; import {BeaconConfig, chainConfigToJson} from "@lodestar/config"; diff --git a/packages/beacon-node/src/network/core/types.ts b/packages/beacon-node/src/network/core/types.ts index 1763b2e6bd61..57992c6d4a7f 100644 --- a/packages/beacon-node/src/network/core/types.ts +++ b/packages/beacon-node/src/network/core/types.ts @@ -1,5 +1,5 @@ -import {PeerScoreStatsDump} from "@chainsafe/libp2p-gossipsub/score"; -import {PublishOpts} from "@chainsafe/libp2p-gossipsub/types"; +import type {PeerScoreStatsDump} from "@libp2p/gossipsub/score"; +import type {PublishOpts} from "@libp2p/gossipsub/types"; import {routes} from "@lodestar/api"; import {SpecJson} from "@lodestar/config"; import {LoggerNodeOpts} from "@lodestar/logger/node"; diff --git a/packages/beacon-node/src/network/discv5/utils.ts b/packages/beacon-node/src/network/discv5/utils.ts index 961607991af0..d0a91a1930cd 100644 --- a/packages/beacon-node/src/network/discv5/utils.ts +++ b/packages/beacon-node/src/network/discv5/utils.ts @@ -4,7 +4,7 @@ import {IClock} from "../../util/clock.js"; import {ENRKey} from "../metadata.js"; export enum ENRRelevance { - no_tcp = "no_tcp", + no_transport = "no_transport", no_eth2 = "no_eth2", // biome-ignore lint/style/useNamingConvention: Need to use the this name for network convention unknown_forkDigest = "unknown_forkDigest", @@ -13,10 +13,11 @@ export enum ENRRelevance { } export function enrRelevance(enr: ENR, config: BeaconConfig, clock: IClock): ENRRelevance { - // We are not interested in peers that don't advertise their tcp addr + // We are not interested in peers that don't advertise at least one transport (tcp or quic) const multiaddrTCP = enr.getLocationMultiaddr(ENRKey.tcp); - if (!multiaddrTCP) { - return ENRRelevance.no_tcp; + const multiaddrQUIC = enr.getLocationMultiaddr(ENRKey.quic); + if (!multiaddrTCP && !multiaddrQUIC) { + return ENRRelevance.no_transport; } // Check if the ENR.eth2 field matches and is of interest diff --git a/packages/beacon-node/src/network/events.ts b/packages/beacon-node/src/network/events.ts index 20d34b9966ea..545625a43972 100644 --- a/packages/beacon-node/src/network/events.ts +++ b/packages/beacon-node/src/network/events.ts @@ -1,5 +1,6 @@ import {EventEmitter} from "node:events"; -import {PeerId, TopicValidatorResult} from "@libp2p/interface"; +import type {TopicValidatorResult} from "@libp2p/gossipsub"; +import type {PeerId} from "@libp2p/interface"; import {CustodyIndex, Status} from "@lodestar/types"; import {PeerIdStr} from "../util/peerId.js"; import {StrictEventEmitterSingleArg} from "../util/strictEvents.js"; diff --git a/packages/beacon-node/src/network/gossip/encoding.ts b/packages/beacon-node/src/network/gossip/encoding.ts index 59b5ff665fe7..9b3f658ad073 100644 --- a/packages/beacon-node/src/network/gossip/encoding.ts +++ b/packages/beacon-node/src/network/gossip/encoding.ts @@ -1,9 +1,9 @@ -import {Message} from "@libp2p/interface"; +import type {Message} from "@libp2p/gossipsub"; +import type {RPC} from "@libp2p/gossipsub/message"; +import type {DataTransform} from "@libp2p/gossipsub/types"; // snappyjs is better for compression for smaller payloads import xxhashFactory from "xxhash-wasm"; import {digest} from "@chainsafe/as-sha256"; -import {RPC} from "@chainsafe/libp2p-gossipsub/message"; -import {DataTransform} from "@chainsafe/libp2p-gossipsub/types"; import snappyWasm from "@chainsafe/snappy-wasm"; import {ForkName} from "@lodestar/params"; import {intToBytes} from "@lodestar/utils"; @@ -24,12 +24,28 @@ const decoder = new snappyWasm.Decoder(); // Shared buffer to convert msgId to string const sharedMsgIdBuf = Buffer.alloc(20); +// Cache topic -> seed to avoid per-message allocations on the hot path. +// Topics are a fixed set per fork (changes only at fork boundaries). +const topicSeedCache = new Map(); + /** * The function used to generate a gossipsub message id * We use the first 8 bytes of SHA256(data) for content addressing */ export function fastMsgIdFn(rpcMsg: RPC.Message): string { if (rpcMsg.data) { + if (rpcMsg.topic) { + // Use topic-derived seed to prevent cross-topic deduplication of identical messages. + // SyncCommitteeMessages are published to multiple sync_committee_{subnet} topics with + // identical data, so hashing only the data incorrectly deduplicates across subnets. + // See https://github.com/ChainSafe/lodestar/issues/8294 + let topicSeed = topicSeedCache.get(rpcMsg.topic); + if (topicSeed === undefined) { + topicSeed = xxhash.h64Raw(Buffer.from(rpcMsg.topic), h64Seed); + topicSeedCache.set(rpcMsg.topic, topicSeed); + } + return xxhash.h64Raw(rpcMsg.data, topicSeed).toString(16); + } return xxhash.h64Raw(rpcMsg.data, h64Seed).toString(16); } return "0000000000000000"; diff --git a/packages/beacon-node/src/network/gossip/gossipsub.ts b/packages/beacon-node/src/network/gossip/gossipsub.ts index 2018591f781e..f5d5b0a22683 100644 --- a/packages/beacon-node/src/network/gossip/gossipsub.ts +++ b/packages/beacon-node/src/network/gossip/gossipsub.ts @@ -1,10 +1,18 @@ +import { + type GossipSub, + type GossipSubEvents, + type PublishResult, + StrictNoSign, + type TopicValidatorResult, + gossipsub, +} from "@libp2p/gossipsub"; +import type {MetricsRegister, TopicLabel, TopicStrToLabel} from "@libp2p/gossipsub/metrics"; +import type {PeerScoreParams, PeerScoreStatsDump} from "@libp2p/gossipsub/score"; +import type {AddrInfo, PublishOpts, TopicStr} from "@libp2p/gossipsub/types"; +import type {PeerId} from "@libp2p/interface"; import {peerIdFromString} from "@libp2p/peer-id"; -import {multiaddr} from "@multiformats/multiaddr"; +import {type Multiaddr, multiaddr} from "@multiformats/multiaddr"; import {ENR} from "@chainsafe/enr"; -import {GossipSub, GossipsubEvents} from "@chainsafe/libp2p-gossipsub"; -import {MetricsRegister, TopicLabel, TopicStrToLabel} from "@chainsafe/libp2p-gossipsub/metrics"; -import {PeerScoreParams} from "@chainsafe/libp2p-gossipsub/score"; -import {AddrInfo, SignaturePolicy, TopicStr} from "@chainsafe/libp2p-gossipsub/types"; import {routes} from "@lodestar/api"; import {BeaconConfig, ForkBoundary} from "@lodestar/config"; import {ATTESTATION_SUBNET_COUNT, SLOTS_PER_EPOCH, SYNC_COMMITTEE_SUBNET_COUNT} from "@lodestar/params"; @@ -69,6 +77,24 @@ export type Eth2GossipsubOpts = { export type ForkBoundaryLabel = string; +// Many of the internal properties we need are not available on the public interface, +// so we create an extended type here to avoid excessive type assertions throughout the codebase. +// Mind that any updates to the gossipsub package may require updates to this type. +type GossipSubInternal = GossipSub & { + mesh: Map>; + peers: Map; + score: {score: (peerIdStr: string) => number}; + direct: Set; + topics: Map>; + start: () => Promise; + stop: () => Promise; + publish: (topic: TopicStr, data: Uint8Array, opts?: PublishOpts) => Promise; + getMeshPeers: (topic: TopicStr) => string[]; + dumpPeerScoreStats: () => PeerScoreStatsDump; + getScore: (peerIdStr: string) => number; + reportMessageValidationResult: (msgId: string, propagationSource: string, acceptance: TopicValidatorResult) => void; +}; + /** * Wrapper around js-libp2p-gossipsub with the following extensions: * - Eth2 message id @@ -82,13 +108,14 @@ export type ForkBoundaryLabel = string; * * See https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/p2p-interface.md#the-gossip-domain-gossipsub */ -export class Eth2Gossipsub extends GossipSub { +export class Eth2Gossipsub { readonly scoreParams: Partial; private readonly config: BeaconConfig; private readonly logger: Logger; private readonly peersData: PeersData; private readonly events: NetworkEventBus; private readonly libp2p: Libp2p; + private readonly gossipsub: GossipSubInternal; // Internal caches private readonly gossipTopicCache: GossipTopicCache; @@ -103,9 +130,6 @@ export class Eth2Gossipsub extends GossipSub { let metrics: Eth2GossipsubMetrics | null = null; if (metricsRegister) { metrics = createEth2GossipsubMetrics(metricsRegister); - metrics.gossipMesh.peersByType.addCollect(() => - this.onScrapeLodestarMetrics(metrics as Eth2GossipsubMetrics, networkConfig) - ); } // Parse direct peers from multiaddr strings to AddrInfo objects @@ -113,8 +137,8 @@ export class Eth2Gossipsub extends GossipSub { // Gossipsub parameters defined here: // https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/p2p-interface.md#the-gossip-domain-gossipsub - super(modules.libp2p.services.components, { - globalSignaturePolicy: SignaturePolicy.StrictNoSign, + const gossipsubInstance = gossipsub({ + globalSignaturePolicy: StrictNoSign, allowPublishToZeroTopicPeers: allowPublishToZeroPeers, D: gossipsubD ?? GOSSIP_D, Dlo: gossipsubDLow ?? GOSSIP_D_LOW, @@ -155,7 +179,12 @@ export class Eth2Gossipsub extends GossipSub { // This should be large enough to not send IDONTWANT for "small" messages // See https://github.com/ChainSafe/lodestar/pull/7077#issuecomment-2383679472 idontwantMinDataSize: 16829, - }); + })(modules.libp2p.services.components) as GossipSubInternal; + + if (metrics) { + metrics.gossipMesh.peersByType.addCollect(() => this.onScrapeLodestarMetrics(metrics, networkConfig)); + } + this.gossipsub = gossipsubInstance; this.scoreParams = scoreParams; this.config = config; this.logger = logger; @@ -164,7 +193,7 @@ export class Eth2Gossipsub extends GossipSub { this.libp2p = modules.libp2p; this.gossipTopicCache = gossipTopicCache; - this.addEventListener("gossipsub:message", this.onGossipsubMessage.bind(this)); + this.gossipsub.addEventListener("gossipsub:message", this.onGossipsubMessage.bind(this)); this.events.on(NetworkEvent.gossipMessageValidationResult, this.onValidationResult.bind(this)); // Having access to this data is CRUCIAL for debugging. While this is a massive log, it must not be deleted. @@ -174,6 +203,38 @@ export class Eth2Gossipsub extends GossipSub { } } + async start(): Promise { + await this.gossipsub.start(); + } + + async stop(): Promise { + await this.gossipsub.stop(); + } + + get mesh(): Map> { + return this.gossipsub.mesh; + } + + getTopics(): TopicStr[] { + return this.gossipsub.getTopics(); + } + + getMeshPeers(topic: TopicStr): string[] { + return this.gossipsub.getMeshPeers(topic); + } + + publish(topic: TopicStr, data: Uint8Array, opts?: PublishOpts): Promise { + return this.gossipsub.publish(topic, data, opts); + } + + dumpPeerScoreStats(): PeerScoreStatsDump { + return this.gossipsub.dumpPeerScoreStats(); + } + + getScore(peerIdStr: string): number { + return this.gossipsub.getScore(peerIdStr); + } + /** * Subscribe to a `GossipTopic` */ @@ -183,7 +244,7 @@ export class Eth2Gossipsub extends GossipSub { this.gossipTopicCache.setTopic(topicStr, topic); this.logger.verbose("Subscribe to gossipsub topic", {topic: topicStr}); - this.subscribe(topicStr); + this.gossipsub.subscribe(topicStr); } /** @@ -192,15 +253,14 @@ export class Eth2Gossipsub extends GossipSub { unsubscribeTopic(topic: GossipTopic): void { const topicStr = stringifyGossipTopic(this.config, topic); this.logger.verbose("Unsubscribe to gossipsub topic", {topic: topicStr}); - this.unsubscribe(topicStr); + this.gossipsub.unsubscribe(topicStr); } private onScrapeLodestarMetrics(metrics: Eth2GossipsubMetrics, networkConfig: NetworkConfig): void { - const mesh = this.mesh; - // biome-ignore lint/complexity/useLiteralKeys: `topics` is a private attribute - const topics = this["topics"] as Map>; - const peers = this.peers; - const score = this.score; + const mesh = this.gossipsub.mesh; + const topics = this.gossipsub.topics; + const peers = this.gossipsub.peers; + const score = this.gossipsub.score; const meshPeersByClient = new Map(); const meshPeerIdStrs = new Set(); @@ -305,7 +365,7 @@ export class Eth2Gossipsub extends GossipSub { metrics.gossipPeer.score.set(gossipScores); } - private onGossipsubMessage(event: GossipsubEvents["gossipsub:message"]): void { + private onGossipsubMessage(event: GossipSubEvents["gossipsub:message"]): void { const {propagationSource, msgId, msg} = event.detail; // Also validates that the topicStr is known @@ -341,7 +401,7 @@ export class Eth2Gossipsub extends GossipSub { // Without this we'll have huge event loop lag // See https://github.com/ChainSafe/lodestar/issues/5604 callInNextEventLoop(() => { - this.reportMessageValidationResult(data.msgId, data.propagationSource, data.acceptance); + this.gossipsub.reportMessageValidationResult(data.msgId, data.propagationSource, data.acceptance); }); } @@ -379,7 +439,7 @@ export class Eth2Gossipsub extends GossipSub { } // Add to direct peers set only after addresses are stored - this.direct.add(peerIdStr); + this.gossipsub.direct.add(peerIdStr); this.logger.info("Added direct peer via API", {peerId: peerIdStr}); return peerIdStr; @@ -389,7 +449,7 @@ export class Eth2Gossipsub extends GossipSub { * Remove a peer from direct peers. */ removeDirectPeer(peerIdStr: string): boolean { - const removed = this.direct.delete(peerIdStr); + const removed = this.gossipsub.direct.delete(peerIdStr); if (removed) { this.logger.info("Removed direct peer via API", {peerId: peerIdStr}); } @@ -400,7 +460,7 @@ export class Eth2Gossipsub extends GossipSub { * Get list of current direct peer IDs. */ getDirectPeers(): string[] { - return Array.from(this.direct); + return Array.from(this.gossipsub.direct); } } @@ -477,19 +537,24 @@ export function parseDirectPeers(directPeerStrs: routes.lodestar.DirectPeer[], l const enr = ENR.decodeTxt(peerStr); const peerId = enr.peerId; - // Get TCP multiaddr from ENR - const multiaddrTCP = enr.getLocationMultiaddr("tcp"); - if (!multiaddrTCP) { - logger.warn("ENR does not contain TCP multiaddr", {enr: peerStr}); + // Get all available transport multiaddrs from ENR + const addrs = [enr.getLocationMultiaddr("quic"), enr.getLocationMultiaddr("tcp")].filter( + (a): a is Multiaddr => a != null + ); + if (addrs.length === 0) { + logger.warn("ENR does not contain any transport multiaddr", {enr: peerStr}); continue; } directPeers.push({ id: peerId, - addrs: [multiaddrTCP], + addrs, }); - logger.info("Added direct peer from ENR", {peerId: peerId.toString(), addr: multiaddrTCP.toString()}); + logger.info("Added direct peer from ENR", { + peerId: peerId.toString(), + addrs: addrs.map((a) => a.toString()).join(", "), + }); } catch (e) { logger.warn("Failed to parse direct peer ENR", {enr: peerStr}, e as Error); } @@ -498,7 +563,8 @@ export function parseDirectPeers(directPeerStrs: routes.lodestar.DirectPeer[], l try { const ma = multiaddr(peerStr); - const peerIdStr = ma.getPeerId(); + const peerIdComponent = ma.getComponents().findLast((component) => component.name === "p2p"); + const peerIdStr = peerIdComponent?.value; if (!peerIdStr) { logger.warn("Direct peer multiaddr must contain /p2p/ component with peer ID", {multiaddr: peerStr}); continue; diff --git a/packages/beacon-node/src/network/gossip/interface.ts b/packages/beacon-node/src/network/gossip/interface.ts index f7544b91f67b..76584a0ebc51 100644 --- a/packages/beacon-node/src/network/gossip/interface.ts +++ b/packages/beacon-node/src/network/gossip/interface.ts @@ -1,6 +1,6 @@ -import {Message, TopicValidatorResult} from "@libp2p/interface"; -import {Libp2p} from "libp2p"; -import {PeerIdStr} from "@chainsafe/libp2p-gossipsub/types"; +import type {Message, TopicValidatorResult} from "@libp2p/gossipsub"; +import type {PeerIdStr} from "@libp2p/gossipsub/types"; +import type {Libp2p} from "libp2p"; import {BeaconConfig, ForkBoundary} from "@lodestar/config"; import { AttesterSlashing, diff --git a/packages/beacon-node/src/network/gossip/scoringParameters.ts b/packages/beacon-node/src/network/gossip/scoringParameters.ts index b9f2f24adf03..669d188ae8c9 100644 --- a/packages/beacon-node/src/network/gossip/scoringParameters.ts +++ b/packages/beacon-node/src/network/gossip/scoringParameters.ts @@ -1,9 +1,9 @@ import { - PeerScoreParams, - PeerScoreThresholds, - TopicScoreParams, + type PeerScoreParams, + type PeerScoreThresholds, + type TopicScoreParams, defaultTopicScoreParams, -} from "@chainsafe/libp2p-gossipsub/score"; +} from "@libp2p/gossipsub/score"; import {BeaconConfig} from "@lodestar/config"; import {ATTESTATION_SUBNET_COUNT, PTC_SIZE, SLOTS_PER_EPOCH, TARGET_AGGREGATORS_PER_COMMITTEE} from "@lodestar/params"; import {computeCommitteeCount} from "@lodestar/state-transition"; diff --git a/packages/beacon-node/src/network/interface.ts b/packages/beacon-node/src/network/interface.ts index f86ffa1dd9e7..a5cb1926a776 100644 --- a/packages/beacon-node/src/network/interface.ts +++ b/packages/beacon-node/src/network/interface.ts @@ -1,5 +1,5 @@ -import {Identify} from "@libp2p/identify"; -import { +import type {Identify} from "@libp2p/identify"; +import type { ComponentLogger, ConnectionGater, ConnectionProtector, @@ -16,7 +16,7 @@ import { } from "@libp2p/interface"; import type {AddressManager, ConnectionManager, Registrar, TransportManager} from "@libp2p/interface-internal"; import type {Datastore} from "interface-datastore"; -import {Libp2p as ILibp2p} from "libp2p"; +import type {Libp2p as ILibp2p} from "libp2p"; import { AttesterSlashing, DataColumnSidecar, diff --git a/packages/beacon-node/src/network/libp2p/index.ts b/packages/beacon-node/src/network/libp2p/index.ts index ec47acd06d90..95fa001bfcbd 100644 --- a/packages/beacon-node/src/network/libp2p/index.ts +++ b/packages/beacon-node/src/network/libp2p/index.ts @@ -1,15 +1,16 @@ import {bootstrap} from "@libp2p/bootstrap"; import {identify} from "@libp2p/identify"; -import {PrivateKey} from "@libp2p/interface"; +import type {PrivateKey} from "@libp2p/interface"; import {mdns} from "@libp2p/mdns"; import {mplex} from "@libp2p/mplex"; import {prometheusMetrics} from "@libp2p/prometheus-metrics"; import {tcp} from "@libp2p/tcp"; -import {createLibp2p} from "libp2p"; +import {Libp2pInit, createLibp2p} from "libp2p"; import {Registry} from "prom-client"; import {ENR} from "@chainsafe/enr"; import {noise} from "@chainsafe/libp2p-noise"; import {asCrypto, defaultCrypto} from "@chainsafe/libp2p-noise/crypto"; +import {quic} from "@chainsafe/libp2p-quic"; import {Libp2p, LodestarComponents} from "../interface.js"; import {NetworkOptions, defaultNetworkOptions} from "../options.js"; import {Eth2PeerDataStore} from "../peers/datastore.js"; @@ -21,11 +22,14 @@ export type NodeJsLibp2pOpts = { metricsRegistry?: Registry; }; -export async function getDiscv5Multiaddrs(bootEnrs: string[]): Promise { +export async function getDiscv5Multiaddrs(bootEnrs: string[], quicEnabled?: boolean): Promise { const bootMultiaddrs = []; for (const enrStr of bootEnrs) { const enr = ENR.decodeTxt(enrStr); - const multiaddrWithPeerId = (await enr.getFullMultiaddr("tcp"))?.toString(); + // Prefer QUIC over TCP when available + const quicMultiaddr = quicEnabled ? (await enr.getFullMultiaddr("quic"))?.toString() : undefined; + const tcpMultiaddr = (await enr.getFullMultiaddr("tcp"))?.toString(); + const multiaddrWithPeerId = quicMultiaddr ?? tcpMultiaddr; if (multiaddrWithPeerId) { bootMultiaddrs.push(multiaddrWithPeerId); } @@ -39,6 +43,7 @@ export async function createNodeJsLibp2p( nodeJsLibp2pOpts: NodeJsLibp2pOpts = {} ): Promise { const localMultiaddrs = networkOpts.localMultiaddrs || defaultNetworkOptions.localMultiaddrs; + const disconnectThreshold = networkOpts.disconnectThreshold ?? defaultNetworkOptions.disconnectThreshold; const {peerStoreDir, disablePeerDiscovery} = nodeJsLibp2pOpts; let datastore: undefined | Eth2PeerDataStore = undefined; @@ -52,7 +57,9 @@ export async function createNodeJsLibp2p( const bootMultiaddrs = [ ...(networkOpts.bootMultiaddrs ?? defaultNetworkOptions.bootMultiaddrs ?? []), // Append discv5.bootEnrs to bootMultiaddrs if requested - ...(networkOpts.connectToDiscv5Bootnodes ? await getDiscv5Multiaddrs(networkOpts.discv5?.bootEnrs ?? []) : []), + ...(networkOpts.connectToDiscv5Bootnodes + ? await getDiscv5Multiaddrs(networkOpts.discv5?.bootEnrs ?? [], networkOpts.quic) + : []), ]; if ((bootMultiaddrs.length ?? 0) > 0) { @@ -63,6 +70,35 @@ export async function createNodeJsLibp2p( peerDiscovery.push(mdns()); } } + const transports: Libp2pInit["transports"] = []; + if (networkOpts.tcp ?? true) { + transports.unshift( + tcp({ + // Reject connections when the server's connection count gets high + maxConnections: networkOpts.maxPeers, + // socket option: the maximum length of the queue of pending connections + // https://nodejs.org/dist/latest-v18.x/docs/api/net.html#serverlisten + // it's not safe if we increase this number + backlog: 5, + closeServerOnMaxConnections: { + closeAbove: networkOpts.maxPeers ?? Infinity, + listenBelow: networkOpts.maxPeers ?? Infinity, + }, + }) + ); + } + if (networkOpts.quic) { + transports.unshift( + quic({ + handshakeTimeout: 5_000, + maxIdleTimeout: 10_000, + keepAliveInterval: 5_000, + maxConcurrentStreamLimit: 256, + maxStreamData: 10_000_000, + maxConnectionData: 15_000_000, + }) + ); + } const noiseCrypto = { ...defaultCrypto, @@ -74,26 +110,18 @@ export async function createNodeJsLibp2p( return createLibp2p({ privateKey, + nodeInfo: { + name: "lodestar", + version: networkOpts.version ?? "unknown", + userAgent: networkOpts.private ? "" : networkOpts.version ? `lodestar/${networkOpts.version}` : "lodestar", + }, addresses: { listen: localMultiaddrs, announce: [], }, connectionEncrypters: [noise({crypto: noiseCrypto})], - // Reject connections when the server's connection count gets high - transports: [ - tcp({ - maxConnections: networkOpts.maxPeers, - // socket option: the maximum length of the queue of pending connections - // https://nodejs.org/dist/latest-v18.x/docs/api/net.html#serverlisten - // it's not safe if we increase this number - backlog: 5, - closeServerOnMaxConnections: { - closeAbove: networkOpts.maxPeers ?? Infinity, - listenBelow: networkOpts.maxPeers ?? Infinity, - }, - }), - ], - streamMuxers: [mplex({maxInboundStreams: 256, disconnectThreshold: networkOpts.disconnectThreshold})], + transports, + streamMuxers: [mplex({disconnectThreshold})], peerDiscovery, metrics: nodeJsLibp2pOpts.metrics ? prometheusMetrics({ @@ -124,7 +152,6 @@ export async function createNodeJsLibp2p( datastore, services: { identify: identify({ - agentVersion: networkOpts.private ? "" : networkOpts.version ? `lodestar/${networkOpts.version}` : "lodestar", runOnConnectionOpen: false, }), // individual components are specified because the components object is a Proxy diff --git a/packages/beacon-node/src/network/metadata.ts b/packages/beacon-node/src/network/metadata.ts index dfc844ea5fa5..705220cd158e 100644 --- a/packages/beacon-node/src/network/metadata.ts +++ b/packages/beacon-node/src/network/metadata.ts @@ -11,6 +11,7 @@ import {NetworkConfig} from "./networkConfig.js"; export enum ENRKey { tcp = "tcp", + quic = "quic", eth2 = "eth2", attnets = "attnets", syncnets = "syncnets", diff --git a/packages/beacon-node/src/network/network.ts b/packages/beacon-node/src/network/network.ts index c6a02f5fb707..bab137be1203 100644 --- a/packages/beacon-node/src/network/network.ts +++ b/packages/beacon-node/src/network/network.ts @@ -1,7 +1,7 @@ -import {PeerId, PrivateKey} from "@libp2p/interface"; +import type {PeerScoreStatsDump} from "@libp2p/gossipsub/score"; +import type {PublishOpts} from "@libp2p/gossipsub/types"; +import type {PeerId, PrivateKey} from "@libp2p/interface"; import {peerIdFromPrivateKey} from "@libp2p/peer-id"; -import {PeerScoreStatsDump} from "@chainsafe/libp2p-gossipsub/score"; -import {PublishOpts} from "@chainsafe/libp2p-gossipsub/types"; import {routes} from "@lodestar/api"; import {BeaconConfig} from "@lodestar/config"; import {LoggerNode} from "@lodestar/logger/node"; @@ -544,7 +544,8 @@ export class Network implements INetwork { this.config.getForkSeq(this.clock.currentSlot) >= ForkSeq.altair ? [Version.V2] : [Version.V2, Version.V1], request ), - request + request, + this.chain.serializedCache ); } @@ -748,7 +749,7 @@ export class Network implements INetwork { try { // messages SHOULD be broadcast after SYNC_MESSAGE_DUE_BPS of slot has transpired - // https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/p2p-interface.md#sync-committee + // https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/altair/light-client/p2p-interface.md#sync-committee await this.waitForSyncMessageCutoff(finalityUpdate.signatureSlot); await this.publishLightClientFinalityUpdate(finalityUpdate); } catch (e) { @@ -765,7 +766,7 @@ export class Network implements INetwork { try { // messages SHOULD be broadcast after SYNC_MESSAGE_DUE_BPS of slot has transpired - // https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/p2p-interface.md#sync-committee + // https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/altair/light-client/p2p-interface.md#sync-committee await this.waitForSyncMessageCutoff(optimisticUpdate.signatureSlot); await this.publishLightClientOptimisticUpdate(optimisticUpdate); } catch (e) { diff --git a/packages/beacon-node/src/network/options.ts b/packages/beacon-node/src/network/options.ts index e3989ebd9502..6f458ae131ef 100644 --- a/packages/beacon-node/src/network/options.ts +++ b/packages/beacon-node/src/network/options.ts @@ -29,7 +29,6 @@ export interface NetworkOptions useWorker?: boolean; maxYoungGenerationSizeMb?: number; disableLightClientServer?: boolean; - /** * During E2E tests observe a lot of following `missing stream`: * @@ -48,11 +47,14 @@ export interface NetworkOptions * We need to increase this only for the testing purpose */ disconnectThreshold?: number; + quic?: boolean; + tcp?: boolean; } export const defaultNetworkOptions: NetworkOptions = { maxPeers: 210, // Allow some room above targetPeers for new inbound peers targetPeers: 200, + // In CLI usage this is typically overridden; when unset it serves as a fallback default (e.g. programmatic usage/tests) localMultiaddrs: ["/ip4/0.0.0.0/tcp/9000", "/ip6/::/tcp/9000"], bootMultiaddrs: [], /** disabled by default */ @@ -67,9 +69,14 @@ export const defaultNetworkOptions: NetworkOptions = { slotsToSubscribeBeforeAggregatorDuty: 2, // This will enable the light client server by default disableLightClientServer: false, + quic: false, + tcp: true, // specific option for fulu // - this is the same to TARGET_SUBNET_PEERS // - for fusaka-devnets, we have 25-30 peers per subnet // - for public testnets or mainnet, average number of peers per group is SAMPLES_PER_SLOT * targetPeers / NUMBER_OF_CUSTODY_GROUPS = 6.25 so this should not be an issue targetGroupPeers: 6, + // Keep this high enough for normal req/resp bursts on stable connections. + // libp2p-mplex default (5) is too low and can cause frequent connection resets. + disconnectThreshold: 50, }; diff --git a/packages/beacon-node/src/network/peers/datastore.ts b/packages/beacon-node/src/network/peers/datastore.ts index f57e66bf5caa..d21d145cec1f 100644 --- a/packages/beacon-node/src/network/peers/datastore.ts +++ b/packages/beacon-node/src/network/peers/datastore.ts @@ -8,6 +8,9 @@ type MemoryItem = { data: Uint8Array; }; +// biome-ignore lint/suspicious/noExplicitAny: used below (copied from upstream) +type AwaitGenerator = Generator | AsyncGenerator; + /** * Before libp2p 0.35, peerstore stays in memory and periodically write to db after n dirty items * This has a memory issue because all peer data stays in memory and loaded at startup time @@ -93,7 +96,7 @@ export class Eth2PeerDataStore extends BaseDatastore { * This throws error if not found * see https://github.com/ipfs/js-datastore-level/blob/38f44058dd6be858e757a1c90b8edb31590ec0bc/src/index.js#L102 */ - async get(key: Key): Promise { + async get(key: Key, options?: AbortOptions): Promise { const keyStr = key.toString(); const memoryItem = this._memoryDatastore.get(keyStr); if (memoryItem) { @@ -102,16 +105,16 @@ export class Eth2PeerDataStore extends BaseDatastore { } // this throws error if not found - const dbValue = await this._dbDatastore.get(key); + const dbValue = await this._dbDatastore.get(key, options); // don't call this._memoryDatastore.set directly // we want to get through prune() logic with fromDb as true await this._put(key, dbValue, true); return dbValue; } - async has(key: Key): Promise { + async has(key: Key, options?: AbortOptions): Promise { try { - await this.get(key); + await this.get(key, options); } catch (err) { // this is the same to how js-datastore-level handles notFound error // https://github.com/ipfs/js-datastore-level/blob/38f44058dd6be858e757a1c90b8edb31590ec0bc/src/index.js#L121 @@ -121,26 +124,26 @@ export class Eth2PeerDataStore extends BaseDatastore { return true; } - async delete(key: Key): Promise { + async delete(key: Key, options?: AbortOptions): Promise { this._memoryDatastore.delete(key.toString()); - await this._dbDatastore.delete(key); + await this._dbDatastore.delete(key, options); } - async *_all(q: Query): AsyncIterable { + async *_all(q: Query, options?: AbortOptions): AwaitGenerator { for (const [key, value] of this._memoryDatastore.entries()) { yield { key: new Key(key), value: value.data, }; } - yield* this._dbDatastore.query(q); + yield* this._dbDatastore.query(q, options); } - async *_allKeys(q: KeyQuery): AsyncIterable { + async *_allKeys(q: KeyQuery, options?: AbortOptions): AwaitGenerator { for (const key of this._memoryDatastore.keys()) { yield new Key(key); } - yield* this._dbDatastore.queryKeys(q); + yield* this._dbDatastore.queryKeys(q, options); } private async _addDirtyItem(keyStr: string): Promise { diff --git a/packages/beacon-node/src/network/peers/discover.ts b/packages/beacon-node/src/network/peers/discover.ts index e39a61465f56..3da865f8bbc7 100644 --- a/packages/beacon-node/src/network/peers/discover.ts +++ b/packages/beacon-node/src/network/peers/discover.ts @@ -59,6 +59,7 @@ export enum DiscoveredPeerStatus { cached = "cached", dropped = "dropped", no_multiaddrs = "no_multiaddrs", + transport_incompatible = "transport_incompatible", peer_cooling_down = "peer_cooling_down", } @@ -88,7 +89,8 @@ export type SubnetDiscvQueryMs = { type CachedENR = { peerId: PeerId; - multiaddrTCP: Multiaddr; + multiaddrTCP?: Multiaddr; + multiaddrQUIC?: Multiaddr; subnets: Record; addedUnixMs: number; // custodyGroups is null for pre-fulu @@ -114,6 +116,7 @@ export class PeerDiscovery { attnets: new Map(), syncnets: new Map(), }; + private transports: string[]; private custodyGroupQueries: CustodyGroupQueries; @@ -180,6 +183,17 @@ export class PeerDiscovery { } }); } + + // Transport tags vary by library: @libp2p/tcp uses '@libp2p/tcp', @chainsafe/libp2p-quic uses 'quic' + // Normalize to simple 'tcp' / 'quic' strings for matching + this.transports = libp2p.services.components.transportManager + .getTransports() + .map((t) => t[Symbol.toStringTag]) + .map((tag) => { + if (tag?.includes("tcp")) return "tcp"; + if (tag?.includes("quic")) return "quic"; + return tag; + }); } static async init(modules: PeerDiscoveryModules, opts: PeerDiscoveryOpts): Promise { @@ -372,10 +386,15 @@ export class PeerDiscovery { return; } + // Select multiaddrs by protocol rather than index — libp2p discovery events + // don't guarantee ordering or number of addresses + const multiaddrTCP = multiaddrs.find((ma) => ma.toString().includes("/tcp/")); + const multiaddrQUIC = multiaddrs.find((ma) => ma.toString().includes("/quic-v1")); + const attnets = zeroAttnets; const syncnets = zeroSyncnets; - const status = this.handleDiscoveredPeer(id, multiaddrs[0], attnets, syncnets, undefined); + const status = this.handleDiscoveredPeer(id, multiaddrTCP, multiaddrQUIC, attnets, syncnets, undefined); this.logger.debug("Discovered peer via libp2p", {peer: prettyPrintPeerId(id), status}); this.metrics?.discovery.discoveredStatus.inc({status}); }; @@ -388,13 +407,15 @@ export class PeerDiscovery { this.randomNodeQuery.count++; } const peerId = enr.peerId; - // tcp multiaddr is known to be be present, checked inside the worker + // At least one transport is known to be present, checked inside the worker const multiaddrTCP = enr.getLocationMultiaddr(ENRKey.tcp); - if (!multiaddrTCP) { - this.logger.warn("Discv5 worker sent enr without tcp multiaddr", {enr: enr.encodeTxt()}); + const multiaddrQUIC = enr.getLocationMultiaddr(ENRKey.quic); + if (!multiaddrTCP && !multiaddrQUIC) { + this.logger.warn("Discv5 worker sent enr without any transport multiaddr", {enr: enr.encodeTxt()}); this.metrics?.discovery.discoveredStatus.inc({status: DiscoveredPeerStatus.no_multiaddrs}); return; } + // Are this fields mandatory? const attnetsBytes = enr.kvs.get(ENRKey.attnets); // 64 bits const syncnetsBytes = enr.kvs.get(ENRKey.syncnets); // 4 bits @@ -414,7 +435,7 @@ export class PeerDiscovery { const syncnets = syncnetsBytes ? deserializeEnrSubnets(syncnetsBytes, SYNC_COMMITTEE_SUBNET_COUNT) : zeroSyncnets; const custodyGroupCount = custodyGroupCountBytes ? bytesToInt(custodyGroupCountBytes, "be") : undefined; - const status = this.handleDiscoveredPeer(peerId, multiaddrTCP, attnets, syncnets, custodyGroupCount); + const status = this.handleDiscoveredPeer(peerId, multiaddrTCP, multiaddrQUIC, attnets, syncnets, custodyGroupCount); this.logger.debug("Discovered peer via discv5", { peer: prettyPrintPeerId(peerId), status, @@ -428,7 +449,8 @@ export class PeerDiscovery { */ private handleDiscoveredPeer( peerId: PeerId, - multiaddrTCP: Multiaddr, + multiaddrTCP: Multiaddr | undefined, + multiaddrQUIC: Multiaddr | undefined, attnets: boolean[], syncnets: boolean[], custodySubnetCount?: number @@ -454,6 +476,13 @@ export class PeerDiscovery { return DiscoveredPeerStatus.already_connected; } + // ignore peers if they don't share any transport with us + const hasTcpMatch = this.transports.includes("tcp") && multiaddrTCP; + const hasQuicMatch = this.transports.includes("quic") && multiaddrQUIC; + if (!hasTcpMatch && !hasQuicMatch) { + return DiscoveredPeerStatus.transport_incompatible; + } + // Ignore dialing peers if ( this.libp2p.services.components.connectionManager @@ -469,6 +498,7 @@ export class PeerDiscovery { const cachedPeer: CachedENR = { peerId, multiaddrTCP, + multiaddrQUIC, subnets: {attnets, syncnets}, addedUnixMs: Date.now(), // for pre-fulu, custodyGroups is null @@ -566,19 +596,24 @@ export class PeerDiscovery { // are not successful. this.peersToConnect = Math.max(this.peersToConnect - 1, 0); - const {peerId, multiaddrTCP} = cachedPeer; + const {peerId, multiaddrTCP, multiaddrQUIC} = cachedPeer; // Must add the multiaddrs array to the address book before dialing // https://github.com/libp2p/js-libp2p/blob/aec8e3d3bb1b245051b60c2a890550d262d5b062/src/index.js#L638 - const peer = await this.libp2p.peerStore.merge(peerId, {multiaddrs: [multiaddrTCP]}); + const peer = await this.libp2p.peerStore.merge(peerId, { + multiaddrs: [multiaddrQUIC, multiaddrTCP].filter(Boolean) as Multiaddr[], + }); if (peer.addresses.length === 0) { this.metrics?.discovery.notDialReason.inc({reason: NotDialReason.no_multiaddrs}); return; } - // Note: PeerDiscovery adds the multiaddrTCP beforehand + // Note: PeerDiscovery adds the multiaddrs beforehand const peerIdShort = prettyPrintPeerId(peerId); - this.logger.debug("Dialing discovered peer", {peer: peerIdShort}); + this.logger.debug("Dialing discovered peer", { + peer: peerIdShort, + addresses: peer.addresses.map((a) => a.multiaddr.toString()).join(", "), + }); this.metrics?.discovery.dialAttempts.inc(); const timer = this.metrics?.discovery.dialTime.startTimer(); diff --git a/packages/beacon-node/src/network/peers/peerManager.ts b/packages/beacon-node/src/network/peers/peerManager.ts index 597542ef552e..ca8c8892dd6d 100644 --- a/packages/beacon-node/src/network/peers/peerManager.ts +++ b/packages/beacon-node/src/network/peers/peerManager.ts @@ -9,6 +9,7 @@ import {prettyPrintIndices, toHex, withTimeout} from "@lodestar/utils"; import {GOODBYE_KNOWN_CODES, GoodByeReasonCode, Libp2pEvent} from "../../constants/index.js"; import {IClock} from "../../util/clock.js"; import {computeColumnsForCustodyGroup, getCustodyGroups} from "../../util/dataColumns.js"; +import {callInNextEventLoop} from "../../util/eventLoop.js"; import {NetworkCoreMetrics} from "../core/metrics.js"; import {LodestarDiscv5Opts} from "../discv5/types.js"; import {INetworkEventBus, NetworkEvent, NetworkEventData} from "../events.js"; @@ -161,7 +162,6 @@ export class PeerManager { // A single map of connected peers with all necessary data to handle PINGs, STATUS, and metrics private connectedPeers: Map; - private opts: PeerManagerOpts; private intervals: NodeJS.Timeout[] = []; @@ -196,10 +196,12 @@ export class PeerManager { this.lastStatus = this.statusCache.get(); - // A connection may already be open before listeners are attached (e.g. worker startup race). - // Seed those peers and trigger status/ping on the next macrotask so startup listeners are ready. + // A connection may already be open before listeners are attached. + // Seed those peers so they are tracked in connectedPeers immediately. this.bootstrapAlreadyOpenConnections(); - setTimeout(() => this.pingAndStatusTimeouts(), 0); + // Defer status/ping to the next event loop tick so the heartbeat interval and + // event listeners are fully registered before we begin handshakes. + callInNextEventLoop(() => this.pingAndStatusTimeouts()); // On start-up will connected to existing peers in libp2p.peerStore, same as autoDial behaviour this.heartbeat(); @@ -477,6 +479,14 @@ export class PeerManager { clientAgent, custodyColumns, }); + + // Identify peer after status proves the connection is usable. + // This is the only place we trigger identify — avoids wasted streams on + // peers that close identify right after connection open or turn out to be + // irrelevant. + if (peerData?.agentVersion === null) { + void this.identifyPeer(peer.toString(), prettyPrintPeerId(peer), getConnection(this.libp2p, peer.toString())); + } } } @@ -702,11 +712,7 @@ export class PeerManager { for (const {value: connections} of getConnectionsMap(this.libp2p).values()) { for (const connection of connections) { - const peerIdStr = connection.remotePeer.toString(); - if (this.connectedPeers.has(peerIdStr)) { - continue; - } - + // trackLibp2pConnection handles deduplication via overwriteExisting: false if (this.trackLibp2pConnection(connection, {overwriteExisting: false, triggerHandshakeNow: false})) { bootstrapped++; } @@ -714,7 +720,7 @@ export class PeerManager { } if (bootstrapped > 0) { - this.logger.info("Bootstrapped already-open libp2p peers", {bootstrapped}); + this.logger.verbose("Bootstrapped already-open libp2p peers", {bootstrapped}); } } @@ -754,21 +760,26 @@ export class PeerManager { // NOTE: libp2p may emit two "peer:connect" events: One for inbound, one for outbound // If that happens, it's okay. Only the "outbound" connection triggers immediate action const now = Date.now(); + const existingPeerData = this.connectedPeers.get(remotePeerStr); const nodeId = computeNodeId(remotePeer); const peerData: PeerData = { - lastReceivedMsgUnixTsMs: direction === "outbound" ? 0 : now, + // Keep existing timestamps if this peer already had another open connection. + // libp2p may emit multiple connection:open events per peer. + lastReceivedMsgUnixTsMs: existingPeerData?.lastReceivedMsgUnixTsMs ?? (direction === "outbound" ? 0 : now), // If inbound, request after STATUS_INBOUND_GRACE_PERIOD - lastStatusUnixTsMs: direction === "outbound" ? 0 : now - STATUS_INTERVAL_MS + STATUS_INBOUND_GRACE_PERIOD, - connectedUnixTsMs: now, - relevantStatus: RelevantPeerStatus.Unknown, + lastStatusUnixTsMs: + existingPeerData?.lastStatusUnixTsMs ?? + (direction === "outbound" ? 0 : now - STATUS_INTERVAL_MS + STATUS_INBOUND_GRACE_PERIOD), + connectedUnixTsMs: existingPeerData?.connectedUnixTsMs ?? now, + relevantStatus: existingPeerData?.relevantStatus ?? RelevantPeerStatus.Unknown, direction, nodeId, peerId: remotePeer, - status: null, - metadata: null, - agentVersion: null, - agentClient: null, - encodingPreference: null, + status: existingPeerData?.status ?? null, + metadata: existingPeerData?.metadata ?? null, + agentVersion: existingPeerData?.agentVersion ?? null, + agentClient: existingPeerData?.agentClient ?? null, + encodingPreference: existingPeerData?.encodingPreference ?? null, }; this.connectedPeers.set(remotePeerStr, peerData); @@ -777,26 +788,6 @@ export class PeerManager { void this.requestStatus(remotePeer, this.statusCache.get()); } - this.libp2p.services.identify - .identify(connection) - .then((result) => { - const agentVersion = result.agentVersion; - if (agentVersion) { - peerData.agentVersion = agentVersion; - peerData.agentClient = getKnownClientFromAgentVersion(agentVersion); - } - }) - .catch((err) => { - if (connection.status !== "open") { - this.logger.debug("Peer disconnected during identify protocol", { - peerId: remotePeerPrettyStr, - error: (err as Error).message, - }); - } else { - this.logger.debug("Error setting agentVersion for the peer", {peerId: remotePeerPrettyStr}, err); - } - }); - return true; } @@ -823,6 +814,19 @@ export class PeerManager { const {direction, status, remotePeer} = evt.detail; const peerIdStr = remotePeer.toString(); + const openConnections = + getConnectionsMap(this.libp2p) + .get(peerIdStr) + ?.value.filter((connection) => connection.status === "open") ?? []; + if (openConnections.length > 0) { + this.logger.debug("Ignoring peer disconnect event while another connection is still open", { + peerId: prettyPrintPeerIdStr(peerIdStr), + direction, + status, + }); + return; + } + let logMessage = "onLibp2pPeerDisconnect"; const logContext: Record = { peerId: prettyPrintPeerIdStr(peerIdStr), @@ -857,6 +861,27 @@ export class PeerManager { } } + private async identifyPeer(peerIdStr: string, peerIdPretty: string, connection?: Connection): Promise { + if (!connection || connection.status !== "open") { + this.logger.debug("Peer has no open connection for identify", {peerId: peerIdPretty}); + return; + } + + try { + const result = await this.libp2p.services.identify.identify(connection); + const agentVersion = result.agentVersion; + if (agentVersion) { + const connectedPeerData = this.connectedPeers.get(peerIdStr); + if (connectedPeerData) { + connectedPeerData.agentVersion = agentVersion; + connectedPeerData.agentClient = getKnownClientFromAgentVersion(agentVersion); + } + } + } catch (e) { + this.logger.debug("Error setting agentVersion for the peer", {peerId: peerIdPretty}, e as Error); + } + } + private async goodbyeAndDisconnect(peer: PeerId, goodbye: GoodByeReasonCode): Promise { const reason = GOODBYE_KNOWN_CODES[goodbye.toString()] || ""; const peerIdStr = peer.toString(); diff --git a/packages/beacon-node/src/network/peers/utils/prioritizePeers.ts b/packages/beacon-node/src/network/peers/utils/prioritizePeers.ts index 057398d19bc2..571af140214b 100644 --- a/packages/beacon-node/src/network/peers/utils/prioritizePeers.ts +++ b/packages/beacon-node/src/network/peers/utils/prioritizePeers.ts @@ -1,4 +1,4 @@ -import {Direction, PeerId} from "@libp2p/interface"; +import type {MessageStreamDirection, PeerId} from "@libp2p/interface"; import {BitArray} from "@chainsafe/ssz"; import {ChainConfig} from "@lodestar/config"; import {ATTESTATION_SUBNET_COUNT, SYNC_COMMITTEE_SUBNET_COUNT} from "@lodestar/params"; @@ -95,7 +95,7 @@ function computeStatusScore(ours: Status, theirs: Status | null, opts: Prioritiz type PeerInfo = { id: PeerId; - direction: Direction | null; + direction: MessageStreamDirection | null; statusScore: StatusScore; attnets: phase0.AttestationSubnets; syncnets: altair.SyncSubnets; @@ -137,7 +137,7 @@ export enum ExcessPeerDisconnectReason { export function prioritizePeers( connectedPeersInfo: { id: PeerId; - direction: Direction | null; + direction: MessageStreamDirection | null; status: Status | null; attnets: phase0.AttestationSubnets | null; syncnets: altair.SyncSubnets | null; diff --git a/packages/beacon-node/src/network/processor/extractSlotRootFns.ts b/packages/beacon-node/src/network/processor/extractSlotRootFns.ts index 1f0fb321890b..e1bb301a725b 100644 --- a/packages/beacon-node/src/network/processor/extractSlotRootFns.ts +++ b/packages/beacon-node/src/network/processor/extractSlotRootFns.ts @@ -1,6 +1,7 @@ -import {ForkName} from "@lodestar/params"; +import {ForkName, isForkPostGloas} from "@lodestar/params"; import {SlotOptionalRoot, SlotRootHex} from "@lodestar/types"; import { + getBeaconBlockRootFromDataColumnSidecarSerialized, getBlockRootFromBeaconAttestationSerialized, getBlockRootFromSignedAggregateAndProofSerialized, getBlockRootFromSignedExecutionPayloadEnvelopeSerialized, @@ -54,13 +55,14 @@ export function createExtractBlockSlotRootFns(): ExtractSlotRootFns { } return {slot}; }, - [GossipType.data_column_sidecar]: (data: Uint8Array): SlotOptionalRoot | null => { - const slot = getSlotFromDataColumnSidecarSerialized(data); - + [GossipType.data_column_sidecar]: (data: Uint8Array, fork: ForkName): SlotOptionalRoot | null => { + const slot = getSlotFromDataColumnSidecarSerialized(data, fork); if (slot === null) { return null; } - return {slot}; + + const root = isForkPostGloas(fork) ? getBeaconBlockRootFromDataColumnSidecarSerialized(data) : null; + return root !== null ? {slot, root} : {slot}; }, [GossipType.execution_payload]: (data: Uint8Array): SlotRootHex | null => { const slot = getSlotFromSignedExecutionPayloadEnvelopeSerialized(data); diff --git a/packages/beacon-node/src/network/processor/gossipHandlers.ts b/packages/beacon-node/src/network/processor/gossipHandlers.ts index 236dc6ecbd46..5b2d06df027e 100644 --- a/packages/beacon-node/src/network/processor/gossipHandlers.ts +++ b/packages/beacon-node/src/network/processor/gossipHandlers.ts @@ -2,6 +2,7 @@ import {routes} from "@lodestar/api"; import {BeaconConfig, ChainForkConfig} from "@lodestar/config"; import { ForkName, + ForkPostDeneb, ForkPostElectra, ForkPreElectra, ForkSeq, @@ -31,6 +32,7 @@ import { IBlockInput, isBlockInputColumns, } from "../../chain/blocks/blockInput/index.js"; +import {PayloadEnvelopeInputSource} from "../../chain/blocks/payloadEnvelopeInput/index.js"; import type {BidInfo} from "../../chain/blocks/payloadEnvelopeInput.js"; import {BlobSidecarValidation} from "../../chain/blocks/types.js"; import {ChainEvent} from "../../chain/emitter.js"; @@ -44,6 +46,8 @@ import { BlockGossipError, DataColumnSidecarErrorCode, DataColumnSidecarGossipError, + ExecutionPayloadEnvelopeError, + ExecutionPayloadEnvelopeErrorCode, GossipAction, GossipActionError, SyncCommitteeError, @@ -74,6 +78,7 @@ import {OpSource} from "../../chain/validatorMonitor.js"; import {Metrics} from "../../metrics/index.js"; import {kzgCommitmentToVersionedHash} from "../../util/blobs.js"; import {ClockEvent} from "../../util/clock.js"; +import {getBlobKzgCommitments} from "../../util/dataColumns.js"; import {INetworkCore} from "../core/index.js"; import {NetworkEventBus} from "../events.js"; import { @@ -459,12 +464,11 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand chain.getBlobsTracker.triggerGetBlobs(blockInput); } else { metrics?.blockInputFetchStats.totalDataAvailableBlockInputs.inc(); - // blobKzgCommitments is removed from BeaconBlockBody post-Gloas (EIP-7732) - if (config.getForkSeq(slot) < ForkSeq.gloas) { - metrics?.blockInputFetchStats.totalDataAvailableBlockInputBlobs.inc( - (signedBlock.message as deneb.BeaconBlock).body.blobKzgCommitments.length - ); - } + const blobCount = getBlobKzgCommitments( + blockInput.forkName, + signedBlock as SignedBeaconBlock + ).length; + metrics?.blockInputFetchStats.totalDataAvailableBlockInputBlobs.inc(blobCount); } chain @@ -657,6 +661,13 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand }); }); } + + // TODO GLOAS: In Gloas, also add column to PayloadEnvelopeInput and notify the payload processor: + // const payloadInput = chain.seenPayloadEnvelopeInput.get(blockRootHex); + // if (payloadInput) { + // payloadInput.addColumn({columnSidecar, source: BlockInputSource.gossip, seenTimestampSec, peerIdStr}); + // chain.processExecutionPayload(payloadInput, {validSignature: true}); + // } }, [GossipType.beacon_aggregate_and_proof]: async ({ @@ -808,9 +819,9 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand const {serializedData} = gossipData; const syncCommittee = sszDeserialize(topic, serializedData); const {subnet} = topic; - let indexInSubcommittee = 0; + let indicesInSubcommittee: number[] = [0]; try { - indexInSubcommittee = (await validateGossipSyncCommittee(chain, syncCommittee, subnet)).indexInSubcommittee; + indicesInSubcommittee = (await validateGossipSyncCommittee(chain, syncCommittee, subnet)).indicesInSubcommittee; } catch (e) { if (e instanceof SyncCommitteeError && e.action === GossipAction.REJECT) { chain.persistInvalidSszValue(ssz.altair.SyncCommitteeMessage, syncCommittee, "gossip_reject"); @@ -818,11 +829,12 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand throw e; } - // Handler - + // Handler — add for ALL positions this validator holds in the subcommittee try { - const insertOutcome = chain.syncCommitteeMessagePool.add(subnet, syncCommittee, indexInSubcommittee); - metrics?.opPool.syncCommitteeMessagePoolInsertOutcome.inc({insertOutcome}); + for (const indexInSubcommittee of indicesInSubcommittee) { + const insertOutcome = chain.syncCommitteeMessagePool.add(subnet, syncCommittee, indexInSubcommittee); + metrics?.opPool.syncCommitteeMessagePoolInsertOutcome.inc({insertOutcome}); + } } catch (e) { logger.debug("Error adding to syncCommittee pool", {subnet}, e as Error); } @@ -867,6 +879,7 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand [GossipType.execution_payload]: async ({ gossipData, topic, + peerIdStr, seenTimestampSec, }: GossipHandlerParamGeneric) => { const {serializedData} = gossipData; @@ -885,19 +898,36 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand return; } - const envelopeInput = chain.seenPayloadEnvelopeCache.get(blockRootHex); - - // Pass envelopeInput so validation uses bid info from cache when available. - // Blocks imported via unknown-block sync may not have a cache entry — validation - // falls back to fork-choice metadata in that case. - await validateGossipExecutionPayloadEnvelope(chain, executionPayloadEnvelope, envelopeInput); - envelopeInput?.setEnvelope(executionPayloadEnvelope); + await validateGossipExecutionPayloadEnvelope(chain, executionPayloadEnvelope); const slot = executionPayloadEnvelope.message.slot; const delaySec = seenTimestampSec - computeTimeAtSlot(config, slot, chain.genesisTime); metrics?.gossipExecutionPayloadEnvelope.elapsedTimeTillReceived.observe({source: OpSource.gossip}, delaySec); + chain.validatorMonitor?.registerExecutionPayloadEnvelope(OpSource.gossip, delaySec, executionPayloadEnvelope); + + const payloadInput = chain.seenPayloadEnvelopeInputCache.get(blockRootHex); + + if (!payloadInput) { + // This shouldn't happen because beacon block should have been imported and thus payload input should have been created. + throw new ExecutionPayloadEnvelopeError(GossipAction.REJECT, { + code: ExecutionPayloadEnvelopeErrorCode.PAYLOAD_ENVELOPE_INPUT_MISSING, + blockRoot: blockRootHex, + }); + } + + chain.serializedCache.set(executionPayloadEnvelope, serializedData); + + payloadInput.addPayloadEnvelope({ + envelope: executionPayloadEnvelope, + source: PayloadEnvelopeInputSource.gossip, + seenTimestampSec, + peerIdStr, + }); - await chain.importExecutionPayloadEnvelope(executionPayloadEnvelope); + // TODO GLOAS: Emit execution_payload_gossip event for gossip receipt. + chain.processExecutionPayload(payloadInput, {validSignature: true}).catch((e) => { + chain.logger.debug("Error processing execution payload from gossip", {slot, root: blockRootHex}, e as Error); + }); }, [GossipType.payload_attestation_message]: async ({ gossipData, @@ -917,6 +947,11 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand } catch (e) { logger.error("Error adding to payloadAttestation pool", {}, e as Error); } + chain.forkChoice.notifyPtcMessages( + toRootHex(payloadAttestationMessage.data.beaconBlockRoot), + [validationResult.validatorCommitteeIndex], + payloadAttestationMessage.data.payloadPresent + ); }, [GossipType.execution_payload_bid]: async ({ gossipData, diff --git a/packages/beacon-node/src/network/processor/gossipValidatorFn.ts b/packages/beacon-node/src/network/processor/gossipValidatorFn.ts index cf6a7be07be2..19442c7173c4 100644 --- a/packages/beacon-node/src/network/processor/gossipValidatorFn.ts +++ b/packages/beacon-node/src/network/processor/gossipValidatorFn.ts @@ -1,4 +1,4 @@ -import {TopicValidatorResult} from "@libp2p/interface"; +import {TopicValidatorResult} from "@libp2p/gossipsub"; import {ChainForkConfig} from "@lodestar/config"; import {Logger} from "@lodestar/utils"; import {AttestationError, GossipAction, GossipActionError} from "../../chain/errors/index.js"; @@ -11,7 +11,7 @@ import { GossipValidatorBatchFn, GossipValidatorFn, } from "../gossip/interface.js"; -import {prettyPrintPeerIdStr} from "../util.ts"; +import {prettyPrintPeerIdStr} from "../util.js"; export type ValidatorFnModules = { config: ChainForkConfig; diff --git a/packages/beacon-node/src/network/processor/types.ts b/packages/beacon-node/src/network/processor/types.ts index b6c01a253f2e..49ec7b0243e5 100644 --- a/packages/beacon-node/src/network/processor/types.ts +++ b/packages/beacon-node/src/network/processor/types.ts @@ -1,4 +1,4 @@ -import {Message} from "@libp2p/interface"; +import type {Message} from "@libp2p/gossipsub"; import {ForkName} from "@lodestar/params"; import {Slot, SlotOptionalRoot} from "@lodestar/types"; import {PeerIdStr} from "../../util/peerId.js"; diff --git a/packages/beacon-node/src/network/reqresp/ReqRespBeaconNode.ts b/packages/beacon-node/src/network/reqresp/ReqRespBeaconNode.ts index 3673ffbbb730..5aba7277cbac 100644 --- a/packages/beacon-node/src/network/reqresp/ReqRespBeaconNode.ts +++ b/packages/beacon-node/src/network/reqresp/ReqRespBeaconNode.ts @@ -19,7 +19,7 @@ import {callInNextEventLoop} from "../../util/eventLoop.js"; import {NetworkCoreMetrics} from "../core/metrics.js"; import {INetworkEventBus, NetworkEvent} from "../events.js"; import {MetadataController} from "../metadata.js"; -import {ClientKind} from "../peers/client.ts"; +import {ClientKind} from "../peers/client.js"; import {PeersData} from "../peers/peersData.js"; import {IPeerRpcScoreStore, PeerAction} from "../peers/score/index.js"; import {StatusCache} from "../statusCache.js"; @@ -58,7 +58,7 @@ export type ReqRespBeaconNodeOpts = ReqRespOpts & {disableLightClientServer?: bo * Implementation of Ethereum Consensus p2p Req/Resp domain. * For the spec that this code is based on, see: * https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/p2p-interface.md#the-reqresp-domain - * https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/p2p-interface.md#the-reqresp-domain + * https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/altair/light-client/p2p-interface.md#the-reqresp-domain */ export class ReqRespBeaconNode extends ReqResp { private readonly metadataController: MetadataController; diff --git a/packages/beacon-node/src/network/reqresp/handlers/beaconBlocksByRange.ts b/packages/beacon-node/src/network/reqresp/handlers/beaconBlocksByRange.ts index 8199bc3f6035..33709cbbbc7f 100644 --- a/packages/beacon-node/src/network/reqresp/handlers/beaconBlocksByRange.ts +++ b/packages/beacon-node/src/network/reqresp/handlers/beaconBlocksByRange.ts @@ -6,7 +6,7 @@ import {computeEpochAtSlot} from "@lodestar/state-transition"; import {deneb, phase0} from "@lodestar/types"; import {IBeaconChain} from "../../../chain/index.js"; import {IBeaconDb} from "../../../db/index.js"; -import {prettyPrintPeerId} from "../../util.ts"; +import {prettyPrintPeerId} from "../../util.js"; // TODO: Unit test @@ -47,9 +47,10 @@ export async function* onBeaconBlocksByRange( // Non-finalized range of blocks if (endSlot > finalizedSlot) { - const head = chain.forkChoice.getHead(); + const headBlock = chain.forkChoice.getHead(); + const headRoot = headBlock.blockRoot; // TODO DENEB: forkChoice should mantain an array of canonical blocks, and change only on reorg - const headChain = chain.forkChoice.getAllAncestorBlocks(head); + const headChain = chain.forkChoice.getAllAncestorBlocks(headRoot, headBlock.payloadStatus); // getAllAncestorBlocks response includes the head node, so it's the full chain. // Iterate head chain with ascending block numbers diff --git a/packages/beacon-node/src/network/reqresp/handlers/blobSidecarsByRange.ts b/packages/beacon-node/src/network/reqresp/handlers/blobSidecarsByRange.ts index bba2cb97d2af..ef6a59a2b5ce 100644 --- a/packages/beacon-node/src/network/reqresp/handlers/blobSidecarsByRange.ts +++ b/packages/beacon-node/src/network/reqresp/handlers/blobSidecarsByRange.ts @@ -34,9 +34,10 @@ export async function* onBlobSidecarsByRange( // Non-finalized range of blobs if (endSlot > finalizedSlot) { - const head = chain.forkChoice.getHead(); + const headBlock = chain.forkChoice.getHead(); + const headRoot = headBlock.blockRoot; // TODO DENEB: forkChoice should mantain an array of canonical blocks, and change only on reorg - const headChain = chain.forkChoice.getAllAncestorBlocks(head); + const headChain = chain.forkChoice.getAllAncestorBlocks(headRoot, headBlock.payloadStatus); // Iterate head chain with ascending block numbers for (let i = headChain.length - 1; i >= 0; i--) { diff --git a/packages/beacon-node/src/network/reqresp/handlers/dataColumnSidecarsByRange.ts b/packages/beacon-node/src/network/reqresp/handlers/dataColumnSidecarsByRange.ts index 2b99ebe68b2e..ebb1d62d68fd 100644 --- a/packages/beacon-node/src/network/reqresp/handlers/dataColumnSidecarsByRange.ts +++ b/packages/beacon-node/src/network/reqresp/handlers/dataColumnSidecarsByRange.ts @@ -7,7 +7,7 @@ import {ColumnIndex, fulu} from "@lodestar/types"; import {fromHex} from "@lodestar/utils"; import {IBeaconChain} from "../../../chain/index.js"; import {IBeaconDb} from "../../../db/index.js"; -import {prettyPrintPeerId} from "../../util.ts"; +import {prettyPrintPeerId} from "../../util.js"; import { handleColumnSidecarUnavailability, validateRequestedDataColumns, @@ -78,8 +78,9 @@ export async function* onDataColumnSidecarsByRange( // Non-finalized range of columns if (endSlot > finalizedSlot) { - const head = chain.forkChoice.getHead(); - const headChain = chain.forkChoice.getAllAncestorBlocks(head); + const headBlock = chain.forkChoice.getHead(); + const headRoot = headBlock.blockRoot; + const headChain = chain.forkChoice.getAllAncestorBlocks(headRoot, headBlock.payloadStatus); // Iterate head chain with ascending block numbers for (let i = headChain.length - 1; i >= 0; i--) { diff --git a/packages/beacon-node/src/network/reqresp/handlers/dataColumnSidecarsByRoot.ts b/packages/beacon-node/src/network/reqresp/handlers/dataColumnSidecarsByRoot.ts index aa51c73b1d97..7748c940e236 100644 --- a/packages/beacon-node/src/network/reqresp/handlers/dataColumnSidecarsByRoot.ts +++ b/packages/beacon-node/src/network/reqresp/handlers/dataColumnSidecarsByRoot.ts @@ -6,7 +6,7 @@ import {toRootHex} from "@lodestar/utils"; import {IBeaconChain} from "../../../chain/index.js"; import {IBeaconDb} from "../../../db/index.js"; import {DataColumnSidecarsByRootRequest} from "../../../util/types.js"; -import {prettyPrintPeerId} from "../../util.ts"; +import {prettyPrintPeerId} from "../../util.js"; import { handleColumnSidecarUnavailability, validateRequestedDataColumns, diff --git a/packages/beacon-node/src/network/reqresp/handlers/executionPayloadEnvelopesByRange.ts b/packages/beacon-node/src/network/reqresp/handlers/executionPayloadEnvelopesByRange.ts index 460d7a0b8190..c457526f4a28 100644 --- a/packages/beacon-node/src/network/reqresp/handlers/executionPayloadEnvelopesByRange.ts +++ b/packages/beacon-node/src/network/reqresp/handlers/executionPayloadEnvelopesByRange.ts @@ -28,7 +28,7 @@ export async function* onExecutionPayloadEnvelopesByRange( // Non-finalized range: from chain cache if (endSlot > finalizedSlot) { const head = chain.forkChoice.getHead(); - const headChain = chain.forkChoice.getAllAncestorBlocks(head); + const headChain = chain.forkChoice.getAllAncestorBlocks(head.blockRoot, head.payloadStatus); for (let i = headChain.length - 1; i >= 0; i--) { const block = headChain[i]; if (block.slot >= startSlot && block.slot < endSlot) { diff --git a/packages/beacon-node/src/network/reqresp/score.ts b/packages/beacon-node/src/network/reqresp/score.ts index f32a6e791b84..57818fef6780 100644 --- a/packages/beacon-node/src/network/reqresp/score.ts +++ b/packages/beacon-node/src/network/reqresp/score.ts @@ -46,7 +46,6 @@ export function onOutgoingReqRespError(e: RequestError, method: ReqRespMethod): : PeerAction.LowToleranceError; // TODO: Detect SSZDecodeError and return PeerAction.Fatal - case RequestErrorCode.TTFB_TIMEOUT: case RequestErrorCode.RESP_TIMEOUT: switch (method) { case ReqRespMethod.Ping: diff --git a/packages/beacon-node/src/network/reqresp/utils/collect.ts b/packages/beacon-node/src/network/reqresp/utils/collect.ts index 6e528cbf545d..5f80a28b42b5 100644 --- a/packages/beacon-node/src/network/reqresp/utils/collect.ts +++ b/packages/beacon-node/src/network/reqresp/utils/collect.ts @@ -1,6 +1,6 @@ import {Type} from "@chainsafe/ssz"; import {RequestError, RequestErrorCode, ResponseIncoming} from "@lodestar/reqresp"; -import {SerializedCache} from "../../../util/serializedCache.ts"; +import {SerializedCache} from "../../../util/serializedCache.js"; import {ResponseTypeGetter} from "../types.js"; /** diff --git a/packages/beacon-node/src/network/reqresp/utils/collectSequentialBlocksInRange.ts b/packages/beacon-node/src/network/reqresp/utils/collectSequentialBlocksInRange.ts index 5e8df776eb6c..b44fb07ca843 100644 --- a/packages/beacon-node/src/network/reqresp/utils/collectSequentialBlocksInRange.ts +++ b/packages/beacon-node/src/network/reqresp/utils/collectSequentialBlocksInRange.ts @@ -1,7 +1,7 @@ import {ResponseIncoming} from "@lodestar/reqresp"; import {SignedBeaconBlock, phase0} from "@lodestar/types"; import {LodestarError} from "@lodestar/utils"; -import {SerializedCache} from "../../../util/serializedCache.ts"; +import {SerializedCache} from "../../../util/serializedCache.js"; import {ReqRespMethod, responseSszTypeByMethod} from "../types.js"; import {sszDeserializeResponse} from "./collect.js"; diff --git a/packages/beacon-node/src/network/util.ts b/packages/beacon-node/src/network/util.ts index 38a10a2ac052..0a905f2c60c8 100644 --- a/packages/beacon-node/src/network/util.ts +++ b/packages/beacon-node/src/network/util.ts @@ -23,7 +23,7 @@ export function getConnection(libp2p: Libp2p, peerIdStr: string): Connection | u return getConnectionsMap(libp2p).get(peerIdStr)?.value[0] ?? undefined; } -// https://github.com/ChainSafe/js-libp2p-gossipsub/blob/3475242ed254f7647798ab7f36b21909f6cb61da/src/index.ts#L2009 +// https://github.com/libp2p/js-libp2p/blob/f87cba928991736d9646b3e054c367f55cab315c/packages/gossipsub/src/gossipsub.ts#L2076 export function isPublishToZeroPeersError(e: Error): boolean { - return e.message.includes("PublishError.InsufficientPeers"); + return e.message.includes("PublishError.NoPeersSubscribedToTopic"); } diff --git a/packages/beacon-node/src/node/nodejs.ts b/packages/beacon-node/src/node/nodejs.ts index 18e7961c7604..56e33c3179de 100644 --- a/packages/beacon-node/src/node/nodejs.ts +++ b/packages/beacon-node/src/node/nodejs.ts @@ -2,12 +2,11 @@ import {setMaxListeners} from "node:events"; import {PrivateKey} from "@libp2p/interface"; import {Registry} from "prom-client"; import {hasher} from "@chainsafe/persistent-merkle-tree"; -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {BeaconApiMethods} from "@lodestar/api/beacon/server"; import {BeaconConfig} from "@lodestar/config"; import type {LoggerNode} from "@lodestar/logger/node"; import {ZERO_HASH_HEX} from "@lodestar/params"; -import {CachedBeaconStateAllForks, Index2PubkeyCache, isExecutionCachedStateType} from "@lodestar/state-transition"; +import {CachedBeaconStateAllForks, PubkeyCache, isExecutionCachedStateType} from "@lodestar/state-transition"; import {phase0} from "@lodestar/types"; import {sleep, toRootHex} from "@lodestar/utils"; import {ProcessShutdownCallback} from "@lodestar/validator"; @@ -47,8 +46,7 @@ export type BeaconNodeModules = { export type BeaconNodeInitModules = { opts: IBeaconNodeOptions; config: BeaconConfig; - pubkey2index: PubkeyIndexMap; - index2pubkey: Index2PubkeyCache; + pubkeyCache: PubkeyCache; db: IBeaconDb; logger: LoggerNode; processShutdownCallback: ProcessShutdownCallback; @@ -150,8 +148,7 @@ export class BeaconNode { static async init({ opts, config, - pubkey2index, - index2pubkey, + pubkeyCache, db, logger, processShutdownCallback, @@ -240,8 +237,7 @@ export class BeaconNode { privateKey, config, clock, - pubkey2index, - index2pubkey, + pubkeyCache, dataDir, db, dbName: opts.db.name, @@ -364,9 +360,12 @@ export class BeaconNode { if (this.restApi) await this.restApi.close(); await this.network.close(); if (this.metricsServer) await this.metricsServer.close(); - if (this.monitoring) this.monitoring.close(); + if (this.monitoring) await this.monitoring.close(); await this.chain.persistToDisk(); await this.chain.close(); + // Abort signal last: close() calls above clear intervals/timeouts so no new + // operations get scheduled. If we aborted first, a still-pending interval could + // fire and schedule a new operation after abort, leaving it stuck and delaying shutdown. if (this.controller) this.controller.abort(); await sleep(DELAY_BEFORE_CLOSING_DB_MS); await this.db.close(); diff --git a/packages/beacon-node/src/node/notifier.ts b/packages/beacon-node/src/node/notifier.ts index 7762dd265e30..e4ca82fae788 100644 --- a/packages/beacon-node/src/node/notifier.ts +++ b/packages/beacon-node/src/node/notifier.ts @@ -171,7 +171,7 @@ function getHeadExecutionInfo( // or the parent block's FULL variant to show resolved execution payload info. let payloadBlockForDisplay = headInfo; if ( - headInfo.executionStatus === ExecutionStatus.PendingEnvelope || + headInfo.executionStatus === ExecutionStatus.PayloadSeparated || headInfo.payloadStatus === PayloadStatus.PENDING || headInfo.payloadStatus === PayloadStatus.EMPTY ) { @@ -198,13 +198,10 @@ function getHeadExecutionInfo( const executionPayloadHashInfo = payloadBlockForDisplay.executionPayloadBlockHash; const executionPayloadNumberInfo = payloadBlockForDisplay.executionPayloadNumber; - const bidHashInfo = payloadBlockForDisplay.blockHashFromBid - ? ` bid:${prettyBytesShort(payloadBlockForDisplay.blockHashFromBid)}` - : ""; return [ `exec-block: ${executionStatusStr}/${payloadStatusStr}(${executionPayloadNumberInfo} ${prettyBytesShort( executionPayloadHashInfo - )}${bidHashInfo})`, + )})`, ]; } diff --git a/packages/beacon-node/src/sync/backfill/backfill.ts b/packages/beacon-node/src/sync/backfill/backfill.ts index 52b0bd8646e6..c129c17facfd 100644 --- a/packages/beacon-node/src/sync/backfill/backfill.ts +++ b/packages/beacon-node/src/sync/backfill/backfill.ts @@ -127,7 +127,7 @@ export class BackfillSync extends (EventEmitter as {new (): BackfillSyncEmitter} private wsValidated = false; /** - * From https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/weak-subjectivity.md + * From https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/weak-subjectivity.md * * * If diff --git a/packages/beacon-node/src/sync/constants.ts b/packages/beacon-node/src/sync/constants.ts index 7bef3598b181..25e24f620db5 100644 --- a/packages/beacon-node/src/sync/constants.ts +++ b/packages/beacon-node/src/sync/constants.ts @@ -59,7 +59,7 @@ export const BATCH_BUFFER_SIZE = Math.ceil(10 / EPOCHS_PER_BATCH); /** * Maximum number of concurrent requests to perform with a SyncChain. - * This is according to the spec https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/p2p-interface.md + * This is according to the spec https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/p2p-interface.md */ export const MAX_CONCURRENT_REQUESTS = 2; diff --git a/packages/beacon-node/src/sync/range/range.ts b/packages/beacon-node/src/sync/range/range.ts index 9eb3e76e4b1d..8feef52a49b2 100644 --- a/packages/beacon-node/src/sync/range/range.ts +++ b/packages/beacon-node/src/sync/range/range.ts @@ -207,6 +207,7 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) { logger: this.logger, peerIdStr: peer.peerId, batchBlocks, + peerDasMetrics: this.chain.metrics?.peerDas, ...batch.getRequestsForPeer(peer), }); const cached = cacheByRangeResponses({ diff --git a/packages/beacon-node/src/sync/unknownBlock.ts b/packages/beacon-node/src/sync/unknownBlock.ts index 60f0a1f835a9..afd020b5c7be 100644 --- a/packages/beacon-node/src/sync/unknownBlock.ts +++ b/packages/beacon-node/src/sync/unknownBlock.ts @@ -543,7 +543,6 @@ export class BlockInputSync { childParentBlockHash?: RootHex; parentRoot?: RootHex; parentPayloadStatus?: PayloadStatus; - parentBlockHashFromBid?: RootHex; wantsFullParent?: boolean; } { const block = pendingBlock.blockInput.getBlock(); @@ -559,7 +558,7 @@ export class BlockInputSync { const parentRoot = pendingBlock.blockInput.parentRootHex; const parentBlock = this.chain.forkChoice.getBlockHexDefaultStatus(parentRoot); - if (!parentBlock || parentBlock.blockHashFromBid == null) { + if (!parentBlock || parentBlock.executionPayloadBlockHash == null) { return { shouldRetry: false, forkName, @@ -568,9 +567,13 @@ export class BlockInputSync { }; } - const parentBlockHashFromBid = parentBlock.blockHashFromBid; const parentPayloadStatus = parentBlock.payloadStatus; - const wantsFullParent = childParentBlockHash === parentBlockHashFromBid; + // The child extends FULL if its parentBlockHash does NOT match the default (PENDING/EMPTY) + // variant's executionPayloadBlockHash. For PENDING/EMPTY variants, executionPayloadBlockHash + // is the parentBlockHash from the parent's bid. For FULL, it's the actual payload block hash + // (which equals the bid's blockHash). If the child points to the FULL variant's hash, + // it won't match the default variant's hash. + const wantsFullParent = childParentBlockHash !== parentBlock.executionPayloadBlockHash; return { shouldRetry: wantsFullParent && parentPayloadStatus !== PayloadStatus.FULL, @@ -578,7 +581,6 @@ export class BlockInputSync { childParentBlockHash, parentRoot, parentPayloadStatus, - parentBlockHashFromBid, wantsFullParent, }; } @@ -634,7 +636,7 @@ export class BlockInputSync { * From a set of shuffled peers: * - fetch the block * - from deneb, fetch all missing blobs - * - from peerDAS, fetch sampled colmns + * - from peerDAS, fetch sampled columns * TODO: this means we only have block root, and nothing else. Consider to reflect this in the function name * prefulu, will attempt a max of `MAX_ATTEMPTS_PER_BLOCK` on different peers, postfulu we may attempt more as defined in `getMaxDownloadAttempts()` function * Also verifies the received block root + returns the peer that provided the block for future downscoring. @@ -642,10 +644,7 @@ export class BlockInputSync { private async fetchBlockInput(cacheItem: BlockInputSyncCacheItem): Promise { const rootHex = getBlockInputSyncCacheItemRootHex(cacheItem); const excludedPeers = new Set(); - const defaultPendingColumns = - this.config.getForkSeq(this.chain.clock.currentSlot) >= ForkSeq.fulu - ? new Set(this.network.custodyConfig.sampledColumns) - : null; + const defaultPendingColumns = new Set(this.network.custodyConfig.sampledColumns); const fetchStartSec = Date.now() / 1000; let slot = isPendingBlockInput(cacheItem) ? cacheItem.blockInput.slot : undefined; @@ -659,14 +658,10 @@ export class BlockInputSync { isPendingBlockInput(cacheItem) && isBlockInputColumns(cacheItem.blockInput) ? new Set(cacheItem.blockInput.getMissingSampledColumnMeta().missing) : defaultPendingColumns; - // pendingDataColumns is null pre-fulu const peerMeta = this.peerBalancer.bestPeerForPendingColumns(pendingColumns, excludedPeers); if (peerMeta === null) { // no more peer with needed columns to try, throw error - let message = `Error fetching UnknownBlockRoot slot=${slot} root=${rootHex} after ${i}: cannot find peer`; - if (pendingColumns) { - message += ` with needed columns=${prettyPrintIndices(Array.from(pendingColumns))}`; - } + const message = `Error fetching UnknownBlockRoot slot=${slot} root=${rootHex} after ${i}: cannot find peer with needed columns=${prettyPrintIndices(Array.from(pendingColumns))}`; this.metrics?.blockInputSync.fetchTimeSec.observe( {result: FetchResult.FailureTriedAllPeers}, Date.now() / 1000 - fetchStartSec @@ -803,7 +798,7 @@ export class BlockInputSync { // TODO(fulu): why is this commented out here? // // this.knownBadBlocks.add(block.blockRootHex); - // for (const peerIdStr of block.peerIdStrs) { + // for (const peerIdStr of block.peerIdStrings) { // // TODO: Refactor peerRpcScores to work with peerIdStr only // this.network.reportPeer(peerIdStr, PeerAction.LowToleranceError, "BadBlockByRoot"); // } @@ -882,11 +877,11 @@ export class UnknownBlockPeerBalancer { } /** - * called from fetchUnknownBlockRoot() where we only have block root and nothing else + * called from fetchBlockInput() where we only have block root and nothing else * excludedPeers are the peers that we requested already so we don't want to try again * pendingColumns is empty for prefulu, or the 1st time we we download a block by root */ - bestPeerForPendingColumns(pendingColumns: Set | null, excludedPeers: Set): PeerSyncMeta | null { + bestPeerForPendingColumns(pendingColumns: Set, excludedPeers: Set): PeerSyncMeta | null { const eligiblePeers = this.filterPeers(pendingColumns, excludedPeers); if (eligiblePeers.length === 0) { return null; @@ -903,37 +898,6 @@ export class UnknownBlockPeerBalancer { return this.peersMeta.get(bestPeerId) ?? null; } - /** - * called from fetchUnavailableBlockInput() where we have either BlockInput or NullBlockInput - * excludedPeers are the peers that we requested already so we don't want to try again - */ - bestPeerForBlockInput(blockInput: IBlockInput, excludedPeers: Set): PeerSyncMeta | null { - const eligiblePeers: PeerIdStr[] = []; - - if (isBlockInputColumns(blockInput)) { - const pendingDataColumns: Set = new Set(blockInput.getMissingSampledColumnMeta().missing); - // there could be no pending column in case when block is still missing - eligiblePeers.push(...this.filterPeers(pendingDataColumns, excludedPeers)); - } else { - // prefulu - eligiblePeers.push(...this.filterPeers(null, excludedPeers)); - } - - if (eligiblePeers.length === 0) { - return null; - } - - const sortedEligiblePeers = sortBy( - shuffle(eligiblePeers), - // prefer peers with least active req - (peerId) => this.activeRequests.get(peerId) ?? 0 - ); - - const bestPeerId = sortedEligiblePeers[0]; - this.onRequest(bestPeerId); - return this.peersMeta.get(bestPeerId) ?? null; - } - /** * Consumers don't need to call this method directly, it is called internally by bestPeer*() methods * make this public for testing @@ -957,8 +921,7 @@ export class UnknownBlockPeerBalancer { return totalActiveRequests; } - // pendingDataColumns could be null for prefulu - private filterPeers(pendingDataColumns: Set | null, excludedPeers: Set): PeerIdStr[] { + private filterPeers(pendingDataColumns: Set, excludedPeers: Set): PeerIdStr[] { let maxColumnCount = 0; const considerPeers: {peerId: PeerIdStr; columnCount: number}[] = []; for (const [peerId, syncMeta] of this.peersMeta.entries()) { @@ -973,13 +936,12 @@ export class UnknownBlockPeerBalancer { continue; } - if (pendingDataColumns === null || pendingDataColumns.size === 0) { - // prefulu, no pending columns + if (pendingDataColumns.size === 0) { considerPeers.push({peerId, columnCount: 0}); continue; } - // postfulu, find peers that have custody columns that we need + // find peers that have custody columns that we need const {custodyColumns: peerColumns} = syncMeta; // check if the peer has all needed columns // get match diff --git a/packages/beacon-node/src/sync/utils/downloadByRange.ts b/packages/beacon-node/src/sync/utils/downloadByRange.ts index 6a4abb43c06b..65dbeea30f6c 100644 --- a/packages/beacon-node/src/sync/utils/downloadByRange.ts +++ b/packages/beacon-node/src/sync/utils/downloadByRange.ts @@ -21,6 +21,7 @@ import { import {SeenBlockInput} from "../../chain/seenCache/seenGossipBlockInput.js"; import {validateBlockBlobSidecars} from "../../chain/validation/blobSidecar.js"; import {validateBlockDataColumnSidecars} from "../../chain/validation/dataColumnSidecar.js"; +import {BeaconMetrics} from "../../metrics/metrics/beacon.js"; import {INetwork} from "../../network/index.js"; import {getBlobKzgCommitments} from "../../util/dataColumns.js"; import {PeerIdStr} from "../../util/peerId.js"; @@ -46,6 +47,7 @@ export type DownloadAndCacheByRangeProps = DownloadByRangeRequests & { logger: Logger; peerIdStr: string; batchBlocks?: IBlockInput[]; + peerDasMetrics?: BeaconMetrics["peerDas"] | null; }; export type CacheByRangeResponsesProps = { @@ -237,6 +239,7 @@ export async function downloadByRange({ blobsRequest, columnsRequest, envelopesRequest, + peerDasMetrics, }: DownloadAndCacheByRangeProps): Promise> { let response: DownloadByRangeResponses; try { @@ -263,6 +266,7 @@ export async function downloadByRange({ blobsRequest, columnsRequest, envelopesRequest, + peerDasMetrics, ...response, }); @@ -350,10 +354,12 @@ export async function validateResponses({ blobSidecars, columnSidecars, signedEnvelopes, + peerDasMetrics, }: DownloadByRangeRequests & DownloadByRangeResponses & { config: ChainForkConfig; batchBlocks?: IBlockInput[]; + peerDasMetrics?: BeaconMetrics["peerDas"] | null; }): Promise> { // Blocks are always required for blob/column validation // If a blocksRequest is provided, blocks have just been downloaded @@ -432,7 +438,8 @@ export async function validateResponses({ config, columnsRequest, blocksForDataValidation, - columnSidecars + columnSidecars, + peerDasMetrics ); validatedResponses.validatedColumnSidecars = validatedColumnSidecarsResult.result; warnings = validatedColumnSidecarsResult.warnings; @@ -728,7 +735,8 @@ export async function validateColumnsByRangeResponse( config: ChainForkConfig, request: fulu.DataColumnSidecarsByRangeRequest, blocks: ValidatedBlock[], - columnSidecars: DataColumnSidecars + columnSidecars: DataColumnSidecars, + peerDasMetrics?: BeaconMetrics["peerDas"] | null ): Promise> { const warnings: DownloadByRangeError[] = []; @@ -887,7 +895,8 @@ export async function validateColumnsByRangeResponse( blockRoot, blobCount, columnSidecars, - getBlobKzgCommitments(forkName, block as SignedBeaconBlock) + getBlobKzgCommitments(forkName, block as SignedBeaconBlock), + peerDasMetrics ).then(() => ({ blockRoot, columnSidecars, diff --git a/packages/beacon-node/src/sync/utils/downloadByRoot.ts b/packages/beacon-node/src/sync/utils/downloadByRoot.ts index 0b7a593196a1..7ad742c7f137 100644 --- a/packages/beacon-node/src/sync/utils/downloadByRoot.ts +++ b/packages/beacon-node/src/sync/utils/downloadByRoot.ts @@ -6,7 +6,7 @@ import {LodestarError, byteArrayEquals, fromHex, prettyPrintIndices, toHex, toRo import {isBlockInputBlobs, isBlockInputColumns} from "../../chain/blocks/blockInput/blockInput.js"; import {BlockInputSource, IBlockInput} from "../../chain/blocks/blockInput/types.js"; import {ChainEventEmitter} from "../../chain/emitter.js"; -import {IBeaconChain} from "../../chain/interface.ts"; +import {IBeaconChain} from "../../chain/interface.js"; import {validateBlockBlobSidecars} from "../../chain/validation/blobSidecar.js"; import {validateBlockDataColumnSidecars} from "../../chain/validation/dataColumnSidecar.js"; import {INetwork} from "../../network/interface.js"; @@ -446,7 +446,8 @@ export async function fetchAndValidateColumns({ blockRoot, blobCount, columnSidecars, - getBlobKzgCommitments(forkName, block) + getBlobKzgCommitments(forkName, block), + chain?.metrics?.peerDas ); return {result: columnSidecars, warnings: warnings.length > 0 ? warnings : null}; diff --git a/packages/beacon-node/src/sync/utils/remoteSyncType.ts b/packages/beacon-node/src/sync/utils/remoteSyncType.ts index 9024d8cb7367..68f958a203a6 100644 --- a/packages/beacon-node/src/sync/utils/remoteSyncType.ts +++ b/packages/beacon-node/src/sync/utils/remoteSyncType.ts @@ -1,7 +1,7 @@ import {IForkChoice} from "@lodestar/fork-choice"; import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {Slot, Status} from "@lodestar/types"; -import {IBeaconChain} from "../../chain/interface.ts"; +import {IBeaconChain} from "../../chain/interface.js"; import {ChainTarget} from "../range/utils/index.js"; /** The type of peer relative to our current state */ diff --git a/packages/beacon-node/src/util/clock.ts b/packages/beacon-node/src/util/clock.ts index bd530ef61d4a..52ade3e79079 100644 --- a/packages/beacon-node/src/util/clock.ts +++ b/packages/beacon-node/src/util/clock.ts @@ -92,15 +92,17 @@ export class Clock extends EventEmitter implements IClock { } return slot; } - /** * If it's too close to next slot given MAXIMUM_GOSSIP_CLOCK_DISPARITY, return currentSlot + 1. * Otherwise return currentSlot + * + * Spec: phase0/p2p-interface.md - gossip validation uses `current_time + MAXIMUM_GOSSIP_CLOCK_DISPARITY < message_time` + * to reject future messages (strict `<`), so the boundary (exactly equal) is accepted, hence `<=` here. */ get currentSlotWithGossipDisparity(): Slot { const currentSlot = this.currentSlot; const nextSlotTime = computeTimeAtSlot(this.config, currentSlot + 1, this.genesisTime) * 1000; - return nextSlotTime - Date.now() < this.config.MAXIMUM_GOSSIP_CLOCK_DISPARITY ? currentSlot + 1 : currentSlot; + return nextSlotTime - Date.now() <= this.config.MAXIMUM_GOSSIP_CLOCK_DISPARITY ? currentSlot + 1 : currentSlot; } get currentEpoch(): Epoch { @@ -121,6 +123,9 @@ export class Clock extends EventEmitter implements IClock { /** * Check if a slot is current slot given MAXIMUM_GOSSIP_CLOCK_DISPARITY. + * + * Uses `<=` for disparity checks because the spec rejects with strict `<` + * (phase0/p2p-interface.md), meaning the boundary (exactly equal) is accepted. */ isCurrentSlotGivenGossipDisparity(slot: Slot): boolean { const currentSlot = this.currentSlot; @@ -129,12 +134,12 @@ export class Clock extends EventEmitter implements IClock { } const nextSlotTime = computeTimeAtSlot(this.config, currentSlot + 1, this.genesisTime) * 1000; // we're too close to next slot, accept next slot - if (nextSlotTime - Date.now() < this.config.MAXIMUM_GOSSIP_CLOCK_DISPARITY) { + if (nextSlotTime - Date.now() <= this.config.MAXIMUM_GOSSIP_CLOCK_DISPARITY) { return slot === currentSlot + 1; } const currentSlotTime = computeTimeAtSlot(this.config, currentSlot, this.genesisTime) * 1000; // we've just passed the current slot, accept previous slot - if (Date.now() - currentSlotTime < this.config.MAXIMUM_GOSSIP_CLOCK_DISPARITY) { + if (Date.now() - currentSlotTime <= this.config.MAXIMUM_GOSSIP_CLOCK_DISPARITY) { return slot === currentSlot - 1; } return false; diff --git a/packages/beacon-node/src/util/dataColumns.ts b/packages/beacon-node/src/util/dataColumns.ts index 69bca00577e4..ac16f9683d3e 100644 --- a/packages/beacon-node/src/util/dataColumns.ts +++ b/packages/beacon-node/src/util/dataColumns.ts @@ -425,7 +425,8 @@ export async function recoverDataColumnSidecars( partialSidecars.set(columnSidecar.index, columnSidecar); } - const timer = metrics?.recoverDataColumnSidecars.recoverTime.startTimer(); + const timer = metrics?.peerDas.dataColumnsReconstructionTime.startTimer(); + // if this function throws, we catch at the consumer side const fullSidecars = await dataColumnMatrixRecovery(partialSidecars).catch(() => null); timer?.(); @@ -435,6 +436,7 @@ export async function recoverDataColumnSidecars( if (blockInput.getAllColumns().length === NUMBER_OF_COLUMNS) { // either gossip or getBlobsV2 resolved availability while we were recovering + metrics?.dataColumns.alreadyAdded.inc(fullSidecars.length); return DataColumnReconstructionCode.SuccessLate; } @@ -458,8 +460,10 @@ export async function recoverDataColumnSidecars( sidecarsToPublish.push(columnSidecar); } } + metrics?.peerDas.reconstructedColumns.inc(sidecarsToPublish.length); + metrics?.dataColumns.bySource.inc({source: BlockInputSource.recovery}, sidecarsToPublish.length); emitter.emit(ChainEvent.publishDataColumns, sidecarsToPublish); - + // TODO: Can we record dataColumns.sentPeersPerSubnet metric somehow return DataColumnReconstructionCode.SuccessResolved; } diff --git a/packages/beacon-node/src/util/execution.ts b/packages/beacon-node/src/util/execution.ts index c05393769829..18fcf070c0d5 100644 --- a/packages/beacon-node/src/util/execution.ts +++ b/packages/beacon-node/src/util/execution.ts @@ -173,16 +173,21 @@ export async function getDataColumnSidecarsFromExecution( } let dataColumnSidecars: fulu.DataColumnSidecars; - const cellsAndProofs = await getCellsAndProofs(blobs); - if (blockInput.hasBlock()) { - dataColumnSidecars = getDataColumnSidecarsFromBlock( - config, - blockInput.getBlock() as fulu.SignedBeaconBlock, - cellsAndProofs - ); - } else { - const firstSidecar = blockInput.getAllColumns()[0]; - dataColumnSidecars = getDataColumnSidecarsFromColumnSidecar(firstSidecar, cellsAndProofs); + const compTimer = metrics?.peerDas.dataColumnSidecarComputationTime.startTimer(); + try { + const cellsAndProofs = await getCellsAndProofs(blobs); + if (blockInput.hasBlock()) { + dataColumnSidecars = getDataColumnSidecarsFromBlock( + config, + blockInput.getBlock() as fulu.SignedBeaconBlock, + cellsAndProofs + ); + } else { + const firstSidecar = blockInput.getAllColumns()[0]; + dataColumnSidecars = getDataColumnSidecarsFromColumnSidecar(firstSidecar, cellsAndProofs); + } + } finally { + compTimer?.(); } // Publish columns if and only if subscribed to them @@ -191,13 +196,15 @@ export async function getDataColumnSidecarsFromExecution( // for columns that we already seen, it will be ignored through `ignoreDuplicatePublishError` gossip option emitter.emit(ChainEvent.publishDataColumns, sampledColumns); + // TODO: Can we record dataColumns.sentPeersPerSubnet metric here somehow // add all sampled columns to the block input, even if we didn't sample them const seenTimestampSec = Date.now() / 1000; + let alreadyAddedColumnsCount = 0; for (const columnSidecar of sampledColumns) { if (blockInput.hasColumn(columnSidecar.index)) { // columns may have been added while waiting - // TODO(fulu): add metrics for this condition + alreadyAddedColumnsCount++; continue; } @@ -217,7 +224,11 @@ export async function getDataColumnSidecarsFromExecution( }); } } + metrics?.dataColumns.alreadyAdded.inc(alreadyAddedColumnsCount); - metrics?.dataColumns.bySource.inc({source: BlockInputSource.engine}, previouslyMissingColumns.length); + metrics?.dataColumns.bySource.inc( + {source: BlockInputSource.engine}, + previouslyMissingColumns.length - alreadyAddedColumnsCount + ); return DataColumnEngineResult.SuccessResolved; } diff --git a/packages/beacon-node/src/util/serializedCache.ts b/packages/beacon-node/src/util/serializedCache.ts index 45785eb807fb..777ac4dd0c3b 100644 --- a/packages/beacon-node/src/util/serializedCache.ts +++ b/packages/beacon-node/src/util/serializedCache.ts @@ -4,7 +4,7 @@ * This is a thin wrapper around WeakMap */ export class SerializedCache { - map: WeakMap = new WeakMap(); + private map: WeakMap = new WeakMap(); get(obj: object): Uint8Array | undefined { return this.map.get(obj); @@ -15,11 +15,13 @@ export class SerializedCache { } /** - * Replace the internal WeakMap to force GC of all cached entries. - * Must only be called after all DB writes that may read from this cache have completed, + * Delete cached serialized entries for the provided object references. + * Must only be called after all DB writes that read from this cache for these objects have completed, * otherwise cached serialized bytes will be unavailable and data will be re-serialized unnecessarily. */ - clear(): void { - this.map = new WeakMap(); + delete(objs: object[]): void { + for (const obj of objs) { + this.map.delete(obj); + } } } diff --git a/packages/beacon-node/src/util/sszBytes.ts b/packages/beacon-node/src/util/sszBytes.ts index b8377b249d70..855ee7fd2574 100644 --- a/packages/beacon-node/src/util/sszBytes.ts +++ b/packages/beacon-node/src/util/sszBytes.ts @@ -8,6 +8,7 @@ import { ForkSeq, MAX_COMMITTEES_PER_SLOT, isForkPostElectra, + isForkPostGloas, } from "@lodestar/params"; import {BLSSignature, CommitteeIndex, RootHex, Slot, ValidatorIndex, ssz} from "@lodestar/types"; @@ -398,23 +399,102 @@ export function getSlotFromBlobSidecarSerialized(data: Uint8Array): Slot | null } /** + * Pre-Gloas DataColumnSidecar: * { - index: ColumnIndex [ fixed - 8 bytes], - column: DataColumn BYTES_PER_FIELD_ELEMENT * FIELD_ELEMENTS_PER_CELL * , - kzgCommitments: denebSsz.BlobKzgCommitments, - kzgProofs: denebSsz.KZGProofs, - signedBlockHeader: phase0Ssz.SignedBeaconBlockHeader, - kzgCommitmentsInclusionProof: KzgCommitmentsInclusionProof, + * index: ColumnIndex [fixed - 8 bytes], + * column: DataColumn (offset - 4 bytes), + * kzgCommitments: (offset - 4 bytes), + * kzgProofs: (offset - 4 bytes), + * signedBlockHeader: (offset - 4 bytes) -> slot at variable offset after fixed header + * kzgCommitmentsInclusionProof: (offset - 4 bytes), + * } + * Post-Gloas DataColumnSidecar: + * { + * index: ColumnIndex [8 bytes], + * column: DataColumn (offset - 4 bytes), + * kzgProofs: (offset - 4 bytes), + * slot: Slot [8 bytes] - at offset 16, + * beaconBlockRoot: Root [32 bytes] - at offset 24, + * } + */ +const SLOT_BYTES_POSITION_IN_SIGNED_DATA_COLUMN_SIDECAR_PRE_GLOAS = 20; +const SLOT_BYTES_POSITION_IN_SIGNED_DATA_COLUMN_SIDECAR_POST_GLOAS = 16; +const BEACON_BLOCK_ROOT_POSITION_IN_GLOAS_DATA_COLUMN_SIDECAR = 24; + +export function getSlotFromDataColumnSidecarSerialized(data: Uint8Array, fork: ForkName): Slot | null { + const offset = isForkPostGloas(fork) + ? SLOT_BYTES_POSITION_IN_SIGNED_DATA_COLUMN_SIDECAR_POST_GLOAS + : SLOT_BYTES_POSITION_IN_SIGNED_DATA_COLUMN_SIDECAR_PRE_GLOAS; + + if (data.length < offset + SLOT_SIZE) { + return null; + } + + return getSlotFromOffset(data, offset); +} + +export function getBeaconBlockRootFromDataColumnSidecarSerialized(data: Uint8Array): RootHex | null { + if (data.length < BEACON_BLOCK_ROOT_POSITION_IN_GLOAS_DATA_COLUMN_SIDECAR + ROOT_SIZE) { + return null; } + + blockRootBuf.set( + data.subarray( + BEACON_BLOCK_ROOT_POSITION_IN_GLOAS_DATA_COLUMN_SIDECAR, + BEACON_BLOCK_ROOT_POSITION_IN_GLOAS_DATA_COLUMN_SIDECAR + ROOT_SIZE + ) + ); + return "0x" + blockRootBuf.toString("hex"); +} + +/** + * SignedExecutionPayloadEnvelope SSZ Layout: + * ├─ 4 bytes: message offset (points to byte 100) + * ├─ 96 bytes: signature + * └─ ExecutionPayloadEnvelope (starts at byte 100): + * ├─ 4 bytes: payload offset + * ├─ 4 bytes: executionRequests offset + * ├─ 8 bytes: builderIndex (offset 108-115) + * ├─ 32 bytes: beaconBlockRoot (offset 116-147) + * ├─ 8 bytes: slot (offset 148-155) + * └─ 32 bytes: stateRoot (offset 156-187) */ +const SIGNED_EXECUTION_PAYLOAD_ENVELOPE_MESSAGE_OFFSET = 4; +const SIGNED_EXECUTION_PAYLOAD_ENVELOPE_SIGNATURE_SIZE = 96; +const EXECUTION_PAYLOAD_ENVELOPE_PAYLOAD_OFFSET = 4; +const EXECUTION_PAYLOAD_ENVELOPE_REQUESTS_OFFSET = 4; +const EXECUTION_PAYLOAD_ENVELOPE_BUILDER_INDEX_SIZE = 8; + +const BEACON_BLOCK_ROOT_OFFSET_IN_SIGNED_EXECUTION_PAYLOAD_ENVELOPE = + SIGNED_EXECUTION_PAYLOAD_ENVELOPE_MESSAGE_OFFSET + + SIGNED_EXECUTION_PAYLOAD_ENVELOPE_SIGNATURE_SIZE + + EXECUTION_PAYLOAD_ENVELOPE_PAYLOAD_OFFSET + + EXECUTION_PAYLOAD_ENVELOPE_REQUESTS_OFFSET + + EXECUTION_PAYLOAD_ENVELOPE_BUILDER_INDEX_SIZE; // 116 + +const SLOT_OFFSET_IN_SIGNED_EXECUTION_PAYLOAD_ENVELOPE = + BEACON_BLOCK_ROOT_OFFSET_IN_SIGNED_EXECUTION_PAYLOAD_ENVELOPE + ROOT_SIZE; // 148 + +export function getSlotFromExecutionPayloadEnvelopeSerialized(data: Uint8Array): Slot | null { + if (data.length < SLOT_OFFSET_IN_SIGNED_EXECUTION_PAYLOAD_ENVELOPE + SLOT_SIZE) { + return null; + } -const SLOT_BYTES_POSITION_IN_SIGNED_DATA_COLUMN_SIDECAR = 20; -export function getSlotFromDataColumnSidecarSerialized(data: Uint8Array): Slot | null { - if (data.length < SLOT_BYTES_POSITION_IN_SIGNED_DATA_COLUMN_SIDECAR + SLOT_SIZE) { + return getSlotFromOffset(data, SLOT_OFFSET_IN_SIGNED_EXECUTION_PAYLOAD_ENVELOPE); +} + +export function getBeaconBlockRootFromExecutionPayloadEnvelopeSerialized(data: Uint8Array): RootHex | null { + if (data.length < BEACON_BLOCK_ROOT_OFFSET_IN_SIGNED_EXECUTION_PAYLOAD_ENVELOPE + ROOT_SIZE) { return null; } - return getSlotFromOffset(data, SLOT_BYTES_POSITION_IN_SIGNED_DATA_COLUMN_SIDECAR); + blockRootBuf.set( + data.subarray( + BEACON_BLOCK_ROOT_OFFSET_IN_SIGNED_EXECUTION_PAYLOAD_ENVELOPE, + BEACON_BLOCK_ROOT_OFFSET_IN_SIGNED_EXECUTION_PAYLOAD_ENVELOPE + ROOT_SIZE + ) + ); + return "0x" + blockRootBuf.toString("hex"); } /** diff --git a/packages/beacon-node/src/util/workerEvents.ts b/packages/beacon-node/src/util/workerEvents.ts index 24941bd31699..d67017089763 100644 --- a/packages/beacon-node/src/util/workerEvents.ts +++ b/packages/beacon-node/src/util/workerEvents.ts @@ -1,5 +1,5 @@ import {MessagePort, Worker} from "node:worker_threads"; -import {Message} from "@libp2p/interface"; +import type {Message} from "@libp2p/gossipsub"; import {Thread} from "@chainsafe/threads"; import {Logger} from "@lodestar/logger"; import {sleep} from "@lodestar/utils"; diff --git a/packages/beacon-node/test/e2e/api/impl/beacon/block/endpoint.test.ts b/packages/beacon-node/test/e2e/api/impl/beacon/block/endpoint.test.ts index de4e0a693c0b..b80dd73b9b63 100644 --- a/packages/beacon-node/test/e2e/api/impl/beacon/block/endpoint.test.ts +++ b/packages/beacon-node/test/e2e/api/impl/beacon/block/endpoint.test.ts @@ -1,6 +1,8 @@ import {afterAll, beforeAll, describe, expect, it, vi} from "vitest"; import {ApiClient, WireFormat, getClient} from "@lodestar/api"; import {createBeaconConfig} from "@lodestar/config"; +import {getConfig} from "@lodestar/config/test-utils"; +import {LogLevel, testLogger} from "@lodestar/logger/test-utils"; import {ForkName} from "@lodestar/params"; import { SignedBeaconBlock, @@ -10,8 +12,6 @@ import { isExecutionPayloadHeader, } from "@lodestar/types"; import {BeaconNode} from "../../../../../../src/node/nodejs.js"; -import {getConfig} from "../../../../../utils/config.js"; -import {LogLevel, testLogger} from "../../../../../utils/logger.js"; import {getDevBeaconNode} from "../../../../../utils/node/beacon.js"; describe("beacon block api", () => { diff --git a/packages/beacon-node/test/e2e/api/impl/beacon/node/endpoints.test.ts b/packages/beacon-node/test/e2e/api/impl/beacon/node/endpoints.test.ts index 52508af1ea66..183595dcfee6 100644 --- a/packages/beacon-node/test/e2e/api/impl/beacon/node/endpoints.test.ts +++ b/packages/beacon-node/test/e2e/api/impl/beacon/node/endpoints.test.ts @@ -3,9 +3,9 @@ import {routes} from "@lodestar/api"; import {ApiClient, getClient} from "@lodestar/api/beacon"; import {createBeaconConfig} from "@lodestar/config"; import {chainConfig as chainConfigDef} from "@lodestar/config/default"; +import {LogLevel, testLogger} from "@lodestar/logger/test-utils"; import {sleep} from "@lodestar/utils"; import {BeaconNode} from "../../../../../../src/node/nodejs.js"; -import {LogLevel, testLogger} from "../../../../../utils/logger.js"; import {getDevBeaconNode} from "../../../../../utils/node/beacon.js"; import {getAndInitDevValidators} from "../../../../../utils/node/validator.js"; diff --git a/packages/beacon-node/test/e2e/api/impl/beacon/state/endpoint.test.ts b/packages/beacon-node/test/e2e/api/impl/beacon/state/endpoint.test.ts index 25d4dee79ee0..1338780cf59d 100644 --- a/packages/beacon-node/test/e2e/api/impl/beacon/state/endpoint.test.ts +++ b/packages/beacon-node/test/e2e/api/impl/beacon/state/endpoint.test.ts @@ -2,10 +2,10 @@ import {afterAll, beforeAll, describe, expect, it} from "vitest"; import {ApiClient, getClient} from "@lodestar/api"; import {createBeaconConfig} from "@lodestar/config"; import {chainConfig as chainConfigDef} from "@lodestar/config/default"; +import {LogLevel, testLogger} from "@lodestar/logger/test-utils"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {computeCommitteeCount} from "@lodestar/state-transition"; import {BeaconNode} from "../../../../../../src/node/nodejs.js"; -import {LogLevel, testLogger} from "../../../../../utils/logger.js"; import {getDevBeaconNode} from "../../../../../utils/node/beacon.js"; describe("beacon state api", () => { diff --git a/packages/beacon-node/test/e2e/api/impl/config.test.ts b/packages/beacon-node/test/e2e/api/impl/config.test.ts index 2bb042f1da32..c9f208a41266 100644 --- a/packages/beacon-node/test/e2e/api/impl/config.test.ts +++ b/packages/beacon-node/test/e2e/api/impl/config.test.ts @@ -3,7 +3,7 @@ import {chainConfig} from "@lodestar/config/default"; import {ForkName, activePreset} from "@lodestar/params"; import {fetch} from "@lodestar/utils"; import {specConstants} from "../../../../src/api/impl/config/constants.js"; -import {ethereumConsensusSpecsTests} from "../../../spec/specTestVersioning.js"; +import specTestVersion from "../../../spec-tests-version.json" with {type: "json"}; const CONSTANT_NAMES_SKIP_LIST = new Set([ // This constant is an array, so it's skipped due to not being just a string. @@ -14,7 +14,7 @@ const CONSTANT_NAMES_SKIP_LIST = new Set([ describe("api / impl / config", () => { it("Ensure all constants are exposed", async () => { - const constantNames = await downloadRemoteConstants(ethereumConsensusSpecsTests.specVersion); + const constantNames = await downloadRemoteConstants(specTestVersion.ethereumConsensusSpecsTests.specVersion); const constantsInCode = new Set([ // Constants for API only diff --git a/packages/beacon-node/test/e2e/api/impl/lightclient/endpoint.test.ts b/packages/beacon-node/test/e2e/api/impl/lightclient/endpoint.test.ts index 50b8e219b824..b968ed34fdc5 100644 --- a/packages/beacon-node/test/e2e/api/impl/lightclient/endpoint.test.ts +++ b/packages/beacon-node/test/e2e/api/impl/lightclient/endpoint.test.ts @@ -1,6 +1,7 @@ import {afterEach, beforeEach, describe, expect, it} from "vitest"; import {HttpHeader, getClient, routes} from "@lodestar/api"; import {ChainConfig, createBeaconConfig} from "@lodestar/config"; +import {LogLevel, TestLoggerOpts, testLogger} from "@lodestar/logger/test-utils"; import {ForkName} from "@lodestar/params"; import {CachedBeaconStateAltair} from "@lodestar/state-transition"; import {phase0} from "@lodestar/types"; @@ -8,7 +9,6 @@ import {sleep} from "@lodestar/utils"; import {Validator} from "@lodestar/validator"; import {BeaconNode} from "../../../../../src/node/nodejs.js"; import {waitForEvent} from "../../../../utils/events/resolver.js"; -import {LogLevel, TestLoggerOpts, testLogger} from "../../../../utils/logger.js"; import {getDevBeaconNode} from "../../../../utils/node/beacon.js"; import {getAndInitDevValidators} from "../../../../utils/node/validator.js"; diff --git a/packages/beacon-node/test/e2e/api/lodestar/lodestar.test.ts b/packages/beacon-node/test/e2e/api/lodestar/lodestar.test.ts index 53389b38f917..774ba8e1c569 100644 --- a/packages/beacon-node/test/e2e/api/lodestar/lodestar.test.ts +++ b/packages/beacon-node/test/e2e/api/lodestar/lodestar.test.ts @@ -2,12 +2,12 @@ import {afterEach, describe, expect, it, vi} from "vitest"; import {getClient} from "@lodestar/api"; import {ChainConfig, createBeaconConfig} from "@lodestar/config"; import {chainConfig as chainConfigDef} from "@lodestar/config/default"; +import {LogLevel, TestLoggerOpts, testLogger} from "@lodestar/logger/test-utils"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {phase0} from "@lodestar/types"; import {BeaconNode} from "../../../../src/index.js"; import {ClockEvent} from "../../../../src/util/clock.js"; import {waitForEvent} from "../../../utils/events/resolver.js"; -import {LogLevel, TestLoggerOpts, testLogger} from "../../../utils/logger.js"; import {getDevBeaconNode} from "../../../utils/node/beacon.js"; describe("api / impl / validator", () => { diff --git a/packages/beacon-node/test/e2e/chain/bls/multithread.test.ts b/packages/beacon-node/test/e2e/chain/bls/multithread.test.ts index 12e5130c0208..aa3fb09998ed 100644 --- a/packages/beacon-node/test/e2e/chain/bls/multithread.test.ts +++ b/packages/beacon-node/test/e2e/chain/bls/multithread.test.ts @@ -1,9 +1,9 @@ import {afterEach, beforeAll, beforeEach, describe, expect, it} from "vitest"; import {PublicKey, SecretKey} from "@chainsafe/blst"; -import {ISignatureSet, SignatureSetType} from "@lodestar/state-transition"; +import {testLogger} from "@lodestar/logger/test-utils"; +import {ISignatureSet, SignatureSetType, createPubkeyCache} from "@lodestar/state-transition"; import {VerifySignatureOpts} from "../../../../src/chain/bls/interface.js"; import {BlsMultiThreadWorkerPool} from "../../../../src/chain/bls/multithread/index.js"; -import {testLogger} from "../../../utils/logger.js"; describe("chain / bls / multithread queue", () => { const logger = testLogger(); @@ -13,7 +13,7 @@ describe("chain / bls / multithread queue", () => { const sets: ISignatureSet[] = []; const sameMessageSets: {publicKey: PublicKey; signature: Uint8Array}[] = []; const sameMessage = Buffer.alloc(32, 100); - const index2pubkey: PublicKey[] = []; + const pubkeyCache = createPubkeyCache(); beforeAll(() => { for (let i = 0; i < 3; i++) { @@ -31,7 +31,7 @@ describe("chain / bls / multithread queue", () => { publicKey: pk, signature: sk.sign(sameMessage).toBytes(), }); - index2pubkey.push(pk); + pubkeyCache.set(pubkeyCache.size, pk.toBytes()); } }); @@ -49,7 +49,7 @@ describe("chain / bls / multithread queue", () => { }); async function initializePool(): Promise { - const pool = new BlsMultiThreadWorkerPool({}, {logger, metrics: null, index2pubkey}); + const pool = new BlsMultiThreadWorkerPool({}, {logger, metrics: null, pubkeyCache}); // await terminating all workers afterEachCallbacks.push(() => pool.close()); // Wait until initialized diff --git a/packages/beacon-node/test/e2e/chain/lightclient.test.ts b/packages/beacon-node/test/e2e/chain/lightclient.test.ts index 8f9c5e7f0612..9f20a2a904a6 100644 --- a/packages/beacon-node/test/e2e/chain/lightclient.test.ts +++ b/packages/beacon-node/test/e2e/chain/lightclient.test.ts @@ -6,11 +6,11 @@ import {BeaconConfig, ChainConfig} from "@lodestar/config"; import {Lightclient} from "@lodestar/light-client"; import {LightClientRestTransport} from "@lodestar/light-client/transport"; import {TimestampFormatCode} from "@lodestar/logger"; +import {LogLevel, TestLoggerOpts, testLogger} from "@lodestar/logger/test-utils"; import {EPOCHS_PER_SYNC_COMMITTEE_PERIOD, SLOTS_PER_EPOCH} from "@lodestar/params"; import {computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {LightClientHeader} from "@lodestar/types"; import {HeadEventData} from "../../../src/chain/index.js"; -import {LogLevel, TestLoggerOpts, testLogger} from "../../utils/logger.js"; import {getDevBeaconNode} from "../../utils/node/beacon.js"; import {getAndInitDevValidators} from "../../utils/node/validator.js"; diff --git a/packages/beacon-node/test/e2e/chain/proposerBoostReorg.test.ts b/packages/beacon-node/test/e2e/chain/proposerBoostReorg.test.ts index 51e7a0ac5198..460fcc13314f 100644 --- a/packages/beacon-node/test/e2e/chain/proposerBoostReorg.test.ts +++ b/packages/beacon-node/test/e2e/chain/proposerBoostReorg.test.ts @@ -2,13 +2,13 @@ import {afterEach, describe, expect, it, vi} from "vitest"; import {routes} from "@lodestar/api"; import {ChainConfig} from "@lodestar/config"; import {TimestampFormatCode} from "@lodestar/logger"; +import {LogLevel, TestLoggerOpts, testLogger} from "@lodestar/logger/test-utils"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {RootHex, Slot} from "@lodestar/types"; import {toHexString} from "@lodestar/utils"; import {ReorgEventData} from "../../../src/chain/emitter.js"; import {TimelinessForkChoice} from "../../mocks/fork-choice/timeliness.js"; import {waitForEvent} from "../../utils/events/resolver.js"; -import {LogLevel, TestLoggerOpts, testLogger} from "../../utils/logger.js"; import {getDevBeaconNode} from "../../utils/node/beacon.js"; import {getAndInitDevValidators} from "../../utils/node/validator.js"; diff --git a/packages/beacon-node/test/e2e/chain/stateCache/nHistoricalStates.test.ts b/packages/beacon-node/test/e2e/chain/stateCache/nHistoricalStates.test.ts index 3b23a35a81fc..40182ecb5d9a 100644 --- a/packages/beacon-node/test/e2e/chain/stateCache/nHistoricalStates.test.ts +++ b/packages/beacon-node/test/e2e/chain/stateCache/nHistoricalStates.test.ts @@ -3,13 +3,13 @@ import {afterEach, describe, expect, it, vi} from "vitest"; import {routes} from "@lodestar/api"; import {ChainConfig} from "@lodestar/config"; import {TimestampFormatCode} from "@lodestar/logger"; +import {LogLevel, TestLoggerOpts, testLogger} from "@lodestar/logger/test-utils"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {Slot, phase0} from "@lodestar/types"; import {ChainEvent, ReorgEventData} from "../../../../src/chain/emitter.js"; import {CacheItemType} from "../../../../src/chain/stateCache/types.js"; import {ReorgedForkChoice} from "../../../mocks/fork-choice/reorg.js"; import {waitForEvent} from "../../../utils/events/resolver.js"; -import {LogLevel, TestLoggerOpts, testLogger} from "../../../utils/logger.js"; import {connect, onPeerConnect} from "../../../utils/network.js"; import {getDevBeaconNode} from "../../../utils/node/beacon.js"; import {getAndInitDevValidators} from "../../../utils/node/validator.js"; diff --git a/packages/beacon-node/test/e2e/doppelganger/doppelganger.test.ts b/packages/beacon-node/test/e2e/doppelganger/doppelganger.test.ts index 063d9a44b326..2ddc38507d91 100644 --- a/packages/beacon-node/test/e2e/doppelganger/doppelganger.test.ts +++ b/packages/beacon-node/test/e2e/doppelganger/doppelganger.test.ts @@ -2,13 +2,13 @@ import {afterEach, describe, expect, it} from "vitest"; import {fromHexString} from "@chainsafe/ssz"; import {routes} from "@lodestar/api/beacon"; import {ChainConfig} from "@lodestar/config"; +import {LogLevel, TestLoggerOpts, testLogger} from "@lodestar/logger/test-utils"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {BLSPubkey, Epoch, Slot, phase0, ssz} from "@lodestar/types"; import {Validator} from "@lodestar/validator"; import {BeaconNode} from "../../../src/node/index.js"; import {ClockEvent} from "../../../src/util/clock.js"; import {waitForEvent} from "../../utils/events/resolver.js"; -import {LogLevel, TestLoggerOpts, testLogger} from "../../utils/logger.js"; import {connect} from "../../utils/network.js"; import {getDevBeaconNode} from "../../utils/node/beacon.js"; import {getAndInitDevValidators} from "../../utils/node/validator.js"; diff --git a/packages/beacon-node/test/e2e/network/mdns.test.ts b/packages/beacon-node/test/e2e/network/mdns.test.ts index d5403369b03c..66023433d16f 100644 --- a/packages/beacon-node/test/e2e/network/mdns.test.ts +++ b/packages/beacon-node/test/e2e/network/mdns.test.ts @@ -5,6 +5,7 @@ import {afterEach, beforeEach, describe, expect, it, vi} from "vitest"; import {SignableENR} from "@chainsafe/enr"; import {createBeaconConfig} from "@lodestar/config"; import {config} from "@lodestar/config/default"; +import {testLogger} from "@lodestar/logger/test-utils"; import {computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {ssz} from "@lodestar/types"; import {GossipHandlers} from "../../../src/network/gossip/index.js"; @@ -13,7 +14,6 @@ import {NetworkOptions, defaultNetworkOptions} from "../../../src/network/option import {getMockedBeaconChain} from "../../mocks/mockedBeaconChain.js"; import {getMockedBeaconDb} from "../../mocks/mockedBeaconDb.js"; import {memoOnce} from "../../utils/cache.js"; -import {testLogger} from "../../utils/logger.js"; import {createNetworkModules, onPeerConnect} from "../../utils/network.js"; import {generateState, zeroProtoBlock} from "../../utils/state.js"; diff --git a/packages/beacon-node/test/e2e/network/onWorker/dataSerialization.test.ts b/packages/beacon-node/test/e2e/network/onWorker/dataSerialization.test.ts index cbef30901e57..3d96df834caf 100644 --- a/packages/beacon-node/test/e2e/network/onWorker/dataSerialization.test.ts +++ b/packages/beacon-node/test/e2e/network/onWorker/dataSerialization.test.ts @@ -1,4 +1,4 @@ -import {TopicValidatorResult} from "@libp2p/interface"; +import {TopicValidatorResult} from "@libp2p/gossipsub"; import {afterAll, beforeAll, describe, expect, it} from "vitest"; import {BitArray} from "@chainsafe/ssz"; import {routes} from "@lodestar/api"; diff --git a/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts b/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts index ae1f0e86946a..64f2e87f47fc 100644 --- a/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts +++ b/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts @@ -4,10 +4,12 @@ import {afterEach, describe, expect, it, vi} from "vitest"; import {BitArray} from "@chainsafe/ssz"; import {createBeaconConfig} from "@lodestar/config"; import {config} from "@lodestar/config/default"; +import {testLogger} from "@lodestar/logger/test-utils"; import {phase0, ssz} from "@lodestar/types"; import {sleep} from "@lodestar/utils"; import {Eth2Gossipsub, NetworkEvent, NetworkEventBus, getConnectionsMap} from "../../../../src/network/index.js"; import {NetworkConfig} from "../../../../src/network/networkConfig.js"; +import {ClientKind} from "../../../../src/network/peers/client.js"; import {IReqRespBeaconNodePeerManager, PeerManager, PeerRpcScoreStore} from "../../../../src/network/peers/index.js"; import {PeersData} from "../../../../src/network/peers/peersData.js"; import {ReqRespMethod} from "../../../../src/network/reqresp/ReqRespBeaconNode.js"; @@ -16,7 +18,6 @@ import {IAttnetsService, computeNodeId} from "../../../../src/network/subnets/in import {Clock} from "../../../../src/util/clock.js"; import {CustodyConfig, getCustodyGroups} from "../../../../src/util/dataColumns.js"; import {waitForEvent} from "../../../utils/events/resolver.js"; -import {testLogger} from "../../../utils/logger.js"; import {createNode, getAttnets, getSyncnets} from "../../../utils/network.js"; import {getValidPeerId} from "../../../utils/peer.js"; import {generateState} from "../../../utils/state.js"; @@ -179,7 +180,9 @@ describe("network / peers / PeerManager", () => { direction: "outbound", status: "open", remotePeer: peerId1, - } as Connection; + close: async () => {}, + abort: () => {}, + } as unknown as Connection; it("Should emit peer connected event on relevant peer status", async () => { const {statusCache, libp2p, networkEventBus} = await mockModules(); @@ -259,4 +262,84 @@ describe("network / peers / PeerManager", () => { expect(peerManager["connectedPeers"].get(peerId1.toString())?.metadata).toEqual(remoteMetadata); }); + + it("Should identify peer after successful status", async () => { + const {libp2p, peerManager, statusCache, networkEventBus} = await mockModules(); + + vi.spyOn(libp2p.services.identify, "identify").mockImplementation( + () => Promise.resolve({agentVersion: "Lighthouse/v6.0.1"}) as ReturnType + ); + + const inboundConnection = { + id: "connection-1", + direction: "inbound", + status: "open", + remotePeer: peerId1, + close: async () => {}, + abort: () => {}, + } as unknown as Connection; + + // Connection open does NOT trigger identify + getConnectionsMap(libp2p).set(peerId1.toString(), {key: peerId1, value: [inboundConnection]}); + await peerManager["onLibp2pPeerConnect"](new CustomEvent("evt", {detail: inboundConnection})); + await sleep(0); + expect(libp2p.services.identify.identify).not.toHaveBeenCalled(); + + // Status proves the connection is usable — triggers identify + const remoteStatus = statusCache.get(); + networkEventBus.emit(NetworkEvent.reqRespRequest, { + request: {method: ReqRespMethod.Status, body: remoteStatus}, + peer: peerId1, + peerClient: "Unknown", + }); + await sleep(0); + + expect(libp2p.services.identify.identify).toHaveBeenCalledTimes(1); + const peerData = peerManager["connectedPeers"].get(peerId1.toString()); + expect(peerData?.agentVersion).toBe("Lighthouse/v6.0.1"); + expect(peerData?.agentClient).toBe(ClientKind.Lighthouse); + }); + + it("Should not re-identify after second status if agentVersion is already known", async () => { + const {libp2p, peerManager, statusCache, networkEventBus} = await mockModules(); + + vi.spyOn(libp2p.services.identify, "identify").mockImplementation( + () => Promise.resolve({agentVersion: "Nimbus/v25.0.0"}) as ReturnType + ); + + const inboundConnection = { + id: "connection-1", + direction: "inbound", + status: "open", + remotePeer: peerId1, + close: async () => {}, + abort: () => {}, + } as unknown as Connection; + + getConnectionsMap(libp2p).set(peerId1.toString(), {key: peerId1, value: [inboundConnection]}); + await peerManager["onLibp2pPeerConnect"](new CustomEvent("evt", {detail: inboundConnection})); + + // First status triggers identify + const remoteStatus = statusCache.get(); + networkEventBus.emit(NetworkEvent.reqRespRequest, { + request: {method: ReqRespMethod.Status, body: remoteStatus}, + peer: peerId1, + peerClient: "Unknown", + }); + await sleep(0); + expect(libp2p.services.identify.identify).toHaveBeenCalledTimes(1); + + // Second status should NOT trigger identify again since agentVersion is already known + networkEventBus.emit(NetworkEvent.reqRespRequest, { + request: {method: ReqRespMethod.Status, body: remoteStatus}, + peer: peerId1, + peerClient: "Unknown", + }); + await sleep(0); + expect(libp2p.services.identify.identify).toHaveBeenCalledTimes(1); + + const peerData = peerManager["connectedPeers"].get(peerId1.toString()); + expect(peerData?.agentVersion).toBe("Nimbus/v25.0.0"); + expect(peerData?.agentClient).toBe(ClientKind.Nimbus); + }); }); diff --git a/packages/beacon-node/test/e2e/network/reqresp.test.ts b/packages/beacon-node/test/e2e/network/reqresp.test.ts index 154d5f282296..0d68bc1d3760 100644 --- a/packages/beacon-node/test/e2e/network/reqresp.test.ts +++ b/packages/beacon-node/test/e2e/network/reqresp.test.ts @@ -9,7 +9,6 @@ import {sleep} from "@lodestar/utils"; import {Network, ReqRespBeaconNodeOpts} from "../../../src/network/index.js"; import {GetReqRespHandlerFn, ReqRespMethod} from "../../../src/network/reqresp/types.js"; import {PeerIdStr} from "../../../src/util/peerId.js"; -import {arrToSource} from "../../unit/network/reqresp/utils.js"; import {expectRejectedWithLodestarError} from "../../utils/errors.js"; import {connect, getPeerIdOf, onPeerConnect} from "../../utils/network.js"; import {getNetworkForTest} from "../../utils/networkWithMockDb.js"; @@ -186,7 +185,7 @@ function runTests({useWorker}: {useWorker: boolean}): void { (method) => async function* onRequest() { if (method === ReqRespMethod.LightClientUpdatesByRange) { - yield* arrToSource(lightClientUpdates); + yield* lightClientUpdates; } } ); @@ -242,29 +241,28 @@ function runTests({useWorker}: {useWorker: boolean}): void { ); }); - it("should trigger TTFB_TIMEOUT error if first response is delayed", async () => { - const ttfbTimeoutMs = 250; + it("should trigger a RESP_TIMEOUT error if first response is delayed", async () => { + const respTimeoutMs = 250; const [netA, _, _0, peerIdB] = await createAndConnectPeers( (method) => async function* onRequest() { if (method === ReqRespMethod.BeaconBlocksByRange) { // Wait for too long before sending first response chunk - await sleep(ttfbTimeoutMs * 10, controller.signal); + await sleep(respTimeoutMs * 10, controller.signal); yield wrapBlockAsEncodedPayload(config, config.getForkTypes(0).SignedBeaconBlock.defaultValue()); } }, - {ttfbTimeoutMs} + {respTimeoutMs} ); await expectRejectedWithLodestarError( netA.sendBeaconBlocksByRange(peerIdB, {startSlot: 0, step: 1, count: 1}), - new RequestError({code: RequestErrorCode.TTFB_TIMEOUT}) + new RequestError({code: RequestErrorCode.RESP_TIMEOUT}) ); }); - it("should trigger a RESP_TIMEOUT error if first byte is on time but later delayed", async () => { - const ttfbTimeoutMs = 250; + it("should trigger a RESP_TIMEOUT error if later response is delayed", async () => { const respTimeoutMs = 300; const [netA, _, _0, peerIdB] = await createAndConnectPeers( @@ -277,46 +275,7 @@ function runTests({useWorker}: {useWorker: boolean}): void { yield getEmptyEncodedPayloadSignedBeaconBlock(config); } }, - {ttfbTimeoutMs, respTimeoutMs} - ); - - await expectRejectedWithLodestarError( - netA.sendBeaconBlocksByRange(peerIdB, {startSlot: 0, step: 1, count: 2}), - new RequestError({code: RequestErrorCode.RESP_TIMEOUT}) - ); - }); - - it("should trigger TTFB_TIMEOUT error if respTimeoutMs and ttfbTimeoutMs is the same", async () => { - const ttfbTimeoutMs = 250; - const respTimeoutMs = 250; - - const [netA, _, _0, peerIdB] = await createAndConnectPeers( - (method) => - // biome-ignore lint/correctness/useYield: No need for yield in test context - async function* onRequest() { - if (method === ReqRespMethod.BeaconBlocksByRange) { - await sleep(100000000, controller.signal); - } - }, - {respTimeoutMs, ttfbTimeoutMs} - ); - - await expectRejectedWithLodestarError( - netA.sendBeaconBlocksByRange(peerIdB, {startSlot: 0, step: 1, count: 2}), - new RequestError({code: RequestErrorCode.TTFB_TIMEOUT}) - ); - }); - - it("should trigger a RESP_TIMEOUT error if first byte is on time but sleep infinite", async () => { - const [netA, _, _0, peerIdB] = await createAndConnectPeers( - (method) => - async function* onRequest() { - if (method === ReqRespMethod.BeaconBlocksByRange) { - yield getEmptyEncodedPayloadSignedBeaconBlock(config); - await sleep(100000000, controller.signal); - } - }, - {respTimeoutMs: 250, ttfbTimeoutMs: 250} + {respTimeoutMs} ); await expectRejectedWithLodestarError( diff --git a/packages/beacon-node/test/e2e/network/reqrespEncode.test.ts b/packages/beacon-node/test/e2e/network/reqrespEncode.test.ts index 64b128eee389..23d9c04bbcf7 100644 --- a/packages/beacon-node/test/e2e/network/reqrespEncode.test.ts +++ b/packages/beacon-node/test/e2e/network/reqrespEncode.test.ts @@ -1,15 +1,17 @@ import {generateKeyPair} from "@libp2p/crypto/keys"; -import {PrivateKey} from "@libp2p/interface"; +import type {PrivateKey} from "@libp2p/interface"; import {mplex} from "@libp2p/mplex"; import {peerIdFromPrivateKey} from "@libp2p/peer-id"; import {tcp} from "@libp2p/tcp"; -import {Multiaddr, multiaddr} from "@multiformats/multiaddr"; -import all from "it-all"; -import {Libp2p, createLibp2p} from "libp2p"; +import {byteStream} from "@libp2p/utils"; +import type {Multiaddr} from "@multiformats/multiaddr"; +import type {Libp2p} from "libp2p"; +import {createLibp2p} from "libp2p"; import {afterEach, describe, expect, it} from "vitest"; import {noise} from "@chainsafe/libp2p-noise"; import {createBeaconConfig} from "@lodestar/config"; import {config} from "@lodestar/config/default"; +import {testLogger} from "@lodestar/logger/test-utils"; import {ForkName, GENESIS_EPOCH} from "@lodestar/params"; import {ssz} from "@lodestar/types"; import {fromHex, sleep, toHex} from "@lodestar/utils"; @@ -27,11 +29,8 @@ import {GetReqRespHandlerFn} from "../../../src/network/reqresp/types.js"; import {LocalStatusCache} from "../../../src/network/statusCache.js"; import {computeNodeId} from "../../../src/network/subnets/index.js"; import {CustodyConfig} from "../../../src/util/dataColumns.js"; -import {testLogger} from "../../utils/logger.js"; describe("reqresp encoder", () => { - let port = 60000; - const afterEachCallbacks: (() => Promise | void)[] = []; afterEach(async () => { while (afterEachCallbacks.length > 0) { @@ -41,18 +40,20 @@ describe("reqresp encoder", () => { }); async function getLibp2p(privateKey?: PrivateKey) { - const listen = `/ip4/127.0.0.1/tcp/${port++}`; const libp2p = await createLibp2p({ privateKey, transports: [tcp()], - streamMuxers: [mplex()], + // Increase disconnectThreshold to prevent mplex from closing the connection + // when it receives messages for already-closed streams + streamMuxers: [mplex({disconnectThreshold: Infinity})], connectionEncrypters: [noise()], addresses: { - listen: [listen], + listen: ["/ip4/127.0.0.1/tcp/0"], }, }); afterEachCallbacks.push(() => libp2p.stop()); - return {libp2p, multiaddr: multiaddr(`${listen}/p2p/${libp2p.peerId.toString()}`)}; + const listenMultiaddr = libp2p.getMultiaddrs()[0]; + return {libp2p, multiaddr: listenMultiaddr}; } async function getReqResp(getHandler?: GetReqRespHandlerFn) { @@ -107,17 +108,33 @@ describe("reqresp encoder", () => { expectedChunks: string[]; }) { const stream = await dialer.dialProtocol(toMultiaddr, protocol); + // Use byteStream to read response - it attaches event listeners immediately, + // avoiding race conditions with the async iterator where remoteCloseWrite + // events can be lost if the server responds in the same macrotask + const bytes = byteStream(stream); + if (requestChunks) { - await stream.sink(requestChunks.map(fromHex)); + for (const chunk of requestChunks) { + await bytes.write(fromHex(chunk)); + } + } + + const chunks: Uint8Array[] = []; + while (true) { + const chunk = await bytes.read({signal: AbortSignal.timeout(2000)}); + if (chunk === null) break; + chunks.push(chunk.subarray()); } - const chunks = await all(stream.source); + // Abort for fast cleanup instead of graceful close which can be slow + stream.abort(new Error("test done")); + const join = (c: string[]): string => c.join("").replace(/0x/g, ""); - const chunksHex = chunks.map((chunk) => toHex(chunk.slice(0, chunk.byteLength))); + const chunksHex = chunks.map((chunk) => toHex(chunk)); expect(join(chunksHex)).toEqual(join(expectedChunks)); } - it("assert correct handler switch between metadata v2 and v1", async () => { + it("assert correct handler for metadata v3", async () => { const {multiaddr: serverMultiaddr, reqresp} = await getReqResp(); reqresp.registerProtocolsAtBoundary({fork: ForkName.phase0, epoch: GENESIS_EPOCH}); await sleep(0); // Sleep to resolve register handler promises @@ -126,20 +143,34 @@ describe("reqresp encoder", () => { reqresp["metadataController"].attnets.set(8, true); reqresp["metadataController"].syncnets.set(1, true); - const privateKey = await generateKeyPair("secp256k1"); - const {libp2p: dialer} = await getLibp2p(privateKey); + const {libp2p: dialer} = await getLibp2p(); await dialProtocol({ dialer, toMultiaddr: serverMultiaddr, - protocol: "/eth2/beacon_chain/req/metadata/1/ssz_snappy", - expectedChunks: ["0x00", "0x10", "0xff060000734e615070590114000077b18d3800000000000000000101000000000000"], + protocol: "/eth2/beacon_chain/req/metadata/3/ssz_snappy", + expectedChunks: [ + "0x00", + "0x19", + "0xff060000734e61507059001b000082e4dd0e1900000d01400101000000000000020400000000000000", + ], }); + }); + + it("assert correct handler for metadata v1", async () => { + const {multiaddr: serverMultiaddr, reqresp} = await getReqResp(); + reqresp.registerProtocolsAtBoundary({fork: ForkName.phase0, epoch: GENESIS_EPOCH}); + await sleep(0); // Sleep to resolve register handler promises + + reqresp["metadataController"].attnets.set(0, true); + reqresp["metadataController"].attnets.set(8, true); + reqresp["metadataController"].syncnets.set(1, true); + const {libp2p: dialer} = await getLibp2p(); await dialProtocol({ dialer, toMultiaddr: serverMultiaddr, - protocol: "/eth2/beacon_chain/req/metadata/2/ssz_snappy", - expectedChunks: ["0x00", "0x11", "0xff060000734e615070590013000080f865931100000d0120010100000000000002"], + protocol: "/eth2/beacon_chain/req/metadata/1/ssz_snappy", + expectedChunks: ["0x00", "0x10", "0xff060000734e615070590114000077b18d3800000000000000000101000000000000"], }); }); diff --git a/packages/beacon-node/test/e2e/sync/finalizedSync.test.ts b/packages/beacon-node/test/e2e/sync/finalizedSync.test.ts index 99431b7cb893..b08b4621fce4 100644 --- a/packages/beacon-node/test/e2e/sync/finalizedSync.test.ts +++ b/packages/beacon-node/test/e2e/sync/finalizedSync.test.ts @@ -3,11 +3,11 @@ import {fromHexString} from "@chainsafe/ssz"; import {routes} from "@lodestar/api"; import {ChainConfig} from "@lodestar/config"; import {TimestampFormatCode} from "@lodestar/logger"; +import {LogLevel, TestLoggerOpts, testLogger} from "@lodestar/logger/test-utils"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {phase0} from "@lodestar/types"; import {ChainEvent} from "../../../src/chain/index.js"; import {waitForEvent} from "../../utils/events/resolver.js"; -import {LogLevel, TestLoggerOpts, testLogger} from "../../utils/logger.js"; import {connect, onPeerConnect} from "../../utils/network.js"; import {getDevBeaconNode} from "../../utils/node/beacon.js"; import {getAndInitDevValidators} from "../../utils/node/validator.js"; diff --git a/packages/beacon-node/test/e2e/sync/unknownBlockSync.test.ts b/packages/beacon-node/test/e2e/sync/unknownBlockSync.test.ts index 7403cb5d7c11..b69c34db3ca9 100644 --- a/packages/beacon-node/test/e2e/sync/unknownBlockSync.test.ts +++ b/packages/beacon-node/test/e2e/sync/unknownBlockSync.test.ts @@ -3,6 +3,7 @@ import {fromHexString} from "@chainsafe/ssz"; import {routes} from "@lodestar/api"; import {ChainConfig} from "@lodestar/config"; import {TimestampFormatCode} from "@lodestar/logger"; +import {LogLevel, TestLoggerOpts, testLogger} from "@lodestar/logger/test-utils"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {fulu} from "@lodestar/types"; import {retry} from "@lodestar/utils"; @@ -12,7 +13,6 @@ import {ChainEvent} from "../../../src/chain/emitter.js"; import {BlockError, BlockErrorCode} from "../../../src/chain/errors/index.js"; import {INTEROP_BLOCK_HASH} from "../../../src/node/utils/interop/state.js"; import {waitForEvent} from "../../utils/events/resolver.js"; -import {LogLevel, TestLoggerOpts, testLogger} from "../../utils/logger.js"; import {connect, onPeerConnect} from "../../utils/network.js"; import {getDevBeaconNode} from "../../utils/node/beacon.js"; import {getAndInitDevValidators} from "../../utils/node/validator.js"; diff --git a/packages/beacon-node/test/fixtures/capella.ts b/packages/beacon-node/test/fixtures/capella.ts index 0fed040a9f39..484d9f9df4d0 100644 --- a/packages/beacon-node/test/fixtures/capella.ts +++ b/packages/beacon-node/test/fixtures/capella.ts @@ -1,17 +1,19 @@ -import {CachedBeaconStateAltair, Index2PubkeyCache} from "@lodestar/state-transition"; +import {CachedBeaconStateAltair, PubkeyCache} from "@lodestar/state-transition"; import {capella} from "@lodestar/types"; export function generateBlsToExecutionChanges( - index2pubkey: Index2PubkeyCache, + pubkeyCache: PubkeyCache, state: CachedBeaconStateAltair, count: number ): capella.SignedBLSToExecutionChange[] { const result: capella.SignedBLSToExecutionChange[] = []; for (const validatorIndex of state.epochCtx.proposers) { + const pubkey = pubkeyCache.getOrThrow(validatorIndex); + result.push({ message: { - fromBlsPubkey: index2pubkey[validatorIndex].toBytes(), + fromBlsPubkey: pubkey.toBytes(), toExecutionAddress: Buffer.alloc(20), validatorIndex, }, diff --git a/packages/beacon-node/test/mocks/mockedBeaconChain.ts b/packages/beacon-node/test/mocks/mockedBeaconChain.ts index 39a98d95b6df..12d26de6f90b 100644 --- a/packages/beacon-node/test/mocks/mockedBeaconChain.ts +++ b/packages/beacon-node/test/mocks/mockedBeaconChain.ts @@ -1,8 +1,8 @@ import {Mock, Mocked, vi} from "vitest"; -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {BeaconConfig, ChainForkConfig} from "@lodestar/config"; import {config as defaultConfig} from "@lodestar/config/default"; import {EpochDifference, ForkChoice, ProtoBlock} from "@lodestar/fork-choice"; +import {createPubkeyCache} from "@lodestar/state-transition"; import {Logger} from "@lodestar/utils"; import {BeaconProposerCache} from "../../src/chain/beaconProposerCache.js"; import {BeaconChain} from "../../src/chain/chain.js"; @@ -144,8 +144,7 @@ vi.mock("../../src/chain/chain.js", async (importActual) => { // @ts-expect-error seenBlockInputCache: new SeenBlockInput(), shufflingCache: new ShufflingCache(), - pubkey2index: new PubkeyIndexMap(), - index2pubkey: [], + pubkeyCache: createPubkeyCache(), produceCommonBlockBody: vi.fn(), getProposerHead: vi.fn(), produceBlock: vi.fn(), diff --git a/packages/beacon-node/test/perf/api/impl/validator/attester.test.ts b/packages/beacon-node/test/perf/api/impl/validator/attester.test.ts index 2fbd607b309d..eeba4bd72083 100644 --- a/packages/beacon-node/test/perf/api/impl/validator/attester.test.ts +++ b/packages/beacon-node/test/perf/api/impl/validator/attester.test.ts @@ -1,5 +1,5 @@ import {beforeAll, bench, describe} from "@chainsafe/benchmark"; -import {generatePerfTestCachedStatePhase0, numValidators} from "../../../../../../state-transition/test/perf/util.js"; +import {generatePerfTestCachedStatePhase0, numValidators} from "@lodestar/state-transition/test-utils"; import {getPubkeysForIndices} from "../../../../../src/api/impl/validator/utils.js"; import {linspace} from "../../../../../src/util/numpy.js"; @@ -32,7 +32,7 @@ describe("api / impl / validator", () => { noThreshold: true, fn: () => { for (let i = 0; i < reqCount; i++) { - const pubkey = state.epochCtx.index2pubkey[i]; + const pubkey = state.epochCtx.pubkeyCache.getOrThrow(i); pubkey.toBytes(); } }, diff --git a/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts b/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts index 4817e5e61765..192d96f29d7a 100644 --- a/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts +++ b/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts @@ -12,8 +12,8 @@ import { getBlockRootAtSlot, newFilledArray, } from "@lodestar/state-transition"; +import {generatePerfTestCachedStateAltair} from "@lodestar/state-transition/test-utils"; import {ssz} from "@lodestar/types"; -import {generatePerfTestCachedStateAltair} from "../../../../../state-transition/test/perf/util.js"; import {AggregatedAttestationPool} from "../../../../src/chain/opPools/aggregatedAttestationPool.js"; import {ShufflingCache} from "../../../../src/chain/shufflingCache.js"; @@ -71,8 +71,6 @@ describe(`getAttestationsForBlock vc=${vc}`, () => { parentBlockHash: null, payloadStatus: 2, // PayloadStatus.FULL - builderIndex: null, - blockHashFromBid: null, }, originalState.slot ); @@ -101,10 +99,9 @@ describe(`getAttestationsForBlock vc=${vc}`, () => { parentBlockHash: null, payloadStatus: 2, // PayloadStatus.FULL - builderIndex: null, - blockHashFromBid: null, }, - slot + slot, + null ); } diff --git a/packages/beacon-node/test/perf/chain/opPools/opPool.test.ts b/packages/beacon-node/test/perf/chain/opPools/opPool.test.ts index b3f0bf44a3d5..814dad30e27b 100644 --- a/packages/beacon-node/test/perf/chain/opPools/opPool.test.ts +++ b/packages/beacon-node/test/perf/chain/opPools/opPool.test.ts @@ -8,9 +8,9 @@ import { MAX_PROPOSER_SLASHINGS, MAX_VOLUNTARY_EXITS, } from "@lodestar/params"; -import {CachedBeaconStateAltair, Index2PubkeyCache} from "@lodestar/state-transition"; +import {CachedBeaconStateAltair, PubkeyCache} from "@lodestar/state-transition"; +import {generatePerfTestCachedStateAltair} from "@lodestar/state-transition/test-utils"; import {ssz} from "@lodestar/types"; -import {generatePerfTestCachedStateAltair} from "../../../../../state-transition/test/perf/util.js"; import {BlockType} from "../../../../src/chain/interface.js"; import {OpPool} from "../../../../src/chain/opPools/opPool.js"; import {generateBlsToExecutionChanges} from "../../../fixtures/capella.js"; @@ -38,8 +38,8 @@ describe("opPool", () => { fillAttesterSlashing(pool, originalState, MAX_ATTESTER_SLASHINGS); fillProposerSlashing(pool, originalState, MAX_PROPOSER_SLASHINGS); fillVoluntaryExits(pool, originalState, MAX_VOLUNTARY_EXITS); - // TODO: feed index2pubkey separately instead of getting from originalState - fillBlsToExecutionChanges(originalState.epochCtx.index2pubkey, pool, originalState, MAX_BLS_TO_EXECUTION_CHANGES); + // TODO: feed pubkeyCache separately instead of getting from originalState + fillBlsToExecutionChanges(originalState.epochCtx.pubkeyCache, pool, originalState, MAX_BLS_TO_EXECUTION_CHANGES); return pool; }, @@ -57,8 +57,8 @@ describe("opPool", () => { fillAttesterSlashing(pool, originalState, maxItemsInPool); fillProposerSlashing(pool, originalState, maxItemsInPool); fillVoluntaryExits(pool, originalState, maxItemsInPool); - // TODO: feed index2pubkey separately instead of getting from originalState - fillBlsToExecutionChanges(originalState.epochCtx.index2pubkey, pool, originalState, maxItemsInPool); + // TODO: feed pubkeyCache separately instead of getting from originalState + fillBlsToExecutionChanges(originalState.epochCtx.pubkeyCache, pool, originalState, maxItemsInPool); return pool; }, @@ -105,12 +105,12 @@ function fillVoluntaryExits(pool: OpPool, state: CachedBeaconStateAltair, count: // This does not set the `withdrawalCredentials` for the validator // So it will be in the pool but not returned from `getSlashingsAndExits` function fillBlsToExecutionChanges( - index2pubkey: Index2PubkeyCache, + pubkeyCache: PubkeyCache, pool: OpPool, state: CachedBeaconStateAltair, count: number ): OpPool { - for (const blsToExecution of generateBlsToExecutionChanges(index2pubkey, state, count)) { + for (const blsToExecution of generateBlsToExecutionChanges(pubkeyCache, state, count)) { pool.insertBlsToExecutionChange(blsToExecution); } diff --git a/packages/beacon-node/test/perf/chain/produceBlock/produceBlockBody.test.ts b/packages/beacon-node/test/perf/chain/produceBlock/produceBlockBody.test.ts index 304190626a9c..4f615e6c6a49 100644 --- a/packages/beacon-node/test/perf/chain/produceBlock/produceBlockBody.test.ts +++ b/packages/beacon-node/test/perf/chain/produceBlock/produceBlockBody.test.ts @@ -2,14 +2,14 @@ import {generateKeyPair} from "@libp2p/crypto/keys"; import {afterAll, beforeAll, bench, describe} from "@chainsafe/benchmark"; import {config} from "@lodestar/config/default"; import {LevelDbController} from "@lodestar/db/controller/level"; +import {testLogger} from "@lodestar/logger/test-utils"; import {CachedBeaconStateAltair} from "@lodestar/state-transition"; +import {generatePerfTestCachedStateAltair} from "@lodestar/state-transition/test-utils"; import {defaultOptions as defaultValidatorOptions} from "@lodestar/validator"; -import {generatePerfTestCachedStateAltair} from "../../../../../state-transition/test/perf/util.js"; import {BeaconChain} from "../../../../src/chain/index.js"; import {BlockType, produceBlockBody} from "../../../../src/chain/produceBlock/produceBlockBody.js"; import {ExecutionEngineDisabled} from "../../../../src/execution/engine/index.js"; import {ArchiveMode, BeaconDb} from "../../../../src/index.js"; -import {testLogger} from "../../../utils/logger.js"; const logger = testLogger(); @@ -38,8 +38,7 @@ describe("produceBlockBody", () => { { privateKey: await generateKeyPair("secp256k1"), config: state.config, - pubkey2index: state.epochCtx.pubkey2index, - index2pubkey: state.epochCtx.index2pubkey, + pubkeyCache: state.epochCtx.pubkeyCache, db, dataDir: ".", dbName: ".", @@ -68,7 +67,7 @@ describe("produceBlockBody", () => { beforeEach: async () => { const head = chain.forkChoice.getHead(); const proposerIndex = state.epochCtx.getBeaconProposer(state.slot); - const proposerPubKey = state.epochCtx.index2pubkey[proposerIndex].toBytes(); + const proposerPubKey = state.epochCtx.pubkeyCache.getOrThrow(proposerIndex).toBytes(); return {chain, state, head, proposerIndex, proposerPubKey}; }, diff --git a/packages/beacon-node/test/perf/chain/validation/aggregateAndProof.test.ts b/packages/beacon-node/test/perf/chain/validation/aggregateAndProof.test.ts index 35dfcc589225..cdd23b07ea73 100644 --- a/packages/beacon-node/test/perf/chain/validation/aggregateAndProof.test.ts +++ b/packages/beacon-node/test/perf/chain/validation/aggregateAndProof.test.ts @@ -1,6 +1,6 @@ import {bench, describe} from "@chainsafe/benchmark"; +import {generateTestCachedBeaconStateOnlyValidators} from "@lodestar/state-transition/test-utils"; import {ssz} from "@lodestar/types"; -import {generateTestCachedBeaconStateOnlyValidators} from "../../../../../state-transition/test/perf/util.js"; import {validateApiAggregateAndProof, validateGossipAggregateAndProof} from "../../../../src/chain/validation/index.js"; import {getAggregateAndProofValidData} from "../../../utils/validationData/aggregateAndProof.js"; diff --git a/packages/beacon-node/test/perf/chain/validation/attestation.test.ts b/packages/beacon-node/test/perf/chain/validation/attestation.test.ts index 94c6c381b21c..2ddc6ba7e67f 100644 --- a/packages/beacon-node/test/perf/chain/validation/attestation.test.ts +++ b/packages/beacon-node/test/perf/chain/validation/attestation.test.ts @@ -1,7 +1,7 @@ import assert from "node:assert"; import {bench, describe} from "@chainsafe/benchmark"; +import {generateTestCachedBeaconStateOnlyValidators} from "@lodestar/state-transition/test-utils"; import {ssz} from "@lodestar/types"; -import {generateTestCachedBeaconStateOnlyValidators} from "../../../../../state-transition/test/perf/util.js"; import {validateGossipAttestationsSameAttData} from "../../../../src/chain/validation/index.js"; import {getAttDataFromAttestationSerialized} from "../../../../src/util/sszBytes.js"; import {getAttestationValidData} from "../../../utils/validationData/attestation.js"; diff --git a/packages/beacon-node/test/perf/chain/verifyImportBlocks.test.ts b/packages/beacon-node/test/perf/chain/verifyImportBlocks.test.ts index 247781efe7a3..af039398fcaf 100644 --- a/packages/beacon-node/test/perf/chain/verifyImportBlocks.test.ts +++ b/packages/beacon-node/test/perf/chain/verifyImportBlocks.test.ts @@ -2,12 +2,11 @@ import {generateKeyPair} from "@libp2p/crypto/keys"; import {afterAll, beforeAll, bench, describe, setBenchOpts} from "@chainsafe/benchmark"; import {config} from "@lodestar/config/default"; import {LevelDbController} from "@lodestar/db/controller/level"; +import {testLogger} from "@lodestar/logger/test-utils"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; +import {getNetworkCachedBlock, getNetworkCachedState, rangeSyncTest} from "@lodestar/state-transition/test-utils"; import {sleep, toHex} from "@lodestar/utils"; import {defaultOptions as defaultValidatorOptions} from "@lodestar/validator"; -import {rangeSyncTest} from "../../../../state-transition/test/perf/params.js"; -import {beforeValue} from "../../../../state-transition/test/utils/beforeValueBenchmark.js"; -import {getNetworkCachedBlock, getNetworkCachedState} from "../../../../state-transition/test/utils/testFileCache.js"; import {BlockInputPreData} from "../../../src/chain/blocks/blockInput/blockInput.js"; import {BlockInputSource} from "../../../src/chain/blocks/blockInput/types.js"; import {AttestationImportOpt} from "../../../src/chain/blocks/types.js"; @@ -15,7 +14,7 @@ import {BeaconChain} from "../../../src/chain/index.js"; import {ExecutionEngineDisabled} from "../../../src/execution/engine/index.js"; import {ArchiveMode, BeaconDb} from "../../../src/index.js"; import {linspace} from "../../../src/util/numpy.js"; -import {testLogger} from "../../utils/logger.js"; +import {beforeValue} from "../../utils/beforeValueBenchmark.js"; // Define this params in `packages/state-transition/test/perf/params.ts` // to trigger Github actions CI cache @@ -91,8 +90,7 @@ describe.skip("verify+import blocks - range sync perf test", () => { { privateKey: await generateKeyPair("secp256k1"), config: state.config, - pubkey2index: state.epochCtx.pubkey2index, - index2pubkey: state.epochCtx.index2pubkey, + pubkeyCache: state.epochCtx.pubkeyCache, db, dataDir: ".", dbName: ".", diff --git a/packages/beacon-node/test/perf/network/noise/sendData.test.ts b/packages/beacon-node/test/perf/network/noise/sendData.test.ts index 804d2416d620..62c6f8d0949d 100644 --- a/packages/beacon-node/test/perf/network/noise/sendData.test.ts +++ b/packages/beacon-node/test/perf/network/noise/sendData.test.ts @@ -1,11 +1,8 @@ import {generateKeyPair} from "@libp2p/crypto/keys"; -import {Upgrader} from "@libp2p/interface"; +import type {Upgrader} from "@libp2p/interface"; import {defaultLogger} from "@libp2p/logger"; import {peerIdFromPrivateKey} from "@libp2p/peer-id"; -import drain from "it-drain"; -import {duplexPair} from "it-pair/duplex"; -import {pipe} from "it-pipe"; -import {Uint8ArrayList} from "uint8arraylist"; +import {streamPair} from "@libp2p/utils"; import {bench, describe} from "@chainsafe/benchmark"; import {noise} from "@chainsafe/libp2p-noise"; @@ -34,24 +31,29 @@ describe("network / noise / sendData", () => { const noiseA = noise()({logger: defaultLogger(), privateKey: privateKeyA, peerId: peerA, upgrader}); const noiseB = noise()({logger: defaultLogger(), privateKey: privateKeyB, peerId: peerB, upgrader}); - const [inboundConnection, outboundConnection] = duplexPair(); + const [outboundConnection, inboundConnection] = await streamPair(); const [outbound, inbound] = await Promise.all([ noiseA.secureOutbound(outboundConnection, {remotePeer: peerB}), noiseB.secureInbound(inboundConnection, {remotePeer: peerA}), ]); - return {connA: outbound.conn, connB: inbound.conn, data: new Uint8Array(messageLength)}; + return {connA: outbound.connection, connB: inbound.connection, data: new Uint8Array(messageLength)}; }, fn: async ({connA, connB, data}) => { await Promise.all([ - // - pipe(connB.source, connB.sink), - pipe(function* () { + (async () => { for (let i = 0; i < numberOfMessages; i++) { - yield data; + if (!connA.send(data)) { + await connA.onDrain(); + } } - }, connA.sink), - pipe(connB.source, drain), + await connA.close(); + })(), + (async () => { + for await (const _chunk of connB) { + // Drain inbound messages + } + })(), ]); }, }); diff --git a/packages/beacon-node/test/perf/util/blobs.test.ts b/packages/beacon-node/test/perf/util/blobs.test.ts index 8072667161c8..b430612be7de 100644 --- a/packages/beacon-node/test/perf/util/blobs.test.ts +++ b/packages/beacon-node/test/perf/util/blobs.test.ts @@ -2,10 +2,10 @@ import {bench, describe} from "@chainsafe/benchmark"; import {createChainForkConfig} from "@lodestar/config"; import {NUMBER_OF_COLUMNS} from "@lodestar/params"; import {ssz} from "@lodestar/types"; -import {reconstructBlobs} from "../../../src/util/blobs.ts"; -import {getDataColumnSidecarsFromBlock} from "../../../src/util/dataColumns.ts"; -import {kzg} from "../../../src/util/kzg.ts"; -import {generateRandomBlob} from "../../utils/kzg.ts"; +import {reconstructBlobs} from "../../../src/util/blobs.js"; +import {getDataColumnSidecarsFromBlock} from "../../../src/util/dataColumns.js"; +import {kzg} from "../../../src/util/kzg.js"; +import {generateRandomBlob} from "../../utils/kzg.js"; describe("reconstructBlobs", () => { const config = createChainForkConfig({ diff --git a/packages/beacon-node/test/sim/electra-interop.test.ts b/packages/beacon-node/test/sim/electra-interop.test.ts index 02272874415e..b44ce153297c 100644 --- a/packages/beacon-node/test/sim/electra-interop.test.ts +++ b/packages/beacon-node/test/sim/electra-interop.test.ts @@ -3,6 +3,7 @@ import fs from "node:fs"; import {afterAll, afterEach, describe, it, vi} from "vitest"; import {ChainConfig} from "@lodestar/config"; import {TimestampFormatCode} from "@lodestar/logger"; +import {TestLoggerOpts, testLogger} from "@lodestar/logger/test-utils"; import {ForkName, SLOTS_PER_EPOCH, UNSET_DEPOSIT_REQUESTS_START_INDEX} from "@lodestar/params"; import {CachedBeaconStateElectra} from "@lodestar/state-transition"; import {Epoch, Slot, electra} from "@lodestar/types"; @@ -15,7 +16,6 @@ import {bytesToData, dataToBytes} from "../../src/execution/engine/utils.js"; import {initializeExecutionEngine} from "../../src/execution/index.js"; import {BeaconNode} from "../../src/index.js"; import {ClockEvent} from "../../src/util/clock.js"; -import {TestLoggerOpts, testLogger} from "../utils/logger.js"; import {getDevBeaconNode} from "../utils/node/beacon.js"; import {simTestInfoTracker} from "../utils/node/simTest.js"; import {getAndInitDevValidators} from "../utils/node/validator.js"; @@ -364,7 +364,7 @@ describe("executionEngine / ExecutionEngineHttp", () => { if (headState.validators.length !== 33 || headState.balances.length !== 33) { throw Error("New validator is not reflected in the beacon state at slot 5"); } - if (epochCtx.index2pubkey.length !== 33 || epochCtx.pubkey2index.size !== 33) { + if (epochCtx.pubkeyCache.size !== 33) { throw Error("Pubkey cache is not updated"); } @@ -391,7 +391,7 @@ describe("executionEngine / ExecutionEngineHttp", () => { if (headState.validators.length !== 33 || headState.balances.length !== 33) { throw Error("New validator is not reflected in the beacon state."); } - if (epochCtx.index2pubkey.length !== 33 || epochCtx.pubkey2index.size !== 33) { + if (epochCtx.pubkeyCache.size !== 33) { throw Error("New validator is not in pubkey cache"); } diff --git a/packages/beacon-node/test/spec-tests-version.json b/packages/beacon-node/test/spec-tests-version.json new file mode 120000 index 000000000000..bc2e3fd621b2 --- /dev/null +++ b/packages/beacon-node/test/spec-tests-version.json @@ -0,0 +1 @@ +../../../spec-tests-version.json \ No newline at end of file diff --git a/packages/beacon-node/test/spec/presets/epoch_processing.test.ts b/packages/beacon-node/test/spec/presets/epoch_processing.test.ts index 37c0e421688e..67a5f226957d 100644 --- a/packages/beacon-node/test/spec/presets/epoch_processing.test.ts +++ b/packages/beacon-node/test/spec/presets/epoch_processing.test.ts @@ -1,5 +1,6 @@ import path from "node:path"; import {expect} from "vitest"; +import {getConfig} from "@lodestar/config/test-utils"; import {ACTIVE_PRESET} from "@lodestar/params"; import { BeaconStateAllForks, @@ -12,7 +13,6 @@ import { import * as epochFns from "@lodestar/state-transition/epoch"; import {ssz} from "@lodestar/types"; import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState.js"; -import {getConfig} from "../../utils/config.js"; import {assertCorrectProgressiveBalances} from "../config.js"; import {ethereumConsensusSpecsTests} from "../specTestVersioning.js"; import {expectEqualBeaconState, inputTypeSszTreeViewDU} from "../utils/expectEqualBeaconState.js"; @@ -55,7 +55,7 @@ const epochTransitionFns: Record = { }; /** - * https://github.com/ethereum/consensus-specs/blob/dev/tests/formats/epoch_processing/README.md + * https://github.com/ethereum/consensus-specs/blob/v1.6.1/tests/formats/epoch_processing/README.md */ type EpochTransitionCacheingTestCase = { meta?: {bls_setting?: bigint}; @@ -87,7 +87,7 @@ const epochProcessing = if (testcase.post === undefined) { // If post.ssz_snappy is not value, the sub-transition processing is aborted - // https://github.com/ethereum/consensus-specs/blob/dev/tests/formats/epoch_processing/README.md#postssz_snappy + // https://github.com/ethereum/consensus-specs/blob/v1.6.1/tests/formats/epoch_processing/README.md#postssz_snappy expect(() => epochTransitionFn(state, epochTransitionCache)).toThrow(); } else { epochTransitionFn(state, epochTransitionCache); diff --git a/packages/beacon-node/test/spec/presets/finality.test.ts b/packages/beacon-node/test/spec/presets/finality.test.ts index 3c3e91062a23..1c86dd6c9cab 100644 --- a/packages/beacon-node/test/spec/presets/finality.test.ts +++ b/packages/beacon-node/test/spec/presets/finality.test.ts @@ -1,4 +1,5 @@ import path from "node:path"; +import {getConfig} from "@lodestar/config/test-utils"; import {ACTIVE_PRESET, ForkName} from "@lodestar/params"; import { BeaconStateAllForks, @@ -8,7 +9,6 @@ import { } from "@lodestar/state-transition"; import {altair, bellatrix, ssz} from "@lodestar/types"; import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState.js"; -import {getConfig} from "../../utils/config.js"; import {assertCorrectProgressiveBalances} from "../config.js"; import {ethereumConsensusSpecsTests} from "../specTestVersioning.js"; import {expectEqualBeaconState, inputTypeSszTreeViewDU} from "../utils/expectEqualBeaconState.js"; @@ -70,7 +70,7 @@ export function generateBlocksSZZTypeMapping(fork: ForkName, n: number): BlocksS * ``` * {blocks_count: 16} * ``` - * https://github.com/ethereum/consensus-specs/blob/dev/tests/formats/finality/README.md + * https://github.com/ethereum/consensus-specs/blob/v1.6.1/tests/formats/finality/README.md */ type FinalityTestCase = { [k: string]: altair.SignedBeaconBlock | unknown | null | undefined; diff --git a/packages/beacon-node/test/spec/presets/fork_choice.test.ts b/packages/beacon-node/test/spec/presets/fork_choice.test.ts index 498262674428..f7c841a32a74 100644 --- a/packages/beacon-node/test/spec/presets/fork_choice.test.ts +++ b/packages/beacon-node/test/spec/presets/fork_choice.test.ts @@ -1,10 +1,11 @@ import path from "node:path"; import {generateKeyPair} from "@libp2p/crypto/keys"; import {expect} from "vitest"; -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {toHexString} from "@chainsafe/ssz"; import {createBeaconConfig} from "@lodestar/config"; +import {getConfig} from "@lodestar/config/test-utils"; import {CheckpointWithHex, ForkChoice} from "@lodestar/fork-choice"; +import {testLogger} from "@lodestar/logger/test-utils"; import { ACTIVE_PRESET, ForkPostDeneb, @@ -17,8 +18,8 @@ import { import {InputType} from "@lodestar/spec-test-util"; import { BeaconStateAllForks, - Index2PubkeyCache, createCachedBeaconState, + createPubkeyCache, isExecutionStateType, signedBlockToSignedHeader, syncPubkeys, @@ -53,8 +54,6 @@ import {computePreFuluKzgCommitmentsInclusionProof} from "../../../src/util/blob import {ClockEvent} from "../../../src/util/clock.js"; import {ClockStopped} from "../../mocks/clock.js"; import {getMockedBeaconDb} from "../../mocks/mockedBeaconDb.js"; -import {getConfig} from "../../utils/config.js"; -import {testLogger} from "../../utils/logger.js"; import {assertCorrectProgressiveBalances} from "../config.js"; import {ethereumConsensusSpecsTests} from "../specTestVersioning.js"; import {specTestIterator} from "../utils/specTestIterator.js"; @@ -97,15 +96,13 @@ const forkChoiceTest = }); const beaconConfig = createBeaconConfig(config, anchorState.genesisValidatorsRoot); - const pubkey2index = new PubkeyIndexMap(); - const index2pubkey: Index2PubkeyCache = []; - syncPubkeys(anchorState.validators.getAllReadonlyValues(), pubkey2index, index2pubkey); + const pubkeyCache = createPubkeyCache(); + syncPubkeys(pubkeyCache, anchorState.validators.getAllReadonlyValues()); const cachedState = createCachedBeaconState( anchorState, { config: beaconConfig, - pubkey2index, - index2pubkey, + pubkeyCache, }, {skipSyncPubkeys: true} ); @@ -133,8 +130,7 @@ const forkChoiceTest = { privateKey: await generateKeyPair("secp256k1"), config: beaconConfig, - pubkey2index, - index2pubkey, + pubkeyCache, db: getMockedBeaconDb(), dataDir: ".", dbName: ",", @@ -249,7 +245,9 @@ const forkChoiceTest = blockRoot, (signedBlock as SignedBeaconBlock).message.body.blobKzgCommitments .length, - columns + columns, + undefined, + chain.metrics?.peerDas ); blockImport = BlockInputColumns.createFromBlock({ diff --git a/packages/beacon-node/test/spec/presets/genesis.test.ts b/packages/beacon-node/test/spec/presets/genesis.test.ts index 3f2a76851a7e..530666207646 100644 --- a/packages/beacon-node/test/spec/presets/genesis.test.ts +++ b/packages/beacon-node/test/spec/presets/genesis.test.ts @@ -1,5 +1,6 @@ import path from "node:path"; import {expect} from "vitest"; +import {getConfig} from "@lodestar/config/test-utils"; import {ACTIVE_PRESET, ForkName} from "@lodestar/params"; import {InputType} from "@lodestar/spec-test-util"; import { @@ -10,7 +11,6 @@ import { } from "@lodestar/state-transition"; import {ExecutionPayloadHeader, Root, TimeSeconds, phase0, ssz, sszTypesFor} from "@lodestar/types"; import {bnToNum} from "@lodestar/utils"; -import {getConfig} from "../../utils/config.js"; import {ethereumConsensusSpecsTests} from "../specTestVersioning.js"; import {expectEqualBeaconState} from "../utils/expectEqualBeaconState.js"; import {specTestIterator} from "../utils/specTestIterator.js"; diff --git a/packages/beacon-node/test/spec/presets/light_client/sync.ts b/packages/beacon-node/test/spec/presets/light_client/sync.ts index 3b788fecd4e8..96f1d219c6ae 100644 --- a/packages/beacon-node/test/spec/presets/light_client/sync.ts +++ b/packages/beacon-node/test/spec/presets/light_client/sync.ts @@ -1,12 +1,12 @@ import {expect} from "vitest"; import {ChainConfig, createBeaconConfig} from "@lodestar/config"; import {LightclientSpec, toLightClientUpdateSummary} from "@lodestar/light-client/spec"; +import {testLogger} from "@lodestar/logger/test-utils"; import {isForkPostAltair} from "@lodestar/params"; import {InputType} from "@lodestar/spec-test-util"; import {computeSyncPeriodAtSlot} from "@lodestar/state-transition"; import {RootHex, Slot, altair, phase0, ssz, sszTypesFor} from "@lodestar/types"; import {fromHex, toHex} from "@lodestar/utils"; -import {testLogger} from "../../../utils/logger.js"; import {TestRunnerFn} from "../../utils/types.js"; // https://github.com/ethereum/consensus-specs/blob/da3f5af919be4abb5a6db5a80b235deb8b4b5cba/tests/formats/light_client/single_merkle_proof.md diff --git a/packages/beacon-node/test/spec/presets/networking.test.ts b/packages/beacon-node/test/spec/presets/networking.test.ts index eb5c550f143c..577794bbb224 100644 --- a/packages/beacon-node/test/spec/presets/networking.test.ts +++ b/packages/beacon-node/test/spec/presets/networking.test.ts @@ -1,12 +1,14 @@ +import fs from "node:fs"; import path from "node:path"; +import {expect, it} from "vitest"; import {config} from "@lodestar/config/default"; import {ACTIVE_PRESET} from "@lodestar/params"; -import {InputType} from "@lodestar/spec-test-util"; -import {bigIntToBytes} from "@lodestar/utils"; +import {bigIntToBytes, loadYaml} from "@lodestar/utils"; import {computeColumnsForCustodyGroup, getCustodyGroups} from "../../../src/util/dataColumns.js"; import {ethereumConsensusSpecsTests} from "../specTestVersioning.js"; -import {specTestIterator} from "../utils/specTestIterator.js"; -import {RunnerType, TestRunnerFn} from "../utils/types.js"; +import {isGossipValidationHandler, runGossipValidationTest} from "../utils/gossipValidation.js"; +import {readdirSyncSpec, specTestIterator} from "../utils/specTestIterator.js"; +import {RunnerType, TestRunnerCustom} from "../utils/types.js"; type ComputeColumnForCustodyGroupInput = { custody_group: number; @@ -28,30 +30,50 @@ const networkingFns: Record = { }, }; -const networking: TestRunnerFn = (_fork, testName) => { - return { - testFunction: (testcase) => { - const networkingFn = networkingFns[testName]; - if (networkingFn === undefined) { - throw Error(`No networkingFn for ${testName}`); - } - - return networkingFn(testcase.meta); - }, - options: { - inputTypes: {meta: InputType.YAML}, - getExpected: (testCase) => testCase.meta.result.map(Number), - // Do not manually skip tests here, do it in packages/beacon-node/test/spec/presets/index.test.ts - }, - }; -}; - type NetworkingTestCase = { meta: { result: number[]; }; }; +function loadNetworkingTestMeta(testCaseDir: string): NetworkingTestCase["meta"] { + return loadYaml(fs.readFileSync(path.join(testCaseDir, "meta.yaml"), "utf8")); +} + +function runNetworkingFnTests(testHandler: string, testSuite: string, testSuiteDirpath: string): void { + const networkingFn = networkingFns[testHandler]; + if (networkingFn === undefined) { + throw Error(`No networkingFn for ${testHandler}`); + } + + for (const testCaseName of readdirSyncSpec(testSuiteDirpath)) { + const testCaseDir = path.join(testSuiteDirpath, testCaseName); + it(testCaseName, () => { + const meta = loadNetworkingTestMeta(testCaseDir); + const actual = networkingFn(meta); + expect(actual).toEqualWithMessage( + meta.result.map(Number), + `Unexpected networking result for ${testHandler}/${testSuite}/${testCaseName}` + ); + }); + } +} + +const networking: TestRunnerCustom = (fork, testHandler, testSuite, testSuiteDirpath) => { + if (isGossipValidationHandler(testHandler)) { + for (const testCaseName of readdirSyncSpec(testSuiteDirpath)) { + const testCaseDir = path.join(testSuiteDirpath, testCaseName); + it(testCaseName, async () => { + await runGossipValidationTest(fork, testHandler, testCaseDir); + }, 30_000); + } + } else if (networkingFns[testHandler] !== undefined) { + runNetworkingFnTests(testHandler, testSuite, testSuiteDirpath); + } else { + throw new Error(`No runner for networking handler ${testHandler}`); + } +}; + specTestIterator(path.join(ethereumConsensusSpecsTests.outputDir, "tests", ACTIVE_PRESET), { - networking: {type: RunnerType.default, fn: networking}, + networking: {type: RunnerType.custom, fn: networking}, }); diff --git a/packages/beacon-node/test/spec/presets/operations.test.ts b/packages/beacon-node/test/spec/presets/operations.test.ts index 99019c685272..48fd66e28c50 100644 --- a/packages/beacon-node/test/spec/presets/operations.test.ts +++ b/packages/beacon-node/test/spec/presets/operations.test.ts @@ -1,4 +1,5 @@ import path from "node:path"; +import {getConfig} from "@lodestar/config/test-utils"; import {ACTIVE_PRESET, ForkName, ForkSeq} from "@lodestar/params"; import {InputType} from "@lodestar/spec-test-util"; import { @@ -14,7 +15,6 @@ import { import * as blockFns from "@lodestar/state-transition/block"; import {AttesterSlashing, altair, bellatrix, capella, electra, gloas, phase0, ssz, sszTypesFor} from "@lodestar/types"; import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState.js"; -import {getConfig} from "../../utils/config.js"; import {ethereumConsensusSpecsTests} from "../specTestVersioning.js"; import {expectEqualBeaconState, inputTypeSszTreeViewDU} from "../utils/expectEqualBeaconState.js"; import {specTestIterator} from "../utils/specTestIterator.js"; @@ -75,17 +75,19 @@ const operationFns: Record> = signed_envelope: gloas.SignedExecutionPayloadEnvelope; execution: {execution_valid: boolean}; } - ) => { + ): CachedBeaconStateAllForks | void => { const fork = state.config.getForkSeq(state.slot); if (fork >= ForkSeq.gloas) { - blockFns.processExecutionPayloadEnvelope(state as CachedBeaconStateGloas, testCase.signed_envelope, true); - } else { - blockFns.processExecutionPayload(fork, state as CachedBeaconStateBellatrix, testCase.body, { - executionPayloadStatus: testCase.execution.execution_valid - ? ExecutionPayloadStatus.valid - : ExecutionPayloadStatus.invalid, + return blockFns.processExecutionPayloadEnvelope(state as CachedBeaconStateGloas, testCase.signed_envelope, { + verifySignature: true, + verifyStateRoot: true, }); } + blockFns.processExecutionPayload(fork, state as CachedBeaconStateBellatrix, testCase.body, { + executionPayloadStatus: testCase.execution.execution_valid + ? ExecutionPayloadStatus.valid + : ExecutionPayloadStatus.invalid, + }); }, bls_to_execution_change: (state, testCase: {address_change: capella.SignedBLSToExecutionChange}) => { @@ -120,7 +122,7 @@ const operationFns: Record> = }, }; -export type BlockProcessFn = (state: T, testCase: any) => void; +export type BlockProcessFn = (state: T, testCase: any) => T | void; export type OperationsTestCase = { meta?: {bls_setting?: bigint}; @@ -141,7 +143,12 @@ const operations: TestRunnerFn = (fork, const epoch = (state.fork as phase0.Fork).epoch; const cachedState = createCachedBeaconStateTest(state, getConfig(fork, epoch)); - operationFn(cachedState, testcase); + const postState = operationFn(cachedState, testcase); + // processExecutionPayloadEnvelope returns the postState, other operations mutate the state in-place and return void + if (postState !== undefined) { + postState.commit(); + return postState; + } state.commit(); return state; }, diff --git a/packages/beacon-node/test/spec/presets/rewards.test.ts b/packages/beacon-node/test/spec/presets/rewards.test.ts index eda1837b9e2d..1ca70f297159 100644 --- a/packages/beacon-node/test/spec/presets/rewards.test.ts +++ b/packages/beacon-node/test/spec/presets/rewards.test.ts @@ -1,12 +1,12 @@ import path from "node:path"; import {expect} from "vitest"; import {VectorCompositeType} from "@chainsafe/ssz"; +import {getConfig} from "@lodestar/config/test-utils"; import {ACTIVE_PRESET} from "@lodestar/params"; import {BeaconStateAllForks, beforeProcessEpoch} from "@lodestar/state-transition"; import {getRewardsAndPenalties} from "@lodestar/state-transition/epoch"; import {ssz} from "@lodestar/types"; import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState.js"; -import {getConfig} from "../../utils/config.js"; import {assertCorrectProgressiveBalances} from "../config.js"; import {ethereumConsensusSpecsTests} from "../specTestVersioning.js"; import {inputTypeSszTreeViewDU} from "../utils/expectEqualBeaconState.js"; diff --git a/packages/beacon-node/test/spec/presets/sanity.test.ts b/packages/beacon-node/test/spec/presets/sanity.test.ts index 4731bdc22cd9..1d35ab48ecf8 100644 --- a/packages/beacon-node/test/spec/presets/sanity.test.ts +++ b/packages/beacon-node/test/spec/presets/sanity.test.ts @@ -1,4 +1,5 @@ import path from "node:path"; +import {getConfig} from "@lodestar/config/test-utils"; import {ACTIVE_PRESET, ForkName} from "@lodestar/params"; import {InputType} from "@lodestar/spec-test-util"; import { @@ -11,7 +12,6 @@ import { import {SignedBeaconBlock, deneb, ssz} from "@lodestar/types"; import {bnToNum} from "@lodestar/utils"; import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState.js"; -import {getConfig} from "../../utils/config.js"; import {assertCorrectProgressiveBalances} from "../config.js"; import {ethereumConsensusSpecsTests} from "../specTestVersioning.js"; import {expectEqualBeaconState, inputTypeSszTreeViewDU} from "../utils/expectEqualBeaconState.js"; diff --git a/packages/beacon-node/test/spec/presets/ssz_static.test.ts b/packages/beacon-node/test/spec/presets/ssz_static.test.ts index 3a4af2c789a9..0a7d825b6a0f 100644 --- a/packages/beacon-node/test/spec/presets/ssz_static.test.ts +++ b/packages/beacon-node/test/spec/presets/ssz_static.test.ts @@ -19,7 +19,7 @@ import {RunnerType} from "../utils/types.js"; // | serialized.ssz_snappy // | value.yaml // -// Docs: https://github.com/ethereum/consensus-specs/blob/master/tests/formats/ssz_static/core.md +// Docs: https://github.com/ethereum/consensus-specs/blob/v1.6.1/tests/formats/ssz_static/core.md type Types = Record>; diff --git a/packages/beacon-node/test/spec/specTestVersioning.ts b/packages/beacon-node/test/spec/specTestVersioning.ts index 93833af14b17..d4726ca26e23 100644 --- a/packages/beacon-node/test/spec/specTestVersioning.ts +++ b/packages/beacon-node/test/spec/specTestVersioning.ts @@ -1,30 +1,15 @@ import path from "node:path"; import {fileURLToPath} from "node:url"; -import {DownloadTestsOptions} from "@lodestar/spec-test-util/downloadTests"; +import specTestVersions from "../spec-tests-version.json" with {type: "json"}; -// WARNING! Don't move or rename this file !!! -// -// This file is used to generate the cache ID for spec tests download in Github Actions CI -// It's path is hardcoded in: `.github/workflows/test-spec.yml` -// -// The contents of this file MUST include the URL, version and target path, and nothing else. - -// Global variable __dirname no longer available in ES6 modules. -// Solutions: https://stackoverflow.com/questions/46745014/alternative-for-dirname-in-node-js-when-using-es6-modules const __dirname = path.dirname(fileURLToPath(import.meta.url)); -export const ethereumConsensusSpecsTests: DownloadTestsOptions = { - specVersion: "v1.7.0-alpha.2", - // Target directory is the host package root: 'packages/*/spec-tests' - outputDir: path.join(__dirname, "../../spec-tests"), - specTestsRepoUrl: "https://github.com/ethereum/consensus-specs", - testsToDownload: ["general", "mainnet", "minimal"], +export const ethereumConsensusSpecsTests = { + ...specTestVersions.ethereumConsensusSpecsTests, + outputDir: path.join(__dirname, "../../", specTestVersions.ethereumConsensusSpecsTests.outputDirBase), }; -export const blsSpecTests: DownloadTestsOptions = { - specVersion: "v0.1.1", - // Target directory is the host package root: 'packages/*/spec-tests-bls' - outputDir: path.join(__dirname, "../../spec-tests-bls"), - specTestsRepoUrl: "https://github.com/ethereum/bls12-381-tests", - testsToDownload: ["bls_tests_yaml"], +export const blsSpecTests = { + ...specTestVersions.blsSpecTests, + outputDir: path.join(__dirname, "../../", specTestVersions.blsSpecTests.outputDirBase), }; diff --git a/packages/beacon-node/test/spec/utils/gossipValidation.ts b/packages/beacon-node/test/spec/utils/gossipValidation.ts new file mode 100644 index 000000000000..3228838ca11a --- /dev/null +++ b/packages/beacon-node/test/spec/utils/gossipValidation.ts @@ -0,0 +1,549 @@ +import {EventEmitter} from "node:events"; +import fs from "node:fs"; +import path from "node:path"; +import {generateKeyPair} from "@libp2p/crypto/keys"; +import snappy from "snappy"; +import {expect} from "vitest"; +import {createBeaconConfig} from "@lodestar/config"; +import {getConfig} from "@lodestar/config/test-utils"; +import {testLogger} from "@lodestar/logger/test-utils"; +import {ForkName} from "@lodestar/params"; +import { + BeaconStateAllForks, + computeEpochAtSlot, + computeStartSlotAtEpoch, + createCachedBeaconState, + createPubkeyCache, + isExecutionStateType, + syncPubkeys, +} from "@lodestar/state-transition"; +import {RootHex, sszTypesFor} from "@lodestar/types"; +import {fromHex, loadYaml, toHex, toRootHex} from "@lodestar/utils"; +import {BlockInputPreData, BlockInputSource} from "../../../src/chain/blocks/blockInput/index.js"; +import {AttestationImportOpt, BlobSidecarValidation} from "../../../src/chain/blocks/types.js"; +import {GossipAction, GossipActionError} from "../../../src/chain/errors/gossipValidation.js"; +import {BeaconChain, ChainEvent} from "../../../src/chain/index.js"; +import {defaultChainOptions} from "../../../src/chain/options.js"; +import {validateGossipAggregateAndProof} from "../../../src/chain/validation/aggregateAndProof.js"; +import {GossipAttestation, validateGossipAttestationsSameAttData} from "../../../src/chain/validation/attestation.js"; +import {validateGossipAttesterSlashing} from "../../../src/chain/validation/attesterSlashing.js"; +import {validateGossipBlock} from "../../../src/chain/validation/block.js"; +import {validateGossipProposerSlashing} from "../../../src/chain/validation/proposerSlashing.js"; +import {validateGossipVoluntaryExit} from "../../../src/chain/validation/voluntaryExit.js"; +import {ZERO_HASH_HEX} from "../../../src/constants/constants.js"; +import {ExecutionEngineMockBackend} from "../../../src/execution/engine/mock.js"; +import {getExecutionEngineFromBackend} from "../../../src/execution/index.js"; +import {GossipType} from "../../../src/network/gossip/interface.js"; +import type {IClock} from "../../../src/util/clock.js"; +import {getBeaconAttestationGossipIndex, getSlotFromBeaconAttestationSerialized} from "../../../src/util/sszBytes.js"; +import {getMockedBeaconDb} from "../../mocks/mockedBeaconDb.js"; +import {assertCorrectProgressiveBalances} from "../config.js"; + +/** + * A test clock that models gossip clock disparity from a millisecond timestamp. + * Unlike ClockStopped which returns exact slot values, this clock computes + * currentSlotWithGossipDisparity correctly for spec conformance tests. + */ +class GossipTestClock extends EventEmitter implements IClock { + genesisTime: number; + private currentTimeMs: number; + private secondsPerSlot: number; + private maxDisparityMs: number; + + constructor(genesisTimeSec: number, secondsPerSlot: number, maxDisparityMs: number) { + super(); + this.genesisTime = genesisTimeSec; + this.currentTimeMs = genesisTimeSec * 1000; + this.secondsPerSlot = secondsPerSlot; + this.maxDisparityMs = maxDisparityMs; + } + + get currentSlot(): number { + return Math.floor((this.currentTimeMs / 1000 - this.genesisTime) / this.secondsPerSlot); + } + + get currentSlotWithGossipDisparity(): number { + // Model: if we're within maxDisparityMs of next slot, return next slot + // Spec: current_time_ms + MAXIMUM_GOSSIP_CLOCK_DISPARITY >= block_time_ms + // This means: nextSlotTimeMs - currentTimeMs <= maxDisparityMs + const slot = this.currentSlot; + const nextSlotTimeMs = (this.genesisTime + (slot + 1) * this.secondsPerSlot) * 1000; + if (nextSlotTimeMs - this.currentTimeMs <= this.maxDisparityMs) { + return slot + 1; + } + return slot; + } + + get currentEpoch(): number { + return computeEpochAtSlot(this.currentSlot); + } + + slotWithFutureTolerance(toleranceSec: number): number { + return Math.floor((this.currentTimeMs / 1000 + toleranceSec - this.genesisTime) / this.secondsPerSlot); + } + + slotWithPastTolerance(toleranceSec: number): number { + return Math.floor((this.currentTimeMs / 1000 - toleranceSec - this.genesisTime) / this.secondsPerSlot); + } + + isCurrentSlotGivenGossipDisparity(slot: number): boolean { + const current = this.currentSlot; + if (slot === current) return true; + const nextSlotTimeMs = (this.genesisTime + (current + 1) * this.secondsPerSlot) * 1000; + if (nextSlotTimeMs - this.currentTimeMs <= this.maxDisparityMs) { + return slot === current + 1; + } + const currentSlotTimeMs = (this.genesisTime + current * this.secondsPerSlot) * 1000; + if (this.currentTimeMs - currentSlotTimeMs <= this.maxDisparityMs) { + return slot === current - 1; + } + return false; + } + + async waitForSlot(): Promise { + // Not used in tests + } + + secFromSlot(slot: number, toSec?: number): number { + const slotTimeSec = this.genesisTime + slot * this.secondsPerSlot; + return (toSec ?? this.currentTimeMs / 1000) - slotTimeSec; + } + + msFromSlot(slot: number, toMs?: number): number { + const slotTimeMs = (this.genesisTime + slot * this.secondsPerSlot) * 1000; + return (toMs ?? this.currentTimeMs) - slotTimeMs; + } + + /** Set the current time in milliseconds since genesis */ + setCurrentTimeMs(ms: number): void { + this.currentTimeMs = this.genesisTime * 1000 + ms; + } + + /** Also support setSlot for block import phases */ + setSlot(slot: number): void { + this.currentTimeMs = (this.genesisTime + slot * this.secondsPerSlot) * 1000; + } +} + +interface MetaYaml { + topic: GossipType; + blocks?: {block: string; failed?: boolean}[]; + finalized_checkpoint?: {epoch: number; root?: string; block?: string}; + current_time_ms?: number; + messages: { + offset_ms?: number; + subnet_id?: number; + message: string; + expected: "valid" | "ignore" | "reject"; + reason?: string; + }[]; +} + +const gossipTopicByHandler = { + gossip_beacon_block: GossipType.beacon_block, + gossip_beacon_aggregate_and_proof: GossipType.beacon_aggregate_and_proof, + gossip_beacon_attestation: GossipType.beacon_attestation, + gossip_proposer_slashing: GossipType.proposer_slashing, + gossip_attester_slashing: GossipType.attester_slashing, + gossip_voluntary_exit: GossipType.voluntary_exit, +} as const satisfies Record; + +export function isGossipValidationHandler(topicHandler: string): topicHandler is keyof typeof gossipTopicByHandler { + return topicHandler in gossipTopicByHandler; +} + +function getGossipTopic(topicHandler: string): GossipType { + if (!isGossipValidationHandler(topicHandler)) { + throw Error(`Unsupported gossip test handler ${topicHandler}`); + } + return gossipTopicByHandler[topicHandler]; +} + +function loadMeta(testCaseDir: string): MetaYaml { + const raw = fs.readFileSync(path.join(testCaseDir, "meta.yaml"), "utf8"); + return loadYaml(raw); +} + +function loadSszSnappy(testCaseDir: string, name: string): Uint8Array { + const compressed = fs.readFileSync(path.join(testCaseDir, `${name}.ssz_snappy`)); + const decompressed = snappy.uncompressSync(compressed); + return typeof decompressed === "string" ? Buffer.from(decompressed) : decompressed; +} + +function loadState(testCaseDir: string, fork: ForkName): BeaconStateAllForks { + const bytes = loadSszSnappy(testCaseDir, "state"); + return sszTypesFor(fork).BeaconState.deserializeToViewDU(bytes); +} + +type FinalizedCheckpoint = {epoch: number; rootHex: RootHex}; + +function loadBlockRootHex(testCaseDir: string, fork: ForkName, name: string): RootHex { + const signedBlock = sszTypesFor(fork).SignedBeaconBlock.deserialize(loadSszSnappy(testCaseDir, name)); + return toHex(sszTypesFor(fork).BeaconBlock.hashTreeRoot(signedBlock.message)); +} + +function resolveFinalizedCheckpoint( + meta: MetaYaml, + testCaseDir: string, + fork: ForkName, + blockRootsByName: Map +): FinalizedCheckpoint | null { + const cp = meta.finalized_checkpoint; + if (!cp) return null; + + let rootHex: RootHex | null = null; + if (cp.root) { + rootHex = toRootHex(fromHex(cp.root)); + } + if (cp.block) { + const blockRootHex = blockRootsByName.get(cp.block) ?? loadBlockRootHex(testCaseDir, fork, cp.block); + blockRootsByName.set(cp.block, blockRootHex); + if (rootHex !== null && rootHex !== blockRootHex) { + throw new Error(`finalized_checkpoint.root does not match root of ${cp.block}`); + } + rootHex = blockRootHex; + } + + if (rootHex === null) { + throw new Error("finalized_checkpoint must include either root or block"); + } + + if (cp.epoch == null) { + throw new Error("finalized_checkpoint must include an epoch"); + } + return {epoch: Number(cp.epoch), rootHex}; +} + +function setFinalizedCheckpoint(chain: BeaconChain, checkpoint: FinalizedCheckpoint): void { + const checkpointWithHex = { + epoch: checkpoint.epoch, + root: fromHex(checkpoint.rootHex), + rootHex: checkpoint.rootHex, + }; + + const forkChoice = chain.forkChoice as unknown as { + fcStore: { + finalizedCheckpoint: typeof checkpointWithHex; + unrealizedFinalizedCheckpoint: typeof checkpointWithHex; + }; + protoArray: { + finalizedEpoch: number; + finalizedRoot: RootHex; + }; + updateHead?: () => unknown; + }; + + forkChoice.fcStore.finalizedCheckpoint = checkpointWithHex; + forkChoice.fcStore.unrealizedFinalizedCheckpoint = checkpointWithHex; + forkChoice.protoArray.finalizedEpoch = checkpoint.epoch; + forkChoice.protoArray.finalizedRoot = checkpoint.rootHex; + forkChoice.updateHead?.(); +} + +function isDescendantAtFinalizedCheckpoint( + chain: BeaconChain, + blockRootHex: RootHex, + checkpoint: FinalizedCheckpoint +): boolean { + try { + const finalizedSlot = computeStartSlotAtEpoch(checkpoint.epoch); + return chain.forkChoice.getAncestor(blockRootHex, finalizedSlot).blockRoot === checkpoint.rootHex; + } catch { + return false; + } +} + +function mapErrorToResult(e: unknown): "valid" | "ignore" | "reject" { + if (e instanceof GossipActionError) { + return e.action === GossipAction.IGNORE ? "ignore" : "reject"; + } + // Some validation paths throw raw errors instead of GossipActionError + // (e.g., validator index out of range → TypeError on undefined access). + if (e instanceof TypeError || e instanceof RangeError || e instanceof Error) { + return "reject"; + } + throw e; +} + +export async function runGossipValidationTest( + fork: ForkName, + topicHandler: string, + testCaseDir: string +): Promise { + const meta = loadMeta(testCaseDir); + const topic = getGossipTopic(topicHandler); + if (meta.topic !== topic) { + throw Error(`Gossip test topic mismatch for ${topicHandler}: expected ${topic}, got ${meta.topic}`); + } + + const anchorState = loadState(testCaseDir, fork); + const config = getConfig(fork); + const beaconConfig = createBeaconConfig(config, anchorState.genesisValidatorsRoot); + + const genesisTimeSec = Number(anchorState.genesisTime); + const clock = new GossipTestClock( + genesisTimeSec, + beaconConfig.SLOT_DURATION_MS / 1000, + beaconConfig.MAXIMUM_GOSSIP_CLOCK_DISPARITY + ); + + const controller = new AbortController(); + const executionEngineBackend = new ExecutionEngineMockBackend({ + onlyPredefinedResponses: false, + genesisBlockHash: isExecutionStateType(anchorState) + ? toHex(anchorState.latestExecutionPayloadHeader.blockHash) + : ZERO_HASH_HEX, + }); + const executionEngine = getExecutionEngineFromBackend(executionEngineBackend, { + signal: controller.signal, + logger: testLogger("executionEngine"), + }); + + const pubkeyCache = createPubkeyCache(); + syncPubkeys(pubkeyCache, anchorState.validators.getAllReadonlyValues()); + const cachedState = createCachedBeaconState( + anchorState, + {config: beaconConfig, pubkeyCache}, + {skipSyncPubkeys: true} + ); + + const chain = new BeaconChain( + { + ...defaultChainOptions, + // Disable non-spec maxSkipSlots check for conformance tests + maxSkipSlots: undefined, + blsVerifyAllMainThread: true, + disableArchiveOnCheckpoint: true, + disableLightClientServerOnImportBlockHead: true, + disableOnBlockError: true, + disablePrepareNextSlot: true, + assertCorrectProgressiveBalances, + proposerBoost: true, + proposerBoostReorg: true, + }, + { + privateKey: await generateKeyPair("secp256k1"), + config: beaconConfig, + pubkeyCache, + db: getMockedBeaconDb(), + dataDir: ".", + dbName: ",", + logger: testLogger("spec-gossip"), + processShutdownCallback: () => {}, + clock, + metrics: null, + validatorMonitor: null, + anchorState: cachedState, + isAnchorStateFinalized: true, + executionEngine, + executionBuilder: undefined, + } + ); + + chain.emitter.removeAllListeners(ChainEvent.forkChoiceFinalized); + + try { + const blockRootsByName = new Map(); + + if (meta.blocks) { + for (const blockEntry of meta.blocks) { + const signedBlock = sszTypesFor(fork).SignedBeaconBlock.deserialize( + loadSszSnappy(testCaseDir, blockEntry.block) + ); + const slot = signedBlock.message.slot; + const blockRootHex = toHex(beaconConfig.getForkTypes(slot).BeaconBlock.hashTreeRoot(signedBlock.message)); + blockRootsByName.set(blockEntry.block, blockRootHex); + + if (blockEntry.failed) continue; + + // Skip genesis block — it's already the anchor state + if (slot === 0) continue; + + clock.setSlot(slot); + chain.forkChoice.updateTime(slot); + + const blockImport = BlockInputPreData.createFromBlock({ + forkName: fork, + block: signedBlock, + blockRootHex, + source: BlockInputSource.gossip, + seenTimestampSec: 0, + daOutOfRange: false, + }); + + await chain.processBlock(blockImport, { + seenTimestampSec: 0, + validBlobSidecars: BlobSidecarValidation.Full, + importAttestations: AttestationImportOpt.Force, + validSignatures: false, + }); + } + } + + const finalizedCheckpoint = resolveFinalizedCheckpoint(meta, testCaseDir, fork, blockRootsByName); + if (finalizedCheckpoint) { + setFinalizedCheckpoint(chain, finalizedCheckpoint); + } + + const failedBlockRoots = new Set( + (meta.blocks ?? []) + .filter((blockEntry) => blockEntry.failed === true) + .map((blockEntry) => { + const rootHex = blockRootsByName.get(blockEntry.block); + if (!rootHex) throw new Error(`Missing cached root for block ${blockEntry.block}`); + return rootHex; + }) + ); + + const baseCurrentTimeMs = Number(meta.current_time_ms ?? 0); + for (const message of meta.messages) { + const messageTimeMs = baseCurrentTimeMs + Number(message.offset_ms ?? 0); + clock.setCurrentTimeMs(messageTimeMs); + + let result: "valid" | "ignore" | "reject"; + try { + await validateMessageForTopic(chain, fork, topic, testCaseDir, message, failedBlockRoots, finalizedCheckpoint); + result = "valid"; + } catch (e) { + result = mapErrorToResult(e); + } + + expect(result).toEqualWithMessage( + message.expected, + `Unexpected gossip result for ${topicHandler}/${path.basename(testCaseDir)}/${message.message}` + ); + } + } finally { + controller.abort(); + await chain.close(); + } +} + +async function validateMessageForTopic( + chain: BeaconChain, + fork: ForkName, + topic: GossipType, + testCaseDir: string, + message: MetaYaml["messages"][number], + failedBlockRoots: Set, + finalizedCheckpoint: FinalizedCheckpoint | null +): Promise { + const bytes = rejectOnInvalidSerializedBytes(() => loadSszSnappy(testCaseDir, message.message)); + + switch (topic) { + case GossipType.beacon_block: { + const signedBlock = rejectOnInvalidSerializedBytes(() => sszTypesFor(fork).SignedBeaconBlock.deserialize(bytes)); + const parentRootHex = toRootHex(signedBlock.message.parentRoot); + + if (failedBlockRoots.has(parentRootHex)) { + throw new GossipActionError(GossipAction.REJECT, {code: "SPEC_PARENT_BLOCK_FAILED"}); + } + + if ( + finalizedCheckpoint !== null && + !isDescendantAtFinalizedCheckpoint(chain, parentRootHex, finalizedCheckpoint) + ) { + throw new GossipActionError(GossipAction.REJECT, {code: "SPEC_FINALIZED_NOT_ANCESTOR"}); + } + + await validateGossipBlock(chain.config, chain, signedBlock, fork); + chain.seenBlockProposers.add(signedBlock.message.slot, signedBlock.message.proposerIndex); + break; + } + + case GossipType.beacon_aggregate_and_proof: { + const aggregate = rejectOnInvalidSerializedBytes(() => + sszTypesFor(fork).SignedAggregateAndProof.deserialize(bytes) + ); + const beaconBlockRootHex = toRootHex(aggregate.message.aggregate.data.beaconBlockRoot); + + if (failedBlockRoots.has(beaconBlockRootHex)) { + throw new GossipActionError(GossipAction.REJECT, {code: "SPEC_BLOCK_FAILED_VALIDATION"}); + } + + if ( + finalizedCheckpoint !== null && + !isDescendantAtFinalizedCheckpoint(chain, beaconBlockRootHex, finalizedCheckpoint) + ) { + throw new GossipActionError(GossipAction.IGNORE, {code: "SPEC_FINALIZED_NOT_ANCESTOR"}); + } + + await validateGossipAggregateAndProof(fork, chain, aggregate, bytes); + break; + } + + case GossipType.beacon_attestation: { + const attestation = rejectOnInvalidSerializedBytes(() => sszTypesFor(fork).Attestation.deserialize(bytes)); + const beaconBlockRootHex = toRootHex(attestation.data.beaconBlockRoot); + + if (failedBlockRoots.has(beaconBlockRootHex)) { + throw new GossipActionError(GossipAction.REJECT, {code: "SPEC_BLOCK_FAILED_VALIDATION"}); + } + + if ( + finalizedCheckpoint !== null && + !isDescendantAtFinalizedCheckpoint(chain, beaconBlockRootHex, finalizedCheckpoint) + ) { + throw new GossipActionError(GossipAction.IGNORE, {code: "SPEC_FINALIZED_NOT_ANCESTOR"}); + } + + const attDataBase64 = getBeaconAttestationGossipIndex(fork, bytes); + const attSlot = getSlotFromBeaconAttestationSerialized(fork, bytes); + if (attDataBase64 == null || attSlot == null) { + throw new GossipActionError(GossipAction.REJECT, {code: "SPEC_INVALID_ATTESTATION_SERIALIZATION"}); + } + + const gossipAttestation: GossipAttestation = { + attestation: null, + serializedData: bytes, + attSlot, + attDataBase64, + subnet: Number(message.subnet_id ?? 0), + }; + + const batchResult = await validateGossipAttestationsSameAttData(fork, chain, [gossipAttestation]); + const first = batchResult.results[0]; + if (first?.err) throw first.err; + break; + } + + case GossipType.proposer_slashing: { + const slashing = rejectOnInvalidSerializedBytes(() => sszTypesFor(fork).ProposerSlashing.deserialize(bytes)); + await validateGossipProposerSlashing(chain, slashing); + // Mirror gossip handler: insert into opPool so duplicate detection works + chain.opPool.insertProposerSlashing(slashing); + break; + } + + case GossipType.attester_slashing: { + const slashing = rejectOnInvalidSerializedBytes(() => sszTypesFor(fork).AttesterSlashing.deserialize(bytes)); + await validateGossipAttesterSlashing(chain, slashing); + // Mirror gossip handler: insert into opPool + fork choice + chain.opPool.insertAttesterSlashing(fork, slashing); + chain.forkChoice.onAttesterSlashing(slashing); + break; + } + + case GossipType.voluntary_exit: { + const exit = rejectOnInvalidSerializedBytes(() => sszTypesFor(fork).SignedVoluntaryExit.deserialize(bytes)); + await validateGossipVoluntaryExit(chain, exit); + // Mirror gossip handler: insert into opPool so duplicate detection works + chain.opPool.insertVoluntaryExit(exit); + break; + } + + default: + throw new Error(`Unknown gossip topic: ${topic}`); + } +} + +function rejectOnInvalidSerializedBytes(fn: () => T): T { + try { + return fn(); + } catch (e) { + if (e instanceof Error) { + throw new GossipActionError(GossipAction.REJECT, {code: "SPEC_INVALID_SERIALIZED_BYTES"}); + } + throw e; + } +} diff --git a/packages/beacon-node/test/spec/utils/specTestIterator.ts b/packages/beacon-node/test/spec/utils/specTestIterator.ts index 9e72a5f06ca3..fead70d52f61 100644 --- a/packages/beacon-node/test/spec/utils/specTestIterator.ts +++ b/packages/beacon-node/test/spec/utils/specTestIterator.ts @@ -59,7 +59,7 @@ const coveredTestRunners = [ // ], // ``` export const defaultSkipOpts: SkipOpts = { - skippedForks: ["eip7805"], + skippedForks: ["eip7805", "heze"], skippedTestSuites: [ // Merge transition tests are skipped because we no longer support performing the merge transition. // All networks have already completed the merge, so this code path is no longer needed. @@ -103,7 +103,7 @@ export const defaultSkipOpts: SkipOpts = { * tests / mainnet / altair / ssz_static / Validator / ssz_random / case_0/roots.yaml * tests / mainnet / altair / fork / fork / pyspec_tests / altair_fork_random_0/meta.yaml * ``` - * Ref: https://github.com/ethereum/consensus-specs/tree/dev/tests/formats#test-structure + * Ref: https://github.com/ethereum/consensus-specs/blob/v1.6.1/tests/formats/README.md#test-structure */ export function specTestIterator( configDirpath: string, diff --git a/packages/beacon-node/test/spec/utils/sszTestCaseParser.ts b/packages/beacon-node/test/spec/utils/sszTestCaseParser.ts index 90dca88b67ca..f608dd10b768 100644 --- a/packages/beacon-node/test/spec/utils/sszTestCaseParser.ts +++ b/packages/beacon-node/test/spec/utils/sszTestCaseParser.ts @@ -18,7 +18,7 @@ export type ValidTestCaseData = { * | serialized.ssz_snappy * | value.yaml * - * Docs: https://github.com/ethereum/consensus-specs/blob/master/tests/formats/ssz_static/core.md + * Docs: https://github.com/ethereum/consensus-specs/blob/v1.6.1/tests/formats/ssz_static/core.md */ export function parseSszStaticTestcase(dirpath: string): ValidTestCaseData { return parseSszValidTestcase(dirpath, "roots.yaml"); @@ -33,7 +33,7 @@ export function parseSszStaticTestcase(dirpath: string): ValidTestCaseData { * | serialized.ssz_snappy * | value.yaml * - * Docs: https://github.com/ethereum/eth2.0-specs/blob/master/tests/formats/ssz_generic/README.md + * Docs: https://github.com/ethereum/consensus-specs/blob/v1.6.1/tests/formats/ssz_generic/README.md */ export function parseSszGenericValidTestcase(dirpath: string): ValidTestCaseData { return parseSszValidTestcase(dirpath, "meta.yaml"); @@ -71,7 +71,7 @@ export function parseSszValidTestcase(dirpath: string, metaFilename: string): Va * | vec_bool_0 * | serialized.ssz_snappy * - * Docs: https://github.com/ethereum/eth2.0-specs/blob/master/tests/formats/ssz_generic/README.md + * Docs: https://github.com/ethereum/consensus-specs/blob/v1.6.1/tests/formats/ssz_generic/README.md */ export function parseSszGenericInvalidTestcase(dirpath: string) { // The serialized value is stored in serialized.ssz_snappy diff --git a/packages/beacon-node/test/unit-minimal/chain/stateCache/persistentCheckpointsCache.test.ts b/packages/beacon-node/test/unit-minimal/chain/stateCache/persistentCheckpointsCache.test.ts index c4f9cf0cbd94..4b2ada3b872a 100644 --- a/packages/beacon-node/test/unit-minimal/chain/stateCache/persistentCheckpointsCache.test.ts +++ b/packages/beacon-node/test/unit-minimal/chain/stateCache/persistentCheckpointsCache.test.ts @@ -1,6 +1,7 @@ import {beforeAll, beforeEach, describe, expect, it} from "vitest"; import {createBeaconConfig} from "@lodestar/config"; import {chainConfig as chainConfigDef} from "@lodestar/config/default"; +import {testLogger} from "@lodestar/logger/test-utils"; import {ACTIVE_PRESET, PresetName, SLOTS_PER_EPOCH, SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params"; import {CachedBeaconStateAllForks, computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {RootHex, phase0} from "@lodestar/types"; @@ -13,7 +14,6 @@ import { } from "../../../../src/chain/stateCache/persistentCheckpointsCache.js"; import {CheckpointHexPayload} from "../../../../src/chain/stateCache/types.js"; import {getTestDatastore} from "../../../utils/chain/stateCache/datastore.js"; -import {testLogger} from "../../../utils/logger.js"; import {generateCachedState} from "../../../utils/state.js"; describe("PersistentCheckpointStateCache", () => { @@ -57,7 +57,7 @@ describe("PersistentCheckpointStateCache", () => { cp1 = {epoch: 21, root: root1}; cp2 = {epoch: 22, root: root2}; [cp0aHex, cp0bHex, cp1Hex, cp2Hex] = [cp0a, cp0b, cp1, cp2].map((cp) => toCheckpointHexPayload(cp, true)); - persistent0bKey = toHexString(checkpointToDatastoreKey(cp0b)); + persistent0bKey = toHexString(checkpointToDatastoreKey(cp0b, true)); const allStates = [cp0a, cp0b, cp1, cp2] .map((cp) => generateCachedState({slot: cp.epoch * SLOTS_PER_EPOCH})) .map((state, i) => { @@ -1039,7 +1039,7 @@ describe("PersistentCheckpointStateCache", () => { }); async function assertPersistedCheckpointState(cps: phase0.Checkpoint[], stateBytesArr: Uint8Array[]): Promise { - const persistedKeys = cps.map((cp) => toHexString(checkpointToDatastoreKey(cp))); + const persistedKeys = cps.map((cp) => toHexString(checkpointToDatastoreKey(cp, true))); expect(Array.from(fileApisBuffer.keys())).toStrictEqual(persistedKeys); for (const [i, persistedKey] of persistedKeys.entries()) { expect(fileApisBuffer.get(persistedKey)).toStrictEqual(stateBytesArr[i]); diff --git a/packages/beacon-node/test/unit/api/impl/beacon/state/utils.test.ts b/packages/beacon-node/test/unit/api/impl/beacon/state/utils.test.ts index 45f9d6d760ea..28e0530daf8e 100644 --- a/packages/beacon-node/test/unit/api/impl/beacon/state/utils.test.ts +++ b/packages/beacon-node/test/unit/api/impl/beacon/state/utils.test.ts @@ -8,50 +8,50 @@ import {generateCachedAltairState} from "../../../../../utils/state.js"; describe("beacon state api utils", () => { describe("getStateValidatorIndex", () => { const state = generateCachedAltairState(); - const pubkey2index = state.epochCtx.pubkey2index; + const pubkeyCache = state.epochCtx.pubkeyCache; it("should return valid: false on invalid input", () => { // "invalid validator id number" - expect(getStateValidatorIndex("foo", state, pubkey2index).valid).toBe(false); + expect(getStateValidatorIndex("foo", state, pubkeyCache).valid).toBe(false); // "invalid hex" - expect(getStateValidatorIndex("0xfoo", state, pubkey2index).valid).toBe(false); + expect(getStateValidatorIndex("0xfoo", state, pubkeyCache).valid).toBe(false); }); it("should return valid: false on validator indices / pubkeys not in the state", () => { // "validator id not in state" - expect(getStateValidatorIndex(String(state.validators.length), state, pubkey2index).valid).toBe(false); + expect(getStateValidatorIndex(String(state.validators.length), state, pubkeyCache).valid).toBe(false); // "validator pubkey not in state" expect( getStateValidatorIndex( "0xa99af0913a2834ef4959637e8d7c4e17f0b63adc587d36ab43510452db3102d0771a4554ea4118a33913827d5ee80b76", state, - pubkey2index + pubkeyCache ).valid ).toBe(false); }); it("should return valid: true on validator indices / pubkeys in the state", () => { const index = state.validators.length - 1; - const resp1 = getStateValidatorIndex(String(index), state, pubkey2index); + const resp1 = getStateValidatorIndex(String(index), state, pubkeyCache); if (resp1.valid) { expect(resp1.validatorIndex).toBe(index); } else { expect.fail("validator index should be found - validator index as string input"); } - const resp2 = getStateValidatorIndex(index, state, pubkey2index); + const resp2 = getStateValidatorIndex(index, state, pubkeyCache); if (resp2.valid) { expect(resp2.validatorIndex).toBe(index); } else { expect.fail("validator index should be found - validator index as number input"); } const pubkey = state.validators.get(index).pubkey; - const resp3 = getStateValidatorIndex(pubkey, state, pubkey2index); + const resp3 = getStateValidatorIndex(pubkey, state, pubkeyCache); if (resp3.valid) { expect(resp3.validatorIndex).toBe(index); } else { expect.fail("validator index should be found - Uint8Array input"); } - const resp4 = getStateValidatorIndex(toHexString(pubkey), state, pubkey2index); + const resp4 = getStateValidatorIndex(toHexString(pubkey), state, pubkeyCache); if (resp4.valid) { expect(resp4.validatorIndex).toBe(index); } else { diff --git a/packages/beacon-node/test/unit/chain/archiveStore/blockArchiver.test.ts b/packages/beacon-node/test/unit/chain/archiveStore/blockArchiver.test.ts index 56808e548ec0..cc8730ba24b8 100644 --- a/packages/beacon-node/test/unit/chain/archiveStore/blockArchiver.test.ts +++ b/packages/beacon-node/test/unit/chain/archiveStore/blockArchiver.test.ts @@ -3,14 +3,13 @@ import {fromHexString, toHexString} from "@chainsafe/ssz"; import {createChainForkConfig} from "@lodestar/config"; import {config as defaultConfig} from "@lodestar/config/default"; import {PayloadStatus} from "@lodestar/fork-choice"; -import {ZERO_HASH} from "@lodestar/params"; +import {testLogger} from "@lodestar/logger/test-utils"; import {computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {ssz} from "@lodestar/types"; import {archiveBlocks} from "../../../../src/chain/archiveStore/utils/archiveBlocks.js"; import {ZERO_HASH_HEX} from "../../../../src/constants/index.js"; import {MockedBeaconChain, getMockedBeaconChain} from "../../../mocks/mockedBeaconChain.js"; import {MockedBeaconDb, getMockedBeaconDb} from "../../../mocks/mockedBeaconDb.js"; -import {testLogger} from "../../../utils/logger.js"; import {generateProtoBlock} from "../../../utils/typeGenerator.js"; function toAsyncIterable(items: T[]): AsyncIterable { @@ -70,7 +69,7 @@ describe("block archiver task", () => { forkChoiceStub, lightclientServer, logger, - {epoch: 5, rootHex: ZERO_HASH_HEX, root: ZERO_HASH, payloadStatus: PayloadStatus.EMPTY}, + {epoch: 5, root: fromHexString(ZERO_HASH_HEX), rootHex: ZERO_HASH_HEX, payloadStatus: PayloadStatus.FULL}, currentEpoch ); @@ -143,7 +142,12 @@ describe("block archiver task", () => { forkChoiceStub, lightclientServer, logger, - {epoch: config.FULU_FORK_EPOCH + 1, root: ZERO_HASH, rootHex: ZERO_HASH_HEX, payloadStatus: PayloadStatus.EMPTY}, + { + epoch: config.FULU_FORK_EPOCH + 1, + root: fromHexString(ZERO_HASH_HEX), + rootHex: ZERO_HASH_HEX, + payloadStatus: PayloadStatus.FULL, + }, currentEpoch ); diff --git a/packages/beacon-node/test/unit/chain/bls/bls.test.ts b/packages/beacon-node/test/unit/chain/bls/bls.test.ts index 5a325c492228..1e749d91e3b9 100644 --- a/packages/beacon-node/test/unit/chain/bls/bls.test.ts +++ b/packages/beacon-node/test/unit/chain/bls/bls.test.ts @@ -1,19 +1,22 @@ import {beforeEach, describe, expect, it} from "vitest"; import {PublicKey, SecretKey, Signature} from "@chainsafe/blst"; -import {ISignatureSet, SignatureSetType} from "@lodestar/state-transition"; +import {testLogger} from "@lodestar/logger/test-utils"; +import {ISignatureSet, SignatureSetType, createPubkeyCache} from "@lodestar/state-transition"; import {BlsMultiThreadWorkerPool} from "../../../../src/chain/bls/multithread/index.js"; import {BlsSingleThreadVerifier} from "../../../../src/chain/bls/singleThread.js"; -import {testLogger} from "../../../utils/logger.js"; describe("BlsVerifier ", () => { // take time for creating thread pool const numKeys = 3; const secretKeys = Array.from({length: numKeys}, (_, i) => SecretKey.fromKeygen(Buffer.alloc(32, i))); - // Create a mock index2pubkey that maps indices to public keys - const index2pubkey = secretKeys.map((sk) => sk.toPublicKey()); + // Create a mock pubkeyCache that maps indices to public keys + const pubkeyCache = createPubkeyCache(); + for (const [i, sk] of secretKeys.entries()) { + pubkeyCache.set(i, sk.toPublicKey().toBytes()); + } const verifiers = [ - new BlsSingleThreadVerifier({metrics: null, index2pubkey}), - new BlsMultiThreadWorkerPool({}, {metrics: null, logger: testLogger(), index2pubkey}), + new BlsSingleThreadVerifier({metrics: null, pubkeyCache}), + new BlsMultiThreadWorkerPool({}, {metrics: null, logger: testLogger(), pubkeyCache}), ]; for (const verifier of verifiers) { diff --git a/packages/beacon-node/test/unit/chain/forkChoice/forkChoice.test.ts b/packages/beacon-node/test/unit/chain/forkChoice/forkChoice.test.ts index 01b13ab422db..6aeef48dbf23 100644 --- a/packages/beacon-node/test/unit/chain/forkChoice/forkChoice.test.ts +++ b/packages/beacon-node/test/unit/chain/forkChoice/forkChoice.test.ts @@ -1,7 +1,7 @@ import {beforeAll, beforeEach, describe, expect, it, vi} from "vitest"; import {toHexString} from "@chainsafe/ssz"; import {config} from "@lodestar/config/default"; -import {CheckpointWithHex, ExecutionStatus, ForkChoice} from "@lodestar/fork-choice"; +import {CheckpointWithHex, ExecutionStatus, ForkChoice, PayloadStatus} from "@lodestar/fork-choice"; import {FAR_FUTURE_EPOCH, MAX_EFFECTIVE_BALANCE} from "@lodestar/params"; import { CachedBeaconStateAllForks, @@ -200,22 +200,19 @@ describe("LodestarForkChoice", () => { forkChoice.onBlock(block20.message, state20, blockDelaySec, currentSlot, executionStatus, dataAvailabilityStatus); forkChoice.onBlock(block24.message, state24, blockDelaySec, currentSlot, executionStatus, dataAvailabilityStatus); forkChoice.onBlock(block28.message, state28, blockDelaySec, currentSlot, executionStatus, dataAvailabilityStatus); - const block16Summary = forkChoice.getBlockHexDefaultStatus(hashBlock(block16.message)); - const block24Summary = forkChoice.getBlockHexDefaultStatus(hashBlock(block24.message)); - if (!block16Summary || !block24Summary) throw new Error("Expected block summaries to exist"); - expect(forkChoice.getAllAncestorBlocks(block16Summary)).toHaveLength(3); - expect(forkChoice.getAllAncestorBlocks(block24Summary)).toHaveLength(5); + expect(forkChoice.getAllAncestorBlocks(hashBlock(block16.message), PayloadStatus.FULL)).toHaveLength(3); + expect(forkChoice.getAllAncestorBlocks(hashBlock(block24.message), PayloadStatus.FULL)).toHaveLength(5); expect(forkChoice.getBlockHexDefaultStatus(hashBlock(block08.message))).not.toBeNull(); expect(forkChoice.getBlockHexDefaultStatus(hashBlock(block12.message))).not.toBeNull(); expect(forkChoice.hasBlockHex(hashBlock(block08.message))).toBe(true); expect(forkChoice.hasBlockHex(hashBlock(block12.message))).toBe(true); forkChoice.onBlock(block32.message, state32, blockDelaySec, currentSlot, executionStatus, dataAvailabilityStatus); forkChoice.prune(hashBlock(block16.message)); - expect(forkChoice.getAllAncestorBlocks(block16Summary).length).toBeWithMessage( + expect(forkChoice.getAllAncestorBlocks(hashBlock(block16.message), PayloadStatus.FULL).length).toBeWithMessage( 0, "getAllAncestorBlocks should not return finalized block" ); - expect(forkChoice.getAllAncestorBlocks(block24Summary)).toHaveLength(2); + expect(forkChoice.getAllAncestorBlocks(hashBlock(block24.message), PayloadStatus.FULL)).toHaveLength(2); expect(forkChoice.getBlockHexDefaultStatus(hashBlock(block08.message))).toBe(null); expect(forkChoice.getBlockHexDefaultStatus(hashBlock(block12.message))).toBe(null); expect(forkChoice.hasBlockHex(hashBlock(block08.message))).toBe(false); @@ -277,10 +274,11 @@ describe("LodestarForkChoice", () => { .forwarditerateAncestorBlocks() .filter( (summary) => - summary.slot < childBlock.message.slot && !forkChoice.isDescendant(summary.blockRoot, childBlockRoot) + summary.slot < childBlock.message.slot && + !forkChoice.isDescendant(summary.blockRoot, summary.payloadStatus, childBlockRoot, PayloadStatus.FULL) ); // compare to getAllNonAncestorBlocks api - expect(forkChoice.getAllNonAncestorBlocks(childBlockRoot)).toEqual(nonCanonicalSummaries); + expect(forkChoice.getAllNonAncestorBlocks(childBlockRoot, PayloadStatus.FULL)).toEqual(nonCanonicalSummaries); }); /** diff --git a/packages/beacon-node/test/unit/chain/regen/regen.test.ts b/packages/beacon-node/test/unit/chain/regen/regen.test.ts index 1436100eb352..1e8ace689d7f 100644 --- a/packages/beacon-node/test/unit/chain/regen/regen.test.ts +++ b/packages/beacon-node/test/unit/chain/regen/regen.test.ts @@ -1,6 +1,7 @@ import {beforeEach, describe, expect, it} from "vitest"; import {createBeaconConfig} from "@lodestar/config"; import {chainConfig as chainConfigDef} from "@lodestar/config/default"; +import {testLogger} from "@lodestar/logger/test-utils"; import {SLOTS_PER_EPOCH, SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params"; import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {RegenCaller} from "../../../../src/chain/regen/interface.js"; @@ -8,7 +9,6 @@ import {processSlotsToNearestCheckpoint} from "../../../../src/chain/regen/regen import {FIFOBlockStateCache} from "../../../../src/chain/stateCache/fifoBlockStateCache.js"; import {PersistentCheckpointStateCache} from "../../../../src/chain/stateCache/persistentCheckpointsCache.js"; import {getTestDatastore} from "../../../utils/chain/stateCache/datastore.js"; -import {testLogger} from "../../../utils/logger.js"; import {generateCachedState} from "../../../utils/state.js"; describe("regen", () => { diff --git a/packages/beacon-node/test/unit/chain/seenCache/seenBlockInput.test.ts b/packages/beacon-node/test/unit/chain/seenCache/seenBlockInput.test.ts index f49cbe7ca831..1a121e70bf8e 100644 --- a/packages/beacon-node/test/unit/chain/seenCache/seenBlockInput.test.ts +++ b/packages/beacon-node/test/unit/chain/seenCache/seenBlockInput.test.ts @@ -1,11 +1,12 @@ import {generateKeyPair} from "@libp2p/crypto/keys"; import {beforeEach, describe, expect, it} from "vitest"; import {PayloadStatus} from "@lodestar/fork-choice"; -import {ForkName, ForkPostFulu, ForkPreGloas} from "@lodestar/params"; +import {testLogger} from "@lodestar/logger/test-utils"; +import {ForkName} from "@lodestar/params"; import {signedBlockToSignedHeader} from "@lodestar/state-transition"; -import {SignedBeaconBlock} from "@lodestar/types"; import {toRootHex} from "@lodestar/utils"; import { + BlockInputPreData, BlockInputSource, IBlockInput, isBlockInputBlobs, @@ -17,18 +18,19 @@ import {SeenBlockInput} from "../../../../src/chain/seenCache/seenGossipBlockInp import {computeNodeIdFromPrivateKey} from "../../../../src/network/subnets/index.js"; import {Clock} from "../../../../src/util/clock.js"; import {CustodyConfig} from "../../../../src/util/dataColumns.js"; +import {SerializedCache} from "../../../../src/util/serializedCache.js"; import { config, generateBlock, generateBlockWithBlobSidecars, generateChainOfBlocks, } from "../../../utils/blocksAndData.js"; -import {testLogger} from "../../../utils/logger.js"; describe("SeenBlockInputCache", async () => { let cache: SeenBlockInput; let abortController: AbortController; let chainEvents: ChainEventEmitter; + let serializedCache: SerializedCache; const privateKey = await generateKeyPair("secp256k1"); const nodeId = computeNodeIdFromPrivateKey(privateKey); @@ -38,6 +40,7 @@ describe("SeenBlockInputCache", async () => { beforeEach(() => { chainEvents = new ChainEventEmitter(); abortController = new AbortController(); + serializedCache = new SerializedCache(); const signal = abortController.signal; const genesisTime = Math.floor(Date.now() / 1000); cache = new SeenBlockInput({ @@ -46,6 +49,7 @@ describe("SeenBlockInputCache", async () => { clock: new Clock({config, genesisTime, signal}), chainEvents, signal, + serializedCache, logger, metrics: null, }); @@ -121,6 +125,21 @@ describe("SeenBlockInputCache", async () => { expect(cache.get(rootHex)).toBeUndefined(); }); + it("should remove serialized cache entries for the evicted BlockInput", () => { + const {block, rootHex} = generateBlock({forkName: ForkName.capella}); + cache.getByBlock({ + block, + blockRootHex: rootHex, + source: BlockInputSource.gossip, + seenTimestampSec: Date.now() / 1000, + }); + serializedCache.set(block, new Uint8Array([1])); + + expect(serializedCache.get(block)).toBeDefined(); + cache.remove(rootHex); + expect(serializedCache.get(block)).toBeUndefined(); + }); + it("should not throw an error if BlockInput not in cache", () => { const {block, blockRoot, rootHex} = generateBlock({forkName: ForkName.capella}); const blockInput = cache.getByBlock({ @@ -179,6 +198,34 @@ describe("SeenBlockInputCache", async () => { expect(cache.get(childRootHex)).toBeUndefined(); expect(cache.get(parentRootHex)).toBeUndefined(); }); + + it("should remove serialized cache entries for the pruned BlockInputs", () => { + const blocks = generateChainOfBlocks({forkName: ForkName.capella, count: 2}); + const parentBlock = blocks[0].block; + const parentRootHex = blocks[0].rootHex; + const childBlock = blocks[1].block; + const childRootHex = blocks[1].rootHex; + + cache.getByBlock({ + block: parentBlock, + blockRootHex: parentRootHex, + source: BlockInputSource.gossip, + seenTimestampSec: Date.now() / 1000, + }); + cache.getByBlock({ + block: childBlock, + blockRootHex: childRootHex, + source: BlockInputSource.gossip, + seenTimestampSec: Date.now() / 1000, + }); + serializedCache.set(parentBlock, new Uint8Array([1])); + serializedCache.set(childBlock, new Uint8Array([2])); + + cache.prune(childRootHex); + + expect(serializedCache.get(parentBlock)).toBeUndefined(); + expect(serializedCache.get(childBlock)).toBeUndefined(); + }); }); describe("onFinalized()", () => { @@ -215,6 +262,9 @@ describe("SeenBlockInputCache", async () => { }); it("should remove all BlockInputs in slots before the checkpoint", () => { + serializedCache.set(parentBlockInput.getBlock(), new Uint8Array([1])); + serializedCache.set(childBlockInput.getBlock(), new Uint8Array([2])); + chainEvents.emit(ChainEvent.forkChoiceFinalized, { epoch: config.DENEB_FORK_EPOCH, root, @@ -223,6 +273,8 @@ describe("SeenBlockInputCache", async () => { }); expect(cache.get(childRootHex)).toBeUndefined(); expect(cache.get(parentRootHex)).toBeUndefined(); + expect(serializedCache.get(parentBlockInput.getBlock())).toBeUndefined(); + expect(serializedCache.get(childBlockInput.getBlock())).toBeUndefined(); }); it("should not remove BlockInputs in slots after the checkpoint", () => { @@ -311,9 +363,10 @@ describe("SeenBlockInputCache", async () => { source: BlockInputSource.gossip, seenTimestampSec: Date.now() / 1000, }); + expect(isBlockInputPreDeneb(blockInput)).toBeTruthy(); expect(() => - blockInput.addBlock({ - block: block as SignedBeaconBlock, + (blockInput as BlockInputPreData).addBlock({ + block, blockRootHex: rootHex, source: BlockInputSource.gossip, seenTimestampSec: Date.now() / 1000, diff --git a/packages/beacon-node/test/unit/chain/shufflingCache.test.ts b/packages/beacon-node/test/unit/chain/shufflingCache.test.ts index b8a9602ea6ce..63cf2f53a77c 100644 --- a/packages/beacon-node/test/unit/chain/shufflingCache.test.ts +++ b/packages/beacon-node/test/unit/chain/shufflingCache.test.ts @@ -1,5 +1,5 @@ import {beforeEach, describe, expect, it} from "vitest"; -import {generateTestCachedBeaconStateOnlyValidators} from "../../../../state-transition/test/perf/util.js"; +import {generateTestCachedBeaconStateOnlyValidators} from "@lodestar/state-transition/test-utils"; import {ShufflingCache} from "../../../src/chain/shufflingCache.js"; describe("ShufflingCache", () => { diff --git a/packages/beacon-node/test/unit/chain/validation/aggregateAndProof.test.ts b/packages/beacon-node/test/unit/chain/validation/aggregateAndProof.test.ts index e25fda1a14bb..6cf39e084adf 100644 --- a/packages/beacon-node/test/unit/chain/validation/aggregateAndProof.test.ts +++ b/packages/beacon-node/test/unit/chain/validation/aggregateAndProof.test.ts @@ -1,8 +1,8 @@ import {describe, it} from "vitest"; import {BitArray, toHexString} from "@chainsafe/ssz"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; +import {generateTestCachedBeaconStateOnlyValidators} from "@lodestar/state-transition/test-utils"; import {phase0, ssz} from "@lodestar/types"; -import {generateTestCachedBeaconStateOnlyValidators} from "../../../../../state-transition/test/perf/util.js"; import {AttestationErrorCode} from "../../../../src/chain/errors/index.js"; import {IBeaconChain} from "../../../../src/chain/index.js"; import {validateApiAggregateAndProof, validateGossipAggregateAndProof} from "../../../../src/chain/validation/index.js"; diff --git a/packages/beacon-node/test/unit/chain/validation/attestation/validateAttestation.test.ts b/packages/beacon-node/test/unit/chain/validation/attestation/validateAttestation.test.ts index fcfa71655fa7..99c5d63c9bf2 100644 --- a/packages/beacon-node/test/unit/chain/validation/attestation/validateAttestation.test.ts +++ b/packages/beacon-node/test/unit/chain/validation/attestation/validateAttestation.test.ts @@ -1,9 +1,9 @@ import {describe, expect, it} from "vitest"; import {BitArray} from "@chainsafe/ssz"; import {ForkName, SLOTS_PER_EPOCH} from "@lodestar/params"; +import {generateTestCachedBeaconStateOnlyValidators} from "@lodestar/state-transition/test-utils"; import {ssz} from "@lodestar/types"; import {LodestarError} from "@lodestar/utils"; -import {generateTestCachedBeaconStateOnlyValidators} from "../../../../../../state-transition/test/perf/util.js"; import {AttestationErrorCode, GossipErrorCode} from "../../../../../src/chain/errors/index.js"; import {IBeaconChain} from "../../../../../src/chain/index.js"; import { diff --git a/packages/beacon-node/test/unit/chain/validation/attestation/validateGossipAttestationsSameAttData.test.ts b/packages/beacon-node/test/unit/chain/validation/attestation/validateGossipAttestationsSameAttData.test.ts index fa3e155af54d..7f8d858f9c51 100644 --- a/packages/beacon-node/test/unit/chain/validation/attestation/validateGossipAttestationsSameAttData.test.ts +++ b/packages/beacon-node/test/unit/chain/validation/attestation/validateGossipAttestationsSameAttData.test.ts @@ -1,7 +1,7 @@ import {afterEach, beforeEach, describe, expect, it, vi} from "vitest"; import {PublicKey, SecretKey} from "@chainsafe/blst"; import {ForkName} from "@lodestar/params"; -import {SignatureSetType} from "@lodestar/state-transition"; +import {SignatureSetType, createPubkeyCache} from "@lodestar/state-transition"; import {ssz} from "@lodestar/types"; import {BlsSingleThreadVerifier} from "../../../../../src/chain/bls/singleThread.js"; import {AttestationError, AttestationErrorCode, GossipAction} from "../../../../../src/chain/errors/index.js"; @@ -56,22 +56,22 @@ describe("validateGossipAttestationsSameAttData", () => { return keypair; } - // Build index2pubkey cache for test - const index2pubkey: PublicKey[] = []; + // Build pubkeyCache for test + const pubkeyCache = createPubkeyCache(); for (let i = 0; i < 10; i++) { - index2pubkey.push(getKeypair(i).publicKey); + pubkeyCache.set(i, getKeypair(i).publicKey.toBytes()); } // Add a special keypair for invalid signatures - index2pubkey[2023] = getKeypair(2023).publicKey; + pubkeyCache.set(2023, getKeypair(2023).publicKey.toBytes()); let chain: IBeaconChain; const signingRoot = Buffer.alloc(32, 1); beforeEach(() => { chain = { - bls: new BlsSingleThreadVerifier({metrics: null, index2pubkey}), + bls: new BlsSingleThreadVerifier({metrics: null, pubkeyCache}), seenAttesters: new SeenAttesters(), - index2pubkey, + pubkeyCache, opts: { minSameMessageSignatureSetsToBatch: 2, } as IBeaconChain["opts"], diff --git a/packages/beacon-node/test/unit/chain/validatorMonitor.test.ts b/packages/beacon-node/test/unit/chain/validatorMonitor.test.ts index 8b4a5687b43e..fd194ab1ee51 100644 --- a/packages/beacon-node/test/unit/chain/validatorMonitor.test.ts +++ b/packages/beacon-node/test/unit/chain/validatorMonitor.test.ts @@ -1,8 +1,8 @@ import {describe, expect, it, vi} from "vitest"; import {createChainForkConfig, defaultChainConfig} from "@lodestar/config"; +import {testLogger} from "@lodestar/logger/test-utils"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {createValidatorMonitor} from "../../../src/chain/validatorMonitor.js"; -import {testLogger} from "../../utils/logger.js"; describe("ValidatorMonitor", () => { // Use phase0 config (no altair) to avoid needing full state with block roots diff --git a/packages/beacon-node/test/unit/db/api/repositories/blockArchive.test.ts b/packages/beacon-node/test/unit/db/api/repositories/blockArchive.test.ts index 86c8416eaf13..eeab033952af 100644 --- a/packages/beacon-node/test/unit/db/api/repositories/blockArchive.test.ts +++ b/packages/beacon-node/test/unit/db/api/repositories/blockArchive.test.ts @@ -3,11 +3,11 @@ import {afterEach, beforeEach, describe, expect, it, vi} from "vitest"; import {config} from "@lodestar/config/default"; import {encodeKey} from "@lodestar/db"; import {LevelDbController} from "@lodestar/db/controller/level"; +import {testLogger} from "@lodestar/logger/test-utils"; import {ssz} from "@lodestar/types"; import {intToBytes} from "@lodestar/utils"; import {Bucket, getBucketNameByValue} from "../../../../../src/db/buckets.js"; import {BlockArchiveRepository} from "../../../../../src/db/repositories/index.js"; -import {testLogger} from "../../../../utils/logger.js"; describe("block archive repository", () => { const testDir = "./.tmp_block_archive_unit_test"; diff --git a/packages/beacon-node/test/unit/db/api/repositories/dataColumn.test.ts b/packages/beacon-node/test/unit/db/api/repositories/dataColumn.test.ts index 150e79bbe2a6..a2252a5638a1 100644 --- a/packages/beacon-node/test/unit/db/api/repositories/dataColumn.test.ts +++ b/packages/beacon-node/test/unit/db/api/repositories/dataColumn.test.ts @@ -2,6 +2,7 @@ import {rimraf} from "rimraf"; import {afterEach, beforeEach, describe, expect, it} from "vitest"; import {createChainForkConfig} from "@lodestar/config"; import {LevelDbController} from "@lodestar/db/controller/level"; +import {testLogger} from "@lodestar/logger/test-utils"; import {NUMBER_OF_COLUMNS} from "@lodestar/params"; import {Root, fulu, ssz} from "@lodestar/types"; import {fromAsync, toHex} from "@lodestar/utils"; @@ -9,7 +10,6 @@ import {DataColumnSidecarRepository} from "../../../../../src/db/repositories/da import {DataColumnSidecarArchiveRepository} from "../../../../../src/db/repositories/dataColumnSidecarArchive.js"; import {getDataColumnSidecarsFromBlock} from "../../../../../src/util/dataColumns.js"; import {kzg} from "../../../../../src/util/kzg.js"; -import {testLogger} from "../../../../utils/logger.js"; /* eslint-disable @typescript-eslint/naming-convention */ const config = createChainForkConfig({ diff --git a/packages/beacon-node/test/unit/db/api/repository.test.ts b/packages/beacon-node/test/unit/db/api/repository.test.ts index 3bed3675456d..36da34839947 100644 --- a/packages/beacon-node/test/unit/db/api/repository.test.ts +++ b/packages/beacon-node/test/unit/db/api/repository.test.ts @@ -1,4 +1,3 @@ -import all from "it-all"; import {MockedObject, afterEach, beforeEach, describe, expect, it, vi} from "vitest"; import {ContainerType} from "@chainsafe/ssz"; import {config} from "@lodestar/config/default"; @@ -145,7 +144,7 @@ describe("database repository", () => { } controller.valuesStream.mockReturnValue(sample()); - const result = await all(repository.valuesStream()); + const result = await Array.fromAsync(repository.valuesStream()); expect(result.length).toBe(2); }); }); diff --git a/packages/beacon-node/test/unit/metrics/server/http.test.ts b/packages/beacon-node/test/unit/metrics/server/http.test.ts index 3172ae72cd0e..247d9cfbfff5 100644 --- a/packages/beacon-node/test/unit/metrics/server/http.test.ts +++ b/packages/beacon-node/test/unit/metrics/server/http.test.ts @@ -1,7 +1,7 @@ import {afterAll, describe, it} from "vitest"; +import {testLogger} from "@lodestar/logger/test-utils"; import {fetch} from "@lodestar/utils"; import {HttpMetricsServer, getHttpMetricsServer} from "../../../../src/metrics/index.js"; -import {testLogger} from "../../../utils/logger.js"; import {createMetricsTest} from "../utils.js"; describe("HttpMetricsServer", () => { diff --git a/packages/beacon-node/test/unit/network/discv5/utils.test.ts b/packages/beacon-node/test/unit/network/discv5/utils.test.ts new file mode 100644 index 000000000000..55ae813e810e --- /dev/null +++ b/packages/beacon-node/test/unit/network/discv5/utils.test.ts @@ -0,0 +1,65 @@ +import {generateKeyPair} from "@libp2p/crypto/keys"; +import {multiaddr} from "@multiformats/multiaddr"; +import {describe, expect, it} from "vitest"; +import {type ENR, SignableENR} from "@chainsafe/enr"; +import {config} from "@lodestar/config/test-utils"; +import {ssz} from "@lodestar/types"; +import {ENRRelevance, enrRelevance} from "../../../../src/network/discv5/utils.js"; +import {ENRKey, getENRForkID} from "../../../../src/network/metadata.js"; +import {ClockStatic} from "../../../utils/clock.js"; + +describe("network / discv5 / enrRelevance", () => { + const clock = new ClockStatic(0); + + async function createEnr(opts: {tcp?: boolean; quic?: boolean; eth2?: boolean}): Promise { + const privateKey = await generateKeyPair("secp256k1"); + const enr = SignableENR.createFromPrivateKey(privateKey); + + if (opts.tcp) { + enr.setLocationMultiaddr(multiaddr("/ip4/192.168.1.1/tcp/9000")); + } + if (opts.quic) { + enr.setLocationMultiaddr(multiaddr("/ip4/192.168.1.1/udp/9001/quic-v1")); + } + if (opts.eth2) { + const enrForkID = getENRForkID(config, 0); + enr.set(ENRKey.eth2, ssz.phase0.ENRForkID.serialize(enrForkID)); + } + + return enr; + } + + it("should return no_transport for ENR without tcp or quic", async () => { + const enr = await createEnr({eth2: true}); + expect(enrRelevance(enr as unknown as ENR, config, clock)).toBe(ENRRelevance.no_transport); + }); + + it("should return relevant for ENR with tcp only", async () => { + const enr = await createEnr({tcp: true, eth2: true}); + expect(enrRelevance(enr as unknown as ENR, config, clock)).toBe(ENRRelevance.relevant); + }); + + it("should return relevant for ENR with quic only", async () => { + const enr = await createEnr({quic: true, eth2: true}); + expect(enrRelevance(enr as unknown as ENR, config, clock)).toBe(ENRRelevance.relevant); + }); + + it("should return relevant for ENR with both tcp and quic", async () => { + const enr = await createEnr({tcp: true, quic: true, eth2: true}); + expect(enrRelevance(enr as unknown as ENR, config, clock)).toBe(ENRRelevance.relevant); + }); + + it("should return no_eth2 for ENR without eth2 field", async () => { + const enr = await createEnr({tcp: true}); + expect(enrRelevance(enr as unknown as ENR, config, clock)).toBe(ENRRelevance.no_eth2); + }); + + it("should return unknown_forkDigest for ENR with unrecognized fork digest", async () => { + const enr = await createEnr({tcp: true}); + // Set a fake eth2 field with an unknown fork digest + const fakeEth2 = new Uint8Array(16); + fakeEth2.set([0xff, 0xff, 0xff, 0xff], 0); + enr.set(ENRKey.eth2, fakeEth2); + expect(enrRelevance(enr as unknown as ENR, config, clock)).toBe(ENRRelevance.unknown_forkDigest); + }); +}); diff --git a/packages/beacon-node/test/unit/network/gossip/directPeers.test.ts b/packages/beacon-node/test/unit/network/gossip/directPeers.test.ts index 17ef0b151984..5c471530853b 100644 --- a/packages/beacon-node/test/unit/network/gossip/directPeers.test.ts +++ b/packages/beacon-node/test/unit/network/gossip/directPeers.test.ts @@ -115,21 +115,50 @@ describe("network / gossip / directPeers", () => { expect(result[0].addrs[0].toString()).toBe("/ip4/192.168.1.1/tcp/9000"); expect(logger.info).toHaveBeenCalledWith("Added direct peer from ENR", { peerId: enr.peerId.toString(), - addr: "/ip4/192.168.1.1/tcp/9000", + addrs: "/ip4/192.168.1.1/tcp/9000", }); }); - it("should skip ENR without TCP multiaddr and log warning", async () => { + it("should parse ENR with both QUIC and TCP multiaddrs", async () => { + const privateKey = await generateKeyPair("secp256k1"); + const enr = SignableENR.createFromPrivateKey(privateKey); + enr.setLocationMultiaddr(multiaddr("/ip4/192.168.1.1/tcp/9000")); + enr.setLocationMultiaddr(multiaddr("/ip4/192.168.1.1/udp/9001/quic-v1")); + const enrStr = enr.encodeTxt(); + + const result = parseDirectPeers([enrStr], logger); + + expect(result).toHaveLength(1); + expect(result[0].id.toString()).toBe(enr.peerId.toString()); + expect(result[0].addrs).toHaveLength(2); + expect(result[0].addrs[0].toString()).toBe("/ip4/192.168.1.1/udp/9001/quic-v1"); + expect(result[0].addrs[1].toString()).toBe("/ip4/192.168.1.1/tcp/9000"); + }); + + it("should parse ENR with only QUIC multiaddr", async () => { + const privateKey = await generateKeyPair("secp256k1"); + const enr = SignableENR.createFromPrivateKey(privateKey); + enr.setLocationMultiaddr(multiaddr("/ip4/192.168.1.1/udp/9001/quic-v1")); + const enrStr = enr.encodeTxt(); + + const result = parseDirectPeers([enrStr], logger); + + expect(result).toHaveLength(1); + expect(result[0].addrs).toHaveLength(1); + expect(result[0].addrs[0].toString()).toBe("/ip4/192.168.1.1/udp/9001/quic-v1"); + }); + + it("should skip ENR without any transport multiaddr and log warning", async () => { const privateKey = await generateKeyPair("secp256k1"); const enr = SignableENR.createFromPrivateKey(privateKey); - // Only set UDP, not TCP + // Only set UDP discovery, not a dialable transport enr.setLocationMultiaddr(multiaddr("/ip4/192.168.1.1/udp/9000")); const enrStr = enr.encodeTxt(); const result = parseDirectPeers([enrStr], logger); expect(result).toHaveLength(0); - expect(logger.warn).toHaveBeenCalledWith("ENR does not contain TCP multiaddr", {enr: enrStr}); + expect(logger.warn).toHaveBeenCalledWith("ENR does not contain any transport multiaddr", {enr: enrStr}); }); it("should skip invalid ENR and log warning", () => { @@ -146,6 +175,7 @@ describe("network / gossip / directPeers", () => { const privateKey = await generateKeyPair("secp256k1"); const enr = SignableENR.createFromPrivateKey(privateKey); enr.setLocationMultiaddr(multiaddr("/ip4/10.0.0.1/tcp/9001")); + enr.setLocationMultiaddr(multiaddr("/ip4/10.0.0.1/udp/9002/quic-v1")); const enrStr = enr.encodeTxt(); const mixedPeers = [`/ip4/192.168.1.1/tcp/9000/p2p/${peerIdStr}`, enrStr]; @@ -156,7 +186,9 @@ describe("network / gossip / directPeers", () => { expect(result[0].id.toString()).toBe(peerIdStr); expect(result[0].addrs[0].toString()).toBe("/ip4/192.168.1.1/tcp/9000"); expect(result[1].id.toString()).toBe(enr.peerId.toString()); - expect(result[1].addrs[0].toString()).toBe("/ip4/10.0.0.1/tcp/9001"); + expect(result[1].addrs).toHaveLength(2); + expect(result[1].addrs[0].toString()).toBe("/ip4/10.0.0.1/udp/9002/quic-v1"); + expect(result[1].addrs[1].toString()).toBe("/ip4/10.0.0.1/tcp/9001"); }); }); }); diff --git a/packages/beacon-node/test/unit/network/gossip/scoringParameters.test.ts b/packages/beacon-node/test/unit/network/gossip/scoringParameters.test.ts index 168e4f346d67..aa05f51a96c4 100644 --- a/packages/beacon-node/test/unit/network/gossip/scoringParameters.test.ts +++ b/packages/beacon-node/test/unit/network/gossip/scoringParameters.test.ts @@ -1,5 +1,5 @@ +import type {TopicScoreParams} from "@libp2p/gossipsub/score"; import {describe, expect, it} from "vitest"; -import {TopicScoreParams} from "@chainsafe/libp2p-gossipsub/score"; import {createBeaconConfig} from "@lodestar/config"; import {mainnetChainConfig} from "@lodestar/config/configs"; import {ATTESTATION_SUBNET_COUNT, ForkName, GENESIS_EPOCH, SLOTS_PER_EPOCH} from "@lodestar/params"; diff --git a/packages/beacon-node/test/unit/network/libp2p/getDiscv5Multiaddrs.test.ts b/packages/beacon-node/test/unit/network/libp2p/getDiscv5Multiaddrs.test.ts new file mode 100644 index 000000000000..d042d172d95c --- /dev/null +++ b/packages/beacon-node/test/unit/network/libp2p/getDiscv5Multiaddrs.test.ts @@ -0,0 +1,56 @@ +import {generateKeyPair} from "@libp2p/crypto/keys"; +import {multiaddr} from "@multiformats/multiaddr"; +import {describe, expect, it} from "vitest"; +import {SignableENR} from "@chainsafe/enr"; +import {getDiscv5Multiaddrs} from "../../../../src/network/libp2p/index.js"; + +describe("network / libp2p / getDiscv5Multiaddrs", () => { + async function createEnrTxt(opts: {tcp?: boolean; quic?: boolean}): Promise { + const privateKey = await generateKeyPair("secp256k1"); + const enr = SignableENR.createFromPrivateKey(privateKey); + if (opts.tcp) { + enr.setLocationMultiaddr(multiaddr("/ip4/10.0.0.1/tcp/9000")); + } + if (opts.quic) { + enr.setLocationMultiaddr(multiaddr("/ip4/10.0.0.1/udp/9001/quic-v1")); + } + return enr.encodeTxt(); + } + + it("should prefer quic multiaddr over tcp when quic is enabled", async () => { + const enrTxt = await createEnrTxt({tcp: true, quic: true}); + const result = await getDiscv5Multiaddrs([enrTxt], true); + expect(result).toHaveLength(1); + expect(result[0]).toContain("/quic-v1"); + expect(result[0]).not.toContain("/tcp/"); + }); + + it("should return tcp by default (quic disabled)", async () => { + const enrTxt = await createEnrTxt({tcp: true, quic: true}); + const result = await getDiscv5Multiaddrs([enrTxt]); + expect(result).toHaveLength(1); + expect(result[0]).toContain("/tcp/"); + expect(result[0]).not.toContain("/quic-v1"); + }); + + it("should return tcp when quic is explicitly disabled", async () => { + const enrTxt = await createEnrTxt({tcp: true, quic: true}); + const result = await getDiscv5Multiaddrs([enrTxt], false); + expect(result).toHaveLength(1); + expect(result[0]).toContain("/tcp/"); + expect(result[0]).not.toContain("/quic-v1"); + }); + + it("should skip ENRs with no transport at all", async () => { + const enrTxt = await createEnrTxt({}); + const result = await getDiscv5Multiaddrs([enrTxt]); + expect(result).toHaveLength(0); + }); + + it("should return quic-only ENR when quic is enabled", async () => { + const enrTxt = await createEnrTxt({quic: true}); + const result = await getDiscv5Multiaddrs([enrTxt], true); + expect(result).toHaveLength(1); + expect(result[0]).toContain("/quic-v1"); + }); +}); diff --git a/packages/beacon-node/test/unit/network/metadata.test.ts b/packages/beacon-node/test/unit/network/metadata.test.ts index b2b63687de9d..e1ae191d05f8 100644 --- a/packages/beacon-node/test/unit/network/metadata.test.ts +++ b/packages/beacon-node/test/unit/network/metadata.test.ts @@ -1,5 +1,7 @@ import {describe, expect, it, vi} from "vitest"; import {createBeaconConfig, createChainForkConfig} from "@lodestar/config"; +import {config} from "@lodestar/config/test-utils"; +import {testLogger} from "@lodestar/logger/test-utils"; import {ZERO_HASH} from "@lodestar/params"; import {ssz} from "@lodestar/types"; import {toHex} from "@lodestar/utils"; @@ -8,8 +10,6 @@ import {NetworkConfig} from "../../../src/network/networkConfig.js"; import {computeNodeId} from "../../../src/network/subnets/index.js"; import {CustodyConfig} from "../../../src/util/dataColumns.js"; import {serializeCgc} from "../../../src/util/metadata.js"; -import {config} from "../../utils/config.js"; -import {testLogger} from "../../utils/logger.js"; import {getValidPeerId} from "../../utils/peer.js"; describe("network / metadata", () => { diff --git a/packages/beacon-node/test/unit/network/reqresp/utils.ts b/packages/beacon-node/test/unit/network/reqresp/utils.ts index 614789f8916c..9b483bd9284d 100644 --- a/packages/beacon-node/test/unit/network/reqresp/utils.ts +++ b/packages/beacon-node/test/unit/network/reqresp/utils.ts @@ -1,5 +1,4 @@ -import {Direction, ReadStatus, Stream, StreamStatus, WriteStatus} from "@libp2p/interface"; -import {logger} from "@libp2p/logger"; +import type {Stream} from "@libp2p/interface"; import {Uint8ArrayList} from "uint8arraylist"; import {expect} from "vitest"; import {toHexString} from "@chainsafe/ssz"; @@ -13,16 +12,6 @@ export function generateRoots(count: number, offset = 0): Root[] { return roots; } -/** - * Helper for it-pipe when first argument is an array. - * it-pipe does not convert the chunks array to a generator and BufferedSource breaks - */ -export async function* arrToSource(arr: T[]): AsyncGenerator { - for (const item of arr) { - yield item; - } -} - /** * Wrapper for type-safety to ensure and array of Buffers is equal with a diff in hex */ @@ -30,34 +19,44 @@ export function expectEqualByteChunks(chunks: Uint8Array[], expectedChunks: Uint expect(chunks.map(toHexString)).toEqual(expectedChunks.map(toHexString)); } +type SourceChunk = Uint8Array | Uint8ArrayList; + +function toUint8ArrayList(chunk: SourceChunk): Uint8ArrayList { + return chunk instanceof Uint8ArrayList ? chunk : new Uint8ArrayList(chunk); +} + +function toUint8Array(chunk: SourceChunk): Uint8Array { + return chunk instanceof Uint8ArrayList ? chunk.subarray() : chunk; +} + /** - * Useful to simulate a LibP2P stream source emitting prepared bytes - * and capture the response with a sink accessible via `this.resultChunks` + * Minimal stream test double for reqresp unit tests. + * It captures sent chunks and yields a provided source for reads. */ -export class MockLibP2pStream implements Stream { - id = "mock"; - log = logger("mock"); - direction: Direction = "inbound"; - timeline = { - open: Date.now(), - }; - status: StreamStatus = "open"; - readStatus: ReadStatus = "ready"; - writeStatus: WriteStatus = "ready"; - metadata = {}; - source: Stream["source"]; - resultChunks: Uint8Array[] = []; - - constructor(requestChunks: Uint8ArrayList[]) { - this.source = arrToSource(requestChunks); - } - sink: Stream["sink"] = async (source) => { - for await (const chunk of source) { - this.resultChunks.push(chunk.subarray()); - } - }; - close: Stream["close"] = async () => {}; - closeRead = async (): Promise => {}; - closeWrite = async (): Promise => {}; - abort: Stream["abort"] = () => this.close(); +export function createMockStream({ + protocol = "", + source = (async function* (): AsyncIterable {})(), +}: { + protocol?: string; + source?: AsyncIterable; +} = {}): {stream: Stream; sentChunks: Uint8Array[]} { + const sentChunks: Uint8Array[] = []; + + const stream = { + protocol, + send(chunk: SourceChunk): boolean { + sentChunks.push(toUint8Array(chunk)); + return true; + }, + async onDrain(): Promise {}, + async close(): Promise {}, + abort(): void {}, + async *[Symbol.asyncIterator](): AsyncGenerator { + for await (const chunk of source) { + yield toUint8ArrayList(chunk); + } + }, + } as unknown as Stream; + + return {stream, sentChunks}; } diff --git a/packages/beacon-node/test/unit/network/subnets/attnetsService.test.ts b/packages/beacon-node/test/unit/network/subnets/attnetsService.test.ts index bb004da434b2..bffe99a496ba 100644 --- a/packages/beacon-node/test/unit/network/subnets/attnetsService.test.ts +++ b/packages/beacon-node/test/unit/network/subnets/attnetsService.test.ts @@ -1,5 +1,6 @@ import {MockedObject, afterEach, beforeEach, describe, expect, it, vi} from "vitest"; import {createBeaconConfig} from "@lodestar/config"; +import {testLogger} from "@lodestar/logger/test-utils"; import {ATTESTATION_SUBNET_COUNT, ForkName, GENESIS_EPOCH, SLOTS_PER_EPOCH} from "@lodestar/params"; import {ZERO_HASH, getCurrentSlot} from "@lodestar/state-transition"; import {SubnetID} from "@lodestar/types"; @@ -11,7 +12,6 @@ import {AttnetsService} from "../../../../src/network/subnets/attnetsService.js" import {CommitteeSubscription} from "../../../../src/network/subnets/interface.js"; import {Clock, IClock} from "../../../../src/util/clock.js"; import {CustodyConfig} from "../../../../src/util/dataColumns.js"; -import {testLogger} from "../../../utils/logger.js"; vi.mock("../../../../src/network/gossip/gossipsub.js"); diff --git a/packages/beacon-node/test/unit/sync/range/chain.test.ts b/packages/beacon-node/test/unit/sync/range/chain.test.ts index ec07e5b7bff7..4f6065e5728d 100644 --- a/packages/beacon-node/test/unit/sync/range/chain.test.ts +++ b/packages/beacon-node/test/unit/sync/range/chain.test.ts @@ -1,5 +1,6 @@ import {afterEach, describe, it} from "vitest"; import {config} from "@lodestar/config/default"; +import {testLogger} from "@lodestar/logger/test-utils"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {Epoch, Slot, phase0, ssz} from "@lodestar/types"; @@ -12,7 +13,6 @@ import {RangeSyncType} from "../../../../src/sync/utils/remoteSyncType.js"; import {Clock} from "../../../../src/util/clock.js"; import {CustodyConfig} from "../../../../src/util/dataColumns.js"; import {linspace} from "../../../../src/util/numpy.js"; -import {testLogger} from "../../../utils/logger.js"; import {validPeerIdStr} from "../../../utils/peer.js"; describe("sync / range / chain", () => { diff --git a/packages/beacon-node/test/unit/sync/unknownBlock.test.ts b/packages/beacon-node/test/unit/sync/unknownBlock.test.ts index 8873e1bcbedf..96700c05e041 100644 --- a/packages/beacon-node/test/unit/sync/unknownBlock.test.ts +++ b/packages/beacon-node/test/unit/sync/unknownBlock.test.ts @@ -4,10 +4,10 @@ import {toHexString} from "@chainsafe/ssz"; import {createChainForkConfig} from "@lodestar/config"; import {config as minimalConfig} from "@lodestar/config/default"; import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; -import {ForkName} from "@lodestar/params"; +import {testLogger} from "@lodestar/logger/test-utils"; import {ssz} from "@lodestar/types"; import {notNullish, sleep} from "@lodestar/utils"; -import {BlockInputColumns, BlockInputPreData} from "../../../src/chain/blocks/blockInput/blockInput.js"; +import {BlockInputPreData} from "../../../src/chain/blocks/blockInput/blockInput.js"; import {BlockInputSource} from "../../../src/chain/blocks/blockInput/types.js"; import {BlockError, BlockErrorCode} from "../../../src/chain/errors/blockError.js"; import {ChainEvent, ChainEventEmitter, IBeaconChain} from "../../../src/chain/index.js"; @@ -21,8 +21,6 @@ import {CustodyConfig} from "../../../src/util/dataColumns.js"; import {PeerIdStr} from "../../../src/util/peerId.js"; import {ClockStopped} from "../../mocks/clock.js"; import {MockedBeaconChain, getMockedBeaconChain} from "../../mocks/mockedBeaconChain.js"; -import {generateBlockWithColumnSidecars} from "../../utils/blocksAndData.js"; -import {testLogger} from "../../utils/logger.js"; import {getRandPeerIdStr, getRandPeerSyncMeta} from "../../utils/peer.js"; describe("sync by UnknownBlockSync", {timeout: 20_000}, () => { @@ -355,7 +353,6 @@ describe("UnknownBlockSync", () => { }); describe("UnknownBlockPeerBalancer", async () => { - const custodyConfig = {sampledColumns: [0, 1, 2, 3]} as CustodyConfig; const peer0 = await getRandPeerSyncMeta("peer-0"); const peer1 = await getRandPeerSyncMeta("peer-1"); const peer2 = await getRandPeerSyncMeta("peer-2"); @@ -414,44 +411,6 @@ describe("UnknownBlockPeerBalancer", async () => { peers[i].custodyColumns = groups; } - const signedBlock = ssz.fulu.SignedBeaconBlock.defaultValue(); - signedBlock.message.body.blobKzgCommitments = [ssz.fulu.KZGCommitment.defaultValue()]; - const {block, rootHex, columnSidecars} = generateBlockWithColumnSidecars({forkName: ForkName.fulu}); - const blockInput = BlockInputColumns.createFromBlock({ - block: block, - blockRootHex: rootHex, - forkName: ForkName.fulu, - daOutOfRange: false, - source: BlockInputSource.gossip, - seenTimestampSec: Math.floor(Date.now() / 1000), - custodyColumns: custodyConfig.custodyColumns, - sampledColumns: custodyConfig.sampledColumns, - }); - - // test cases rely on first 2 columns being known, the rest unknown - for (const sidecar of columnSidecars.slice(0, 2)) { - blockInput.addColumn({ - columnSidecar: sidecar, - blockRootHex: rootHex, - seenTimestampSec: Math.floor(Date.now() / 1000), - source: BlockInputSource.gossip, - }); - } - - it(`bestPeerForBlockInput - test case ${testCaseIndex}`, () => { - for (const [i, activeRequest] of activeRequests.entries()) { - for (let j = 0; j < activeRequest; j++) { - peerBalancer.onRequest(peers[i].peerId); - } - } - const peer = peerBalancer.bestPeerForBlockInput(blockInput, new Set(excludedPeers)); - if (bestPeer) { - expect(peer).toEqual(bestPeer); - } else { - expect(peer).toBeNull(); - } - }); - it(`bestPeerForPendingColumns - test case ${testCaseIndex}`, () => { for (const [i, activeRequest] of activeRequests.entries()) { for (let j = 0; j < activeRequest; j++) { diff --git a/packages/beacon-node/test/unit/util/error.test.ts b/packages/beacon-node/test/unit/util/error.test.ts index 02e99df0cac3..37b4edec010b 100644 --- a/packages/beacon-node/test/unit/util/error.test.ts +++ b/packages/beacon-node/test/unit/util/error.test.ts @@ -9,7 +9,7 @@ function structuredClone(value: T): T { describe("ThreadBoundaryError", () => { it("should clone RequestError through thread boundary", () => { - const requestError = new RequestError({code: RequestErrorCode.TTFB_TIMEOUT}); + const requestError = new RequestError({code: RequestErrorCode.RESP_TIMEOUT}); const threadBoundaryError = toThreadBoundaryError(requestError); const clonedError = structuredClone(threadBoundaryError); expect(clonedError.error).toBeNull(); diff --git a/packages/beacon-node/test/unit/util/itTrigger.test.ts b/packages/beacon-node/test/unit/util/itTrigger.test.ts index ff76cf13eafa..baf5f0f69181 100644 --- a/packages/beacon-node/test/unit/util/itTrigger.test.ts +++ b/packages/beacon-node/test/unit/util/itTrigger.test.ts @@ -1,4 +1,3 @@ -import all from "it-all"; import {describe, expect, it} from "vitest"; import {ItTrigger} from "../../../src/util/itTrigger.js"; @@ -10,7 +9,7 @@ describe("util / itTrigger", () => { itTrigger.trigger(); itTrigger.end(); - const res = await all(itTrigger); + const res = await Array.fromAsync(itTrigger); expect(res).toHaveLength(0); }); @@ -27,7 +26,7 @@ describe("util / itTrigger", () => { }, 5); }, 5); - const res = await all(itTrigger); + const res = await Array.fromAsync(itTrigger); expect(res).toHaveLength(2); }); @@ -43,7 +42,7 @@ describe("util / itTrigger", () => { }, 5); }, 5); - await expect(all(itTrigger)).rejects.toThrow(testError); + await expect(Array.fromAsync(itTrigger)).rejects.toThrow(testError); }); it("ItTrigger as a single thread processor", async () => { diff --git a/packages/beacon-node/test/unit/util/sszBytes.test.ts b/packages/beacon-node/test/unit/util/sszBytes.test.ts index 796101b75d4e..354be3880a09 100644 --- a/packages/beacon-node/test/unit/util/sszBytes.test.ts +++ b/packages/beacon-node/test/unit/util/sszBytes.test.ts @@ -27,6 +27,8 @@ import { getAttDataFromSignedAggregateAndProofPhase0, getAttDataFromSingleAttestationSerialized, getAttesterIndexFromSingleAttestationSerialized, + getBeaconBlockRootFromDataColumnSidecarSerialized, + getBeaconBlockRootFromExecutionPayloadEnvelopeSerialized, getBlobKzgCommitmentsCountFromSignedBeaconBlockSerialized, getBlockRootFromAttestationSerialized, getBlockRootFromSignedAggregateAndProofSerialized, @@ -39,6 +41,8 @@ import { getSlotFromAttestationSerialized, getSlotFromBeaconStateSerialized, getSlotFromBlobSidecarSerialized, + getSlotFromDataColumnSidecarSerialized, + getSlotFromExecutionPayloadEnvelopeSerialized, getSlotFromSignedAggregateAndProofSerialized, getSlotFromSignedBeaconBlockSerialized, getSlotFromSingleAttestationSerialized, @@ -494,3 +498,99 @@ function blobSidecarFromValues(slot: Slot): deneb.BlobSidecar { blobSidecar.signedBlockHeader.message.slot = slot; return blobSidecar; } + +describe("SignedExecutionPayloadEnvelope SSZ serialized picking", () => { + const testCases: {slot: Slot; blockRoot: RootHex}[] = [ + {slot: 0, blockRoot: "0x" + "00".repeat(32)}, + {slot: 1_000_000, blockRoot: "0x" + "aa".repeat(32)}, + {slot: 4_294_967_295, blockRoot: "0x" + "ff".repeat(32)}, // max uint32 + ]; + + for (const {slot, blockRoot} of testCases) { + it(`slot=${slot}`, () => { + const envelope = ssz.gloas.SignedExecutionPayloadEnvelope.defaultValue(); + envelope.message.slot = slot; + envelope.message.beaconBlockRoot = fromHex(blockRoot); + const bytes = ssz.gloas.SignedExecutionPayloadEnvelope.serialize(envelope); + + expect(getSlotFromExecutionPayloadEnvelopeSerialized(bytes)).toBe(slot); + expect(getBeaconBlockRootFromExecutionPayloadEnvelopeSerialized(bytes)).toBe(blockRoot); + }); + } + + it("getSlotFromExecutionPayloadEnvelopeSerialized - invalid data", () => { + // Slot is at offset 148, need at least 156 bytes + const invalidSizes = [0, 50, 100, 155]; + for (const size of invalidSizes) { + expect(getSlotFromExecutionPayloadEnvelopeSerialized(Buffer.alloc(size))).toBeNull(); + } + }); + + it("getBeaconBlockRootFromExecutionPayloadEnvelopeSerialized - invalid data", () => { + // Block root is at offset 116, need at least 148 bytes + const invalidSizes = [0, 50, 100, 147]; + for (const size of invalidSizes) { + expect(getBeaconBlockRootFromExecutionPayloadEnvelopeSerialized(Buffer.alloc(size))).toBeNull(); + } + }); +}); + +describe("DataColumnSidecar SSZ serialized picking (fork-aware)", () => { + describe("Fulu (pre-Gloas)", () => { + const testCases: {slot: Slot}[] = [{slot: 0}, {slot: 500_000}, {slot: 4_294_967_295}]; + + for (const {slot} of testCases) { + it(`slot=${slot}`, () => { + const sidecar = ssz.fulu.DataColumnSidecar.defaultValue(); + sidecar.signedBlockHeader.message.slot = slot; + const bytes = ssz.fulu.DataColumnSidecar.serialize(sidecar); + + expect(getSlotFromDataColumnSidecarSerialized(bytes, ForkName.fulu)).toBe(slot); + }); + } + + it("getSlotFromDataColumnSidecarSerialized - invalid data", () => { + // Slot is at offset 20 for pre-Gloas, need at least 28 bytes + const invalidSizes = [0, 10, 27]; + for (const size of invalidSizes) { + expect(getSlotFromDataColumnSidecarSerialized(Buffer.alloc(size), ForkName.fulu)).toBeNull(); + } + }); + }); + + describe("Gloas", () => { + const testCases: {slot: Slot; blockRoot: RootHex}[] = [ + {slot: 0, blockRoot: "0x" + "00".repeat(32)}, + {slot: 600_000, blockRoot: "0x" + "bb".repeat(32)}, + {slot: 4_294_967_295, blockRoot: "0x" + "ff".repeat(32)}, + ]; + + for (const {slot, blockRoot} of testCases) { + it(`slot=${slot}`, () => { + const sidecar = ssz.gloas.DataColumnSidecar.defaultValue(); + sidecar.slot = slot; + sidecar.beaconBlockRoot = fromHex(blockRoot); + const bytes = ssz.gloas.DataColumnSidecar.serialize(sidecar); + + expect(getSlotFromDataColumnSidecarSerialized(bytes, ForkName.gloas)).toBe(slot); + expect(getBeaconBlockRootFromDataColumnSidecarSerialized(bytes)).toBe(blockRoot); + }); + } + + it("getSlotFromDataColumnSidecarSerialized - invalid data", () => { + // Slot is at offset 16 for Gloas, need at least 24 bytes + const invalidSizes = [0, 10, 23]; + for (const size of invalidSizes) { + expect(getSlotFromDataColumnSidecarSerialized(Buffer.alloc(size), ForkName.gloas)).toBeNull(); + } + }); + + it("getBeaconBlockRootFromDataColumnSidecarSerialized - invalid data", () => { + // Block root is at offset 24 for Gloas, need at least 56 bytes + const invalidSizes = [0, 20, 55]; + for (const size of invalidSizes) { + expect(getBeaconBlockRootFromDataColumnSidecarSerialized(Buffer.alloc(size))).toBeNull(); + } + }); + }); +}); diff --git a/packages/beacon-node/test/utils/beforeValueBenchmark.ts b/packages/beacon-node/test/utils/beforeValueBenchmark.ts new file mode 100644 index 000000000000..5bca88fe4df2 --- /dev/null +++ b/packages/beacon-node/test/utils/beforeValueBenchmark.ts @@ -0,0 +1,36 @@ +import {beforeAll} from "@chainsafe/benchmark"; + +export type LazyValue = {value: T}; + +/** + * Register a callback to compute a value in the before() block of benchmark tests + * ```ts + * const state = beforeValue(() => getState()) + * it("test", () => { + * doTest(state.value) + * }) + * ``` + */ +export function beforeValue(fn: () => T | Promise, timeout?: number): LazyValue { + let value: T = null as unknown as T; + + beforeAll(async () => { + value = await fn(); + }, timeout ?? 300_000); + + return new Proxy<{value: T}>( + {value}, + { + get: (_target, prop) => { + if (prop === "value") { + if (value === null) { + throw Error("beforeValue has not yet run the before() block"); + } + return value; + } + + return undefined; + }, + } + ); +} diff --git a/packages/beacon-node/test/utils/blockInput.ts b/packages/beacon-node/test/utils/blockInput.ts index 34b9d0747f07..23e97c555a30 100644 --- a/packages/beacon-node/test/utils/blockInput.ts +++ b/packages/beacon-node/test/utils/blockInput.ts @@ -91,6 +91,10 @@ export class MockBlockInput implements IBlockInput { return this._timeCompleted ?? 0; } + getSerializedCacheKeys(): object[] { + return this._block ? [this._block] : []; + } + waitForAllData(_timeout: number, _signal?: AbortSignal): Promise { return Promise.resolve(null); } diff --git a/packages/beacon-node/test/utils/chain/stateCache/datastore.ts b/packages/beacon-node/test/utils/chain/stateCache/datastore.ts index 20d1708c1045..c4bb52acc43c 100644 --- a/packages/beacon-node/test/utils/chain/stateCache/datastore.ts +++ b/packages/beacon-node/test/utils/chain/stateCache/datastore.ts @@ -3,8 +3,8 @@ import {CPStateDatastore, checkpointToDatastoreKey} from "../../../../src/chain/ export function getTestDatastore(fileApisBuffer: Map): CPStateDatastore { const datastore: CPStateDatastore = { - write: (cp, stateBytes) => { - const persistentKey = checkpointToDatastoreKey(cp); + write: (cp, stateBytes, payloadPresent) => { + const persistentKey = checkpointToDatastoreKey(cp, payloadPresent); const stringKey = toHexString(persistentKey); if (!fileApisBuffer.has(stringKey)) { fileApisBuffer.set(stringKey, stateBytes); diff --git a/packages/beacon-node/test/utils/db.ts b/packages/beacon-node/test/utils/db.ts index eb0f546f702b..9206d9ca657b 100644 --- a/packages/beacon-node/test/utils/db.ts +++ b/packages/beacon-node/test/utils/db.ts @@ -2,8 +2,8 @@ import childProcess from "node:child_process"; import {ChainForkConfig} from "@lodestar/config"; import {FilterOptions} from "@lodestar/db"; import {LevelDbController} from "@lodestar/db/controller/level"; +import {testLogger} from "@lodestar/logger/test-utils"; import {BeaconDb} from "../../src/index.js"; -import {testLogger} from "./logger.js"; export const TEMP_DB_LOCATION = ".tmpdb"; diff --git a/packages/beacon-node/test/utils/networkWithMockDb.ts b/packages/beacon-node/test/utils/networkWithMockDb.ts index edf388a1fdf2..6bef00294cbf 100644 --- a/packages/beacon-node/test/utils/networkWithMockDb.ts +++ b/packages/beacon-node/test/utils/networkWithMockDb.ts @@ -1,7 +1,7 @@ import {generateKeyPair} from "@libp2p/crypto/keys"; -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {ChainForkConfig, createBeaconConfig} from "@lodestar/config"; -import {Index2PubkeyCache, createCachedBeaconState, syncPubkeys} from "@lodestar/state-transition"; +import {testLogger} from "@lodestar/logger/test-utils"; +import {createCachedBeaconState, createPubkeyCache, syncPubkeys} from "@lodestar/state-transition"; import {ssz} from "@lodestar/types"; import {sleep} from "@lodestar/utils"; import {BeaconChain} from "../../src/chain/chain.js"; @@ -12,7 +12,6 @@ import {NetworkOptions, defaultNetworkOptions} from "../../src/network/options.j import {GetReqRespHandlerFn} from "../../src/network/reqresp/types.js"; import {getMockedBeaconDb} from "../mocks/mockedBeaconDb.js"; import {ClockStatic} from "./clock.js"; -import {testLogger} from "./logger.js"; import {generateState} from "./state.js"; export type NetworkForTestOpts = { @@ -43,15 +42,13 @@ export async function getNetworkForTest( ); const beaconConfig = createBeaconConfig(config, state.genesisValidatorsRoot); - const pubkey2index = new PubkeyIndexMap(); - const index2pubkey: Index2PubkeyCache = []; - syncPubkeys(state.validators.getAllReadonlyValues(), pubkey2index, index2pubkey); + const pubkeyCache = createPubkeyCache(); + syncPubkeys(pubkeyCache, state.validators.getAllReadonlyValues()); const cachedState = createCachedBeaconState( state, { config: beaconConfig, - pubkey2index, - index2pubkey, + pubkeyCache, }, {skipSyncPubkeys: true} ); @@ -73,8 +70,7 @@ export async function getNetworkForTest( { privateKey, config: beaconConfig, - pubkey2index, - index2pubkey, + pubkeyCache, db, dataDir: ".", dbName: ".", diff --git a/packages/beacon-node/test/utils/node/beacon.ts b/packages/beacon-node/test/utils/node/beacon.ts index e97341004b43..4f9b369f3635 100644 --- a/packages/beacon-node/test/utils/node/beacon.ts +++ b/packages/beacon-node/test/utils/node/beacon.ts @@ -4,18 +4,18 @@ import deepmerge from "deepmerge"; import tmp from "tmp"; import {setHasher} from "@chainsafe/persistent-merkle-tree"; import {hasher} from "@chainsafe/persistent-merkle-tree/hasher/hashtree"; -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {ChainConfig, createBeaconConfig, createChainForkConfig} from "@lodestar/config"; import {config as minimalConfig} from "@lodestar/config/default"; import {LevelDbController} from "@lodestar/db/controller/level"; import {LoggerNode} from "@lodestar/logger/node"; +import {testLogger} from "@lodestar/logger/test-utils"; import {ForkSeq, GENESIS_SLOT} from "@lodestar/params"; import { BeaconStateAllForks, - Index2PubkeyCache, computeAnchorCheckpoint, computeEpochAtSlot, createCachedBeaconState, + createPubkeyCache, syncPubkeys, } from "@lodestar/state-transition"; import {phase0, ssz} from "@lodestar/types"; @@ -27,7 +27,6 @@ import {defaultNetworkOptions} from "../../../src/network/options.js"; import {IBeaconNodeOptions, defaultOptions} from "../../../src/node/options.js"; import {InteropStateOpts} from "../../../src/node/utils/interop/state.js"; import {initDevState} from "../../../src/node/utils/state.js"; -import {testLogger} from "../logger.js"; export async function getDevBeaconNode( opts: { @@ -133,15 +132,13 @@ export async function getDevBeaconNode( ); const beaconConfig = createBeaconConfig(config, anchorState.genesisValidatorsRoot); - const pubkey2index = new PubkeyIndexMap(); - const index2pubkey: Index2PubkeyCache = []; - syncPubkeys(anchorState.validators.getAllReadonlyValues(), pubkey2index, index2pubkey); + const pubkeyCache = createPubkeyCache(); + syncPubkeys(pubkeyCache, anchorState.validators.getAllReadonlyValues()); const cachedState = createCachedBeaconState( anchorState, { config: beaconConfig, - pubkey2index, - index2pubkey, + pubkeyCache, }, {skipSyncPubkeys: true} ); @@ -149,8 +146,7 @@ export async function getDevBeaconNode( return BeaconNode.init({ opts: options as IBeaconNodeOptions, config: beaconConfig, - pubkey2index, - index2pubkey, + pubkeyCache, db, logger, processShutdownCallback: () => {}, diff --git a/packages/beacon-node/test/utils/node/p2p.ts b/packages/beacon-node/test/utils/node/p2p.ts index 1a6e1d987b29..c4399f65f8c0 100644 --- a/packages/beacon-node/test/utils/node/p2p.ts +++ b/packages/beacon-node/test/utils/node/p2p.ts @@ -1,10 +1,10 @@ -import {Direction, PeerId} from "@libp2p/interface"; +import type {MessageStreamDirection, PeerId} from "@libp2p/interface"; import {routes} from "@lodestar/api"; export function lodestarNodePeer( peer: PeerId, state: routes.node.PeerState, - direction: Direction | null + direction: MessageStreamDirection | null ): routes.lodestar.LodestarNodePeer { return { peerId: peer.toString(), diff --git a/packages/beacon-node/test/utils/node/validator.ts b/packages/beacon-node/test/utils/node/validator.ts index e77fb1083f01..4bc28beef3b3 100644 --- a/packages/beacon-node/test/utils/node/validator.ts +++ b/packages/beacon-node/test/utils/node/validator.ts @@ -4,11 +4,11 @@ import {SecretKey} from "@chainsafe/blst"; import {ApiClient, ApiError, ApiResponse, HttpStatusCode} from "@lodestar/api"; import {BeaconApiMethods} from "@lodestar/api/beacon/server"; import {LevelDbController} from "@lodestar/db/controller/level"; +import {TestLoggerOpts, testLogger} from "@lodestar/logger/test-utils"; import {interopSecretKey} from "@lodestar/state-transition"; import {mapValues} from "@lodestar/utils"; import {Signer, SignerType, SlashingProtection, Validator, ValidatorProposerConfig} from "@lodestar/validator"; import {BeaconNode} from "../../../src/index.js"; -import {TestLoggerOpts, testLogger} from "../logger.js"; export async function getAndInitDevValidators({ node, diff --git a/packages/beacon-node/test/utils/state.ts b/packages/beacon-node/test/utils/state.ts index eb20c875740e..d5ee1595e637 100644 --- a/packages/beacon-node/test/utils/state.ts +++ b/packages/beacon-node/test/utils/state.ts @@ -1,7 +1,7 @@ import {SecretKey} from "@chainsafe/blst"; -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {ChainForkConfig, createBeaconConfig} from "@lodestar/config"; import {config as minimalConfig} from "@lodestar/config/default"; +import {getConfig} from "@lodestar/config/test-utils"; import {ExecutionStatus, PayloadStatus, ProtoBlock} from "@lodestar/fork-choice"; import {FAR_FUTURE_EPOCH, ForkName, ForkSeq, MAX_EFFECTIVE_BALANCE, SYNC_COMMITTEE_SIZE} from "@lodestar/params"; import { @@ -13,10 +13,10 @@ import { CachedBeaconStateElectra, DataAvailabilityStatus, createCachedBeaconState, + createPubkeyCache, } from "@lodestar/state-transition"; import {BeaconState, altair, bellatrix, electra, ssz} from "@lodestar/types"; import {ZERO_HASH_HEX} from "../../src/constants/constants.js"; -import {getConfig} from "./config.js"; import {generateValidator, generateValidators} from "./validator.js"; /** @@ -114,8 +114,7 @@ export function generateCachedState(opts?: TestBeaconState): CachedBeaconStateAl return createCachedBeaconState(state, { config: createBeaconConfig(config, state.genesisValidatorsRoot), // This is a performance test, there's no need to have a global shared cache of keys - pubkey2index: new PubkeyIndexMap(), - index2pubkey: [], + pubkeyCache: createPubkeyCache(), }); } @@ -128,8 +127,7 @@ export function generateCachedAltairState(opts?: TestBeaconState, altairForkEpoc return createCachedBeaconState(state, { config: createBeaconConfig(config, state.genesisValidatorsRoot), // This is a performance test, there's no need to have a global shared cache of keys - pubkey2index: new PubkeyIndexMap(), - index2pubkey: [], + pubkeyCache: createPubkeyCache(), }); } @@ -142,8 +140,7 @@ export function generateCachedBellatrixState(opts?: TestBeaconState): CachedBeac return createCachedBeaconState(state as BeaconStateBellatrix, { config: createBeaconConfig(config, state.genesisValidatorsRoot), // This is a performance test, there's no need to have a global shared cache of keys - pubkey2index: new PubkeyIndexMap(), - index2pubkey: [], + pubkeyCache: createPubkeyCache(), }); } @@ -155,8 +152,7 @@ export function generateCachedElectraState(opts?: TestBeaconState, electraForkEp const state = generateState(opts, config); return createCachedBeaconState(state as BeaconStateElectra, { config: createBeaconConfig(config, state.genesisValidatorsRoot), - pubkey2index: new PubkeyIndexMap(), - index2pubkey: [], + pubkeyCache: createPubkeyCache(), }); } export const zeroProtoBlock: ProtoBlock = { @@ -180,7 +176,5 @@ export const zeroProtoBlock: ProtoBlock = { ...{executionPayloadBlockHash: null, executionStatus: ExecutionStatus.PreMerge}, dataAvailabilityStatus: DataAvailabilityStatus.PreData, payloadStatus: PayloadStatus.FULL, - builderIndex: null, - blockHashFromBid: null, parentBlockHash: null, }; diff --git a/packages/beacon-node/test/utils/validationData/aggregateAndProof.ts b/packages/beacon-node/test/utils/validationData/aggregateAndProof.ts index a8ac5833fbd5..be91ae47c35c 100644 --- a/packages/beacon-node/test/utils/validationData/aggregateAndProof.ts +++ b/packages/beacon-node/test/utils/validationData/aggregateAndProof.ts @@ -1,7 +1,7 @@ import {DOMAIN_AGGREGATE_AND_PROOF, DOMAIN_SELECTION_PROOF} from "@lodestar/params"; import {computeSigningRoot} from "@lodestar/state-transition"; +import {getSecretKeyFromIndexCached} from "@lodestar/state-transition/test-utils"; import {phase0, ssz} from "@lodestar/types"; -import {getSecretKeyFromIndexCached} from "../../../../state-transition/test/perf/util.js"; import {IBeaconChain} from "../../../src/chain/index.js"; import {SeenAggregators} from "../../../src/chain/seenCache/index.js"; import {signCached} from "../cache.js"; diff --git a/packages/beacon-node/test/utils/validationData/attestation.ts b/packages/beacon-node/test/utils/validationData/attestation.ts index 61ad1190ef7a..f54ea3748d9b 100644 --- a/packages/beacon-node/test/utils/validationData/attestation.ts +++ b/packages/beacon-node/test/utils/validationData/attestation.ts @@ -1,5 +1,6 @@ import {BitArray, toHexString} from "@chainsafe/ssz"; import {ExecutionStatus, IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; +import {testLogger} from "@lodestar/logger/test-utils"; import {DOMAIN_BEACON_ATTESTER} from "@lodestar/params"; import { DataAvailabilityStatus, @@ -7,11 +8,11 @@ import { computeSigningRoot, computeStartSlotAtEpoch, } from "@lodestar/state-transition"; -import {Slot, SubnetID, phase0, ssz} from "@lodestar/types"; import { generateTestCachedBeaconStateOnlyValidators, getSecretKeyFromIndexCached, -} from "../../../../state-transition/test/perf/util.js"; +} from "@lodestar/state-transition/test-utils"; +import {Slot, SubnetID, phase0, ssz} from "@lodestar/types"; import {BlsMultiThreadWorkerPool, BlsSingleThreadVerifier} from "../../../src/chain/bls/index.js"; import {IBeaconChain} from "../../../src/chain/index.js"; import {defaultChainOptions} from "../../../src/chain/options.js"; @@ -23,7 +24,6 @@ import {ShufflingCache} from "../../../src/chain/shufflingCache.js"; import {ZERO_HASH, ZERO_HASH_HEX} from "../../../src/constants/index.js"; import {signCached} from "../cache.js"; import {ClockStatic} from "../clock.js"; -import {testLogger} from "../logger.js"; export type AttestationValidDataOpts = { currentSlot?: Slot; @@ -82,8 +82,6 @@ export function getAttestationValidData(opts: AttestationValidDataOpts): { parentBlockHash: null, payloadStatus: 2, // PayloadStatus.FULL - builderIndex: null, - blockHashFromBid: null, }; const shufflingCache = new ShufflingCache(null, null, {}, [ @@ -171,13 +169,13 @@ export function getAttestationValidData(opts: AttestationValidDataOpts): { seenAggregatedAttestations: new SeenAggregatedAttestations(null), seenAttestationDatas: new SeenAttestationDatas(null, 0, 0), bls: blsVerifyAllMainThread - ? new BlsSingleThreadVerifier({metrics: null, index2pubkey: state.epochCtx.index2pubkey}) + ? new BlsSingleThreadVerifier({metrics: null, pubkeyCache: state.epochCtx.pubkeyCache}) : new BlsMultiThreadWorkerPool( {}, - {logger: testLogger(), metrics: null, index2pubkey: state.epochCtx.index2pubkey} + {logger: testLogger(), metrics: null, pubkeyCache: state.epochCtx.pubkeyCache} ), waitForBlock: () => Promise.resolve(false), - index2pubkey: state.epochCtx.index2pubkey, + pubkeyCache: state.epochCtx.pubkeyCache, shufflingCache, opts: defaultChainOptions, } as Partial as IBeaconChain; diff --git a/packages/beacon-node/tsconfig.json b/packages/beacon-node/tsconfig.json index d1c8b717b54f..19b9527ba0e3 100644 --- a/packages/beacon-node/tsconfig.json +++ b/packages/beacon-node/tsconfig.json @@ -1,7 +1,9 @@ { "extends": "../../tsconfig.json", - "exclude": ["../../node_modules/it-pipe"], - "include": ["src", "test"], + "include": [ + "src", + "test" + ], "compilerOptions": { "outDir": "lib" } diff --git a/packages/cli/package.json b/packages/cli/package.json index de708f1b7f97..80265a3e7b37 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@chainsafe/lodestar", - "version": "1.40.0", + "version": "1.41.0", "description": "Command line interface for lodestar", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -28,12 +28,12 @@ }, "scripts": { "clean": "rm -rf lib && rm -f *.tsbuildinfo", - "build": "tsc -p tsconfig.build.json && pnpm write-git-data", + "build": "tsgo -p tsconfig.build.json && pnpm write-git-data", "build:release": "pnpm clean && pnpm run build", - "build:watch": "tsc -p tsconfig.build.json --watch", + "build:watch": "tsgo -p tsconfig.build.json --watch", "write-git-data": "node lib/util/gitData/writeGitData.js", "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\" lodestar --help", - "check-types": "tsc", + "check-types": "tsgo", "docs:build": "node --loader ts-node/esm ./docsgen/index.ts", "lint": "biome check src/ test/", "lint:fix": "pnpm run lint --write", @@ -60,15 +60,15 @@ "@chainsafe/bls-keygen": "^0.4.0", "@chainsafe/bls-keystore": "^3.1.0", "@chainsafe/blst": "^2.2.0", - "@chainsafe/discv5": "^11.0.4", - "@chainsafe/enr": "^5.0.1", + "@chainsafe/discv5": "^12.0.1", + "@chainsafe/enr": "^6.0.1", "@chainsafe/persistent-merkle-tree": "^1.2.1", "@chainsafe/pubkey-index-map": "^3.0.0", "@chainsafe/ssz": "^1.2.2", "@chainsafe/threads": "^1.11.3", - "@libp2p/crypto": "^5.0.15", - "@libp2p/interface": "^2.7.0", - "@libp2p/peer-id": "^5.1.0", + "@libp2p/crypto": "^5.1.13", + "@libp2p/interface": "^3.1.0", + "@libp2p/peer-id": "^6.0.4", "@lodestar/api": "workspace:^", "@lodestar/beacon-node": "workspace:^", "@lodestar/config": "workspace:^", @@ -80,7 +80,7 @@ "@lodestar/types": "workspace:^", "@lodestar/utils": "workspace:^", "@lodestar/validator": "workspace:^", - "@multiformats/multiaddr": "^12.1.3", + "@multiformats/multiaddr": "^13.0.1", "deepmerge": "^4.3.1", "ethers": "^6.7.0", "find-up": "^6.3.0", @@ -104,7 +104,7 @@ "@types/tmp": "^0.2.3", "@types/yargs": "^17.0.24", "ethereum-cryptography": "^2.2.1", - "fastify": "^5.7.4", + "fastify": "^5.8.1", "tmp": "^0.2.4", "web3": "^4.0.3", "web3-eth-accounts": "^4.0.3" diff --git a/packages/cli/src/cmds/beacon/handler.ts b/packages/cli/src/cmds/beacon/handler.ts index 93f9141a18da..6b617aa16a9c 100644 --- a/packages/cli/src/cmds/beacon/handler.ts +++ b/packages/cli/src/cmds/beacon/handler.ts @@ -2,13 +2,12 @@ import path from "node:path"; import {getHeapStatistics} from "node:v8"; import {SignableENR} from "@chainsafe/enr"; import {hasher} from "@chainsafe/persistent-merkle-tree"; -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {BeaconDb, BeaconNode} from "@lodestar/beacon-node"; import {ChainForkConfig, createBeaconConfig} from "@lodestar/config"; import {LevelDbController} from "@lodestar/db/controller/level"; import {LoggerNode, getNodeLogger} from "@lodestar/logger/node"; import {ACTIVE_PRESET, PresetName} from "@lodestar/params"; -import {Index2PubkeyCache, createCachedBeaconState, syncPubkeys} from "@lodestar/state-transition"; +import {createCachedBeaconState, createPubkeyCache, syncPubkeys} from "@lodestar/state-transition"; import {ErrorAborted, bytesToInt, formatBytes} from "@lodestar/utils"; import {ProcessShutdownCallback} from "@lodestar/validator"; import {BeaconNodeOptions, getBeaconConfigFromArgs} from "../../config/index.js"; @@ -80,15 +79,13 @@ export async function beaconHandler(args: BeaconArgs & GlobalArgs): Promise n.toString(16)) - .join(":"); - for (const networkInterfaces of Object.values(interfaces)) { for (const networkInterface of networkInterfaces || []) { // since node version 18, the netowrkinterface family returns 4 | 6 instead of ipv4 | ipv6, @@ -57,7 +49,7 @@ export function isLocalMultiAddr(multiaddr: Multiaddr | undefined): boolean { /** * Only update the enr if the value has changed */ -function maybeUpdateEnr( +function maybeUpdateEnr( enr: SignableENR, key: T, value: SignableENR[T] | undefined @@ -74,14 +66,18 @@ export function overwriteEnrWithCliArgs( opts?: {newEnr?: boolean; bootnode?: boolean} ): void { const preSeq = enr.seq; - const {port, discoveryPort, port6, discoveryPort6} = parseListenArgs(args); + const {port, discoveryPort, quicPort, port6, discoveryPort6, quicPort6} = parseListenArgs(args); + const tcp = args.tcp ?? true; + const quic = args.quic ?? false; maybeUpdateEnr(enr, "ip", args["enr.ip"] ?? enr.ip); maybeUpdateEnr(enr, "ip6", args["enr.ip6"] ?? enr.ip6); maybeUpdateEnr(enr, "udp", args["enr.udp"] ?? discoveryPort ?? enr.udp); maybeUpdateEnr(enr, "udp6", args["enr.udp6"] ?? discoveryPort6 ?? enr.udp6); if (!opts?.bootnode) { - maybeUpdateEnr(enr, "tcp", args["enr.tcp"] ?? port ?? enr.tcp); - maybeUpdateEnr(enr, "tcp6", args["enr.tcp6"] ?? port6 ?? enr.tcp6); + maybeUpdateEnr(enr, "tcp", tcp ? (args["enr.tcp"] ?? port ?? enr.tcp) : undefined); + maybeUpdateEnr(enr, "tcp6", tcp ? (args["enr.tcp6"] ?? port6 ?? enr.tcp6) : undefined); + maybeUpdateEnr(enr, "quic", quic ? (args["enr.quic"] ?? quicPort ?? enr.quic) : undefined); + maybeUpdateEnr(enr, "quic6", quic ? (args["enr.quic6"] ?? quicPort6 ?? enr.quic6) : undefined); } function testMultiaddrForLocal(mu: Multiaddr, ip4: boolean): void { @@ -100,9 +96,13 @@ export function overwriteEnrWithCliArgs( if (ip4) { enr.delete("ip"); enr.delete("udp"); + enr.delete("tcp"); + enr.delete("quic"); } else { enr.delete("ip6"); enr.delete("udp6"); + enr.delete("tcp6"); + enr.delete("quic6"); } } } diff --git a/packages/cli/src/cmds/beacon/options.ts b/packages/cli/src/cmds/beacon/options.ts index 82660e4eabc4..65dff288119a 100644 --- a/packages/cli/src/cmds/beacon/options.ts +++ b/packages/cli/src/cmds/beacon/options.ts @@ -174,10 +174,12 @@ export const beaconExtraOptions: CliCommandOptions = { type ENRArgs = { "enr.ip"?: string; "enr.tcp"?: number; - "enr.ip6"?: string; "enr.udp"?: number; + "enr.quic"?: number; + "enr.ip6"?: string; "enr.tcp6"?: number; "enr.udp6"?: number; + "enr.quic6"?: number; nat?: boolean; }; @@ -197,6 +199,11 @@ const enrOptions: CliCommandOptions = { type: "number", group: "enr", }, + "enr.quic": { + description: "Override ENR QUIC entry", + type: "number", + group: "enr", + }, "enr.ip6": { description: "Override ENR IPv6 entry", type: "string", @@ -212,6 +219,11 @@ const enrOptions: CliCommandOptions = { type: "number", group: "enr", }, + "enr.quic6": { + description: "Override ENR (IPv6-specific) QUIC entry", + type: "number", + group: "enr", + }, nat: { type: "boolean", description: "Allow configuration of non-local addresses", diff --git a/packages/cli/src/cmds/bootnode/handler.ts b/packages/cli/src/cmds/bootnode/handler.ts index 639e022bea2c..47ba4daeb707 100644 --- a/packages/cli/src/cmds/bootnode/handler.ts +++ b/packages/cli/src/cmds/bootnode/handler.ts @@ -2,7 +2,7 @@ import path from "node:path"; import {PrivateKey} from "@libp2p/interface"; import {Multiaddr, multiaddr} from "@multiformats/multiaddr"; import {Discv5, Discv5EventEmitter} from "@chainsafe/discv5"; -import {ENR, ENRData, SignableENR} from "@chainsafe/enr"; +import {ENR, SignableENR} from "@chainsafe/enr"; import { HttpMetricsServer, IBeaconNodeOptions, @@ -87,7 +87,7 @@ export async function bootnodeHandler(args: BootnodeArgs & GlobalArgs): Promise< void discv5.findRandomNode(); } - discv5.on("multiaddrUpdated", (addr: ENRData) => { + discv5.on("multiaddrUpdated", (addr: Multiaddr) => { logger.info("Advertised socket address updated", {addr: addr.toString()}); }); diff --git a/packages/cli/src/cmds/dev/options.ts b/packages/cli/src/cmds/dev/options.ts index a79aff6e6707..0df90862be44 100644 --- a/packages/cli/src/cmds/dev/options.ts +++ b/packages/cli/src/cmds/dev/options.ts @@ -1,7 +1,7 @@ import {CliCommandOptions, CliOptionDefinition} from "@lodestar/utils"; import {NetworkName} from "../../networks/index.js"; import {beaconNodeOptions, globalOptions} from "../../options/index.js"; -import {parseRange} from "../../util/format.ts"; +import {parseRange} from "../../util/format.js"; import {BeaconArgs, beaconOptions} from "../beacon/options.js"; import {IValidatorCliArgs, validatorOptions} from "../validator/options.js"; diff --git a/packages/cli/src/cmds/validator/handler.ts b/packages/cli/src/cmds/validator/handler.ts index dd92257ed785..f2aafef98717 100644 --- a/packages/cli/src/cmds/validator/handler.ts +++ b/packages/cli/src/cmds/validator/handler.ts @@ -164,6 +164,7 @@ export async function validatorHandler(args: IValidatorCliArgs & GlobalArgs): Pr globalInit: { requestWireFormat: parseWireFormat(args, "http.requestWireFormat"), responseWireFormat: parseWireFormat(args, "http.responseWireFormat"), + timeoutMs: args["http.requestTimeout"], headers: {"User-Agent": `Lodestar/${version}`}, }, }, diff --git a/packages/cli/src/cmds/validator/options.ts b/packages/cli/src/cmds/validator/options.ts index 191aff6adcd8..a73b3821b118 100644 --- a/packages/cli/src/cmds/validator/options.ts +++ b/packages/cli/src/cmds/validator/options.ts @@ -60,6 +60,7 @@ export type IValidatorCliArgs = AccountValidatorArgs & "http.requestWireFormat"?: string; "http.responseWireFormat"?: string; + "http.requestTimeout"?: number; "clock.skipSlots"?: boolean; @@ -333,6 +334,12 @@ export const validatorOptions: CliCommandOptions = { group: "http", }, + "http.requestTimeout": { + type: "number", + description: "Timeout in milliseconds for HTTP requests to the beacon node", + group: "http", + }, + "clock.skipSlots": { hidden: true, description: "Skip slots when tasks take more than one slot to run", diff --git a/packages/cli/src/options/beaconNodeOptions/network.ts b/packages/cli/src/options/beaconNodeOptions/network.ts index 301a0a0e3bbc..faadadefdc26 100644 --- a/packages/cli/src/options/beaconNodeOptions/network.ts +++ b/packages/cli/src/options/beaconNodeOptions/network.ts @@ -7,20 +7,25 @@ import {YargsError} from "../../util/index.js"; export const defaultListenAddress = "0.0.0.0"; export const defaultListenAddress6 = "::"; export const defaultP2pPort = 9000; +export const defaultQuicPort = 9001; export type NetworkArgs = { discv5?: boolean; listenAddress?: string; port?: number; discoveryPort?: number; + quicPort?: number; listenAddress6?: string; port6?: number; discoveryPort6?: number; + quicPort6?: number; bootnodes?: string[]; targetPeers?: number; subscribeAllSubnets?: boolean; slotsToSubscribeBeforeAggregatorDuty?: number; disablePeerScoring?: boolean; + quic?: boolean; + tcp?: boolean; mdns?: boolean; directPeers?: string[]; "network.maxPeers"?: number; @@ -65,43 +70,79 @@ export function parseListenArgs(args: NetworkArgs) { const listenAddress = args.listenAddress ?? (args.listenAddress6 ? undefined : defaultListenAddress); const port = listenAddress ? (args.port ?? defaultP2pPort) : undefined; const discoveryPort = listenAddress ? (args.discoveryPort ?? port) : undefined; + const quicPort = listenAddress ? (args.quicPort ?? (port !== undefined ? port + 1 : defaultQuicPort)) : undefined; // If listenAddress6 is explicitly set, use it // If listenAddress is not set, use defaultListenAddress6 const listenAddress6 = args.listenAddress6 ?? (args.listenAddress ? undefined : defaultListenAddress6); const port6 = listenAddress6 ? (args.port6 ?? args.port ?? defaultP2pPort) : undefined; const discoveryPort6 = listenAddress6 ? (args.discoveryPort6 ?? port6) : undefined; + const quicPort6 = listenAddress6 + ? (args.quicPort6 ?? (port6 !== undefined ? port6 + 1 : defaultQuicPort)) + : undefined; - return {listenAddress, port, discoveryPort, listenAddress6, port6, discoveryPort6}; + return {listenAddress, port, discoveryPort, quicPort, listenAddress6, port6, discoveryPort6, quicPort6}; } export function parseArgs(args: NetworkArgs): IBeaconNodeOptions["network"] { - const {listenAddress, port, discoveryPort, listenAddress6, port6, discoveryPort6} = parseListenArgs(args); + const {listenAddress, port, discoveryPort, quicPort, listenAddress6, port6, discoveryPort6, quicPort6} = + parseListenArgs(args); + const quic = args.quic ?? false; + const tcp = args.tcp ?? true; + + if (!quic && !tcp) { + throw new YargsError("Cannot disable both TCP and QUIC transports"); + } + // validate ip, ip6, ports const muArgs = { listenAddress: listenAddress ? `/ip4/${listenAddress}` : undefined, port: listenAddress ? `/tcp/${port}` : undefined, discoveryPort: listenAddress ? `/udp/${discoveryPort}` : undefined, + quicPort: listenAddress ? `/udp/${quicPort}/quic-v1` : undefined, listenAddress6: listenAddress6 ? `/ip6/${listenAddress6}` : undefined, port6: listenAddress6 ? `/tcp/${port6}` : undefined, discoveryPort6: listenAddress6 ? `/udp/${discoveryPort6}` : undefined, + quicPort6: listenAddress6 ? `/udp/${quicPort6}/quic-v1` : undefined, }; - for (const key of [ + const keysToValidate: (keyof typeof muArgs)[] = [ "listenAddress", "port", "discoveryPort", "listenAddress6", "port6", "discoveryPort6", - ] as (keyof typeof muArgs)[]) { + ]; + + if (quic) { + keysToValidate.push("quicPort"); + keysToValidate.push("quicPort6"); + } + + for (const key of keysToValidate) { validateMultiaddrArg(muArgs, key); } + if (quic) { + if (discoveryPort !== undefined && quicPort !== undefined && discoveryPort === quicPort) { + throw new YargsError( + `discoveryPort and quicPort must not collide, both are UDP. Got discoveryPort=${discoveryPort} quicPort=${quicPort}` + ); + } + if (discoveryPort6 !== undefined && quicPort6 !== undefined && discoveryPort6 === quicPort6) { + throw new YargsError( + `discoveryPort6 and quicPort6 must not collide, both are UDP. Got discoveryPort6=${discoveryPort6} quicPort6=${quicPort6}` + ); + } + } + const bindMu = listenAddress ? `${muArgs.listenAddress}${muArgs.discoveryPort}` : undefined; - const localMu = listenAddress ? `${muArgs.listenAddress}${muArgs.port}` : undefined; + const localMu = listenAddress && tcp ? `${muArgs.listenAddress}${muArgs.port}` : undefined; + const quicMu = listenAddress && quic ? `${muArgs.listenAddress}${muArgs.quicPort}` : undefined; const bindMu6 = listenAddress6 ? `${muArgs.listenAddress6}${muArgs.discoveryPort6}` : undefined; - const localMu6 = listenAddress6 ? `${muArgs.listenAddress6}${muArgs.port6}` : undefined; + const localMu6 = listenAddress6 && tcp ? `${muArgs.listenAddress6}${muArgs.port6}` : undefined; + const quicMu6 = listenAddress6 && quic ? `${muArgs.listenAddress6}${muArgs.quicPort6}` : undefined; const targetPeers = args.targetPeers; const maxPeers = args["network.maxPeers"] ?? (targetPeers !== undefined ? Math.floor(targetPeers * 1.1) : undefined); @@ -137,11 +178,13 @@ export function parseArgs(args: NetworkArgs): IBeaconNodeOptions["network"] { : null, maxPeers: maxPeers ?? defaultOptions.network.maxPeers, targetPeers: targetPeers ?? defaultOptions.network.targetPeers, - localMultiaddrs: [localMu, localMu6].filter(Boolean) as string[], + localMultiaddrs: [quicMu, quicMu6, localMu, localMu6].filter(Boolean) as string[], subscribeAllSubnets: args.subscribeAllSubnets, slotsToSubscribeBeforeAggregatorDuty: args.slotsToSubscribeBeforeAggregatorDuty ?? defaultOptions.network.slotsToSubscribeBeforeAggregatorDuty, disablePeerScoring: args.disablePeerScoring, + quic, + tcp, connectToDiscv5Bootnodes: args["network.connectToDiscv5Bootnodes"], discv5FirstQueryDelayMs: args["network.discv5FirstQueryDelayMs"], dontSendGossipAttestationsToForkchoice: args["network.dontSendGossipAttestationsToForkchoice"], @@ -192,6 +235,13 @@ export const options: CliCommandOptions = { group: "network", }, + quicPort: { + description: "The UDP port that QUIC will listen on. Defaults to `port` + 1", + type: "number", + defaultDescription: "`port` + 1", + group: "network", + }, + listenAddress6: { type: "string", description: "The IPv6 address to listen for p2p UDP and TCP connections", @@ -214,6 +264,13 @@ export const options: CliCommandOptions = { group: "network", }, + quicPort6: { + description: "The UDP port that QUIC will listen on. Defaults to `port6` + 1", + type: "number", + defaultDescription: "`port6` + 1", + group: "network", + }, + bootnodes: { type: "array", description: "Bootnodes for discv5 discovery", @@ -254,6 +311,21 @@ export const options: CliCommandOptions = { group: "network", }, + quic: { + type: "boolean", + description: "Enable QUIC transport", + default: false, + group: "network", + }, + + tcp: { + hidden: true, + type: "boolean", + description: "Enable TCP transport", + default: true, + group: "network", + }, + mdns: { type: "boolean", description: "Enable mdns local peer discovery", diff --git a/packages/cli/test/unit/cmds/initPeerIdAndEnr.test.ts b/packages/cli/test/unit/cmds/initPeerIdAndEnr.test.ts index b07e9b85873e..a02bf8f8bc69 100644 --- a/packages/cli/test/unit/cmds/initPeerIdAndEnr.test.ts +++ b/packages/cli/test/unit/cmds/initPeerIdAndEnr.test.ts @@ -1,11 +1,107 @@ import fs from "node:fs"; +import {generateKeyPair} from "@libp2p/crypto/keys"; import {peerIdFromPrivateKey} from "@libp2p/peer-id"; import tmp from "tmp"; import {afterEach, beforeEach, describe, expect, it} from "vitest"; -import {initPrivateKeyAndEnr} from "../../../src/cmds/beacon/initPeerIdAndEnr.js"; +import {SignableENR} from "@chainsafe/enr"; +import {initPrivateKeyAndEnr, overwriteEnrWithCliArgs} from "../../../src/cmds/beacon/initPeerIdAndEnr.js"; import {BeaconArgs} from "../../../src/cmds/beacon/options.js"; import {testLogger} from "../../utils.js"; +describe("overwriteEnrWithCliArgs", () => { + it("should set tcp but not quic fields by default", async () => { + const privateKey = await generateKeyPair("secp256k1"); + const enr = SignableENR.createFromPrivateKey(privateKey); + const logger = testLogger(); + + overwriteEnrWithCliArgs(enr, {listenAddress: "0.0.0.0", port: 9000, nat: true} as unknown as BeaconArgs, logger); + + expect(enr.tcp).toBe(9000); + expect(enr.quic).toBeUndefined(); + }); + + it("should set both tcp and quic fields when quic is true", async () => { + const privateKey = await generateKeyPair("secp256k1"); + const enr = SignableENR.createFromPrivateKey(privateKey); + const logger = testLogger(); + + overwriteEnrWithCliArgs( + enr, + {listenAddress: "0.0.0.0", port: 9000, quic: true, nat: true} as unknown as BeaconArgs, + logger + ); + + expect(enr.tcp).toBe(9000); + expect(enr.quic).toBe(9001); + }); + + it("should not set tcp fields when tcp is false", async () => { + const privateKey = await generateKeyPair("secp256k1"); + const enr = SignableENR.createFromPrivateKey(privateKey); + const logger = testLogger(); + + overwriteEnrWithCliArgs( + enr, + {listenAddress: "0.0.0.0", port: 9000, tcp: false, quic: true, nat: true} as unknown as BeaconArgs, + logger + ); + + expect(enr.tcp).toBeUndefined(); + expect(enr.tcp6).toBeUndefined(); + expect(enr.quic).toBe(9001); + }); + + it("should not set quic fields when quic is false", async () => { + const privateKey = await generateKeyPair("secp256k1"); + const enr = SignableENR.createFromPrivateKey(privateKey); + const logger = testLogger(); + + overwriteEnrWithCliArgs( + enr, + {listenAddress: "0.0.0.0", port: 9000, quic: false, nat: true} as unknown as BeaconArgs, + logger + ); + + expect(enr.tcp).toBe(9000); + expect(enr.quic).toBeUndefined(); + expect(enr.quic6).toBeUndefined(); + }); + + it("should clear pre-existing tcp fields when tcp is false", async () => { + const privateKey = await generateKeyPair("secp256k1"); + const enr = SignableENR.createFromPrivateKey(privateKey); + enr.tcp = 9000; + const logger = testLogger(); + + overwriteEnrWithCliArgs( + enr, + {listenAddress: "0.0.0.0", port: 9000, tcp: false, nat: true} as unknown as BeaconArgs, + logger + ); + + expect(enr.tcp).toBeUndefined(); + }); + + it("should preserve existing enr quic port when no explicit quicPort given", async () => { + const privateKey = await generateKeyPair("secp256k1"); + const enr = SignableENR.createFromPrivateKey(privateKey); + // Simulate a persisted ENR that already had quic advertised + enr.quic = 9001; + enr.quic6 = 9002; + const logger = testLogger(); + + // No quicPort arg, no enr.quic arg — should fall back to existing enr value + overwriteEnrWithCliArgs( + enr, + {listenAddress: "0.0.0.0", port: 9000, quic: true, nat: true} as unknown as BeaconArgs, + logger + ); + + expect(enr.quic).toBe(9001); + expect(enr.quic6).toBe(9002); + }); +}); + describe("initPeerIdAndEnr", () => { let tmpDir: tmp.DirResult; diff --git a/packages/cli/test/unit/db.test.ts b/packages/cli/test/unit/db.test.ts index b39c9492c9cb..3735ff5a255a 100644 --- a/packages/cli/test/unit/db.test.ts +++ b/packages/cli/test/unit/db.test.ts @@ -1,6 +1,6 @@ import {describe, it} from "vitest"; -import {Bucket as BeaconBucket} from "../../../beacon-node/src/db/buckets.js"; -import {Bucket as ValidatorBucket} from "../../../validator/src/buckets.js"; +import {Bucket as BeaconBucket} from "@lodestar/beacon-node/db"; +import {Bucket as ValidatorBucket} from "@lodestar/validator"; describe("no db bucket overlap", () => { it("beacon and validator dn buckets do not overlap", () => { diff --git a/packages/cli/test/unit/options/beaconNodeOptions.test.ts b/packages/cli/test/unit/options/beaconNodeOptions.test.ts index cef866b30624..a240ad133eae 100644 --- a/packages/cli/test/unit/options/beaconNodeOptions.test.ts +++ b/packages/cli/test/unit/options/beaconNodeOptions.test.ts @@ -2,6 +2,7 @@ import {describe, expect, it} from "vitest"; import {ArchiveMode, IBeaconNodeOptions} from "@lodestar/beacon-node"; import {RecursivePartial} from "@lodestar/utils"; import {BeaconNodeArgs, parseBeaconNodeArgs} from "../../../src/options/beaconNodeOptions/index.js"; +import {NetworkArgs, parseArgs as parseNetworkArgs} from "../../../src/options/beaconNodeOptions/network.js"; describe("options / beaconNodeOptions", () => { it("Should parse BeaconNodeArgs", () => { @@ -67,6 +68,7 @@ describe("options / beaconNodeOptions", () => { listenAddress: "127.0.0.1", port: 9001, discoveryPort: 9002, + quicPort: 9003, bootnodes: [ "enr:-KG4QOtcP9X1FbIMOe17QNMKqDxCpm14jcX5tiOE4_TyMrFqbmhPZHK_ZPG2Gxb1GE2xdtodOfx9-cgvNtxnRyHEmC0ghGV0aDKQ9aX9QgAAAAD__________4JpZIJ2NIJpcIQDE8KdiXNlY3AyNTZrMaEDhpehBDbZjM_L9ek699Y7vhUJ-eAdMyQW_Fil522Y0fODdGNwgiMog3VkcIIjKA", ], @@ -191,11 +193,13 @@ describe("options / beaconNodeOptions", () => { gossipsubDHigh: 6, gossipsubAwaitHandler: true, mdns: false, + quic: false, rateLimitMultiplier: 1, maxGossipTopicConcurrency: 64, useWorker: true, maxYoungGenerationSizeMb: 152, targetGroupPeers: 12, + tcp: true, directPeers: ["/ip4/192.168.1.1/tcp/9000/p2p/16Uiu2HAkuWPWqF4W3aw9oo5Yw79v5muzBaaGTGKMmuqjPfEyfkwu"], }, sync: { @@ -211,3 +215,110 @@ describe("options / beaconNodeOptions", () => { expect(options).toEqual(expectedOptions); }); }); + +describe("options / network / tcp and quic flags", () => { + it("should include only tcp multiaddrs by default", () => { + const result = parseNetworkArgs({listenAddress: "0.0.0.0", port: 9000} as NetworkArgs); + expect(result.localMultiaddrs).toContain("/ip4/0.0.0.0/tcp/9000"); + expect(result.localMultiaddrs).not.toContain("/ip4/0.0.0.0/udp/9001/quic-v1"); + }); + + it("should include both tcp and quic multiaddrs when quic is true", () => { + const result = parseNetworkArgs({listenAddress: "0.0.0.0", port: 9000, quic: true} as NetworkArgs); + expect(result.localMultiaddrs).toContain("/ip4/0.0.0.0/tcp/9000"); + expect(result.localMultiaddrs).toContain("/ip4/0.0.0.0/udp/9001/quic-v1"); + }); + + it("should exclude tcp multiaddrs when tcp is false", () => { + const result = parseNetworkArgs({listenAddress: "0.0.0.0", port: 9000, tcp: false, quic: true} as NetworkArgs); + const tcpAddrs = result.localMultiaddrs.filter((mu) => mu.includes("/tcp/")); + expect(tcpAddrs).toHaveLength(0); + expect(result.localMultiaddrs).toContain("/ip4/0.0.0.0/udp/9001/quic-v1"); + expect(result.tcp).toBe(false); + }); + + it("should exclude quic multiaddrs when quic is false", () => { + const result = parseNetworkArgs({listenAddress: "0.0.0.0", port: 9000, quic: false} as NetworkArgs); + const quicAddrs = result.localMultiaddrs.filter((mu) => mu.includes("/quic")); + expect(quicAddrs).toHaveLength(0); + expect(result.localMultiaddrs).toContain("/ip4/0.0.0.0/tcp/9000"); + expect(result.quic).toBe(false); + }); + + it("should not validate derived quicPort when quic is false", () => { + const result = parseNetworkArgs({listenAddress: "0.0.0.0", port: 65535, quic: false} as NetworkArgs); + expect(result.localMultiaddrs).toContain("/ip4/0.0.0.0/tcp/65535"); + expect(result.localMultiaddrs).not.toContain("/ip4/0.0.0.0/udp/65536/quic-v1"); + }); + + it("should not validate explicit quicPort when quic is false", () => { + const result = parseNetworkArgs({ + listenAddress: "0.0.0.0", + port: 9000, + quic: false, + quicPort: 65536, + } as NetworkArgs); + expect(result.quic).toBe(false); + }); + + it("should exclude ipv6 tcp multiaddrs when tcp is false", () => { + const result = parseNetworkArgs({ + listenAddress: "0.0.0.0", + listenAddress6: "::", + port: 9000, + tcp: false, + quic: true, + } as NetworkArgs); + const tcpAddrs = result.localMultiaddrs.filter((mu) => mu.includes("/tcp/")); + expect(tcpAddrs).toHaveLength(0); + // quic for both ipv4 and ipv6 should still be present + const quicAddrs = result.localMultiaddrs.filter((mu) => mu.includes("/quic")); + expect(quicAddrs).toHaveLength(2); + }); + + it("should pass tcp through to network options", () => { + const result = parseNetworkArgs({listenAddress: "0.0.0.0", port: 9000, tcp: false, quic: true} as NetworkArgs); + expect(result.tcp).toBe(false); + }); + + it("should throw when both TCP and QUIC are disabled", () => { + expect(() => + parseNetworkArgs({listenAddress: "0.0.0.0", port: 9000, tcp: false, quic: false} as NetworkArgs) + ).toThrow("Cannot disable both TCP and QUIC transports"); + }); + + it("should throw when discoveryPort and quicPort collide", () => { + expect(() => + parseNetworkArgs({ + listenAddress: "0.0.0.0", + port: 9000, + discoveryPort: 9001, + quicPort: 9001, + quic: true, + } as NetworkArgs) + ).toThrow(/discoveryPort and quicPort must not collide/); + }); + + it("should not throw on port collision when quic is false", () => { + const result = parseNetworkArgs({ + listenAddress: "0.0.0.0", + port: 9000, + discoveryPort: 9001, + quicPort: 9001, + quic: false, + } as NetworkArgs); + expect(result.quic).toBe(false); + }); + + it("should throw when discoveryPort6 and quicPort6 collide", () => { + expect(() => + parseNetworkArgs({ + listenAddress6: "::", + port6: 9000, + discoveryPort6: 9001, + quicPort6: 9001, + quic: true, + } as NetworkArgs) + ).toThrow(/discoveryPort6 and quicPort6 must not collide/); + }); +}); diff --git a/packages/cli/test/utils/crucible/utils/ports.ts b/packages/cli/test/utils/crucible/utils/ports.ts index 3b6f51ad1429..e512a77d8133 100644 --- a/packages/cli/test/utils/crucible/utils/ports.ts +++ b/packages/cli/test/utils/crucible/utils/ports.ts @@ -15,7 +15,8 @@ export const getNodePorts = ( execution: {p2pPort: number; enginePort: number; httpPort: number}; } => ({ beacon: { - p2pPort: BN_P2P_BASE_PORT + 1 + nodeIndex, + // Stride by 2 to leave room for the QUIC port (p2pPort + 1) + p2pPort: BN_P2P_BASE_PORT + 1 + nodeIndex * 2, httpPort: BN_REST_BASE_PORT + 1 + nodeIndex, }, validator: { diff --git a/packages/cli/test/utils/mockBeaconApiServer.ts b/packages/cli/test/utils/mockBeaconApiServer.ts index 24b8b9c7473b..cac8bb3f90b4 100644 --- a/packages/cli/test/utils/mockBeaconApiServer.ts +++ b/packages/cli/test/utils/mockBeaconApiServer.ts @@ -3,9 +3,9 @@ import {BeaconApiMethods, registerRoutes} from "@lodestar/api/beacon/server"; import {RestApiServer, RestApiServerModules, RestApiServerOpts} from "@lodestar/beacon-node"; import {ChainForkConfig} from "@lodestar/config"; import {config} from "@lodestar/config/default"; +import {testLogger} from "@lodestar/logger/test-utils"; import {ssz} from "@lodestar/types"; import {fromHex, toHex} from "@lodestar/utils"; -import {testLogger} from "../../../beacon-node/test/utils/logger.js"; const ZERO_HASH_HEX = toHex(Buffer.alloc(32, 0)); diff --git a/packages/config/package.json b/packages/config/package.json index 2c296fef32be..552ea9d5a336 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/config", - "version": "1.40.0", + "version": "1.41.0", "description": "Chain configuration required for lodestar", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -29,6 +29,11 @@ "bun": "./src/configs.ts", "types": "./lib/configs.d.ts", "import": "./lib/configs.js" + }, + "./test-utils": { + "bun": "./src/testUtils/index.ts", + "types": "./lib/testUtils/index.d.ts", + "import": "./lib/testUtils/index.js" } }, "files": [ @@ -38,11 +43,11 @@ ], "scripts": { "clean": "rm -rf lib && rm -f *.tsbuildinfo", - "build": "tsc -p tsconfig.build.json", + "build": "tsgo -p tsconfig.build.json", "build:release": "pnpm run clean && pnpm run build", "build:watch": "pnpm run build --watch", "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\"", - "check-types": "tsc", + "check-types": "tsgo", "lint": "biome check src/", "lint:fix": "pnpm run lint --write", "test": "pnpm test:unit", @@ -63,6 +68,7 @@ "@chainsafe/as-sha256": "^1.2.0", "@chainsafe/ssz": "^1.2.2", "@lodestar/params": "workspace:^", + "@lodestar/spec-test-util": "workspace:^", "@lodestar/types": "workspace:^", "@lodestar/utils": "workspace:^" } diff --git a/packages/config/src/chainConfig/configs/mainnet.ts b/packages/config/src/chainConfig/configs/mainnet.ts index 10a2242c6910..db958b3e04e8 100644 --- a/packages/config/src/chainConfig/configs/mainnet.ts +++ b/packages/config/src/chainConfig/configs/mainnet.ts @@ -3,7 +3,7 @@ import {fromHex as b} from "@lodestar/utils"; import {ChainConfig} from "../types.js"; // Mainnet config -// https://github.com/ethereum/consensus-specs/blob/dev/configs/mainnet.yaml +// https://github.com/ethereum/consensus-specs/blob/master/configs/mainnet.yaml export const chainConfig: ChainConfig = { // Extends the mainnet preset diff --git a/packages/config/src/chainConfig/configs/minimal.ts b/packages/config/src/chainConfig/configs/minimal.ts index 200e64ed61af..92d6910b7e35 100644 --- a/packages/config/src/chainConfig/configs/minimal.ts +++ b/packages/config/src/chainConfig/configs/minimal.ts @@ -3,7 +3,7 @@ import {fromHex as b} from "@lodestar/utils"; import {ChainConfig} from "../types.js"; // Minimal config -// https://github.com/ethereum/consensus-specs/blob/dev/configs/minimal.yaml +// https://github.com/ethereum/consensus-specs/blob/master/configs/minimal.yaml export const chainConfig: ChainConfig = { // Extends the minimal preset diff --git a/packages/config/src/chainConfig/networks/chiado.ts b/packages/config/src/chainConfig/networks/chiado.ts index aa5d573cc8b8..f794b55fc337 100644 --- a/packages/config/src/chainConfig/networks/chiado.ts +++ b/packages/config/src/chainConfig/networks/chiado.ts @@ -42,7 +42,7 @@ export const chiadoChainConfig: ChainConfig = { ELECTRA_FORK_EPOCH: 948224, // Thu Mar 06 2025 09:43:40 GMT+0000 // Fulu FULU_FORK_VERSION: b("0x0600006f"), - FULU_FORK_EPOCH: Infinity, + FULU_FORK_EPOCH: 1353216, // Mon Mar 16 2026 09:33:00 GMT+0000 // Gloas GLOAS_FORK_VERSION: b("0x0700006f"), GLOAS_FORK_EPOCH: Infinity, diff --git a/packages/config/src/chainConfig/networks/gnosis.ts b/packages/config/src/chainConfig/networks/gnosis.ts index 3a8a711c9be5..2f97347f7caa 100644 --- a/packages/config/src/chainConfig/networks/gnosis.ts +++ b/packages/config/src/chainConfig/networks/gnosis.ts @@ -33,6 +33,7 @@ export const gnosisChainConfig: ChainConfig = { // Networking MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 16384, + MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS: 16384, // Dec 8, 2021, 13:00 UTC MIN_GENESIS_TIME: 1638968400, @@ -57,7 +58,7 @@ export const gnosisChainConfig: ChainConfig = { ELECTRA_FORK_EPOCH: 1337856, // 2025-04-30T14:03:40.000Z // Fulu FULU_FORK_VERSION: b("0x06000064"), - FULU_FORK_EPOCH: Infinity, + FULU_FORK_EPOCH: 1714688, // 2026-04-14T12:06:20.000Z // Gloas GLOAS_FORK_VERSION: b("0x07000064"), GLOAS_FORK_EPOCH: Infinity, diff --git a/packages/beacon-node/test/utils/config.ts b/packages/config/src/testUtils/config.ts similarity index 92% rename from packages/beacon-node/test/utils/config.ts rename to packages/config/src/testUtils/config.ts index c9bbf215e0fa..a3768bb9adf4 100644 --- a/packages/beacon-node/test/utils/config.ts +++ b/packages/config/src/testUtils/config.ts @@ -1,7 +1,8 @@ -import {ChainForkConfig, createBeaconConfig, createChainForkConfig} from "@lodestar/config"; -import {config as chainConfig} from "@lodestar/config/default"; import {ForkName} from "@lodestar/params"; -import {ZERO_HASH} from "../../src/constants/index.js"; +import {config as chainConfig} from "../default.js"; +import {ChainForkConfig, createBeaconConfig, createChainForkConfig} from "../index.js"; + +export const ZERO_HASH = Buffer.alloc(32, 0); /** default config with ZERO_HASH as genesisValidatorsRoot */ export const config = createBeaconConfig(chainConfig, ZERO_HASH); diff --git a/packages/config/src/testUtils/index.ts b/packages/config/src/testUtils/index.ts new file mode 100644 index 000000000000..d1bc97e80940 --- /dev/null +++ b/packages/config/src/testUtils/index.ts @@ -0,0 +1 @@ +export * from "./config.js"; diff --git a/packages/config/test/e2e/ensure-config-is-synced.test.ts b/packages/config/test/e2e/ensure-config-is-synced.test.ts index 96afac82d784..9380e1e308f6 100644 --- a/packages/config/test/e2e/ensure-config-is-synced.test.ts +++ b/packages/config/test/e2e/ensure-config-is-synced.test.ts @@ -1,9 +1,9 @@ import {describe, expect, it, vi} from "vitest"; import {fetch, fromHex} from "@lodestar/utils"; -import {ethereumConsensusSpecsTests} from "../../../beacon-node/test/spec/specTestVersioning.js"; import {chainConfig as mainnetChainConfig} from "../../src/chainConfig/configs/mainnet.js"; import {chainConfig as minimalChainConfig} from "../../src/chainConfig/configs/minimal.js"; import {ChainConfig} from "../../src/chainConfig/types.js"; +import specTestsVersions from "../spec-tests-version.json" with {type: "json"}; // Not e2e, but slow. Run with e2e tests @@ -61,12 +61,18 @@ describe("Ensure chainConfig is synced", () => { vi.setConfig({testTimeout: 60 * 1000}); it("mainnet chainConfig values match spec", async () => { - const remoteConfig = await downloadRemoteConfig("mainnet", ethereumConsensusSpecsTests.specVersion); + const remoteConfig = await downloadRemoteConfig( + "mainnet", + specTestsVersions.ethereumConsensusSpecsTests.specVersion + ); assertCorrectConfig({...mainnetChainConfig}, remoteConfig); }); it("minimal chainConfig values match spec", async () => { - const remoteConfig = await downloadRemoteConfig("minimal", ethereumConsensusSpecsTests.specVersion); + const remoteConfig = await downloadRemoteConfig( + "minimal", + specTestsVersions.ethereumConsensusSpecsTests.specVersion + ); assertCorrectConfig({...minimalChainConfig}, remoteConfig); }); }); diff --git a/packages/config/test/spec-tests-version.json b/packages/config/test/spec-tests-version.json new file mode 120000 index 000000000000..bc2e3fd621b2 --- /dev/null +++ b/packages/config/test/spec-tests-version.json @@ -0,0 +1 @@ +../../../spec-tests-version.json \ No newline at end of file diff --git a/packages/config/tsconfig.json b/packages/config/tsconfig.json index a0f4f2a31e93..19b9527ba0e3 100644 --- a/packages/config/tsconfig.json +++ b/packages/config/tsconfig.json @@ -1,6 +1,9 @@ { "extends": "../../tsconfig.json", - "include": ["src", "test"], + "include": [ + "src", + "test" + ], "compilerOptions": { "outDir": "lib" } diff --git a/packages/db/package.json b/packages/db/package.json index dba1d1c5af7d..0d74a4ec2135 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/db", - "version": "1.40.0", + "version": "1.41.0", "description": "DB modules of Lodestar", "author": "ChainSafe Systems", "homepage": "https://github.com/ChainSafe/lodestar#readme", @@ -30,11 +30,11 @@ ], "scripts": { "clean": "rm -rf lib && rm -f *.tsbuildinfo", - "build": "tsc -p tsconfig.build.json", + "build": "tsgo -p tsconfig.build.json", "build:watch": "pnpm run build --watch", "build:release": "pnpm clean && pnpm run build", "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\"", - "check-types": "tsc", + "check-types": "tsgo", "lint": "biome check src/ test/", "lint:fix": "pnpm run lint --write", "test": "pnpm test:unit", @@ -46,8 +46,7 @@ "@chainsafe/ssz": "^1.2.2", "@lodestar/config": "workspace:^", "@lodestar/utils": "workspace:^", - "classic-level": "^1.4.1", - "it-all": "^3.0.4" + "classic-level": "^1.4.1" }, "devDependencies": { "@lodestar/logger": "workspace:^" diff --git a/packages/db/test/unit/controller/level.test.ts b/packages/db/test/unit/controller/level.test.ts index 9bd0b68f6dbd..f265e562c149 100644 --- a/packages/db/test/unit/controller/level.test.ts +++ b/packages/db/test/unit/controller/level.test.ts @@ -1,6 +1,5 @@ import {execSync} from "node:child_process"; import os from "node:os"; -import all from "it-all"; import {afterEach, beforeEach, describe, expect, it} from "vitest"; import {getEnvLogger} from "@lodestar/logger/env"; import {LevelDbController} from "../../../src/index.js"; @@ -168,7 +167,7 @@ describe("LevelDB controller", () => { gte: k1, lte: k2, }); - const result = await all(resultStream); + const result = await Array.fromAsync(resultStream); expect(result.length).toBe(2); }); @@ -180,7 +179,7 @@ describe("LevelDB controller", () => { const result = await db.entries({limit: 3}); expect(result.length).toBe(3); const resultStream = db.entriesStream({limit: 3}); - expect((await all(resultStream)).length).toBe(3); + expect((await Array.fromAsync(resultStream)).length).toBe(3); }); it("test reverse", async () => { diff --git a/packages/era/package.json b/packages/era/package.json index 7947184eeb17..ca0add58b0aa 100644 --- a/packages/era/package.json +++ b/packages/era/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.40.0", + "version": "1.41.0", "type": "module", "exports": { ".": { @@ -27,11 +27,11 @@ ], "scripts": { "clean": "rm -rf lib && rm -f *.tsbuildinfo", - "build": "tsc -p tsconfig.build.json", + "build": "tsgo -p tsconfig.build.json", "build:watch": "pnpm run build --watch", "build:release": "pnpm clean && pnpm run build", "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\"", - "check-types": "tsc", + "check-types": "tsgo", "lint": "biome check src/ test/", "lint:fix": "pnpm run lint --write", "test": "pnpm test:unit", diff --git a/packages/era/src/e2s.ts b/packages/era/src/e2s.ts index 0060cd167d7f..58bd97f0cc4e 100644 --- a/packages/era/src/e2s.ts +++ b/packages/era/src/e2s.ts @@ -1,7 +1,7 @@ import type {FileHandle} from "node:fs/promises"; import {Slot} from "@lodestar/types"; import {byteArrayEquals} from "@lodestar/utils"; -import {readInt48, readUint16, readUint32, writeInt48, writeUint16, writeUint32} from "./util.ts"; +import {readInt48, readUint16, readUint32, writeInt48, writeUint16, writeUint32} from "./util.js"; /** * Known entry types in an E2Store (.e2s) file along with their exact 2-byte codes. diff --git a/packages/era/src/era/reader.ts b/packages/era/src/era/reader.ts index bf7f8df3460b..675a835a9aa3 100644 --- a/packages/era/src/era/reader.ts +++ b/packages/era/src/era/reader.ts @@ -5,15 +5,15 @@ import {ChainForkConfig, createCachedGenesis} from "@lodestar/config"; import {DOMAIN_BEACON_PROPOSER, GENESIS_SLOT, SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params"; import {BeaconState, SignedBeaconBlock, Slot, ssz} from "@lodestar/types"; import {byteArrayEquals} from "@lodestar/utils"; -import {E2STORE_HEADER_SIZE, EntryType, readEntry, readVersion} from "../e2s.ts"; -import {snappyUncompress} from "../util.ts"; +import {E2STORE_HEADER_SIZE, EntryType, readEntry, readVersion} from "../e2s.js"; +import {snappyUncompress} from "../util.js"; import { EraIndices, computeEraNumberFromBlockSlot, parseEraName, readAllEraIndices, readSlotFromBeaconStateBytes, -} from "./util.ts"; +} from "./util.js"; /** * EraReader is responsible for reading and validating ERA files. diff --git a/packages/era/src/era/util.ts b/packages/era/src/era/util.ts index b49f7fdb4aee..554fba589b97 100644 --- a/packages/era/src/era/util.ts +++ b/packages/era/src/era/util.ts @@ -2,8 +2,8 @@ import type {FileHandle} from "node:fs/promises"; import {ChainForkConfig} from "@lodestar/config"; import {SLOTS_PER_HISTORICAL_ROOT, isForkPostCapella} from "@lodestar/params"; import {BeaconState, Slot, capella, ssz} from "@lodestar/types"; -import {E2STORE_HEADER_SIZE, SlotIndex, readSlotIndex} from "../e2s.ts"; -import {readUint48} from "../util.ts"; +import {E2STORE_HEADER_SIZE, SlotIndex, readSlotIndex} from "../e2s.js"; +import {readUint48} from "../util.js"; /** * Parsed components of an .era file name. diff --git a/packages/era/src/era/writer.ts b/packages/era/src/era/writer.ts index d0aeaf95763b..d377ce9f0ac9 100644 --- a/packages/era/src/era/writer.ts +++ b/packages/era/src/era/writer.ts @@ -3,14 +3,14 @@ import {format, parse} from "node:path"; import {ChainForkConfig} from "@lodestar/config"; import {SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params"; import {BeaconState, SignedBeaconBlock, Slot} from "@lodestar/types"; -import {E2STORE_HEADER_SIZE, EntryType, SlotIndex, serializeSlotIndex, writeEntry} from "../e2s.ts"; -import {snappyCompress} from "../util.ts"; +import {E2STORE_HEADER_SIZE, EntryType, SlotIndex, serializeSlotIndex, writeEntry} from "../e2s.js"; +import {snappyCompress} from "../util.js"; import { computeStartBlockSlotFromEraNumber, getShortHistoricalRoot, isSlotInRange, isValidEraStateSlot, -} from "./util.ts"; +} from "./util.js"; enum WriterStateType { InitGroup, diff --git a/packages/era/src/util.ts b/packages/era/src/util.ts index d8afcc4e5540..02df02258162 100644 --- a/packages/era/src/util.ts +++ b/packages/era/src/util.ts @@ -1,5 +1,4 @@ -import {Uint8ArrayList} from "uint8arraylist"; -import {SnappyFramesUncompress, encodeSnappy} from "@lodestar/reqresp/utils"; +import {decodeSnappyFrames, encodeSnappy} from "@lodestar/reqresp/utils"; /** Read 48-bit signed integer (little-endian) at offset. */ export function readInt48(bytes: Uint8Array, offset: number): number { @@ -38,16 +37,7 @@ export function writeUint32(target: Uint8Array, offset: number, v: number): void /** Decompress snappy-framed data */ export function snappyUncompress(compressedData: Uint8Array): Uint8Array { - const decompressor = new SnappyFramesUncompress(); - - const input = new Uint8ArrayList(compressedData); - const result = decompressor.uncompress(input); - - if (result === null) { - throw new Error("Snappy decompression failed - no data returned"); - } - - return result.subarray(); + return decodeSnappyFrames(compressedData).subarray(); } /** Compress data using snappy framing */ diff --git a/packages/era/test/e2e-mainnet/era.readwrite.integration.test.ts b/packages/era/test/e2e-mainnet/era.readwrite.integration.test.ts index e7b783e252df..f8ab9dc42691 100644 --- a/packages/era/test/e2e-mainnet/era.readwrite.integration.test.ts +++ b/packages/era/test/e2e-mainnet/era.readwrite.integration.test.ts @@ -5,7 +5,7 @@ import {beforeAll, describe, expect, it} from "vitest"; import {ChainForkConfig, createChainForkConfig} from "@lodestar/config"; import {mainnetChainConfig} from "@lodestar/config/networks"; import {SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params"; -import {EraReader, EraWriter} from "../../src/era/index.ts"; +import {EraReader, EraWriter} from "../../src/era/index.js"; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); diff --git a/packages/era/test/unit/era.unit.test.ts b/packages/era/test/unit/era.unit.test.ts index ebb927e9fa48..1b6c0006b344 100644 --- a/packages/era/test/unit/era.unit.test.ts +++ b/packages/era/test/unit/era.unit.test.ts @@ -1,5 +1,5 @@ import {assert, describe, it} from "vitest"; -import {E2STORE_HEADER_SIZE, EntryType, parseEntryHeader} from "../../src/e2s.ts"; +import {E2STORE_HEADER_SIZE, EntryType, parseEntryHeader} from "../../src/e2s.js"; function header(type: EntryType, dataLen: number): Uint8Array { const h = new Uint8Array(8); diff --git a/packages/flare/package.json b/packages/flare/package.json index 45ada05108ce..a65332cfbdb2 100644 --- a/packages/flare/package.json +++ b/packages/flare/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/flare", - "version": "1.40.0", + "version": "1.41.0", "description": "Beacon chain debugging tool", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -26,11 +26,11 @@ }, "scripts": { "clean": "rm -rf lib && rm -f *.tsbuildinfo", - "build": "tsc -p tsconfig.build.json", + "build": "tsgo -p tsconfig.build.json", "build:release": "pnpm run clean && pnpm run build", "build:watch": "pnpm run build --watch", "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\" flare --help", - "check-types": "tsc", + "check-types": "tsgo", "lint": "biome check src/", "lint:fix": "pnpm run lint --write", "test": "pnpm test:unit", diff --git a/packages/fork-choice/README.md b/packages/fork-choice/README.md index e51b0aa1d256..eb52ca219b10 100644 --- a/packages/fork-choice/README.md +++ b/packages/fork-choice/README.md @@ -2,7 +2,7 @@ > This package is part of [ChainSafe's Lodestar](https://lodestar.chainsafe.io) project -Lodestar implementation of the [Ethereum Consensus fork choice](https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/fork-choice.md). +Lodestar implementation of the [Ethereum Consensus fork choice](https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/fork-choice.md). ## Usage diff --git a/packages/fork-choice/package.json b/packages/fork-choice/package.json index cdea9f49076e..3f0d0b227ea0 100644 --- a/packages/fork-choice/package.json +++ b/packages/fork-choice/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.40.0", + "version": "1.41.0", "type": "module", "exports": { ".": { @@ -27,11 +27,11 @@ ], "scripts": { "clean": "rm -rf lib && rm -f *.tsbuildinfo", - "build": "tsc -p tsconfig.build.json", + "build": "tsgo -p tsconfig.build.json", "build:watch": "pnpm run build --watch", "build:release": "pnpm clean && pnpm run build", "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\"", - "check-types": "tsc", + "check-types": "tsgo", "lint": "biome check src/ test/", "lint:fix": "pnpm run lint --write", "test": "pnpm test:unit", diff --git a/packages/fork-choice/src/forkChoice/forkChoice.ts b/packages/fork-choice/src/forkChoice/forkChoice.ts index 653c2ff79479..63944884badc 100644 --- a/packages/fork-choice/src/forkChoice/forkChoice.ts +++ b/packages/fork-choice/src/forkChoice/forkChoice.ts @@ -54,7 +54,7 @@ import { NotReorgedReason, ShouldOverrideForkChoiceUpdateResult, } from "./interface.js"; -import {CheckpointWithPayload, IForkChoiceStore, JustifiedBalances, toCheckpointWithPayload} from "./store.js"; +import {CheckpointWithPayloadStatus, IForkChoiceStore, JustifiedBalances, toCheckpointWithPayload} from "./store.js"; export type ForkChoiceOpts = { proposerBoost?: boolean; @@ -453,7 +453,7 @@ export class ForkChoice implements IForkChoice { } // No reorg if parentBlock is "not strong" ie. parentBlock's weight is less than or equal to (REORG_PARENT_WEIGHT_THRESHOLD = 160)% of total attester weight - // https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/fork-choice.md#is_parent_strong + // https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/fork-choice.md#is_parent_strong const parentThreshold = getCommitteeFraction(this.fcStore.justified.totalBalance, { slotsPerEpoch: SLOTS_PER_EPOCH, committeePercent: this.config.REORG_PARENT_WEIGHT_THRESHOLD, @@ -581,11 +581,11 @@ export class ForkChoice implements IForkChoice { return this.protoArray.nodes; } - getFinalizedCheckpoint(): CheckpointWithPayload { + getFinalizedCheckpoint(): CheckpointWithPayloadStatus { return this.fcStore.finalizedCheckpoint; } - getJustifiedCheckpoint(): CheckpointWithPayload { + getJustifiedCheckpoint(): CheckpointWithPayloadStatus { return this.fcStore.justified.checkpoint; } @@ -665,13 +665,16 @@ export class ForkChoice implements IForkChoice { // Check block is a descendant of the finalized block at the checkpoint finalized slot. const blockAncestorNode = this.getAncestor(parentRootHex, finalizedSlot); - const finalizedRoot = this.fcStore.finalizedCheckpoint.rootHex; - if (blockAncestorNode.blockRoot !== finalizedRoot) { + const fcStoreFinalized = this.fcStore.finalizedCheckpoint; + if ( + blockAncestorNode.blockRoot !== fcStoreFinalized.rootHex || + blockAncestorNode.payloadStatus !== fcStoreFinalized.payloadStatus + ) { throw new ForkChoiceError({ code: ForkChoiceErrorCode.INVALID_BLOCK, err: { code: InvalidBlockCode.NOT_FINALIZED_DESCENDANT, - finalizedRoot, + finalizedRoot: fcStoreFinalized.rootHex, blockAncestor: blockAncestorNode.blockRoot, }, }); @@ -721,8 +724,8 @@ export class ForkChoice implements IForkChoice { // This is an optimization. It should reduce the amount of times we run // `process_justification_and_finalization` by approximately 1/3rd when the chain is // performing optimally. - let unrealizedJustifiedCheckpoint: CheckpointWithPayload; - let unrealizedFinalizedCheckpoint: CheckpointWithPayload; + let unrealizedJustifiedCheckpoint: CheckpointWithPayloadStatus; + let unrealizedFinalizedCheckpoint: CheckpointWithPayloadStatus; if (this.opts?.computeUnrealized) { if ( parentBlock.unrealizedJustifiedEpoch === blockEpoch && @@ -830,27 +833,22 @@ export class ForkChoice implements IForkChoice { return parentBlock.executionPayloadNumber; } - // Parent is Gloas. - // If child extends FULL parent (by bid hash), parent execution number advances by 1 - // relative to EMPTY/PENDING path. FULL variant may not exist locally yet. - if (parentBlock.blockHashFromBid !== null && parentBlockHashFromBid === parentBlock.blockHashFromBid) { - const parentFullVariant = this.getBlockHex(parentRootHex, PayloadStatus.FULL); - if (parentFullVariant && parentFullVariant.executionPayloadBlockHash !== null) { - return parentFullVariant.executionPayloadNumber; - } - return parentBlock.executionPayloadNumber + 1; + // Parent is Gloas: get the variant that matches the parentBlockHash from bid + const parentVariant = this.getBlockHexAndBlockHash(parentRootHex, parentBlockHashFromBid); + if (parentVariant && parentVariant.executionPayloadBlockHash !== null) { + return parentVariant.executionPayloadNumber; } - + // Fallback to parent block's number (we know it's post-merge from check above) return parentBlock.executionPayloadNumber; })(), - executionStatus: this.getPostMergeExecStatus(executionStatus), + executionStatus: this.getPostGloasExecStatus(executionStatus), dataAvailabilityStatus, } : isExecutionBlockBodyType(block.body) && isExecutionStateType(state) && isExecutionEnabled(state, block) ? { executionPayloadBlockHash: toRootHex(block.body.executionPayload.blockHash), executionPayloadNumber: block.body.executionPayload.blockNumber, - executionStatus: this.getPostMergeExecStatus(executionStatus), + executionStatus: this.getPreGloasExecStatus(executionStatus), dataAvailabilityStatus, } : { @@ -860,14 +858,10 @@ export class ForkChoice implements IForkChoice { }), payloadStatus: isGloasBeaconBlock(block) ? PayloadStatus.PENDING : PayloadStatus.FULL, - builderIndex: isGloasBeaconBlock(block) ? block.body.signedExecutionPayloadBid.message.builderIndex : null, - blockHashFromBid: isGloasBeaconBlock(block) - ? toRootHex(block.body.signedExecutionPayloadBid.message.blockHash) - : null, parentBlockHash: parentHashHex, }; - this.protoArray.onBlock(protoBlock, currentSlot); + this.protoArray.onBlock(protoBlock, currentSlot, this.proposerBoostRoot); return protoBlock; } @@ -990,8 +984,8 @@ export class ForkChoice implements IForkChoice { * Updates the PTC votes for multiple validators attesting to a block * Spec: gloas/fork-choice.md#new-on_payload_attestation_message */ - notifyPtcMessage(blockRoot: RootHex, ptcIndices: number[], payloadPresent: boolean): void { - this.protoArray.notifyPtcMessage(blockRoot, ptcIndices, payloadPresent); + notifyPtcMessages(blockRoot: RootHex, ptcIndices: number[], payloadPresent: boolean): void { + this.protoArray.notifyPtcMessages(blockRoot, ptcIndices, payloadPresent); } /** @@ -1010,7 +1004,8 @@ export class ForkChoice implements IForkChoice { this.fcStore.currentSlot, executionPayloadBlockHash, executionPayloadNumber, - executionPayloadStateRoot + executionPayloadStateRoot, + this.proposerBoostRoot ); } @@ -1158,8 +1153,13 @@ export class ForkChoice implements IForkChoice { * Always returns `false` if either input roots are unknown. * Still returns `true` if `ancestorRoot===descendantRoot` (and the roots are known) */ - isDescendant(ancestorRoot: RootHex, descendantRoot: RootHex): boolean { - return this.protoArray.isDescendant(ancestorRoot, descendantRoot); + isDescendant( + ancestorRoot: RootHex, + ancestorPayloadStatus: PayloadStatus, + descendantRoot: RootHex, + descendantPayloadStatus: PayloadStatus + ): boolean { + return this.protoArray.isDescendant(ancestorRoot, ancestorPayloadStatus, descendantRoot, descendantPayloadStatus); } /** @@ -1202,16 +1202,16 @@ export class ForkChoice implements IForkChoice { * Iterates backwards through block summaries, starting from a block root. * Return only the non-finalized blocks. */ - iterateAncestorBlocks(blockRoot: RootHex): IterableIterator { - return this.protoArray.iterateAncestorNodes(blockRoot); + iterateAncestorBlocks(blockRoot: RootHex, payloadStatus: PayloadStatus): IterableIterator { + return this.protoArray.iterateAncestorNodes(blockRoot, payloadStatus); } /** * Returns all blocks backwards starting from a block root. * Return only the non-finalized blocks. */ - getAllAncestorBlocks(block: ProtoBlock): ProtoBlock[] { - const blocks = this.protoArray.getAllAncestorNodes(block.blockRoot, block.payloadStatus); + getAllAncestorBlocks(blockRoot: RootHex, payloadStatus: PayloadStatus): ProtoBlock[] { + const blocks = this.protoArray.getAllAncestorNodes(blockRoot, payloadStatus); // the last node is the previous finalized one, it's there to check onBlock finalized checkpoint only. return blocks.slice(0, blocks.length - 1); } @@ -1219,8 +1219,8 @@ export class ForkChoice implements IForkChoice { /** * The same to iterateAncestorBlocks but this gets non-ancestor nodes instead of ancestor nodes. */ - getAllNonAncestorBlocks(blockRoot: RootHex): ProtoBlock[] { - return this.protoArray.getAllNonAncestorNodes(blockRoot); + getAllNonAncestorBlocks(blockRoot: RootHex, payloadStatus: PayloadStatus): ProtoBlock[] { + return this.protoArray.getAllNonAncestorNodes(blockRoot, payloadStatus); } /** @@ -1245,7 +1245,7 @@ export class ForkChoice implements IForkChoice { return this.head; } - for (const block of this.protoArray.iterateAncestorNodes(this.head.blockRoot)) { + for (const block of this.protoArray.iterateAncestorNodes(this.head.blockRoot, this.head.payloadStatus)) { if (block.blockRoot === blockRootHex) { return block; } @@ -1263,7 +1263,7 @@ export class ForkChoice implements IForkChoice { return this.head; } - for (const block of this.protoArray.iterateAncestorNodes(this.head.blockRoot)) { + for (const block of this.protoArray.iterateAncestorNodes(this.head.blockRoot, this.head.payloadStatus)) { if (block.slot === slot) { return block; } @@ -1276,7 +1276,7 @@ export class ForkChoice implements IForkChoice { return this.head; } - for (const block of this.protoArray.iterateAncestorNodes(this.head.blockRoot)) { + for (const block of this.protoArray.iterateAncestorNodes(this.head.blockRoot, this.head.payloadStatus)) { if (slot >= block.slot) { return block; } @@ -1289,24 +1289,16 @@ export class ForkChoice implements IForkChoice { return this.protoArray.nodes; } - // TODO GLOAS: this function is ambiguous, consumer should also provide payload, or it should accept a ProtoBlock instead - // also consumer may want PENDING or EMPTY only - *forwardIterateDescendants(blockRoot: RootHex): IterableIterator { + *forwardIterateDescendants(blockRoot: RootHex, payloadStatus: PayloadStatus): IterableIterator { const rootsInChain = new Set([blockRoot]); - - const blockVariants = this.protoArray.indices.get(blockRoot); - if (blockVariants === undefined) { + const blockIndex = this.protoArray.getNodeIndexByRootAndStatus(blockRoot, payloadStatus); + if (blockIndex === undefined) { throw new ForkChoiceError({ code: ForkChoiceErrorCode.MISSING_PROTO_ARRAY_BLOCK, root: blockRoot, }); } - // Find the minimum index among all variants to start iteration - const blockIndex = Array.isArray(blockVariants) - ? Math.min(...blockVariants.filter((idx) => idx !== undefined)) - : blockVariants; - for (let i = blockIndex + 1; i < this.protoArray.nodes.length; i++) { const node = this.protoArray.nodes[i]; if (rootsInChain.has(node.parentRoot)) { @@ -1467,12 +1459,20 @@ export class ForkChoice implements IForkChoice { return dataAvailabilityStatus; } - private getPostMergeExecStatus( + private getPreGloasExecStatus( executionStatus: MaybeValidExecutionStatus - ): ExecutionStatus.Valid | ExecutionStatus.Syncing | ExecutionStatus.PendingEnvelope { - if (executionStatus === ExecutionStatus.PreMerge) + ): ExecutionStatus.Valid | ExecutionStatus.Syncing { + if (executionStatus === ExecutionStatus.PreMerge || executionStatus === ExecutionStatus.PayloadSeparated) + throw Error( + `Invalid post-merge execution status: expected: ${ExecutionStatus.Syncing} or ${ExecutionStatus.Valid}, got ${executionStatus}` + ); + return executionStatus; + } + + private getPostGloasExecStatus(executionStatus: MaybeValidExecutionStatus): ExecutionStatus.PayloadSeparated { + if (executionStatus !== ExecutionStatus.PayloadSeparated) throw Error( - `Invalid post-merge execution status: expected: ${ExecutionStatus.Syncing}, ${ExecutionStatus.Valid}, or ${ExecutionStatus.PendingEnvelope}, got ${executionStatus}` + `Invalid post-gloas execution status: expected: ${ExecutionStatus.PayloadSeparated}, got ${executionStatus}` ); return executionStatus; } @@ -1498,8 +1498,8 @@ export class ForkChoice implements IForkChoice { * Since this balances are already available the getter is just `() => balances`, without cache interaction */ private updateCheckpoints( - justifiedCheckpoint: CheckpointWithPayload, - finalizedCheckpoint: CheckpointWithPayload, + justifiedCheckpoint: CheckpointWithPayloadStatus, + finalizedCheckpoint: CheckpointWithPayloadStatus, getJustifiedBalances: () => JustifiedBalances ): void { // Update justified checkpoint. @@ -1519,8 +1519,8 @@ export class ForkChoice implements IForkChoice { * Update unrealized checkpoints in store if necessary */ private updateUnrealizedCheckpoints( - unrealizedJustifiedCheckpoint: CheckpointWithPayload, - unrealizedFinalizedCheckpoint: CheckpointWithPayload, + unrealizedJustifiedCheckpoint: CheckpointWithPayloadStatus, + unrealizedFinalizedCheckpoint: CheckpointWithPayloadStatus, getJustifiedBalances: () => JustifiedBalances ): void { if (unrealizedJustifiedCheckpoint.epoch > this.fcStore.unrealizedJustified.checkpoint.epoch) { @@ -1742,7 +1742,7 @@ export class ForkChoice implements IForkChoice { } const existingNextSlot = this.voteNextSlots[validatorIndex]; - if (existingNextSlot === INIT_VOTE_SLOT || nextSlot > existingNextSlot) { + if (existingNextSlot === INIT_VOTE_SLOT || computeEpochAtSlot(nextSlot) > computeEpochAtSlot(existingNextSlot)) { // nextIndex is transfered to currentIndex in computeDeltas() this.voteNextIndices[validatorIndex] = nextIndex; this.voteNextSlots[validatorIndex] = nextSlot; @@ -1872,7 +1872,7 @@ export class ForkChoice implements IForkChoice { } } -// Approximate https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/fork-choice.md#calculate_committee_fraction +// Approximate https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/fork-choice.md#calculate_committee_fraction // Calculates proposer boost score when committeePercent = config.PROPOSER_SCORE_BOOST export function getCommitteeFraction( justifiedTotalActiveBalanceByIncrement: number, @@ -1892,7 +1892,9 @@ export function getCommitteeFraction( * @param checkpointEpoch - The epoch of the checkpoint */ export function getCheckpointPayloadStatus(state: CachedBeaconStateAllForks, checkpointEpoch: number): PayloadStatus { - const fork = state.config.getForkSeq(state.slot); + // Compute checkpoint slot first to determine the correct fork + const checkpointSlot = computeStartSlotAtEpoch(checkpointEpoch); + const fork = state.config.getForkSeq(checkpointSlot); // Pre-Gloas: always FULL if (fork < ForkSeq.gloas) { @@ -1902,7 +1904,6 @@ export function getCheckpointPayloadStatus(state: CachedBeaconStateAllForks, che // For Gloas, check state.execution_payload_availability // - For non-skipped slots at checkpoint: returns false (EMPTY) since payload hasn't arrived yet // - For skipped slots at checkpoint: returns the actual availability status from state - const checkpointSlot = computeStartSlotAtEpoch(checkpointEpoch); const gloasState = state as CachedBeaconStateGloas; const payloadAvailable = gloasState.executionPayloadAvailability.get(checkpointSlot % SLOTS_PER_HISTORICAL_ROOT); diff --git a/packages/fork-choice/src/forkChoice/interface.ts b/packages/fork-choice/src/forkChoice/interface.ts index 9fba4c812360..6093e3ee779b 100644 --- a/packages/fork-choice/src/forkChoice/interface.ts +++ b/packages/fork-choice/src/forkChoice/interface.ts @@ -12,7 +12,7 @@ import { ProtoNode, } from "../protoArray/interface.js"; import {UpdateAndGetHeadOpt} from "./forkChoice.js"; -import {CheckpointWithHex, CheckpointWithPayload} from "./store.js"; +import {CheckpointWithHex, CheckpointWithPayloadStatus} from "./store.js"; export type CheckpointHex = { epoch: Epoch; @@ -25,7 +25,7 @@ export type CheckpointsWithHex = { }; export type CheckpointWithPayloadAndBalance = { - checkpoint: CheckpointWithPayload; + checkpoint: CheckpointWithPayloadStatus; balances: EffectiveBalanceIncrements; }; @@ -124,8 +124,8 @@ export interface IForkChoice { * Retrieve all nodes for the debug API. */ getAllNodes(): ProtoNode[]; - getFinalizedCheckpoint(): CheckpointWithPayload; - getJustifiedCheckpoint(): CheckpointWithPayload; + getFinalizedCheckpoint(): CheckpointWithPayloadStatus; + getJustifiedCheckpoint(): CheckpointWithPayloadStatus; /** * Add `block` to the fork choice DAG. * @@ -189,7 +189,7 @@ export interface IForkChoice { * @param ptcIndices - Array of PTC committee indices that voted * @param payloadPresent - Whether validators attest the payload is present */ - notifyPtcMessage(blockRoot: RootHex, ptcIndices: number[], payloadPresent: boolean): void; + notifyPtcMessages(blockRoot: RootHex, ptcIndices: number[], payloadPresent: boolean): void; /** * Notify fork choice that an execution payload has arrived (Gloas fork) * Creates the FULL variant of a Gloas block when the payload becomes available @@ -246,7 +246,12 @@ export interface IForkChoice { * Always returns `false` if either input roots are unknown. * Still returns `true` if `ancestorRoot===descendantRoot` (and the roots are known) */ - isDescendant(ancestorRoot: RootHex, descendantRoot: RootHex): boolean; + isDescendant( + ancestorRoot: RootHex, + ancestorPayloadStatus: PayloadStatus, + descendantRoot: RootHex, + descendantPayloadStatus: PayloadStatus + ): boolean; /** * Prune items up to a finalized root. */ @@ -255,12 +260,12 @@ export interface IForkChoice { /** * Iterates backwards through ancestor block summaries, starting from a block root */ - iterateAncestorBlocks(blockRoot: RootHex): IterableIterator; - getAllAncestorBlocks(block: ProtoBlock): ProtoBlock[]; + iterateAncestorBlocks(blockRoot: RootHex, payloadStatus: PayloadStatus): IterableIterator; + getAllAncestorBlocks(blockRoot: RootHex, payloadStatus: PayloadStatus): ProtoBlock[]; /** * The same to iterateAncestorBlocks but this gets non-ancestor nodes instead of ancestor nodes. */ - getAllNonAncestorBlocks(blockRoot: RootHex): ProtoBlock[]; + getAllNonAncestorBlocks(blockRoot: RootHex, payloadStatus: PayloadStatus): ProtoBlock[]; /** * Returns both ancestor and non-ancestor blocks in a single traversal. */ @@ -278,7 +283,7 @@ export interface IForkChoice { /** * Iterates forward descendants of blockRoot. Does not yield blockRoot itself */ - forwardIterateDescendants(blockRoot: RootHex): IterableIterator; + forwardIterateDescendants(blockRoot: RootHex, payloadStatus: PayloadStatus): IterableIterator; getBlockSummariesByParentRoot(parentRoot: RootHex): ProtoBlock[]; getBlockSummariesAtSlot(slot: Slot): ProtoBlock[]; /** Returns the distance of common ancestor of nodes to the max of the newNode and the prevNode. */ diff --git a/packages/fork-choice/src/forkChoice/store.ts b/packages/fork-choice/src/forkChoice/store.ts index 8e817987d7e6..6f2129d11f53 100644 --- a/packages/fork-choice/src/forkChoice/store.ts +++ b/packages/fork-choice/src/forkChoice/store.ts @@ -18,7 +18,7 @@ export type CheckpointWithHex = phase0.Checkpoint & {rootHex: RootHex}; * Pre-Gloas: payloadStatus is always FULL (payload embedded in block) * Gloas: determined by state.execution_payload_availability */ -export type CheckpointWithPayload = CheckpointWithHex & {payloadStatus: PayloadStatus}; +export type CheckpointWithPayloadStatus = CheckpointWithHex & {payloadStatus: PayloadStatus}; export type JustifiedBalances = EffectiveBalanceIncrements; @@ -29,7 +29,7 @@ export type JustifiedBalances = EffectiveBalanceIncrements; * @param blockState state that declares justified checkpoint `checkpoint` */ export type JustifiedBalancesGetter = ( - checkpoint: CheckpointWithPayload, + checkpoint: CheckpointWithPayloadStatus, blockState: CachedBeaconStateAllForks ) => JustifiedBalances; @@ -50,8 +50,8 @@ export interface IForkChoiceStore { get justified(): CheckpointWithPayloadAndTotalBalance; set justified(justified: CheckpointWithPayloadAndBalance); unrealizedJustified: CheckpointWithPayloadAndBalance; - finalizedCheckpoint: CheckpointWithPayload; - unrealizedFinalizedCheckpoint: CheckpointWithPayload; + finalizedCheckpoint: CheckpointWithPayloadStatus; + unrealizedFinalizedCheckpoint: CheckpointWithPayloadStatus; justifiedBalancesGetter: JustifiedBalancesGetter; equivocatingIndices: Set; } @@ -62,8 +62,8 @@ export interface IForkChoiceStore { export class ForkChoiceStore implements IForkChoiceStore { private _justified: CheckpointWithPayloadAndTotalBalance; unrealizedJustified: CheckpointWithPayloadAndBalance; - private _finalizedCheckpoint: CheckpointWithPayload; - unrealizedFinalizedCheckpoint: CheckpointWithPayload; + private _finalizedCheckpoint: CheckpointWithPayloadStatus; + unrealizedFinalizedCheckpoint: CheckpointWithPayloadStatus; equivocatingIndices = new Set(); justifiedBalancesGetter: JustifiedBalancesGetter; currentSlot: Slot; @@ -87,8 +87,8 @@ export class ForkChoiceStore implements IForkChoiceStore { */ finalizedPayloadStatus: PayloadStatus, private readonly events?: { - onJustified: (cp: CheckpointWithPayload) => void; - onFinalized: (cp: CheckpointWithPayload) => void; + onJustified: (cp: CheckpointWithPayloadStatus) => void; + onFinalized: (cp: CheckpointWithPayloadStatus) => void; } ) { this.justifiedBalancesGetter = justifiedBalancesGetter; @@ -112,10 +112,10 @@ export class ForkChoiceStore implements IForkChoiceStore { this.events?.onJustified(justified.checkpoint); } - get finalizedCheckpoint(): CheckpointWithPayload { + get finalizedCheckpoint(): CheckpointWithPayloadStatus { return this._finalizedCheckpoint; } - set finalizedCheckpoint(checkpoint: CheckpointWithPayload) { + set finalizedCheckpoint(checkpoint: CheckpointWithPayloadStatus) { const cp = toCheckpointWithPayload(checkpoint, checkpoint.payloadStatus); this._finalizedCheckpoint = cp; this.events?.onFinalized(cp); @@ -136,7 +136,7 @@ export function toCheckpointWithHex(checkpoint: phase0.Checkpoint): CheckpointWi export function toCheckpointWithPayload( checkpoint: phase0.Checkpoint, payloadStatus: PayloadStatus -): CheckpointWithPayload { +): CheckpointWithPayloadStatus { return { ...toCheckpointWithHex(checkpoint), payloadStatus, diff --git a/packages/fork-choice/src/index.ts b/packages/fork-choice/src/index.ts index 99246e5225f8..cf4f75bceffe 100644 --- a/packages/fork-choice/src/index.ts +++ b/packages/fork-choice/src/index.ts @@ -24,7 +24,7 @@ export { export * from "./forkChoice/safeBlocks.js"; export { type CheckpointWithHex, - type CheckpointWithPayload, + type CheckpointWithPayloadStatus, ForkChoiceStore, type IForkChoiceStore, type JustifiedBalancesGetter, @@ -38,5 +38,5 @@ export type { ProtoBlock, ProtoNode, } from "./protoArray/interface.js"; -export {ExecutionStatus, PayloadStatus} from "./protoArray/interface.js"; +export {ExecutionStatus, PayloadStatus, isGloasBlock} from "./protoArray/interface.js"; export {ProtoArray} from "./protoArray/protoArray.js"; diff --git a/packages/fork-choice/src/protoArray/interface.ts b/packages/fork-choice/src/protoArray/interface.ts index a8a2a9047bbb..02e9890297fc 100644 --- a/packages/fork-choice/src/protoArray/interface.ts +++ b/packages/fork-choice/src/protoArray/interface.ts @@ -17,17 +17,27 @@ export const NULL_VOTE_INDEX = 0xffffffff; */ export type VoteIndex = number; +/** + * Execution status of a block in fork choice. + * + * - Valid: Execution payload verified as valid by the EL + * - Syncing: EL is syncing, payload validity unknown (optimistic sync) + * - PreMerge: Block is from before The Merge, no execution payload exists + * - Invalid: Execution payload was invalidated by the EL (post-import status) + * - PayloadSeparated: Gloas beacon block without embedded execution payload. + * The execution payload arrives separately via SignedExecutionPayloadEnvelope. + * Gloas blocks WITH execution payload (FULL variant) use Valid/Invalid/Syncing. + */ export enum ExecutionStatus { Valid = "Valid", Syncing = "Syncing", PreMerge = "PreMerge", Invalid = "Invalid", - /** Post-Gloas blocks with separated payload (bid-only). Pending envelope arrival. */ - PendingEnvelope = "PendingEnvelope", + PayloadSeparated = "PayloadSeparated", } /** - * Payload status for Gloas fork (EIP-7732 separated payloads) + * Payload status for ePBS (Gloas fork) * Spec: gloas/fork-choice.md#constants */ export enum PayloadStatus { @@ -117,12 +127,6 @@ export type ProtoBlock = BlockExtraMeta & { /** Payload status for this node (Gloas fork). Always FULL in pre-gloas */ payloadStatus: PayloadStatus; - // GLOAS: The followings are from bids. They are null in pre-gloas - // Used for execution payload gossip validation - builderIndex: number | null; - // Used for execution payload gossip validation. Not to be confused with executionPayloadBlockHash - blockHashFromBid: RootHex | null; - // Used to determine if this block extends EMPTY or FULL parent variant // Spec: gloas/fork-choice.md#new-get_parent_payload_status parentBlockHash: RootHex | null; diff --git a/packages/fork-choice/src/protoArray/protoArray.ts b/packages/fork-choice/src/protoArray/protoArray.ts index 0c596de5947d..f7486cc03804 100644 --- a/packages/fork-choice/src/protoArray/protoArray.ts +++ b/packages/fork-choice/src/protoArray/protoArray.ts @@ -1,7 +1,8 @@ +import {BitArray} from "@chainsafe/ssz"; import {GENESIS_EPOCH, PTC_SIZE} from "@lodestar/params"; import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {Epoch, RootHex, Slot} from "@lodestar/types"; -import {toRootHex} from "@lodestar/utils"; +import {bitCount, toRootHex} from "@lodestar/utils"; import {ForkChoiceError, ForkChoiceErrorCode} from "../forkChoice/errors.js"; import {LVHExecError, LVHExecErrorCode, ProtoArrayError, ProtoArrayErrorCode} from "./errors.js"; import { @@ -60,14 +61,14 @@ export class ProtoArray { private previousProposerBoost: ProposerBoost | null = null; /** - * PTC (Payload Timeliness Committee) votes per block - * Maps block root to boolean array of size PTC_SIZE (from params: 512 mainnet, 2 minimal) + * PTC (Payload Timeliness Committee) votes per block as bitvectors + * Maps block root to BitArray of PTC_SIZE bits (512 mainnet, 2 minimal) * Spec: gloas/fork-choice.md#modified-store (line 148) * - * ptcVote[blockRoot][i] = true if PTC member i voted payload_present=true + * Bit i is set if PTC member i voted payload_present=true * Used by is_payload_timely() to determine if payload is timely */ - private ptcVote = new Map(); + private ptcVotes = new Map(); constructor({ pruneThreshold, @@ -103,7 +104,8 @@ export class ProtoArray { // We are using the blockROot as the targetRoot, since it always lies on an epoch boundary targetRoot: block.blockRoot, } as ProtoBlock, - currentSlot + currentSlot, + null ); return protoArray; } @@ -167,6 +169,26 @@ export class ProtoArray { return PayloadStatus.PENDING; } + /** + * Get the node index for the default/canonical variant in a single hash lookup. + * - Pre-Gloas blocks: returns the FULL variant index + * - Gloas blocks: returns the PENDING variant index + */ + getDefaultNodeIndex(blockRoot: RootHex): number | undefined { + const variantOrArr = this.indices.get(blockRoot); + if (variantOrArr == null) { + return undefined; + } + + // Pre-Gloas: value is the index directly + if (!Array.isArray(variantOrArr)) { + return variantOrArr; + } + + // Gloas: PENDING is the canonical variant + return variantOrArr[PayloadStatus.PENDING]; + } + /** * Determine which parent payload status a block extends * Spec: gloas/fork-choice.md#new-get_parent_payload_status @@ -176,33 +198,22 @@ export class ProtoArray { * message_block_hash = parent.body.signed_execution_payload_bid.message.block_hash * return PAYLOAD_STATUS_FULL if parent_block_hash == message_block_hash else PAYLOAD_STATUS_EMPTY * - * For post-Gloas blocks, compare child's parent_block_hash with parent's blockHashFromBid - * (message.block_hash), not with parent's executionPayloadBlockHash. + * In lodestar forkchoice, we don't store the full bid, so we compares parent_block_hash in child's bid with executionPayloadBlockHash in parent: + * - If it matches EMPTY variant, return EMPTY + * - If it matches FULL variant, return FULL + * - If no match, throw UNKNOWN_PARENT_BLOCK error + * + * For pre-Gloas blocks: always returns FULL */ getParentPayloadStatus(block: ProtoBlock): PayloadStatus { - const {parentBlockHash} = block; - // Pre-Gloas blocks have payloads embedded, so parents are always FULL + const {parentBlockHash} = block; if (parentBlockHash === null) { return PayloadStatus.FULL; } - const parentVariants = this.indices.get(block.parentRoot); - if (parentVariants === undefined) { - throw new ProtoArrayError({ - code: ProtoArrayErrorCode.UNKNOWN_PARENT_BLOCK, - parentRoot: block.parentRoot, - parentHash: parentBlockHash, - }); - } - - // Transition case: parent is pre-Gloas - if (!Array.isArray(parentVariants)) { - return PayloadStatus.FULL; - } - - const parentPending = this.nodes[parentVariants[PayloadStatus.PENDING]]; - if (!parentPending || parentPending.blockHashFromBid === null) { + const parentBlock = this.getBlockHexAndBlockHash(block.parentRoot, parentBlockHash); + if (parentBlock == null) { throw new ProtoArrayError({ code: ProtoArrayErrorCode.UNKNOWN_PARENT_BLOCK, parentRoot: block.parentRoot, @@ -210,49 +221,32 @@ export class ProtoArray { }); } - return parentBlockHash === parentPending.blockHashFromBid ? PayloadStatus.FULL : PayloadStatus.EMPTY; + return parentBlock.payloadStatus; } /** * Return the parent `ProtoBlock` given its root and block hash. */ getParent(parentRoot: RootHex, parentBlockHash: RootHex | null): ProtoBlock | null { - const parentVariants = this.indices.get(parentRoot); - if (parentVariants === undefined) { - return null; - } - // pre-gloas - if (!Array.isArray(parentVariants)) { - return this.nodes[parentVariants] ?? null; - } - - // post-gloas parent if (parentBlockHash === null) { - throw new ProtoArrayError({ - code: ProtoArrayErrorCode.UNKNOWN_PARENT_BLOCK, - parentRoot, - parentHash: parentBlockHash, - }); - } - - const parentPending = this.nodes[parentVariants[PayloadStatus.PENDING]]; - if (!parentPending || parentPending.blockHashFromBid === null) { - return null; - } - - const wantsFull = parentBlockHash === parentPending.blockHashFromBid; - const desiredStatus = wantsFull ? PayloadStatus.FULL : PayloadStatus.EMPTY; - const desiredIndex = parentVariants[desiredStatus]; - - if (desiredIndex !== undefined) { - return this.nodes[desiredIndex] ?? null; + const parentIndex = this.indices.get(parentRoot); + if (parentIndex === undefined) { + return null; + } + if (Array.isArray(parentIndex)) { + // Gloas block found when pre-gloas expected + throw new ProtoArrayError({ + code: ProtoArrayErrorCode.UNKNOWN_PARENT_BLOCK, + parentRoot, + parentHash: parentBlockHash, + }); + } + return this.nodes[parentIndex] ?? null; } - // FULL may not exist yet if payload envelope hasn't been imported. - // Fall back to EMPTY so block import can proceed optimistically. - const emptyIndex = parentVariants[PayloadStatus.EMPTY]; - return emptyIndex !== undefined ? (this.nodes[emptyIndex] ?? null) : null; + // post-gloas + return this.getBlockHexAndBlockHash(parentRoot, parentBlockHash); } /** @@ -270,7 +264,7 @@ export class ProtoArray { return node.executionPayloadBlockHash === blockHash ? node : null; } - // Post-Gloas, check full variant first (most authoritative) + // Post-Gloas, check empty and full variants const fullNodeIndex = variantIndices[PayloadStatus.FULL]; if (fullNodeIndex !== undefined) { const fullNode = this.nodes[fullNodeIndex]; @@ -279,22 +273,13 @@ export class ProtoArray { } } - // For EMPTY/PENDING variants, executionPayloadBlockHash is the parent's hash, - // not this block's. The child block's bid references this block's execution hash - // via blockHashFromBid, so check that too. This is needed when the envelope - // hasn't been received yet (no FULL variant exists). const emptyNode = this.nodes[variantIndices[PayloadStatus.EMPTY]]; - if (emptyNode && (emptyNode.executionPayloadBlockHash === blockHash || emptyNode.blockHashFromBid === blockHash)) { + if (emptyNode && emptyNode.executionPayloadBlockHash === blockHash) { return emptyNode; } - const pendingNodeIndex = variantIndices[PayloadStatus.PENDING]; - if (pendingNodeIndex !== undefined) { - const pendingNode = this.nodes[pendingNodeIndex]; - if (pendingNode && pendingNode.blockHashFromBid === blockHash) { - return pendingNode; - } - } + // PENDING is the same to EMPTY so not likely we can return it + // also it's only specific for fork-choice return null; } @@ -406,6 +391,7 @@ export class ProtoArray { // We _must_ perform these functions separate from the weight-updating loop above to ensure // that we have a fully coherent set of weights before updating parent // best-child/descendant. + const proposerBoostRoot = proposerBoost?.root ?? null; for (let nodeIndex = this.nodes.length - 1; nodeIndex >= 0; nodeIndex--) { const node = this.nodes[nodeIndex]; if (node === undefined) { @@ -418,7 +404,7 @@ export class ProtoArray { // If the node has a parent, try to update its best-child and best-descendant. const parentIndex = node.parent; if (parentIndex !== undefined) { - this.maybeUpdateBestChildAndDescendant(parentIndex, nodeIndex, currentSlot); + this.maybeUpdateBestChildAndDescendant(parentIndex, nodeIndex, currentSlot, proposerBoostRoot); } } // Post-Gloas: reconcile PENDING.bestDescendant with the deepest branch among EMPTY/FULL. @@ -439,7 +425,7 @@ export class ProtoArray { * * It is only sane to supply an undefined parent for the genesis block */ - onBlock(block: ProtoBlock, currentSlot: Slot): void { + onBlock(block: ProtoBlock, currentSlot: Slot, proposerBoostRoot: RootHex | null): void { // If the block is already known, simply ignore it if (this.hasBlock(block.blockRoot)) { return; @@ -451,9 +437,7 @@ export class ProtoArray { }); } - const isGloas = isGloasBlock(block); - - if (isGloas) { + if (isGloasBlock(block)) { // Gloas: Create PENDING + EMPTY nodes with correct parent relationships // Parent of new PENDING node = parent block's EMPTY or FULL (inter-block edge) // Parent of new EMPTY node = own PENDING node (intra-block edge) @@ -517,7 +501,7 @@ export class ProtoArray { // Update bestChild pointers if (parentIndex !== undefined) { - this.maybeUpdateBestChildAndDescendant(parentIndex, pendingIndex, currentSlot); + this.maybeUpdateBestChildAndDescendant(parentIndex, pendingIndex, currentSlot, proposerBoostRoot); if (pendingNode.executionStatus === ExecutionStatus.Valid) { this.propagateValidExecutionStatusByIndex(parentIndex); @@ -525,11 +509,11 @@ export class ProtoArray { } // Update bestChild for PENDING → EMPTY edge - this.maybeUpdateBestChildAndDescendant(pendingIndex, emptyIndex, currentSlot); + this.maybeUpdateBestChildAndDescendant(pendingIndex, emptyIndex, currentSlot, proposerBoostRoot); // Initialize PTC votes for this block (all false initially) // Spec: gloas/fork-choice.md#modified-on_block (line 645) - this.ptcVote.set(block.blockRoot, new Array(PTC_SIZE).fill(false)); + this.ptcVotes.set(block.blockRoot, BitArray.fromBitLen(PTC_SIZE)); } else { // Pre-Gloas: Only create FULL node (payload embedded in block) const node: ProtoNode = { @@ -550,7 +534,7 @@ export class ProtoArray { // If this node is valid, lets propagate the valid status up the chain // and throw error if we counter invalid, as this breaks consensus if (node.parent !== undefined) { - this.maybeUpdateBestChildAndDescendant(node.parent, nodeIndex, currentSlot); + this.maybeUpdateBestChildAndDescendant(node.parent, nodeIndex, currentSlot, proposerBoostRoot); if (node.executionStatus === ExecutionStatus.Valid) { this.propagateValidExecutionStatusByIndex(node.parent); @@ -571,7 +555,8 @@ export class ProtoArray { currentSlot: Slot, executionPayloadBlockHash: RootHex, executionPayloadNumber: number, - executionPayloadStateRoot: RootHex + executionPayloadStateRoot: RootHex, + proposerBoostRoot: RootHex | null ): void { // First check if block exists const variants = this.indices.get(blockRoot); @@ -621,7 +606,7 @@ export class ProtoArray { weight: 0, bestChild: undefined, bestDescendant: undefined, - executionStatus: ExecutionStatus.Valid, // TODO GLOAS: Review execution status + executionStatus: ExecutionStatus.Valid, executionPayloadBlockHash, executionPayloadNumber, stateRoot: executionPayloadStateRoot, @@ -634,7 +619,7 @@ export class ProtoArray { variants[PayloadStatus.FULL] = fullIndex; // Update bestChild for PENDING node (may now prefer FULL over EMPTY) - this.maybeUpdateBestChildAndDescendant(pendingIndex, fullIndex, currentSlot); + this.maybeUpdateBestChildAndDescendant(pendingIndex, fullIndex, currentSlot, proposerBoostRoot); // Reconcile PENDING bestDescendant: maybeUpdateBestChildAndDescendant may have // set PENDING.bestChild = FULL (via payload-status tiebreaker), but FULL has no @@ -651,8 +636,8 @@ export class ProtoArray { * @param ptcIndices - Array of PTC committee indices that voted (0..PTC_SIZE-1) * @param payloadPresent - Whether the validators attest the payload is present */ - notifyPtcMessage(blockRoot: RootHex, ptcIndices: number[], payloadPresent: boolean): void { - const votes = this.ptcVote.get(blockRoot); + notifyPtcMessages(blockRoot: RootHex, ptcIndices: number[], payloadPresent: boolean): void { + const votes = this.ptcVotes.get(blockRoot); if (votes === undefined) { // Block not found or not a Gloas block, ignore return; @@ -663,8 +648,7 @@ export class ProtoArray { throw new Error(`Invalid PTC index: ${ptcIndex}, must be 0..${PTC_SIZE - 1}`); } - // Update the vote - votes[ptcIndex] = payloadPresent; + votes.set(ptcIndex, payloadPresent); } } @@ -680,7 +664,7 @@ export class ProtoArray { * @param blockRoot - The beacon block root to check */ isPayloadTimely(blockRoot: RootHex): boolean { - const votes = this.ptcVote.get(blockRoot); + const votes = this.ptcVotes.get(blockRoot); if (votes === undefined) { // Block not found or not a Gloas block return false; @@ -694,7 +678,7 @@ export class ProtoArray { } // Count votes for payload_present=true - const yesVotes = votes.filter((v) => v).length; + const yesVotes = bitCount(votes.uint8Array); return yesVotes > PAYLOAD_TIMELY_THRESHOLD; } @@ -734,8 +718,8 @@ export class ProtoArray { // Get proposer boost block // We don't care about variant here, just need proposer boost block info - const defaultStatus = this.getDefaultVariant(proposerBoostRoot); - const proposerBoostBlock = defaultStatus !== undefined ? this.getNode(proposerBoostRoot, defaultStatus) : undefined; + const proposerBoostIndex = this.getDefaultNodeIndex(proposerBoostRoot); + const proposerBoostBlock = proposerBoostIndex !== undefined ? this.getNodeByIndex(proposerBoostIndex) : undefined; if (!proposerBoostBlock) { // Proposer boost block not found, default to extending payload return true; @@ -803,12 +787,8 @@ export class ProtoArray { // if its in fcU. // const {invalidateFromParentBlockRoot, latestValidExecHash} = execResponse; - // TODO GLOAS: verify if getting default variant is correct here - const defaultStatus = this.getDefaultVariant(invalidateFromParentBlockRoot); - const invalidateFromParentIndex = - defaultStatus !== undefined - ? this.getNodeIndexByRootAndStatus(invalidateFromParentBlockRoot, defaultStatus) - : undefined; + // TODO GLOAS: verify if getting the default/canonical node index is correct here + const invalidateFromParentIndex = this.getDefaultNodeIndex(invalidateFromParentBlockRoot); if (invalidateFromParentIndex === undefined) { throw Error(`Unable to find invalidateFromParentBlockRoot=${invalidateFromParentBlockRoot} in forkChoice`); } @@ -843,13 +823,15 @@ export class ProtoArray { // propagate till we keep encountering syncing status while (nodeIndex !== undefined) { const node = this.getNodeFromIndex(nodeIndex); - if ( - node.executionStatus === ExecutionStatus.PreMerge || - node.executionStatus === ExecutionStatus.Valid || - node.executionStatus === ExecutionStatus.PendingEnvelope - ) { + if (node.executionStatus === ExecutionStatus.PreMerge || node.executionStatus === ExecutionStatus.Valid) { break; } + // If PayloadSeparated, that means the node is either PENDING or EMPTY, there could be + // some ancestor still has syncing status. + if (node.executionStatus === ExecutionStatus.PayloadSeparated) { + nodeIndex = node.parent; + continue; + } this.validateNodeByIndex(nodeIndex); nodeIndex = node.parent; } @@ -979,7 +961,7 @@ export class ProtoArray { * For EMPTY/FULL variants from slot n-1: implements tiebreaker logic based on should_extend_payload * For older blocks: returns node.payloadStatus * - * Note: pre-gloas logic won't reach here. Since it is impossible to have two nodes with same weight and root + * Note: pre-gloas logic won't reach here. Pre-Gloas blocks have different roots, so they are always resolved by the weight and root tiebreaker before reaching here. */ private getPayloadStatusTiebreaker(node: ProtoNode, currentSlot: Slot, proposerBoostRoot: RootHex | null): number { // PENDING nodes always return PENDING (no tiebreaker needed) @@ -996,11 +978,11 @@ export class ProtoArray { // For previous slot blocks in Gloas, decide between FULL and EMPTY // based on should_extend_payload if (node.payloadStatus === PayloadStatus.EMPTY) { - return 1; // EMPTY + return PayloadStatus.EMPTY; } // FULL - check should_extend_payload const shouldExtend = this.shouldExtendPayload(node.blockRoot, proposerBoostRoot); - return shouldExtend ? 2 : 0; // Return 2 if extending, else 0 + return shouldExtend ? PayloadStatus.FULL : PayloadStatus.PENDING; } /** @@ -1019,9 +1001,7 @@ export class ProtoArray { } // Get canonical node: FULL for pre-Gloas, PENDING for Gloas - const defaultStatus = this.getDefaultVariant(justifiedRoot); - const justifiedIndex = - defaultStatus !== undefined ? this.getNodeIndexByRootAndStatus(justifiedRoot, defaultStatus) : undefined; + const justifiedIndex = this.getDefaultNodeIndex(justifiedRoot); if (justifiedIndex === undefined) { throw new ProtoArrayError({ code: ProtoArrayErrorCode.JUSTIFIED_NODE_UNKNOWN, @@ -1125,7 +1105,7 @@ export class ProtoArray { this.indices.delete(root); // Prune PTC votes for this block to prevent memory leak // Spec: gloas/fork-choice.md (implicit - finalized blocks don't need PTC votes) - this.ptcVote.delete(root); + this.ptcVotes.delete(root); } // Store nodes prior to finalization @@ -1211,39 +1191,6 @@ export class ProtoArray { * - The child is not the best child and does not become the best child. */ - /** - * Check if we're comparing EMPTY vs FULL variants of the same block from slot n-1. - * - * This is a special case where the spec requires using `get_payload_status_tiebreaker()` - * directly without weight comparison. - * - * Spec: gloas/fork-choice.md#modified-get_weight (lines 413-446) and - * gloas/fork-choice.md#get_payload_status_tiebreaker (lines 331-343) - * - * @returns true if this is the EMPTY vs FULL edge case, false otherwise - */ - private isEmptyVsFullEdgeCase(childNode: ProtoNode, bestChildNode: ProtoNode, currentSlot: Slot): boolean { - // Check if both nodes are: - // 1. The same block root (different payload status variants) - // 2. Both EMPTY or FULL (not PENDING) - // 3. From slot n-1 (previous slot) - if (childNode.blockRoot !== bestChildNode.blockRoot) { - return false; // Different blocks - } - - const childIsEmptyOrFull = childNode.payloadStatus !== PayloadStatus.PENDING; - const bestChildIsEmptyOrFull = bestChildNode.payloadStatus !== PayloadStatus.PENDING; - - if (!childIsEmptyOrFull || !bestChildIsEmptyOrFull) { - return false; // At least one is PENDING - } - - // Check if from slot n-1 - const isFromPreviousSlot = childNode.slot + 1 === currentSlot; - - return isFromPreviousSlot; - } - /** * Reconcile a single PENDING node's bestChild/bestDescendant with the deepest * viable branch among its EMPTY / FULL siblings. @@ -1304,13 +1251,18 @@ export class ProtoArray { let parentIndex = this.nodes[childIndex]?.parent; while (parentIndex !== undefined) { - this.maybeUpdateBestChildAndDescendant(parentIndex, childIndex, currentSlot); + this.maybeUpdateBestChildAndDescendant(parentIndex, childIndex, currentSlot, null); childIndex = parentIndex; parentIndex = this.nodes[childIndex]?.parent; } } - maybeUpdateBestChildAndDescendant(parentIndex: number, childIndex: number, currentSlot: Slot): void { + maybeUpdateBestChildAndDescendant( + parentIndex: number, + childIndex: number, + currentSlot: Slot, + proposerBoostRoot: RootHex | null + ): void { const childNode = this.nodes[childIndex]; if (childNode === undefined) { throw new ProtoArrayError({ @@ -1376,42 +1328,37 @@ export class ProtoArray { // Pre-fulu we pick whichever has higher weight, tie-breaker by root // Post-fulu we pick whichever has higher weight, then tie-breaker by root, then tie-breaker by `getPayloadStatusTiebreaker` - // Edge case: when comparing EMPTY vs FULL variants of the same block from slot n-1, weights are hardcoded to 0 + // Gloas: nodes from previous slot (n-1) with EMPTY/FULL variant have weight hardcoded to 0. // https://github.com/ethereum/consensus-specs/blob/69a2582d5d62c914b24894bdb65f4bd5d4e49ae4/specs/gloas/fork-choice.md?plain=1#L442 - // in this case we use `get_payload_status_tiebreaker()` directly because weights(0) and roots are equal - - // Gloas: Check if this is the EMPTY vs FULL edge case for slot n-1 - // If true, skip weight and root comparison (weights are 0, roots are equal) - const isEdgeCase = this.isEmptyVsFullEdgeCase(childNode, bestChildNode, currentSlot); - - if (!isEdgeCase && childNode.weight !== bestChildNode.weight) { - // Different weights, choose the winner by weight - if (childNode.weight >= bestChildNode.weight) { - newChildAndDescendant = changeToChild; - } else { - newChildAndDescendant = noChange; - } + const childEffectiveWeight = + !isGloasBlock(childNode) || + childNode.payloadStatus === PayloadStatus.PENDING || + childNode.slot + 1 !== currentSlot + ? childNode.weight + : 0; + const bestChildEffectiveWeight = + !isGloasBlock(bestChildNode) || + bestChildNode.payloadStatus === PayloadStatus.PENDING || + bestChildNode.slot + 1 !== currentSlot + ? bestChildNode.weight + : 0; + + if (childEffectiveWeight !== bestChildEffectiveWeight) { + // Different effective weights, choose the winner by weight + newChildAndDescendant = childEffectiveWeight >= bestChildEffectiveWeight ? changeToChild : noChange; break outer; } - if (!isEdgeCase && childNode.blockRoot !== bestChildNode.blockRoot) { + if (childNode.blockRoot !== bestChildNode.blockRoot) { // Different blocks, tie-breaker by root - if (childNode.blockRoot >= bestChildNode.blockRoot) { - newChildAndDescendant = changeToChild; - } else { - newChildAndDescendant = noChange; - } + newChildAndDescendant = childNode.blockRoot >= bestChildNode.blockRoot ? changeToChild : noChange; break outer; } - // Same weight and same root (or edge case), tie-breaker by payload status - const childTiebreaker = this.getPayloadStatusTiebreaker( - childNode, - currentSlot, - null // proposerBoostRoot - ); - - const bestChildTiebreaker = this.getPayloadStatusTiebreaker(bestChildNode, currentSlot, null); + // Same effective weight and same root — Gloas EMPTY vs FULL from n-1, tie-breaker by payload status + // Note: pre-Gloas, each child node of a block has a unique root, so this point should not be reached + const childTiebreaker = this.getPayloadStatusTiebreaker(childNode, currentSlot, proposerBoostRoot); + const bestChildTiebreaker = this.getPayloadStatusTiebreaker(bestChildNode, currentSlot, proposerBoostRoot); if (childTiebreaker > bestChildTiebreaker) { newChildAndDescendant = changeToChild; @@ -1638,11 +1585,8 @@ export class ProtoArray { * For Gloas blocks: returns EMPTY/FULL variants (not PENDING) based on parent payload status * For pre-Gloas blocks: returns FULL variants */ - *iterateAncestorNodes(blockRoot: RootHex): IterableIterator { - // Get canonical node: FULL for pre-Gloas, PENDING for Gloas - const defaultStatus = this.getDefaultVariant(blockRoot); - const startIndex = - defaultStatus !== undefined ? this.getNodeIndexByRootAndStatus(blockRoot, defaultStatus) : undefined; + *iterateAncestorNodes(blockRoot: RootHex, payloadStatus: PayloadStatus): IterableIterator { + const startIndex = this.getNodeIndexByRootAndStatus(blockRoot, payloadStatus); if (startIndex === undefined) { return; } @@ -1695,8 +1639,12 @@ export class ProtoArray { }); } - // Include starting node - const nodes: ProtoNode[] = [node]; + // Exclude PENDING variant from returned ancestors. + const nodes: ProtoNode[] = []; + + if (node.payloadStatus !== PayloadStatus.PENDING) { + nodes.push(node); + } while (node.parent !== undefined) { const parentIndex = this.getParentNodeIndex(node); @@ -1719,13 +1667,8 @@ export class ProtoArray { * For Gloas blocks: returns EMPTY/FULL variants (not PENDING) based on parent payload status * For pre-Gloas blocks: returns FULL variants */ - getAllNonAncestorNodes(blockRoot: RootHex): ProtoNode[] { - // Get canonical node: FULL for pre-Gloas, PENDING for Gloas - const defaultStatus = this.getDefaultVariant(blockRoot); - if (defaultStatus === undefined) { - return []; - } - const startIndex = this.getNodeIndexByRootAndStatus(blockRoot, defaultStatus); + getAllNonAncestorNodes(blockRoot: RootHex, payloadStatus: PayloadStatus): ProtoNode[] { + const startIndex = this.getNodeIndexByRootAndStatus(blockRoot, payloadStatus); if (startIndex === undefined) { return []; } @@ -1785,10 +1728,10 @@ export class ProtoArray { const ancestors: ProtoNode[] = []; const nonAncestors: ProtoNode[] = []; - // Always include starting node — it's the finalized checkpoint and must be archived. - // For Gloas blocks getDefaultVariant() returns PENDING, but the underlying beacon block - // is identical across all variants and must be persisted to the cold DB. - ancestors.push(node); + // Include starting node if it's not PENDING (i.e., pre-Gloas or EMPTY/FULL variant post-Gloas) + if (node.payloadStatus !== PayloadStatus.PENDING) { + ancestors.push(node); + } let nodeIndex = startIndex; while (node.parent !== undefined) { @@ -1819,12 +1762,7 @@ export class ProtoArray { * Uses default variant (PENDING for Gloas, FULL for pre-Gloas) */ hasBlock(blockRoot: RootHex): boolean { - const defaultVariant = this.getDefaultVariant(blockRoot); - if (defaultVariant === undefined) { - return false; - } - const index = this.getNodeIndexByRootAndStatus(blockRoot, defaultVariant); - return index !== undefined; + return this.getDefaultNodeIndex(blockRoot) !== undefined; } /** @@ -1887,26 +1825,28 @@ export class ProtoArray { /** * Returns `true` if the `descendantRoot` has an ancestor with `ancestorRoot`. * Always returns `false` if either input roots are unknown. - * Still returns `true` if `ancestorRoot` === `descendantRoot` (and the roots are known) + * Still returns `true` if `ancestorRoot` === `descendantRoot` and payload statuses match. */ - isDescendant(ancestorRoot: RootHex, descendantRoot: RootHex): boolean { - // We use the default variant (PENDING for Gloas, FULL for pre-Gloas) - // We cannot use FULL/EMPTY variants for Gloas because they may not be canonical - const defaultStatus = this.getDefaultVariant(ancestorRoot); - const ancestorNode = defaultStatus !== undefined ? this.getNode(ancestorRoot, defaultStatus) : undefined; + isDescendant( + ancestorRoot: RootHex, + ancestorPayloadStatus: PayloadStatus, + descendantRoot: RootHex, + descendantPayloadStatus: PayloadStatus + ): boolean { + const ancestorNode = this.getNode(ancestorRoot, ancestorPayloadStatus); if (!ancestorNode) { return false; } - if (ancestorRoot === descendantRoot) { + if (ancestorRoot === descendantRoot && ancestorPayloadStatus === descendantPayloadStatus) { return true; } - for (const node of this.iterateAncestorNodes(descendantRoot)) { + for (const node of this.iterateAncestorNodes(descendantRoot, descendantPayloadStatus)) { if (node.slot < ancestorNode.slot) { return false; } - if (node.blockRoot === ancestorNode.blockRoot) { + if (node.blockRoot === ancestorNode.blockRoot && node.payloadStatus === ancestorNode.payloadStatus) { return true; } } diff --git a/packages/fork-choice/test/perf/forkChoice/util.ts b/packages/fork-choice/test/perf/forkChoice/util.ts index 59fc71630660..85d1918abc4a 100644 --- a/packages/fork-choice/test/perf/forkChoice/util.ts +++ b/packages/fork-choice/test/perf/forkChoice/util.ts @@ -110,11 +110,9 @@ export function initializeForkChoice(opts: Opts): ForkChoice { parentBlockHash: null, payloadStatus: PayloadStatus.FULL, - builderIndex: null, - blockHashFromBid: null, }; - protoArr.onBlock(block, block.slot); + protoArr.onBlock(block, block.slot, null); parentBlockRoot = blockRoot; } diff --git a/packages/fork-choice/test/unit/forkChoice/forkChoice.test.ts b/packages/fork-choice/test/unit/forkChoice/forkChoice.test.ts index ef38ee6a8624..df68c53632dd 100644 --- a/packages/fork-choice/test/unit/forkChoice/forkChoice.test.ts +++ b/packages/fork-choice/test/unit/forkChoice/forkChoice.test.ts @@ -133,8 +133,6 @@ describe("Forkchoice", () => { parentBlockHash: null, payloadStatus: PayloadStatus.FULL, - builderIndex: null, - blockHashFromBid: null, }; }; @@ -142,7 +140,7 @@ describe("Forkchoice", () => { for (let slot = genesisSlot + 1; slot <= tillSlot; slot++) { if (!skippedSlots.includes(slot)) { const block = getBlock(slot, skippedSlots); - protoArr.onBlock(block, block.slot); + protoArr.onBlock(block, block.slot, null); } } }; @@ -150,11 +148,9 @@ describe("Forkchoice", () => { it("getAllAncestorBlocks", () => { // Add block that is a finalized descendant. const block = getBlock(genesisSlot + 1); - protoArr.onBlock(block, block.slot); + protoArr.onBlock(block, block.slot, null); const forkchoice = new ForkChoice(config, fcStore, protoArr, validatorCount, null); - const canonicalBlock = forkchoice.getBlockHexDefaultStatus(getBlockRoot(genesisSlot + 1)); - if (!canonicalBlock) throw new Error("Expected block to exist"); - const summaries = forkchoice.getAllAncestorBlocks(canonicalBlock); + const summaries = forkchoice.getAllAncestorBlocks(getBlockRoot(genesisSlot + 1), PayloadStatus.FULL); // there are 2 blocks in protoArray but iterateAncestorBlocks should only return non-finalized blocks expect(summaries).toHaveLength(1); expect(summaries[0]).toEqual({ @@ -167,18 +163,6 @@ describe("Forkchoice", () => { }); }); - it("getAllAncestorBlocks supports ProtoBlock input", () => { - const block = getBlock(genesisSlot + 1); - protoArr.onBlock(block, block.slot); - const forkchoice = new ForkChoice(config, fcStore, protoArr, validatorCount, null); - const head = forkchoice.getHead(); - - const summaries = forkchoice.getAllAncestorBlocks(head); - - expect(summaries).toHaveLength(1); - expect(summaries[0].blockRoot).toBe(head.blockRoot); - }); - it("getAllAncestorAndNonAncestorBlocks equals getAllAncestorBlocks + getAllNonAncestorBlocks", () => { // Create a simple chain: 0 -> 1 -> 2 -> 3 populateProtoArray(genesisSlot + 3); @@ -188,16 +172,14 @@ describe("Forkchoice", () => { ...getBlock(genesisSlot + 10), parentRoot: finalizedRoot, // Connect directly to genesis }; - protoArr.onBlock(forkBlock, forkBlock.slot); + protoArr.onBlock(forkBlock, forkBlock.slot, null); const forkchoice = new ForkChoice(config, fcStore, protoArr, validatorCount, null); // Test with a block from the canonical chain const canonicalBlockRoot = getBlockRoot(genesisSlot + 3); - const canonicalBlock = forkchoice.getBlockHexDefaultStatus(canonicalBlockRoot); - if (!canonicalBlock) throw new Error("Expected canonical block to exist"); - const canonicalAncestorBlocks = forkchoice.getAllAncestorBlocks(canonicalBlock); - const canonicalNonAncestorBlocks = forkchoice.getAllNonAncestorBlocks(canonicalBlockRoot); + const canonicalAncestorBlocks = forkchoice.getAllAncestorBlocks(canonicalBlockRoot, PayloadStatus.FULL); + const canonicalNonAncestorBlocks = forkchoice.getAllNonAncestorBlocks(canonicalBlockRoot, PayloadStatus.FULL); const canonicalCombined = forkchoice.getAllAncestorAndNonAncestorBlocks(canonicalBlockRoot, PayloadStatus.FULL); expect(canonicalCombined.ancestors).toEqual(canonicalAncestorBlocks); @@ -205,31 +187,14 @@ describe("Forkchoice", () => { // Test with a block from the fork chain const forkBlockRoot = getBlockRoot(genesisSlot + 10); - const forkBlockSummary = forkchoice.getBlockHexDefaultStatus(forkBlockRoot); - if (!forkBlockSummary) throw new Error("Expected fork block to exist"); - const forkAncestorBlocks = forkchoice.getAllAncestorBlocks(forkBlockSummary); - const forkNonAncestorBlocks = forkchoice.getAllNonAncestorBlocks(forkBlockRoot); + const forkAncestorBlocks = forkchoice.getAllAncestorBlocks(forkBlockRoot, PayloadStatus.FULL); + const forkNonAncestorBlocks = forkchoice.getAllNonAncestorBlocks(forkBlockRoot, PayloadStatus.FULL); const forkCombined = forkchoice.getAllAncestorAndNonAncestorBlocks(forkBlockRoot, PayloadStatus.FULL); expect(forkCombined.ancestors).toEqual(forkAncestorBlocks); expect(forkCombined.nonAncestors).toEqual(forkNonAncestorBlocks); }); - it("addLatestMessage updates vote for newer slot within same epoch", () => { - populateProtoArray(genesisSlot + 2); - - const forkchoice = new ForkChoice(config, fcStore, protoArr, validatorCount, null); - forkchoice["addLatestMessage"](0, genesisSlot + 1, getBlockRoot(genesisSlot + 1), PayloadStatus.FULL); - const firstVoteIndex = forkchoice["voteNextIndices"][0]; - - // Same epoch as slot 1 (epoch 0), but newer slot. - // Must update to the newer slot vote. - forkchoice["addLatestMessage"](0, genesisSlot + 2, getBlockRoot(genesisSlot + 2), PayloadStatus.FULL); - - expect(forkchoice["voteNextSlots"][0]).toBe(genesisSlot + 2); - expect(forkchoice["voteNextIndices"][0]).not.toBe(firstVoteIndex); - }); - beforeAll(() => { expect(SLOTS_PER_EPOCH).toBe(32); }); diff --git a/packages/fork-choice/test/unit/forkChoice/getProposerHead.test.ts b/packages/fork-choice/test/unit/forkChoice/getProposerHead.test.ts index ef666a080a39..8c89b5d35dba 100644 --- a/packages/fork-choice/test/unit/forkChoice/getProposerHead.test.ts +++ b/packages/fork-choice/test/unit/forkChoice/getProposerHead.test.ts @@ -52,8 +52,6 @@ describe("Forkchoice / GetProposerHead", () => { parentBlockHash: null, payloadStatus: PayloadStatus.FULL, - builderIndex: null, - blockHashFromBid: null, }; const baseHeadBlock: ProtoBlockWithWeight = { @@ -82,8 +80,6 @@ describe("Forkchoice / GetProposerHead", () => { parentBlockHash: null, payloadStatus: PayloadStatus.FULL, - builderIndex: null, - blockHashFromBid: null, }; const baseParentHeadBlock: ProtoBlockWithWeight = { @@ -111,8 +107,6 @@ describe("Forkchoice / GetProposerHead", () => { parentBlockHash: null, payloadStatus: PayloadStatus.FULL, - builderIndex: null, - blockHashFromBid: null, }; const fcStore: IForkChoiceStore = { @@ -258,8 +252,8 @@ describe("Forkchoice / GetProposerHead", () => { expectedNotReorgedReason, } of testCases) { it(`${id}`, async () => { - protoArr.onBlock(parentBlock, parentBlock.slot); - protoArr.onBlock(headBlock, headBlock.slot); + protoArr.onBlock(parentBlock, parentBlock.slot, null); + protoArr.onBlock(headBlock, headBlock.slot, null); const currentSlot = proposalSlot ?? headBlock.slot + 1; const currentSecFromSlot = secFromSlot ?? 0; diff --git a/packages/fork-choice/test/unit/forkChoice/shouldOverrideForkChoiceUpdate.test.ts b/packages/fork-choice/test/unit/forkChoice/shouldOverrideForkChoiceUpdate.test.ts index 7e0899fe9c46..4cd4544c0240 100644 --- a/packages/fork-choice/test/unit/forkChoice/shouldOverrideForkChoiceUpdate.test.ts +++ b/packages/fork-choice/test/unit/forkChoice/shouldOverrideForkChoiceUpdate.test.ts @@ -52,8 +52,6 @@ describe("Forkchoice / shouldOverrideForkChoiceUpdate", () => { parentBlockHash: null, payloadStatus: PayloadStatus.FULL, - builderIndex: null, - blockHashFromBid: null, }; const baseHeadBlock: ProtoBlockWithWeight = { @@ -82,8 +80,6 @@ describe("Forkchoice / shouldOverrideForkChoiceUpdate", () => { parentBlockHash: null, payloadStatus: PayloadStatus.FULL, - builderIndex: null, - blockHashFromBid: null, }; const baseParentHeadBlock: ProtoBlockWithWeight = { @@ -111,8 +107,6 @@ describe("Forkchoice / shouldOverrideForkChoiceUpdate", () => { parentBlockHash: null, payloadStatus: PayloadStatus.FULL, - builderIndex: null, - blockHashFromBid: null, }; const fcStore: IForkChoiceStore = { @@ -224,8 +218,8 @@ describe("Forkchoice / shouldOverrideForkChoiceUpdate", () => { expectedNotReorgedReason, } of testCases) { it(id, async () => { - protoArr.onBlock(parentBlock, parentBlock.slot); - protoArr.onBlock(headBlock, headBlock.slot); + protoArr.onBlock(parentBlock, parentBlock.slot, null); + protoArr.onBlock(headBlock, headBlock.slot, null); const secFromSlot = 0; const currentSlot = blockSeenSlot ?? headBlock.slot; diff --git a/packages/fork-choice/test/unit/protoArray/executionStatusUpdates.test.ts b/packages/fork-choice/test/unit/protoArray/executionStatusUpdates.test.ts index f2eb43e5024e..8a0a544baeb4 100644 --- a/packages/fork-choice/test/unit/protoArray/executionStatusUpdates.test.ts +++ b/packages/fork-choice/test/unit/protoArray/executionStatusUpdates.test.ts @@ -124,10 +124,9 @@ function setupForkChoice(): ProtoArray { parentBlockHash: null, payloadStatus: PayloadStatus.FULL, - builderIndex: null, - blockHashFromBid: null, }, - block.slot + block.slot, + null ); } diff --git a/packages/fork-choice/test/unit/protoArray/getCommonAncestor.test.ts b/packages/fork-choice/test/unit/protoArray/getCommonAncestor.test.ts index 8cd535b1166e..e8349f4ea426 100644 --- a/packages/fork-choice/test/unit/protoArray/getCommonAncestor.test.ts +++ b/packages/fork-choice/test/unit/protoArray/getCommonAncestor.test.ts @@ -48,8 +48,6 @@ describe("getCommonAncestor", () => { parentBlockHash: null, payloadStatus: PayloadStatus.FULL, - builderIndex: null, - blockHashFromBid: null, }, 0 ); @@ -79,10 +77,9 @@ describe("getCommonAncestor", () => { parentBlockHash: null, payloadStatus: PayloadStatus.FULL, - builderIndex: null, - blockHashFromBid: null, }, - block.slot + block.slot, + null ); } diff --git a/packages/fork-choice/test/unit/protoArray/gloas.test.ts b/packages/fork-choice/test/unit/protoArray/gloas.test.ts index c4c0232aef72..8648633e725d 100644 --- a/packages/fork-choice/test/unit/protoArray/gloas.test.ts +++ b/packages/fork-choice/test/unit/protoArray/gloas.test.ts @@ -21,21 +21,6 @@ describe("Gloas Fork Choice", () => { blockRoot: RootHex, payloadStatus: PayloadStatus ): ProtoNode | undefined { - // const variants = (protoArray as any).indices.get(blockRoot); - // if (!variants) return undefined; - - // // For pre-Gloas, variants[0] contains FULL index - // if (variants.length === 1) { - // // Pre-Gloas block only has FULL variant - // // Only return if requested payloadStatus is FULL - // if (payloadStatus === PayloadStatus.FULL) { - // return (protoArray as any).nodes[variants[0]]; - // } - // return undefined; - // } - - // // For post-Gloas, variants[payloadStatus] contains the index for that status - // const index = variants[payloadStatus]; const index = protoArray.getNodeIndexByRootAndStatus(blockRoot, payloadStatus); if (index === undefined) return undefined; return (protoArray as any).nodes[index]; @@ -45,8 +30,7 @@ describe("Gloas Fork Choice", () => { slot: number, blockRoot: RootHex, parentRoot: RootHex, - parentBlockHash?: RootHex, - blockHashFromBid: RootHex | null = null + parentBlockHash?: RootHex ): ProtoBlock { return { slot, @@ -69,8 +53,6 @@ describe("Gloas Fork Choice", () => { dataAvailabilityStatus: DataAvailabilityStatus.Available, parentBlockHash: parentBlockHash === undefined ? null : parentBlockHash, payloadStatus: PayloadStatus.FULL, - builderIndex: null, - blockHashFromBid, }; } @@ -96,7 +78,7 @@ describe("Gloas Fork Choice", () => { // Add a Gloas block const gloasBlock = createTestBlock(gloasForkSlot, "0x02", genesisRoot, genesisRoot); - protoArray.onBlock(gloasBlock, gloasForkSlot); + protoArray.onBlock(gloasBlock, gloasForkSlot, null); const variants = (protoArray as any).indices.get("0x02"); expect(variants).toBeDefined(); @@ -123,7 +105,7 @@ describe("Gloas Fork Choice", () => { it("creates only FULL nodes for pre-Gloas blocks", () => { const block = createTestBlock(1, "0x02", genesisRoot); - protoArray.onBlock(block, 1); + protoArray.onBlock(block, 1, null); // Should only have FULL variant const fullNode = getNodeByPayloadStatus(protoArray, "0x02", PayloadStatus.FULL); @@ -137,7 +119,7 @@ describe("Gloas Fork Choice", () => { it("getNode() finds pre-Gloas blocks by root (FULL)", () => { const block = createTestBlock(1, "0x02", genesisRoot); - protoArray.onBlock(block, 1); + protoArray.onBlock(block, 1, null); const defaultStatus = protoArray.getDefaultVariant("0x02"); expect(defaultStatus).toBe(PayloadStatus.FULL); @@ -148,7 +130,7 @@ describe("Gloas Fork Choice", () => { it("hasBlock() returns true for pre-Gloas blocks", () => { const block = createTestBlock(1, "0x02", genesisRoot); - protoArray.onBlock(block, 1); + protoArray.onBlock(block, 1, null); expect(protoArray.hasBlock("0x02")).toBe(true); expect(protoArray.hasBlock("0x99")).toBe(false); @@ -170,7 +152,7 @@ describe("Gloas Fork Choice", () => { it("creates PENDING + EMPTY nodes for Gloas blocks", () => { const block = createTestBlock(gloasForkSlot, "0x02", genesisRoot, genesisRoot); - protoArray.onBlock(block, gloasForkSlot); + protoArray.onBlock(block, gloasForkSlot, null); // Should have PENDING variant const pendingNode = getNodeByPayloadStatus(protoArray, "0x02", PayloadStatus.PENDING); @@ -189,7 +171,7 @@ describe("Gloas Fork Choice", () => { it("EMPTY node has PENDING as parent", () => { const block = createTestBlock(gloasForkSlot, "0x02", genesisRoot, genesisRoot); - protoArray.onBlock(block, gloasForkSlot); + protoArray.onBlock(block, gloasForkSlot, null); const emptyNode = getNodeByPayloadStatus(protoArray, "0x02", PayloadStatus.EMPTY); const pendingIndex = protoArray.getNodeIndexByRootAndStatus("0x02", PayloadStatus.PENDING); @@ -199,7 +181,7 @@ describe("Gloas Fork Choice", () => { it("initializes PTC votes for Gloas blocks", () => { const block = createTestBlock(gloasForkSlot, "0x02", genesisRoot, genesisRoot); - protoArray.onBlock(block, gloasForkSlot); + protoArray.onBlock(block, gloasForkSlot, null); // All PTC votes should be false initially const isTimely = protoArray.isPayloadTimely("0x02"); @@ -208,7 +190,7 @@ describe("Gloas Fork Choice", () => { it("does not create PENDING/EMPTY for pre-fork blocks", () => { const block = createTestBlock(gloasForkSlot - 1, "0x02", genesisRoot); - protoArray.onBlock(block, gloasForkSlot - 1); + protoArray.onBlock(block, gloasForkSlot - 1, null); // Should only have FULL (pre-Gloas behavior) const fullNode = getNodeByPayloadStatus(protoArray, "0x02", PayloadStatus.FULL); @@ -234,11 +216,11 @@ describe("Gloas Fork Choice", () => { it("first Gloas block points to FULL parent (Fulu block)", () => { // Add pre-Gloas block const fuluBlock = createTestBlock(gloasForkSlot - 1, "0x02", genesisRoot); - protoArray.onBlock(fuluBlock, gloasForkSlot - 1); + protoArray.onBlock(fuluBlock, gloasForkSlot - 1, null); // Add first Gloas block const gloasBlock = createTestBlock(gloasForkSlot, "0x03", "0x02", "0x02"); - protoArray.onBlock(gloasBlock, gloasForkSlot); + protoArray.onBlock(gloasBlock, gloasForkSlot, null); const gloasPendingNode = getNodeByPayloadStatus(protoArray, "0x03", PayloadStatus.PENDING); const fuluFullIndex = protoArray.getNodeIndexByRootAndStatus("0x02", PayloadStatus.FULL); @@ -250,11 +232,11 @@ describe("Gloas Fork Choice", () => { it("getNode() finds blocks across fork transition", () => { // Add pre-Gloas block const fuluBlock = createTestBlock(gloasForkSlot - 1, "0x02", genesisRoot); - protoArray.onBlock(fuluBlock, gloasForkSlot - 1); + protoArray.onBlock(fuluBlock, gloasForkSlot - 1, null); // Add Gloas block const gloasBlock = createTestBlock(gloasForkSlot, "0x03", "0x02", "0x02"); - protoArray.onBlock(gloasBlock, gloasForkSlot); + protoArray.onBlock(gloasBlock, gloasForkSlot, null); // Should find both blocks with correct default variants const fuluDefaultStatus = protoArray.getDefaultVariant("0x02"); @@ -284,13 +266,13 @@ describe("Gloas Fork Choice", () => { it("creates FULL variant when payload arrives", () => { const block = createTestBlock(gloasForkSlot, "0x02", genesisRoot, genesisRoot); - protoArray.onBlock(block, gloasForkSlot); + protoArray.onBlock(block, gloasForkSlot, null); // FULL should not exist yet expect(getNodeByPayloadStatus(protoArray, "0x02", PayloadStatus.FULL)).toBeUndefined(); // Call onExecutionPayload - protoArray.onExecutionPayload("0x02", gloasForkSlot, "0x02", gloasForkSlot, stateRoot); + protoArray.onExecutionPayload("0x02", gloasForkSlot, "0x02", gloasForkSlot, stateRoot, null); // FULL should now exist const fullNode = getNodeByPayloadStatus(protoArray, "0x02", PayloadStatus.FULL); @@ -300,9 +282,9 @@ describe("Gloas Fork Choice", () => { it("FULL node has PENDING as parent", () => { const block = createTestBlock(gloasForkSlot, "0x02", genesisRoot, genesisRoot); - protoArray.onBlock(block, gloasForkSlot); + protoArray.onBlock(block, gloasForkSlot, null); - protoArray.onExecutionPayload("0x02", gloasForkSlot, "0x02", gloasForkSlot, stateRoot); + protoArray.onExecutionPayload("0x02", gloasForkSlot, "0x02", gloasForkSlot, stateRoot, null); const fullNode = getNodeByPayloadStatus(protoArray, "0x02", PayloadStatus.FULL); const pendingIndex = protoArray.getNodeIndexByRootAndStatus("0x02", PayloadStatus.PENDING); @@ -312,10 +294,10 @@ describe("Gloas Fork Choice", () => { it("is idempotent (calling twice does not create duplicate)", () => { const block = createTestBlock(gloasForkSlot, "0x02", genesisRoot, genesisRoot); - protoArray.onBlock(block, gloasForkSlot); + protoArray.onBlock(block, gloasForkSlot, null); - protoArray.onExecutionPayload("0x02", gloasForkSlot, "0x02", gloasForkSlot, stateRoot); - protoArray.onExecutionPayload("0x02", gloasForkSlot, "0x02", gloasForkSlot, stateRoot); + protoArray.onExecutionPayload("0x02", gloasForkSlot, "0x02", gloasForkSlot, stateRoot, null); + protoArray.onExecutionPayload("0x02", gloasForkSlot, "0x02", gloasForkSlot, stateRoot, null); // Should still only have one FULL node const fullNode = getNodeByPayloadStatus(protoArray, "0x02", PayloadStatus.FULL); @@ -324,19 +306,21 @@ describe("Gloas Fork Choice", () => { it("throws for pre-Gloas blocks", () => { const block = createTestBlock(gloasForkSlot - 1, "0x02", genesisRoot); - protoArray.onBlock(block, gloasForkSlot - 1); + protoArray.onBlock(block, gloasForkSlot - 1, null); // Pre-Gloas block already has FULL expect(getNodeByPayloadStatus(protoArray, "0x02", PayloadStatus.FULL)).toBeDefined(); // Calling onExecutionPayload should throw for pre-Gloas blocks expect(() => - protoArray.onExecutionPayload("0x02", gloasForkSlot - 1, "0x02", gloasForkSlot - 1, stateRoot) + protoArray.onExecutionPayload("0x02", gloasForkSlot - 1, "0x02", gloasForkSlot - 1, stateRoot, null) ).toThrow(); }); it("throws for unknown block", () => { - expect(() => protoArray.onExecutionPayload("0x99", gloasForkSlot, "0x99", gloasForkSlot, stateRoot)).toThrow(); + expect(() => + protoArray.onExecutionPayload("0x99", gloasForkSlot, "0x99", gloasForkSlot, stateRoot, null) + ).toThrow(); }); }); @@ -353,43 +337,43 @@ describe("Gloas Fork Choice", () => { }); }); - it("notifyPtcMessage() updates votes for multiple validators", () => { + it("notifyPtcMessages() updates votes for multiple validators", () => { const block = createTestBlock(gloasForkSlot, "0x02", genesisRoot, genesisRoot); - protoArray.onBlock(block, gloasForkSlot); + protoArray.onBlock(block, gloasForkSlot, null); // Initially not timely (no votes) expect(protoArray.isPayloadTimely("0x02")).toBe(false); // Vote yes from validators at indices 0, 1, 2 - protoArray.notifyPtcMessage("0x02", [0, 1, 2], true); + protoArray.notifyPtcMessages("0x02", [0, 1, 2], true); // Still not timely (need >50% of PTC_SIZE) expect(protoArray.isPayloadTimely("0x02")).toBe(false); }); - it("notifyPtcMessage() validates ptcIndex range", () => { + it("notifyPtcMessages() validates ptcIndex range", () => { const block = createTestBlock(gloasForkSlot, "0x02", genesisRoot, genesisRoot); - protoArray.onBlock(block, gloasForkSlot); + protoArray.onBlock(block, gloasForkSlot, null); - expect(() => protoArray.notifyPtcMessage("0x02", [-1], true)).toThrow(/Invalid PTC index/); - expect(() => protoArray.notifyPtcMessage("0x02", [PTC_SIZE], true)).toThrow(/Invalid PTC index/); - expect(() => protoArray.notifyPtcMessage("0x02", [PTC_SIZE + 1], true)).toThrow(/Invalid PTC index/); - expect(() => protoArray.notifyPtcMessage("0x02", [0, 1, PTC_SIZE], true)).toThrow(/Invalid PTC index/); + expect(() => protoArray.notifyPtcMessages("0x02", [-1], true)).toThrow(/Invalid PTC index/); + expect(() => protoArray.notifyPtcMessages("0x02", [PTC_SIZE], true)).toThrow(/Invalid PTC index/); + expect(() => protoArray.notifyPtcMessages("0x02", [PTC_SIZE + 1], true)).toThrow(/Invalid PTC index/); + expect(() => protoArray.notifyPtcMessages("0x02", [0, 1, PTC_SIZE], true)).toThrow(/Invalid PTC index/); }); - it("notifyPtcMessage() handles unknown block gracefully", () => { + it("notifyPtcMessages() handles unknown block gracefully", () => { // Should not throw for unknown block - expect(() => protoArray.notifyPtcMessage("0x99", [0], true)).not.toThrow(); + expect(() => protoArray.notifyPtcMessages("0x99", [0], true)).not.toThrow(); }); it("isPayloadTimely() returns false when payload not locally available", () => { const block = createTestBlock(gloasForkSlot, "0x02", genesisRoot, genesisRoot); - protoArray.onBlock(block, gloasForkSlot); + protoArray.onBlock(block, gloasForkSlot, null); // Vote yes from majority of PTC const threshold = Math.floor(PTC_SIZE / 2) + 1; const indices = Array.from({length: threshold}, (_, i) => i); - protoArray.notifyPtcMessage("0x02", indices, true); + protoArray.notifyPtcMessages("0x02", indices, true); // Without execution payload (no FULL variant), should return false expect(protoArray.isPayloadTimely("0x02")).toBe(false); @@ -397,15 +381,15 @@ describe("Gloas Fork Choice", () => { it("isPayloadTimely() returns true when threshold met and payload available", () => { const block = createTestBlock(gloasForkSlot, "0x02", genesisRoot, genesisRoot); - protoArray.onBlock(block, gloasForkSlot); + protoArray.onBlock(block, gloasForkSlot, null); // Make execution payload available by creating FULL variant - protoArray.onExecutionPayload("0x02", gloasForkSlot, "0x02", gloasForkSlot, stateRoot); + protoArray.onExecutionPayload("0x02", gloasForkSlot, "0x02", gloasForkSlot, stateRoot, null); // Vote yes from majority of PTC (>50%) const threshold = Math.floor(PTC_SIZE / 2) + 1; const indices = Array.from({length: threshold}, (_, i) => i); - protoArray.notifyPtcMessage("0x02", indices, true); + protoArray.notifyPtcMessages("0x02", indices, true); // Should now be timely expect(protoArray.isPayloadTimely("0x02")).toBe(true); @@ -413,15 +397,15 @@ describe("Gloas Fork Choice", () => { it("isPayloadTimely() returns false when threshold not met", () => { const block = createTestBlock(gloasForkSlot, "0x02", genesisRoot, genesisRoot); - protoArray.onBlock(block, gloasForkSlot); + protoArray.onBlock(block, gloasForkSlot, null); // Make execution payload available by creating FULL variant - protoArray.onExecutionPayload("0x02", gloasForkSlot, "0x02", gloasForkSlot, stateRoot); + protoArray.onExecutionPayload("0x02", gloasForkSlot, "0x02", gloasForkSlot, stateRoot, null); // Vote yes from exactly 50% (not >50%) const threshold = Math.floor(PTC_SIZE / 2); const indices = Array.from({length: threshold}, (_, i) => i); - protoArray.notifyPtcMessage("0x02", indices, true); + protoArray.notifyPtcMessages("0x02", indices, true); // Should not be timely (need >50%, not >=50%) expect(protoArray.isPayloadTimely("0x02")).toBe(false); @@ -429,25 +413,25 @@ describe("Gloas Fork Choice", () => { it("isPayloadTimely() counts only 'true' votes", () => { const block = createTestBlock(gloasForkSlot, "0x02", genesisRoot, genesisRoot); - protoArray.onBlock(block, gloasForkSlot); + protoArray.onBlock(block, gloasForkSlot, null); // Make execution payload available by creating FULL variant - protoArray.onExecutionPayload("0x02", gloasForkSlot, "0x02", gloasForkSlot, stateRoot); + protoArray.onExecutionPayload("0x02", gloasForkSlot, "0x02", gloasForkSlot, stateRoot, null); // Vote mixed yes/no const threshold = Math.floor(PTC_SIZE / 2) + 1; // Vote yes from indices 0..threshold-1 const yesIndices = Array.from({length: threshold}, (_, i) => i); - protoArray.notifyPtcMessage("0x02", yesIndices, true); + protoArray.notifyPtcMessages("0x02", yesIndices, true); // Vote no from indices threshold..PTC_SIZE-1 const noIndices = Array.from({length: PTC_SIZE - threshold}, (_, i) => i + threshold); - protoArray.notifyPtcMessage("0x02", noIndices, false); + protoArray.notifyPtcMessages("0x02", noIndices, false); // Should be timely (threshold met) expect(protoArray.isPayloadTimely("0x02")).toBe(true); // Change some yes votes to no - protoArray.notifyPtcMessage("0x02", [0, 1], false); + protoArray.notifyPtcMessages("0x02", [0, 1], false); // Should no longer be timely expect(protoArray.isPayloadTimely("0x02")).toBe(false); @@ -459,13 +443,13 @@ describe("Gloas Fork Choice", () => { it("does not initialize PTC votes for pre-Gloas blocks", () => { const block = createTestBlock(gloasForkSlot - 1, "0x02", genesisRoot); - protoArray.onBlock(block, gloasForkSlot - 1); + protoArray.onBlock(block, gloasForkSlot - 1, null); // Pre-Gloas blocks should not have PTC tracking expect(protoArray.isPayloadTimely("0x02")).toBe(false); - // notifyPtcMessage should be no-op - expect(() => protoArray.notifyPtcMessage("0x02", [0], true)).not.toThrow(); + // notifyPtcMessages should be no-op + expect(() => protoArray.notifyPtcMessages("0x02", [0], true)).not.toThrow(); }); }); @@ -484,8 +468,8 @@ describe("Gloas Fork Choice", () => { it("intra-block: EMPTY/FULL variants have PENDING as parent", () => { const block = createTestBlock(gloasForkSlot, "0x02", genesisRoot, genesisRoot); - protoArray.onBlock(block, gloasForkSlot); - protoArray.onExecutionPayload("0x02", gloasForkSlot, "0x02", gloasForkSlot, stateRoot); + protoArray.onBlock(block, gloasForkSlot, null); + protoArray.onExecutionPayload("0x02", gloasForkSlot, "0x02", gloasForkSlot, stateRoot, null); const pendingIndex = protoArray.getNodeIndexByRootAndStatus("0x02", PayloadStatus.PENDING); const emptyNode = getNodeByPayloadStatus(protoArray, "0x02", PayloadStatus.EMPTY); @@ -497,13 +481,13 @@ describe("Gloas Fork Choice", () => { it("inter-block: new PENDING extends parent's EMPTY or FULL", () => { // Block A - const blockA = createTestBlock(gloasForkSlot, "0x02Root", genesisRoot, genesisRoot, "0x02Hash"); - protoArray.onBlock(blockA, gloasForkSlot); - protoArray.onExecutionPayload("0x02Root", gloasForkSlot, "0x02Hash", gloasForkSlot, stateRoot); + const blockA = createTestBlock(gloasForkSlot, "0x02Root", genesisRoot, genesisRoot); + protoArray.onBlock(blockA, gloasForkSlot, null); + protoArray.onExecutionPayload("0x02Root", gloasForkSlot, "0x02Hash", gloasForkSlot, stateRoot, null); // Block B extends A's FULL (parentBlockHash matches) const blockB = createTestBlock(gloasForkSlot + 1, "0x03Root", "0x02Root", "0x02Hash"); - protoArray.onBlock(blockB, gloasForkSlot + 1); + protoArray.onBlock(blockB, gloasForkSlot + 1, null); const blockAPending = protoArray.getNodeIndexByRootAndStatus("0x02Root", PayloadStatus.PENDING); const blockAFull = protoArray.getNodeIndexByRootAndStatus("0x02Root", PayloadStatus.FULL); @@ -527,8 +511,8 @@ describe("Gloas Fork Choice", () => { it("EMPTY vs FULL comparison uses explicit tiebreaker for slot n-1 blocks", () => { const blockSlot = gloasForkSlot + 10; const block = createTestBlock(blockSlot, "0x02", genesisRoot, genesisRoot); - protoArray.onBlock(block, blockSlot); - protoArray.onExecutionPayload("0x02", blockSlot, "0x02", blockSlot, stateRoot); + protoArray.onBlock(block, blockSlot, null); + protoArray.onExecutionPayload("0x02", blockSlot, "0x02", blockSlot, stateRoot, null); const emptyIndex = protoArray.getNodeIndexByRootAndStatus("0x02", PayloadStatus.EMPTY); if (emptyIndex === undefined) throw new Error("Expected emptyIndex to exist"); @@ -568,8 +552,8 @@ describe("Gloas Fork Choice", () => { const blockA = createTestBlock(blockSlot, "0x02", genesisRoot, genesisRoot); const blockB = createTestBlock(blockSlot, "0x03", genesisRoot, genesisRoot); - protoArray.onBlock(blockA, blockSlot); - protoArray.onBlock(blockB, blockSlot); + protoArray.onBlock(blockA, blockSlot, null); + protoArray.onBlock(blockB, blockSlot, null); const emptyAIndex = protoArray.getNodeIndexByRootAndStatus("0x02", PayloadStatus.EMPTY); if (emptyAIndex === undefined) throw new Error("Expected emptyAIndex to exist"); @@ -604,8 +588,8 @@ describe("Gloas Fork Choice", () => { it("EMPTY vs FULL from older slots (n-2) uses weight comparison", () => { const blockSlot = gloasForkSlot + 10; const block = createTestBlock(blockSlot, "0x02", genesisRoot, genesisRoot); - protoArray.onBlock(block, blockSlot); - protoArray.onExecutionPayload("0x02", blockSlot, "0x02", blockSlot, stateRoot); + protoArray.onBlock(block, blockSlot, null); + protoArray.onExecutionPayload("0x02", blockSlot, "0x02", blockSlot, stateRoot, null); const emptyIndex = protoArray.getNodeIndexByRootAndStatus("0x02", PayloadStatus.EMPTY); if (emptyIndex === undefined) throw new Error("Expected emptyIndex to exist"); @@ -635,83 +619,5 @@ describe("Gloas Fork Choice", () => { expect(fullNode?.weight).toBe(200); // FULL should be preferred due to higher weight }); - - it("EMPTY vs FULL from current slot uses weight comparison", () => { - const blockSlot = gloasForkSlot + 10; - const block = createTestBlock(blockSlot, "0x02", genesisRoot, genesisRoot); - protoArray.onBlock(block, blockSlot); - protoArray.onExecutionPayload("0x02", blockSlot, "0x02", blockSlot, stateRoot); - - const pendingNode = getNodeByPayloadStatus(protoArray, "0x02", PayloadStatus.PENDING); - if (!pendingNode) throw new Error("Expected pendingNode to exist"); - - const emptyIndex = protoArray.getNodeIndexByRootAndStatus("0x02", PayloadStatus.EMPTY); - if (emptyIndex === undefined) throw new Error("Expected emptyIndex to exist"); - const fullIndex = protoArray.getNodeIndexByRootAndStatus("0x02", PayloadStatus.FULL); - if (fullIndex === undefined) throw new Error("Expected fullIndex to exist"); - - const deltas = new Array(protoArray.length()).fill(0); - deltas[emptyIndex] = 200; - deltas[fullIndex] = 100; - - // currentSlot = blockSlot, so comparison must use weights (not payload tiebreaker) - protoArray.applyScoreChanges({ - deltas, - proposerBoost: null, - justifiedEpoch: genesisEpoch, - justifiedRoot: genesisRoot, - finalizedEpoch: genesisEpoch, - finalizedRoot: genesisRoot, - currentSlot: blockSlot, - }); - - expect(pendingNode.bestChild).toBe(emptyIndex); - }); - - it("reconciled PENDING descendant propagates to ancestors", () => { - const slotA = gloasForkSlot + 10; - const slotB = slotA + 1; - - // A has PENDING+EMPTY initially. - const blockA = createTestBlock(slotA, "0x02", genesisRoot, genesisRoot, "0x02-bid"); - protoArray.onBlock(blockA, slotA); - - // B extends A.EMPTY (parentBlockHash !== A.blockHashFromBid, FULL(A) not available yet). - const blockB = createTestBlock(slotB, "0x03", "0x02", genesisRoot); - protoArray.onBlock(blockB, slotB); - - // Later A payload arrives; FULL(A) is created. - protoArray.onExecutionPayload("0x02", slotB, "0x02-full", slotA, stateRoot); - - // Trigger score recomputation at slot n-1 so EMPTY vs FULL tiebreaker is exercised. - const deltas = new Array(protoArray.nodes.length).fill(0); - protoArray.applyScoreChanges({ - deltas, - proposerBoost: null, - justifiedEpoch: genesisEpoch, - justifiedRoot: genesisRoot, - finalizedEpoch: genesisEpoch, - finalizedRoot: genesisRoot, - currentSlot: slotA + 1, - }); - - const pendingAIndex = protoArray.getNodeIndexByRootAndStatus("0x02", PayloadStatus.PENDING); - if (pendingAIndex === undefined) throw new Error("Expected pendingAIndex to exist"); - const pendingANode = protoArray.nodes[pendingAIndex]; - if (!pendingANode || pendingANode.bestDescendant === undefined) - throw new Error("Expected pending A bestDescendant to exist"); - - const pendingADescendant = protoArray.nodes[pendingANode.bestDescendant]; - expect(pendingADescendant.slot).toBe(slotB); - - const genesisIndex = protoArray.getNodeIndexByRootAndStatus(genesisRoot, PayloadStatus.FULL); - if (genesisIndex === undefined) throw new Error("Expected genesisIndex to exist"); - const genesisNode = protoArray.nodes[genesisIndex]; - if (!genesisNode || genesisNode.bestDescendant === undefined) - throw new Error("Expected genesis bestDescendant to exist"); - - const genesisDescendant = protoArray.nodes[genesisNode.bestDescendant]; - expect(genesisDescendant.slot).toBe(slotB); - }); }); }); diff --git a/packages/fork-choice/test/unit/protoArray/protoArray.test.ts b/packages/fork-choice/test/unit/protoArray/protoArray.test.ts index 95fa1373e44c..f23332f7a292 100644 --- a/packages/fork-choice/test/unit/protoArray/protoArray.test.ts +++ b/packages/fork-choice/test/unit/protoArray/protoArray.test.ts @@ -4,68 +4,6 @@ import {RootHex} from "@lodestar/types"; import {ExecutionStatus, PayloadStatus, ProtoArray} from "../../../src/index.js"; describe("ProtoArray", () => { - it("getAllAncestorNodes includes the starting node", () => { - const genesisSlot = 0; - const genesisEpoch = 0; - const stateRoot = "0"; - const finalizedRoot = "1"; - const parentRoot = "1"; - const childRoot = "2"; - const fc = ProtoArray.initialize( - { - slot: genesisSlot, - stateRoot, - parentRoot, - blockRoot: finalizedRoot, - justifiedEpoch: genesisEpoch, - justifiedRoot: stateRoot, - finalizedEpoch: genesisEpoch, - finalizedRoot: stateRoot, - unrealizedJustifiedEpoch: genesisEpoch, - unrealizedJustifiedRoot: stateRoot, - unrealizedFinalizedEpoch: genesisEpoch, - unrealizedFinalizedRoot: stateRoot, - timeliness: false, - ...{executionPayloadBlockHash: null, executionStatus: ExecutionStatus.PreMerge}, - dataAvailabilityStatus: DataAvailabilityStatus.PreData, - parentBlockHash: null, - payloadStatus: PayloadStatus.FULL, - builderIndex: null, - blockHashFromBid: null, - }, - genesisSlot - ); - - fc.onBlock( - { - slot: genesisSlot + 1, - blockRoot: childRoot, - parentRoot: finalizedRoot, - stateRoot, - targetRoot: finalizedRoot, - justifiedEpoch: genesisEpoch, - justifiedRoot: stateRoot, - finalizedEpoch: genesisEpoch, - finalizedRoot: stateRoot, - unrealizedJustifiedEpoch: genesisEpoch, - unrealizedJustifiedRoot: stateRoot, - unrealizedFinalizedEpoch: genesisEpoch, - unrealizedFinalizedRoot: stateRoot, - timeliness: false, - ...{executionPayloadBlockHash: null, executionStatus: ExecutionStatus.PreMerge}, - dataAvailabilityStatus: DataAvailabilityStatus.PreData, - parentBlockHash: null, - payloadStatus: PayloadStatus.FULL, - builderIndex: null, - blockHashFromBid: null, - }, - genesisSlot + 1 - ); - - const ancestors = fc.getAllAncestorNodes(childRoot, PayloadStatus.FULL); - expect(ancestors.map((node) => node.blockRoot)).toEqual([childRoot, finalizedRoot]); - }); - it("finalized descendant", () => { const genesisSlot = 0; const genesisEpoch = 0; @@ -99,8 +37,6 @@ describe("ProtoArray", () => { parentBlockHash: null, payloadStatus: PayloadStatus.FULL, - builderIndex: null, - blockHashFromBid: null, }, genesisSlot ); @@ -130,10 +66,9 @@ describe("ProtoArray", () => { parentBlockHash: null, payloadStatus: PayloadStatus.FULL, - builderIndex: null, - blockHashFromBid: null, }, - genesisSlot + 1 + genesisSlot + 1, + null ); // Add block that is *not* a finalized descendant. @@ -161,10 +96,9 @@ describe("ProtoArray", () => { parentBlockHash: null, payloadStatus: PayloadStatus.FULL, - builderIndex: null, - blockHashFromBid: null, }, - genesisSlot + 1 + genesisSlot + 1, + null ); // ancestorRoot, descendantRoot, isDescendant @@ -193,7 +127,7 @@ describe("ProtoArray", () => { ]; for (const [ancestorRoot, descendantRoot, isDescendant] of assertions) { - expect(fc.isDescendant(ancestorRoot, descendantRoot)).toBeWithMessage( + expect(fc.isDescendant(ancestorRoot, PayloadStatus.FULL, descendantRoot, PayloadStatus.FULL)).toBeWithMessage( isDescendant, `${descendantRoot} must be ${isDescendant ? "descendant" : "not descendant"} of ${ancestorRoot}` ); diff --git a/packages/light-client/package.json b/packages/light-client/package.json index af18c1d13b87..100c677ff7c5 100644 --- a/packages/light-client/package.json +++ b/packages/light-client/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.40.0", + "version": "1.41.0", "type": "module", "exports": { ".": { @@ -48,13 +48,13 @@ ], "scripts": { "clean": "rm -rf lib && rm -rf dist && rm -f *.tsbuildinfo", - "build": "tsc -p tsconfig.build.json", + "build": "tsgo -p tsconfig.build.json", "build:watch": "pnpm run build --watch", "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\"", "build:bundle": "vite build", "check-bundle": "node -e \"(async function() { await import('./dist/lightclient.min.mjs') })()\"", "build:release": "pnpm clean && pnpm run build && pnpm run build:bundle", - "check-types": "tsc", + "check-types": "tsgo", "lint": "biome check src/ test/", "lint:fix": "pnpm run lint --write", "test": "pnpm test:unit", @@ -79,7 +79,7 @@ "@fastify/cors": "^10.0.1", "@lodestar/state-transition": "workspace:^", "@types/qs": "^6.9.7", - "fastify": "^5.7.4", + "fastify": "^5.8.1", "qs": "^6.11.1", "uint8arrays": "^5.0.1" }, diff --git a/packages/logger/package.json b/packages/logger/package.json index 1350175546a0..5e05a5d08ab6 100644 --- a/packages/logger/package.json +++ b/packages/logger/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.40.0", + "version": "1.41.0", "type": "module", "exports": { ".": { @@ -38,6 +38,11 @@ "bun": "./src/empty.ts", "types": "./lib/empty.d.ts", "import": "./lib/empty.js" + }, + "./test-utils": { + "bun": "./src/testUtils.ts", + "types": "./lib/testUtils.d.ts", + "import": "./lib/testUtils.js" } }, "files": [ @@ -47,11 +52,11 @@ ], "scripts": { "clean": "rm -rf lib && rm -f *.tsbuildinfo", - "build": "tsc -p tsconfig.build.json", + "build": "tsgo -p tsconfig.build.json", "build:watch": "pnpm run build --watch", "build:release": "pnpm clean && pnpm build", "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\"", - "check-types": "tsc", + "check-types": "tsgo", "lint": "biome check src/ test/", "lint:fix": "pnpm run lint --write", "test": "pnpm test:unit && pnpm test:e2e", diff --git a/packages/beacon-node/test/utils/logger.ts b/packages/logger/src/testUtils.ts similarity index 80% rename from packages/beacon-node/test/utils/logger.ts rename to packages/logger/src/testUtils.ts index 7d67dd6fbd16..abf4925d105f 100644 --- a/packages/beacon-node/test/utils/logger.ts +++ b/packages/logger/src/testUtils.ts @@ -1,6 +1,6 @@ -import {getEnvLogLevel} from "@lodestar/logger/env"; -import {LoggerNode, LoggerNodeOpts, getNodeLogger} from "@lodestar/logger/node"; import {LogLevel} from "@lodestar/utils"; +import {getEnvLogLevel} from "./env.js"; +import {LoggerNode, LoggerNodeOpts, getNodeLogger} from "./node.js"; export {LogLevel}; export type TestLoggerOpts = LoggerNodeOpts; diff --git a/packages/params/package.json b/packages/params/package.json index 5e14929685e7..d37fe138c3b2 100644 --- a/packages/params/package.json +++ b/packages/params/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/params", - "version": "1.40.0", + "version": "1.41.0", "description": "Chain parameters required for lodestar", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -43,11 +43,11 @@ ], "scripts": { "clean": "rm -rf lib && rm -f *.tsbuildinfo", - "build": "tsc -p tsconfig.build.json", + "build": "tsgo -p tsconfig.build.json", "build:release": "pnpm clean && pnpm build", "build:watch": "pnpm run build --watch", "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\"", - "check-types": "tsc", + "check-types": "tsgo", "lint": "biome check src/ test/", "lint:fix": "pnpm run lint --write", "test": "pnpm test:unit", diff --git a/packages/params/src/presets/mainnet.ts b/packages/params/src/presets/mainnet.ts index 349f0058031e..4ec676b16471 100644 --- a/packages/params/src/presets/mainnet.ts +++ b/packages/params/src/presets/mainnet.ts @@ -1,7 +1,7 @@ import {BeaconPreset} from "../types.js"; // Mainnet preset -// https://github.com/ethereum/consensus-specs/tree/dev/presets/mainnet +// https://github.com/ethereum/consensus-specs/tree/master/presets/mainnet export const mainnetPreset: BeaconPreset = { // Misc diff --git a/packages/params/src/presets/minimal.ts b/packages/params/src/presets/minimal.ts index 42486961ce2d..08080d83f7b7 100644 --- a/packages/params/src/presets/minimal.ts +++ b/packages/params/src/presets/minimal.ts @@ -1,7 +1,7 @@ import {BeaconPreset} from "../types.js"; // Minimal preset -// https://github.com/ethereum/consensus-specs/tree/dev/presets/minimal +// https://github.com/ethereum/consensus-specs/tree/master/presets/minimal export const minimalPreset: BeaconPreset = { // Misc diff --git a/packages/params/test/e2e/ensure-config-is-synced.test.ts b/packages/params/test/e2e/ensure-config-is-synced.test.ts index ba9849eff410..b1bab52a76f8 100644 --- a/packages/params/test/e2e/ensure-config-is-synced.test.ts +++ b/packages/params/test/e2e/ensure-config-is-synced.test.ts @@ -1,9 +1,9 @@ import axios from "axios"; import {describe, expect, it, vi} from "vitest"; -import {ethereumConsensusSpecsTests} from "../../../beacon-node/test/spec/specTestVersioning.js"; import {BeaconPreset, ForkName} from "../../src/index.js"; import {mainnetPreset} from "../../src/presets/mainnet.js"; import {minimalPreset} from "../../src/presets/minimal.js"; +import specTests from "../spec-tests-version.json" with {type: "json"}; import {loadConfigYaml} from "../yaml.js"; // Not e2e, but slow. Run with e2e tests @@ -17,12 +17,12 @@ describe("Ensure config is synced", () => { vi.setConfig({testTimeout: 60 * 1000}); it("mainnet", async () => { - const remotePreset = await downloadRemoteConfig("mainnet", ethereumConsensusSpecsTests.specVersion); + const remotePreset = await downloadRemoteConfig("mainnet", specTests.ethereumConsensusSpecsTests.specVersion); assertCorrectPreset({...mainnetPreset}, remotePreset); }); it("minimal", async () => { - const remotePreset = await downloadRemoteConfig("minimal", ethereumConsensusSpecsTests.specVersion); + const remotePreset = await downloadRemoteConfig("minimal", specTests.ethereumConsensusSpecsTests.specVersion); assertCorrectPreset({...minimalPreset}, remotePreset); }); }); diff --git a/packages/params/test/spec-tests-version.json b/packages/params/test/spec-tests-version.json new file mode 120000 index 000000000000..bc2e3fd621b2 --- /dev/null +++ b/packages/params/test/spec-tests-version.json @@ -0,0 +1 @@ +../../../spec-tests-version.json \ No newline at end of file diff --git a/packages/params/tsconfig.json b/packages/params/tsconfig.json index a0f4f2a31e93..19b9527ba0e3 100644 --- a/packages/params/tsconfig.json +++ b/packages/params/tsconfig.json @@ -1,6 +1,9 @@ { "extends": "../../tsconfig.json", - "include": ["src", "test"], + "include": [ + "src", + "test" + ], "compilerOptions": { "outDir": "lib" } diff --git a/packages/prover/README.md b/packages/prover/README.md index a191b0624639..7bbfda982fc5 100644 --- a/packages/prover/README.md +++ b/packages/prover/README.md @@ -148,7 +148,7 @@ If your project is using some provider type which is not among above list, you h ## What you need -You will need to go over the [specification](https://github.com/ethereum/beacon-apis). You will also need to have a [basic understanding of lightclient](https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/light-client.md). +You will need to go over the [specification](https://github.com/ethereum/beacon-apis). You will also need to have a [basic understanding of lightclient](https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/altair/light-client/light-client.md). ## Getting started diff --git a/packages/prover/package.json b/packages/prover/package.json index 3e3c6ed187ff..e087c53f801b 100644 --- a/packages/prover/package.json +++ b/packages/prover/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.40.0", + "version": "1.41.0", "type": "module", "exports": { ".": { @@ -36,11 +36,11 @@ ], "scripts": { "clean": "rm -rf lib && rm -f *.tsbuildinfo", - "build": "tsc -p tsconfig.build.json", + "build": "tsgo -p tsconfig.build.json", "build:watch": "pnpm run build --watch", "build:release": "pnpm clean && pnpm run build", "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\"", - "check-types": "tsc", + "check-types": "tsgo", "lint": "biome check src/ test/", "lint:fix": "pnpm run lint --write", "test": "pnpm test:unit && pnpm test:e2e", diff --git a/packages/prover/src/constants.ts b/packages/prover/src/constants.ts index 5a9eefd0f3ca..e04b13de1cf1 100644 --- a/packages/prover/src/constants.ts +++ b/packages/prover/src/constants.ts @@ -1,4 +1,4 @@ -// https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/p2p-interface.md#configuration +// https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/altair/light-client/p2p-interface.md#configuration export const MAX_REQUEST_LIGHT_CLIENT_UPDATES = 128; export const MAX_PAYLOAD_HISTORY = 32; export const VERIFICATION_FAILED_RESPONSE_CODE = -33091; diff --git a/packages/reqresp/README.md b/packages/reqresp/README.md index b28d9641c43f..87aafef978b4 100644 --- a/packages/reqresp/README.md +++ b/packages/reqresp/README.md @@ -7,7 +7,7 @@ > This package is part of [ChainSafe's Lodestar](https://lodestar.chainsafe.io) project -Typescript implementation of the [Ethereum Consensus Req/Resp protocol](https://github.com/ethereum/consensus-specs/blob/v1.4.0/specs/phase0/p2p-interface.md#reqresp) +Typescript implementation of the [Ethereum Consensus Req/Resp protocol](https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/p2p-interface.md#reqresp) ## Usage diff --git a/packages/reqresp/package.json b/packages/reqresp/package.json index 174de78130e3..5373f6b8ee4c 100644 --- a/packages/reqresp/package.json +++ b/packages/reqresp/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.40.0", + "version": "1.41.0", "type": "module", "exports": { ".": { @@ -32,11 +32,11 @@ ], "scripts": { "clean": "rm -rf lib && rm -f *.tsbuildinfo", - "build": "tsc -p tsconfig.build.json", + "build": "tsgo -p tsconfig.build.json", "build:watch": "pnpm run build --watch", "build:release": "pnpm clean && pnpm run build", "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\"", - "check-types": "tsc", + "check-types": "tsgo", "lint": "biome check src/ test/", "lint:fix": "pnpm run lint --write", "test": "pnpm test:unit", @@ -45,12 +45,11 @@ }, "dependencies": { "@chainsafe/fast-crc32c": "^4.2.0", - "@libp2p/interface": "^2.7.0", + "@libp2p/interface": "^3.1.0", + "@libp2p/utils": "^7.0.13", "@lodestar/config": "workspace:^", "@lodestar/params": "workspace:^", "@lodestar/utils": "workspace:^", - "it-all": "^3.0.4", - "it-pipe": "^3.0.1", "snappy": "^7.2.2", "snappyjs": "^0.7.0", "uint8-varint": "^2.0.2", @@ -58,16 +57,15 @@ }, "devDependencies": { "@chainsafe/ssz": "^1.2.2", - "@libp2p/crypto": "^5.1.7", - "@libp2p/logger": "^5.1.21", - "@libp2p/peer-id": "^5.1.8", + "@libp2p/crypto": "^5.1.13", + "@libp2p/logger": "^6.2.2", + "@libp2p/peer-id": "^6.0.4", "@lodestar/logger": "workspace:^", "@lodestar/types": "workspace:^", - "it-stream-types": "^2.0.2", - "libp2p": "2.9.0" + "libp2p": "3.1.6" }, "peerDependencies": { - "libp2p": "~2.9.0" + "libp2p": "^3.1.4" }, "keywords": [ "ethereum", diff --git a/packages/reqresp/src/ReqResp.ts b/packages/reqresp/src/ReqResp.ts index 6e1115041fca..756a735cb0f6 100644 --- a/packages/reqresp/src/ReqResp.ts +++ b/packages/reqresp/src/ReqResp.ts @@ -37,8 +37,8 @@ export interface ReqRespOpts extends SendRequestOpts, ReqRespRateLimiterOpts { /** * Implementation of Ethereum Consensus p2p Req/Resp domain. * For the spec that this code is based on, see: - * https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/p2p-interface.md#the-reqresp-domain - * https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/p2p-interface.md#the-reqresp-domain + * https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/p2p-interface.md#the-reqresp-domain + * https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/altair/light-client/p2p-interface.md#the-reqresp-domain */ export class ReqResp { // protected to be usable by extending class @@ -221,7 +221,7 @@ export class ReqResp { } private getRequestHandler(protocol: MixedProtocol, protocolID: string) { - return async ({connection, stream}: {connection: Connection; stream: Stream}) => { + return async (stream: Stream, connection: Connection) => { if (this.dialOnlyProtocols.get(protocolID)) { throw new Error(`Received request on dial only protocol '${protocolID}'`); } @@ -281,7 +281,7 @@ export class ReqResp { * ``` * /ProtocolPrefix/MessageName/SchemaVersion/Encoding * ``` - * https://github.com/ethereum/consensus-specs/blob/v1.2.0/specs/phase0/p2p-interface.md#protocol-identification + * https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/p2p-interface.md#protocol-identification */ protected formatProtocolID(protocol: Pick): string { return formatProtocolID(this.protocolPrefix, protocol.method, protocol.version, protocol.encoding); diff --git a/packages/reqresp/src/encoders/requestDecode.ts b/packages/reqresp/src/encoders/requestDecode.ts index d50f1ee37b5b..d134ac47f4ea 100644 --- a/packages/reqresp/src/encoders/requestDecode.ts +++ b/packages/reqresp/src/encoders/requestDecode.ts @@ -1,8 +1,8 @@ -import type {Sink} from "it-stream-types"; -import {Uint8ArrayList} from "uint8arraylist"; +import type {Stream} from "@libp2p/interface"; +import {byteStream} from "@libp2p/utils"; import {readEncodedPayload} from "../encodingStrategies/index.js"; import {MixedProtocol} from "../types.js"; -import {BufferedSource} from "../utils/index.js"; +import {drainByteStream} from "../utils/stream.js"; const EMPTY_DATA = new Uint8Array(); @@ -12,18 +12,34 @@ const EMPTY_DATA = new Uint8Array(); * request ::= | * ``` */ -export function requestDecode( - protocol: MixedProtocol -): Sink, Promise> { - return async function requestDecodeSink(source) { - const type = protocol.requestSizes; - if (type === null) { - // method has no body - return EMPTY_DATA; - } +export async function requestDecode( + protocol: MixedProtocol, + stream: Stream, + signal?: AbortSignal +): Promise { + const type = protocol.requestSizes; + if (type === null) { + // method has no body + return EMPTY_DATA; + } - // Request has a single payload, so return immediately - const bufferedSource = new BufferedSource(source as AsyncGenerator); - return readEncodedPayload(bufferedSource, protocol.encoding, type); - }; + // Request has a single payload, so return immediately + const bytes = byteStream(stream); + let requestReadDone = false; + try { + const requestBody = await readEncodedPayload(bytes, protocol.encoding, type, signal); + requestReadDone = true; + return requestBody; + } finally { + try { + if (!requestReadDone) { + // Do not push partial bytes back into the stream on decode failure/abort. + // This stream is consumed by req/resp only once. + drainByteStream(bytes); + } + bytes.unwrap(); + } catch { + // Ignore unwrap errors - stream may already be closed + } + } } diff --git a/packages/reqresp/src/encoders/requestEncode.ts b/packages/reqresp/src/encoders/requestEncode.ts index 752924c10118..76f9ed262ef9 100644 --- a/packages/reqresp/src/encoders/requestEncode.ts +++ b/packages/reqresp/src/encoders/requestEncode.ts @@ -9,7 +9,7 @@ import {MixedProtocol} from "../types.js"; * Requests may contain no payload (e.g. /eth2/beacon_chain/req/metadata/1/) * if so, it would yield no byte chunks */ -export async function* requestEncode(protocol: MixedProtocol, requestBody: Uint8Array): AsyncGenerator { +export function* requestEncode(protocol: MixedProtocol, requestBody: Uint8Array): Generator { const type = protocol.requestSizes; if (type && requestBody !== null) { diff --git a/packages/reqresp/src/encoders/responseDecode.ts b/packages/reqresp/src/encoders/responseDecode.ts index affe77a15134..ed8a2a2ee91e 100644 --- a/packages/reqresp/src/encoders/responseDecode.ts +++ b/packages/reqresp/src/encoders/responseDecode.ts @@ -1,4 +1,6 @@ -import {Uint8ArrayList} from "uint8arraylist"; +import type {Stream} from "@libp2p/interface"; +import type {ByteStream} from "@libp2p/utils"; +import {byteStream} from "@libp2p/utils"; import {ForkName} from "@lodestar/params"; import {readEncodedPayload} from "../encodingStrategies/index.js"; import {RespStatus} from "../interface.js"; @@ -10,7 +12,7 @@ import { MixedProtocol, ResponseIncoming, } from "../types.js"; -import {BufferedSource, decodeErrorMessage} from "../utils/index.js"; +import {decodeErrorMessage, drainByteStream} from "../utils/index.js"; /** * Internal helper type to signal stream ended early @@ -27,22 +29,17 @@ enum StreamStatus { * result ::= "0" | "1" | "2" | ["128" ... "255"] * ``` */ -export function responseDecode( +export async function* responseDecode( protocol: MixedProtocol, - cbs: { - onFirstHeader: () => void; - onFirstResponseChunk: () => void; - } -): (source: AsyncIterable) => AsyncIterable { - return async function* responseDecodeSink(source) { - const bufferedSource = new BufferedSource(source as AsyncGenerator); - - let readFirstHeader = false; - let readFirstResponseChunk = false; + stream: Stream, + opts: {signal?: AbortSignal; getError?: () => Error} = {} +): AsyncIterable { + const bytes = byteStream(stream); + let responseReadDone = false; - // Consumers of `responseDecode()` may limit the number of and break out of the while loop - while (!bufferedSource.isDone) { - const status = await readResultHeader(bufferedSource); + try { + while (true) { + const status = await readResultHeader(bytes, opts.signal); // Stream is only allowed to end at the start of a block // The happens when source ends before readResultHeader() can fetch 1 byte @@ -50,34 +47,41 @@ export function responseDecode( break; } - if (!readFirstHeader) { - cbs.onFirstHeader(); - readFirstHeader = true; - } - // For multiple chunks, only the last chunk is allowed to have a non-zero error // code (i.e. The chunk stream is terminated once an error occurs if (status !== RespStatus.SUCCESS) { - const errorMessage = await readErrorMessage(bufferedSource); + const errorMessage = await readErrorMessage(bytes, opts.signal); throw new ResponseError(status, errorMessage); } - const forkName = await readContextBytes(protocol.contextBytes, bufferedSource); + const forkName = await readContextBytes(protocol.contextBytes, bytes, opts.signal); const typeSizes = protocol.responseSizes(forkName); - const chunkData = await readEncodedPayload(bufferedSource, protocol.encoding, typeSizes); + const chunkData = await readEncodedPayload(bytes, protocol.encoding, typeSizes, opts.signal); yield { data: chunkData, fork: forkName, protocolVersion: protocol.version, }; - - if (!readFirstResponseChunk) { - cbs.onFirstResponseChunk(); - readFirstResponseChunk = true; + } + responseReadDone = true; + } catch (e) { + if (opts.signal?.aborted && opts.getError) { + throw opts.getError(); + } + throw e; + } finally { + try { + if (!responseReadDone) { + // Do not push partial bytes back into the stream on decode failure/abort. + // This stream is consumed by req/resp only once. + drainByteStream(bytes); } + bytes.unwrap(); + } catch { + // Ignore unwrap errors - stream may already be closed } - }; + } } /** @@ -87,18 +91,17 @@ export function responseDecode( * ``` * `` starts with a single-byte response code which determines the contents of the response_chunk */ -export async function readResultHeader(bufferedSource: BufferedSource): Promise { - for await (const buffer of bufferedSource) { - const status = buffer.get(0); - buffer.consume(1); - - // If first chunk had zero bytes status === null, get next - if (status !== null) { - return status; - } - } - - return StreamStatus.Ended; +export async function readResultHeader( + bytes: ByteStream, + signal?: AbortSignal +): Promise { + const chunk = await bytes.read({bytes: 1, signal}).catch((e) => { + if ((e as Error).name === "UnexpectedEOFError") return null; + throw e; + }); + if (chunk === null) return StreamStatus.Ended; + + return chunk.get(0); } /** @@ -108,28 +111,29 @@ export async function readResultHeader(bufferedSource: BufferedSource): Promise< * result ::= "1" | "2" | ["128" ... "255"] * ``` */ -export async function readErrorMessage(bufferedSource: BufferedSource): Promise { - // Read at least 256 or wait for the stream to end - let length: number | undefined; - for await (const buffer of bufferedSource) { - // Wait for next chunk with bytes or for the stream to end - // Note: The entire is expected to be in the same chunk - if (buffer.length >= 256) { - length = 256; +export async function readErrorMessage(bytes: ByteStream, signal?: AbortSignal): Promise { + const chunks: Uint8Array[] = []; + let total = 0; + + while (total < 256) { + const chunk = await bytes.read({signal}).catch((e) => { + if ((e as Error).name === "UnexpectedEOFError") return null; + throw e; + }); + if (chunk === null) { + // If EOF is reached while satisfying a larger read, libp2p v3 may still have + // buffered bytes available. Drain them so error_message matches pre-v3 behavior. + const remaining = drainByteStream(bytes); + if (remaining) { + chunks.push(remaining); + } break; } - length = buffer.length; + chunks.push(chunk.subarray()); + total += chunk.byteLength; } - // biome-ignore lint/complexity/useLiteralKeys: It is a private attribute - const bytes = bufferedSource["buffer"].slice(0, length); - - try { - return decodeErrorMessage(bytes); - } catch (_e) { - // Error message is optional and may not be included in the response stream - return Buffer.prototype.toString.call(bytes, "hex"); - } + return decodeErrorMessage(Buffer.concat(chunks).subarray(0, 256)); } /** @@ -139,14 +143,15 @@ export async function readErrorMessage(bufferedSource: BufferedSource): Promise< */ export async function readContextBytes( contextBytes: ContextBytesFactory, - bufferedSource: BufferedSource + bytes: ByteStream, + signal?: AbortSignal ): Promise { switch (contextBytes.type) { case ContextBytesType.Empty: return ForkName.phase0; case ContextBytesType.ForkDigest: { - const forkDigest = await readContextBytesForkDigest(bufferedSource); + const forkDigest = await readContextBytesForkDigest(bytes, signal); return contextBytes.config.forkDigest2ForkBoundary(forkDigest).fork; } } @@ -155,15 +160,6 @@ export async function readContextBytes( /** * Consumes a stream source to read ``, where it's a fixed-width 4 byte */ -export async function readContextBytesForkDigest(bufferedSource: BufferedSource): Promise { - for await (const buffer of bufferedSource) { - if (buffer.length >= CONTEXT_BYTES_FORK_DIGEST_LENGTH) { - const bytes = buffer.slice(0, CONTEXT_BYTES_FORK_DIGEST_LENGTH); - buffer.consume(CONTEXT_BYTES_FORK_DIGEST_LENGTH); - return bytes; - } - } - - // TODO: Use typed error - throw Error("Source ended while reading context bytes"); +export async function readContextBytesForkDigest(bytes: ByteStream, signal?: AbortSignal): Promise { + return (await bytes.read({bytes: CONTEXT_BYTES_FORK_DIGEST_LENGTH, signal})).subarray(); } diff --git a/packages/reqresp/src/encoders/responseEncode.ts b/packages/reqresp/src/encoders/responseEncode.ts index 90406164830b..c1c6f40ed6cf 100644 --- a/packages/reqresp/src/encoders/responseEncode.ts +++ b/packages/reqresp/src/encoders/responseEncode.ts @@ -14,30 +14,23 @@ const SUCCESS_BUFFER = Buffer.from([RespStatus.SUCCESS]); * ``` * Note: `response` has zero or more chunks (denoted by `<>*`) */ -export function responseEncodeSuccess( +export async function* responseEncodeSuccess( protocol: Protocol, - cbs: {onChunk: (chunkIndex: number) => void} -): (source: AsyncIterable) => AsyncIterable { - return async function* responseEncodeSuccessTransform(source) { - let chunkIndex = 0; + source: AsyncIterable +): AsyncIterable { + for await (const chunk of source) { + // + yield SUCCESS_BUFFER; - for await (const chunk of source) { - // Postfix increment, return 0 as first chunk - cbs.onChunk(chunkIndex++); - - // - yield SUCCESS_BUFFER; - - // - from altair - const contextBytes = getContextBytes(protocol.contextBytes, chunk); - if (contextBytes) { - yield contextBytes as Buffer; - } - - // | - yield* writeEncodedPayload(chunk.data, protocol.encoding); + // - from altair + const contextBytes = getContextBytes(protocol.contextBytes, chunk); + if (contextBytes) { + yield contextBytes; } - }; + + // | + yield* writeEncodedPayload(chunk.data, protocol.encoding); + } } /** @@ -50,20 +43,15 @@ export function responseEncodeSuccess( * Only the last `` is allowed to have a non-zero error code, so this * fn yields exactly one `` and afterwards the stream must be terminated */ -export async function* responseEncodeError( +export function* responseEncodeError( protocol: Pick, status: RpcResponseStatusError, errorMessage: string -): AsyncGenerator { - if (!errorMessage) { - yield Buffer.from([status]); - return; - } - +): Generator { // Combine and into a single chunk for atomic delivery. // Yielding them separately causes a race condition where the stream closes after the // status byte but before the error message arrives on the reader side. - const errorMessageBuffer = await encodeErrorMessageToBuffer(errorMessage, protocol.encoding); + const errorMessageBuffer = encodeErrorMessageToBuffer(errorMessage, protocol.encoding); yield Buffer.concat([Buffer.from([status]), errorMessageBuffer]); } diff --git a/packages/reqresp/src/encodingStrategies/index.ts b/packages/reqresp/src/encodingStrategies/index.ts index 209359a7c22d..c78ea0a45ba9 100644 --- a/packages/reqresp/src/encodingStrategies/index.ts +++ b/packages/reqresp/src/encodingStrategies/index.ts @@ -1,10 +1,11 @@ +import type {Stream} from "@libp2p/interface"; +import type {ByteStream} from "@libp2p/utils"; import {Encoding, TypeSizes} from "../types.js"; -import {BufferedSource} from "../utils/index.js"; import {readSszSnappyPayload} from "./sszSnappy/decode.js"; import {writeSszSnappyPayload} from "./sszSnappy/encode.js"; // For more info about Ethereum Consensus request/response encoding strategies, see: -// https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/p2p-interface.md#encoding-strategies +// https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/p2p-interface.md#encoding-strategies // Supported encoding strategies: // - ssz_snappy @@ -15,13 +16,14 @@ import {writeSszSnappyPayload} from "./sszSnappy/encode.js"; * ``` */ export async function readEncodedPayload( - bufferedSource: BufferedSource, + stream: ByteStream, encoding: Encoding, - type: TypeSizes + type: TypeSizes, + signal?: AbortSignal ): Promise { switch (encoding) { case Encoding.SSZ_SNAPPY: - return readSszSnappyPayload(bufferedSource, type); + return readSszSnappyPayload(stream, type, signal); default: throw Error("Unsupported encoding"); @@ -34,7 +36,7 @@ export async function readEncodedPayload( * | * ``` */ -export async function* writeEncodedPayload(chunkData: Uint8Array, encoding: Encoding): AsyncGenerator { +export function* writeEncodedPayload(chunkData: Uint8Array, encoding: Encoding): Generator { switch (encoding) { case Encoding.SSZ_SNAPPY: yield* writeSszSnappyPayload(chunkData); diff --git a/packages/reqresp/src/encodingStrategies/sszSnappy/decode.ts b/packages/reqresp/src/encodingStrategies/sszSnappy/decode.ts index 254a9268da9e..1e63509ccae3 100644 --- a/packages/reqresp/src/encodingStrategies/sszSnappy/decode.ts +++ b/packages/reqresp/src/encodingStrategies/sszSnappy/decode.ts @@ -1,8 +1,9 @@ +import type {Stream} from "@libp2p/interface"; +import type {ByteStream} from "@libp2p/utils"; import {decode as varintDecode, encodingLength as varintEncodingLength} from "uint8-varint"; import {Uint8ArrayList} from "uint8arraylist"; import {TypeSizes} from "../../types.js"; -import {BufferedSource} from "../../utils/index.js"; -import {SnappyFramesUncompress} from "../../utils/snappyIndex.js"; +import {ChunkType, decodeSnappyFrameData, parseSnappyFrameHeader} from "../../utils/snappyIndex.js"; import {SszSnappyError, SszSnappyErrorCode} from "./errors.js"; import {maxEncodedLen} from "./utils.js"; @@ -15,76 +16,115 @@ export const MAX_VARINT_BYTES = 10; * | * ``` */ -export async function readSszSnappyPayload(bufferedSource: BufferedSource, type: TypeSizes): Promise { - const sszDataLength = await readSszSnappyHeader(bufferedSource, type); - - return readSszSnappyBody(bufferedSource, sszDataLength); +export async function readSszSnappyPayload( + stream: ByteStream, + type: TypeSizes, + signal?: AbortSignal +): Promise { + const sszDataLength = await readSszSnappyHeader(stream, type, signal); + + return readSszSnappyBody(stream, sszDataLength, signal); } /** * Reads `` for ssz-snappy. * encoding-header ::= the length of the raw SSZ bytes, encoded as an unsigned protobuf varint */ -export async function readSszSnappyHeader(bufferedSource: BufferedSource, type: TypeSizes): Promise { - for await (const buffer of bufferedSource) { - // Get next bytes if empty - if (buffer.length === 0) { - continue; - } +export async function readSszSnappyHeader( + stream: ByteStream, + type: TypeSizes, + signal?: AbortSignal +): Promise { + const varintBytes: number[] = []; - let sszDataLength: number; - try { - sszDataLength = varintDecode(buffer.subarray()); - } catch (_e) { - throw new SszSnappyError({code: SszSnappyErrorCode.INVALID_VARINT_BYTES_COUNT, bytes: Infinity}); - } + while (true) { + const byte = await readExactOrSourceAborted(stream, 1, signal); - // MUST validate: the unsigned protobuf varint used for the length-prefix MUST not be longer than 10 bytes - // encodingLength function only returns 1-8 inclusive - const varintBytes = varintEncodingLength(sszDataLength); - buffer.consume(varintBytes); + const value = byte.get(0); - // MUST validate: the length-prefix is within the expected size bounds derived from the payload SSZ type. - const minSize = type.minSize; - const maxSize = type.maxSize; - if (sszDataLength < minSize) { - throw new SszSnappyError({code: SszSnappyErrorCode.UNDER_SSZ_MIN_SIZE, minSize, sszDataLength}); - } - if (sszDataLength > maxSize) { - throw new SszSnappyError({code: SszSnappyErrorCode.OVER_SSZ_MAX_SIZE, maxSize, sszDataLength}); + varintBytes.push(value); + if (varintBytes.length > MAX_VARINT_BYTES) { + throw new SszSnappyError({code: SszSnappyErrorCode.INVALID_VARINT_BYTES_COUNT, bytes: varintBytes.length}); } - return sszDataLength; + // MSB not set => varint terminated + if ((value & 0x80) === 0) break; + } + + let sszDataLength: number; + try { + sszDataLength = varintDecode(Uint8Array.from(varintBytes)); + } catch { + throw new SszSnappyError({code: SszSnappyErrorCode.INVALID_VARINT_BYTES_COUNT, bytes: Infinity}); } - throw new SszSnappyError({code: SszSnappyErrorCode.SOURCE_ABORTED}); + // MUST validate: the unsigned protobuf varint used for the length-prefix MUST not be longer than 10 bytes + // encodingLength function only returns 1-8 inclusive + const varintByteLength = varintEncodingLength(sszDataLength); + if (varintByteLength > MAX_VARINT_BYTES) { + throw new SszSnappyError({code: SszSnappyErrorCode.INVALID_VARINT_BYTES_COUNT, bytes: varintByteLength}); + } + + // MUST validate: the length-prefix is within the expected size bounds derived from the payload SSZ type. + const minSize = type.minSize; + const maxSize = type.maxSize; + if (sszDataLength < minSize) { + throw new SszSnappyError({code: SszSnappyErrorCode.UNDER_SSZ_MIN_SIZE, minSize, sszDataLength}); + } + if (sszDataLength > maxSize) { + throw new SszSnappyError({code: SszSnappyErrorCode.OVER_SSZ_MAX_SIZE, maxSize, sszDataLength}); + } + + return sszDataLength; } /** * Reads `` for ssz-snappy and decompress. * The returned bytes can be SSZ deseralized */ -export async function readSszSnappyBody(bufferedSource: BufferedSource, sszDataLength: number): Promise { - const decompressor = new SnappyFramesUncompress(); +export async function readSszSnappyBody( + stream: ByteStream, + sszDataLength: number, + signal?: AbortSignal +): Promise { const uncompressedData = new Uint8ArrayList(); - let readBytes = 0; + let encodedBytesRead = 0; + const maxBytes = maxEncodedLen(sszDataLength); + let foundIdentifier = false; + + while (uncompressedData.length < sszDataLength) { + const header = await readExactOrSourceAborted(stream, 4, signal); - for await (const buffer of bufferedSource) { // SHOULD NOT read more than max_encoded_len(n) bytes after reading the SSZ length-prefix n from the header - readBytes += buffer.length; - if (readBytes > maxEncodedLen(sszDataLength)) { - throw new SszSnappyError({code: SszSnappyErrorCode.TOO_MUCH_BYTES_READ, readBytes, sszDataLength}); + encodedBytesRead = addEncodedBytesReadOrThrow(encodedBytesRead, header.length, maxBytes, sszDataLength); + + let headerParsed: {type: ChunkType; frameSize: number}; + try { + headerParsed = parseSnappyFrameHeader(header.subarray()); + if (!foundIdentifier && headerParsed.type !== ChunkType.IDENTIFIER) { + throw new Error("malformed input: must begin with an identifier"); + } + } catch (e) { + throw new SszSnappyError({code: SszSnappyErrorCode.DECOMPRESSOR_ERROR, decompressorError: e as Error}); } - // No bytes left to consume, get next - if (buffer.length === 0) { - continue; + if (headerParsed.frameSize > maxBytes - encodedBytesRead) { + throw new SszSnappyError({ + code: SszSnappyErrorCode.TOO_MUCH_BYTES_READ, + readBytes: encodedBytesRead + headerParsed.frameSize, + sszDataLength, + }); } + const frame = await readExactOrSourceAborted(stream, headerParsed.frameSize, signal); + + encodedBytesRead = addEncodedBytesReadOrThrow(encodedBytesRead, frame.length, maxBytes, sszDataLength); - // stream contents can be passed through a buffered Snappy reader to decompress frame by frame try { - const uncompressed = decompressor.uncompress(buffer); - buffer.consume(buffer.length); + if (headerParsed.type === ChunkType.IDENTIFIER) { + foundIdentifier = true; + } + + const uncompressed = decodeSnappyFrameData(headerParsed.type, frame.subarray()); if (uncompressed !== null) { uncompressedData.append(uncompressed); } @@ -96,16 +136,34 @@ export async function readSszSnappyBody(bufferedSource: BufferedSource, sszDataL if (uncompressedData.length > sszDataLength) { throw new SszSnappyError({code: SszSnappyErrorCode.TOO_MANY_BYTES, sszDataLength}); } + } - // Keep reading chunks until `n` SSZ bytes - if (uncompressedData.length < sszDataLength) { - continue; - } + // buffer.length === n + return uncompressedData.subarray(0, sszDataLength); +} - // buffer.length === n - return uncompressedData.subarray(0, sszDataLength); +function addEncodedBytesReadOrThrow( + encodedBytesRead: number, + bytesToAdd: number, + maxBytes: number, + sszDataLength: number +): number { + const nextReadBytes = encodedBytesRead + bytesToAdd; + if (nextReadBytes > maxBytes) { + throw new SszSnappyError({code: SszSnappyErrorCode.TOO_MUCH_BYTES_READ, readBytes: nextReadBytes, sszDataLength}); } + return nextReadBytes; +} - // SHOULD consider invalid: An early EOF before fully reading the declared length-prefix worth of SSZ bytes - throw new SszSnappyError({code: SszSnappyErrorCode.SOURCE_ABORTED}); +async function readExactOrSourceAborted( + stream: ByteStream, + bytes: number, + signal?: AbortSignal +): Promise { + return stream.read({bytes, signal}).catch((e) => { + if ((e as Error).name === "UnexpectedEOFError") { + throw new SszSnappyError({code: SszSnappyErrorCode.SOURCE_ABORTED}); + } + throw e; + }); } diff --git a/packages/reqresp/src/encodingStrategies/sszSnappy/encode.ts b/packages/reqresp/src/encodingStrategies/sszSnappy/encode.ts index edea19b3c198..6f34462d5fa5 100644 --- a/packages/reqresp/src/encodingStrategies/sszSnappy/encode.ts +++ b/packages/reqresp/src/encodingStrategies/sszSnappy/encode.ts @@ -8,12 +8,12 @@ import {encodeSnappy} from "../../utils/snappyIndex.js"; * | * ``` */ -export const writeSszSnappyPayload = encodeSszSnappy as (bytes: Uint8Array) => AsyncGenerator; +export const writeSszSnappyPayload = encodeSszSnappy as (bytes: Uint8Array) => Generator; /** * Buffered Snappy writer */ -export async function* encodeSszSnappy(bytes: Buffer): AsyncGenerator { +export function* encodeSszSnappy(bytes: Buffer): Generator { // MUST encode the length of the raw SSZ bytes, encoded as an unsigned protobuf varint const varint = varintEncode(bytes.length); yield Buffer.from(varint.buffer, varint.byteOffset, varint.byteLength); diff --git a/packages/reqresp/src/encodingStrategies/sszSnappy/errors.ts b/packages/reqresp/src/encodingStrategies/sszSnappy/errors.ts index f9d6c4a8e9b5..bfd3b9417b87 100644 --- a/packages/reqresp/src/encodingStrategies/sszSnappy/errors.ts +++ b/packages/reqresp/src/encodingStrategies/sszSnappy/errors.ts @@ -9,8 +9,6 @@ export enum SszSnappyErrorCode { OVER_SSZ_MAX_SIZE = "SSZ_SNAPPY_ERROR_OVER_SSZ_MAX_SIZE", TOO_MUCH_BYTES_READ = "SSZ_SNAPPY_ERROR_TOO_MUCH_BYTES_READ", DECOMPRESSOR_ERROR = "SSZ_SNAPPY_ERROR_DECOMPRESSOR_ERROR", - DESERIALIZE_ERROR = "SSZ_SNAPPY_ERROR_DESERIALIZE_ERROR", - SERIALIZE_ERROR = "SSZ_SNAPPY_ERROR_SERIALIZE_ERROR", /** Received more bytes than specified sszDataLength */ TOO_MANY_BYTES = "SSZ_SNAPPY_ERROR_TOO_MANY_BYTES", /** Source aborted before reading sszDataLength bytes */ @@ -23,8 +21,6 @@ type SszSnappyErrorType = | {code: SszSnappyErrorCode.OVER_SSZ_MAX_SIZE; maxSize: number; sszDataLength: number} | {code: SszSnappyErrorCode.TOO_MUCH_BYTES_READ; readBytes: number; sszDataLength: number} | {code: SszSnappyErrorCode.DECOMPRESSOR_ERROR; decompressorError: Error} - | {code: SszSnappyErrorCode.DESERIALIZE_ERROR; deserializeError: Error} - | {code: SszSnappyErrorCode.SERIALIZE_ERROR; serializeError: Error} | {code: SszSnappyErrorCode.TOO_MANY_BYTES; sszDataLength: number} | {code: SszSnappyErrorCode.SOURCE_ABORTED}; diff --git a/packages/reqresp/src/encodingStrategies/sszSnappy/index.ts b/packages/reqresp/src/encodingStrategies/sszSnappy/index.ts index 8aaf7fc5e6f8..e8a76688571c 100644 --- a/packages/reqresp/src/encodingStrategies/sszSnappy/index.ts +++ b/packages/reqresp/src/encodingStrategies/sszSnappy/index.ts @@ -1,4 +1,3 @@ -export {SnappyFramesUncompress, encodeSnappy} from "../../utils/snappyIndex.js"; export * from "./decode.js"; export * from "./encode.js"; export * from "./errors.js"; diff --git a/packages/reqresp/src/metrics.ts b/packages/reqresp/src/metrics.ts index 7d8386742979..564e29300dd6 100644 --- a/packages/reqresp/src/metrics.ts +++ b/packages/reqresp/src/metrics.ts @@ -4,7 +4,7 @@ import {RequestErrorCode} from "./request/errors.js"; export type Metrics = ReturnType; /** - * A collection of metrics used throughout the Gossipsub behaviour. + * A collection of metrics used throughout the Req/Resp domain. */ export function getMetrics(register: MetricsRegisterExtra) { // Using function style instead of class to prevent having to re-declare all MetricsPrometheus types. @@ -29,7 +29,6 @@ export function getMetrics(register: MetricsRegisterExtra) { name: "beacon_reqresp_outgoing_request_roundtrip_time_seconds", help: "Histogram of outgoing requests round-trip time", labelNames: ["method"], - // Spec sets RESP_TIMEOUT = 10 sec buckets: [0.1, 0.2, 0.5, 1, 5, 10, 15, 60], }), outgoingErrors: register.gauge<{method: string}>({ @@ -61,7 +60,6 @@ export function getMetrics(register: MetricsRegisterExtra) { name: "beacon_reqresp_incoming_request_handler_time_seconds", help: "Histogram of incoming requests internal handling time", labelNames: ["method"], - // Spec sets RESP_TIMEOUT = 10 sec buckets: [0.1, 0.2, 0.5, 1, 5, 10], }), incomingErrors: register.gauge<{method: string}>({ @@ -69,20 +67,6 @@ export function getMetrics(register: MetricsRegisterExtra) { help: "Counts total failed responses handled per method", labelNames: ["method"], }), - outgoingResponseTTFB: register.histogram<{method: string}>({ - name: "beacon_reqresp_outgoing_response_ttfb_seconds", - help: "Time to first byte (TTFB) for outgoing responses", - labelNames: ["method"], - // Spec sets TTFB_TIMEOUT = 5 sec - buckets: [0.1, 1, 5], - }), - incomingResponseTTFB: register.histogram<{method: string}>({ - name: "beacon_reqresp_incoming_response_ttfb_seconds", - help: "Time to first byte (TTFB) for incoming responses", - labelNames: ["method"], - // Spec sets TTFB_TIMEOUT = 5 sec - buckets: [0.1, 1, 5], - }), dialErrors: register.gauge({ name: "beacon_reqresp_dial_errors_total", help: "Count total dial errors", diff --git a/packages/reqresp/src/request/errors.ts b/packages/reqresp/src/request/errors.ts index 2f5c7722f463..5d84b051f004 100644 --- a/packages/reqresp/src/request/errors.ts +++ b/packages/reqresp/src/request/errors.ts @@ -21,13 +21,9 @@ export enum RequestErrorCode { REQUEST_TIMEOUT = "REQUEST_ERROR_REQUEST_TIMEOUT", /** Error when sending request to responder */ REQUEST_ERROR = "REQUEST_ERROR_REQUEST_ERROR", - /** Reponder did not deliver a full reponse before max maxTotalResponseTimeout() */ - RESPONSE_TIMEOUT = "REQUEST_ERROR_RESPONSE_TIMEOUT", /** A single-response method returned 0 chunks */ EMPTY_RESPONSE = "REQUEST_ERROR_EMPTY_RESPONSE", - /** Time to first byte timeout */ - TTFB_TIMEOUT = "REQUEST_ERROR_TTFB_TIMEOUT", - /** Timeout between `` exceed */ + /** Response transfer timeout exceeded */ RESP_TIMEOUT = "REQUEST_ERROR_RESP_TIMEOUT", /** Request rate limited */ REQUEST_RATE_LIMITED = "REQUEST_ERROR_RATE_LIMITED", @@ -50,7 +46,6 @@ type RequestErrorType = | {code: RequestErrorCode.REQUEST_TIMEOUT} | {code: RequestErrorCode.REQUEST_ERROR; error: Error} | {code: RequestErrorCode.EMPTY_RESPONSE} - | {code: RequestErrorCode.TTFB_TIMEOUT} | {code: RequestErrorCode.RESP_TIMEOUT} | {code: RequestErrorCode.REQUEST_RATE_LIMITED} | {code: RequestErrorCode.REQUEST_SELF_RATE_LIMITED} diff --git a/packages/reqresp/src/request/index.ts b/packages/reqresp/src/request/index.ts index 0e37d36ed0dc..6042a12d738e 100644 --- a/packages/reqresp/src/request/index.ts +++ b/packages/reqresp/src/request/index.ts @@ -1,31 +1,93 @@ +import type {Stream} from "@libp2p/interface"; import {PeerId} from "@libp2p/interface"; -import {pipe} from "it-pipe"; import type {Libp2p} from "libp2p"; -import {Uint8ArrayList} from "uint8arraylist"; import {ErrorAborted, Logger, TimeoutError, withTimeout} from "@lodestar/utils"; import {requestEncode} from "../encoders/requestEncode.js"; import {responseDecode} from "../encoders/responseDecode.js"; import {Metrics} from "../metrics.js"; import {ResponseError} from "../response/index.js"; import {MixedProtocol, ResponseIncoming} from "../types.js"; -import {abortableSource, prettyPrintPeerId} from "../utils/index.js"; +import {prettyPrintPeerId, sendChunks} from "../utils/index.js"; import {RequestError, RequestErrorCode, responseStatusErrorToRequestError} from "./errors.js"; export {RequestError, RequestErrorCode}; -// Default spec values from https://github.com/ethereum/consensus-specs/blob/v1.2.0/specs/phase0/p2p-interface.md#configuration +// https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/p2p-interface.md#the-reqresp-domain export const DEFAULT_DIAL_TIMEOUT = 5 * 1000; // 5 sec export const DEFAULT_REQUEST_TIMEOUT = 5 * 1000; // 5 sec -export const DEFAULT_TTFB_TIMEOUT = 5 * 1000; // 5 sec export const DEFAULT_RESP_TIMEOUT = 10 * 1000; // 10 sec +function getStreamNotFullyConsumedError(): Error { + return new Error("ReqResp stream was not fully consumed"); +} + +function scheduleStreamAbortIfNotClosed(stream: Stream, timeoutMs: number): void { + const onClose = (): void => { + clearTimeout(timeout); + }; + + const timeout = setTimeout(() => { + stream.removeEventListener("close", onClose); + if (stream.status === "open" && stream.remoteWriteStatus === "writable") { + stream.abort(getStreamNotFullyConsumedError()); + } + }, timeoutMs); + + stream.addEventListener("close", onClose, {once: true}); +} + +type ClearableSignal = AbortSignal & {clear: () => void}; + +/** + * Compose an abort signal from an optional parent signal and a timeout, with explicit cleanup. + * + * This replaces a plain `AbortSignal.any([signal, AbortSignal.timeout(timeoutMs)])` to work + * around a memory leak in Node.js where AbortSignal.any() retains references to all source + * signals for the lifetime of the longest-lived signal (the timeout). In a long-running + * req/resp workload this causes unbounded growth of the dependent-signal set. + * + * Upstream issue: https://github.com/nodejs/node/issues/54614 + * Lodestar investigation: https://github.com/ChainSafe/lodestar/issues/8969 + */ +function createRespSignal(signal: AbortSignal | undefined, timeoutMs: number): ClearableSignal { + const timeoutSignal = AbortSignal.timeout(timeoutMs); + const signals = signal ? [signal, timeoutSignal] : [timeoutSignal]; + const controller = new AbortController(); + + const clear = (): void => { + for (const entry of signals) { + entry.removeEventListener("abort", onAbort); + } + }; + + const onAbort = (): void => { + if (controller.signal.aborted) { + return; + } + const reason = signals.find((entry) => entry.aborted)?.reason; + controller.abort(reason); + clear(); + }; + + for (const entry of signals) { + if (entry.aborted) { + onAbort(); + break; + } + entry.addEventListener("abort", onAbort); + } + + const respSignal = controller.signal as ClearableSignal; + respSignal.clear = clear; + + return respSignal; +} + export interface SendRequestOpts { /** The maximum time for complete response transfer. */ respTimeoutMs?: number; /** Non-spec timeout from sending request until write stream closed by responder */ requestTimeoutMs?: number; - /** The maximum time to wait for first byte of request response (time-to-first-byte). */ - ttfbTimeoutMs?: number; /** Non-spec timeout from dialing protocol until stream opened */ dialTimeoutMs?: number; } @@ -64,7 +126,6 @@ export async function* sendRequest( const DIAL_TIMEOUT = opts?.dialTimeoutMs ?? DEFAULT_DIAL_TIMEOUT; const REQUEST_TIMEOUT = opts?.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT; - const TTFB_TIMEOUT = opts?.ttfbTimeoutMs ?? DEFAULT_TTFB_TIMEOUT; const RESP_TIMEOUT = opts?.respTimeoutMs ?? DEFAULT_RESP_TIMEOUT; const peerIdStrShort = prettyPrintPeerId(peerId); @@ -83,17 +144,6 @@ export async function* sendRequest( // the picked protocol in `connection.protocol` const protocolsMap = new Map(protocols.map((protocol, i) => [protocolIDs[i], protocol])); - // As of October 2020 we can't rely on libp2p.dialProtocol timeout to work so - // this function wraps the dialProtocol promise with an extra timeout - // - // > The issue might be: you add the peer's addresses to the AddressBook, - // which will result in autoDial to kick in and dial your peer. In parallel, - // you do a manual dial and it will wait for the previous one without using - // the abort signal: - // - // https://github.com/ChainSafe/lodestar/issues/1597#issuecomment-703394386 - - // DIAL_TIMEOUT: Non-spec timeout from dialing protocol until stream opened const stream = await withTimeout( async (timeoutAndParentSignal) => { const protocolIds = Array.from(protocolsMap.keys()); @@ -112,9 +162,6 @@ export async function* sendRequest( metrics?.outgoingOpenedStreams?.inc({method}); - // TODO: Does the TTFB timer start on opening stream or after receiving request - const timerTTFB = metrics?.outgoingResponseTTFB.startTimer({method}); - // Parse protocol selected by the responder const protocolId = stream.protocol ?? "unknown"; const protocol = protocolsMap.get(protocolId); @@ -126,21 +173,24 @@ export async function* sendRequest( logger.debug("Req sending request", logCtx); // Spec: The requester MUST close the write side of the stream once it finishes writing the request message - // Impl: stream.sink is closed automatically by js-libp2p-mplex when piped source is exhausted // REQUEST_TIMEOUT: Non-spec timeout from sending request until write stream closed by responder - // Note: libp2p.stop() will close all connections, so not necessary to abort this pipe on parent stop - await withTimeout(() => pipe(requestEncode(protocol, requestBody), stream.sink), REQUEST_TIMEOUT, signal).catch( - (e) => { - // Must close the stream read side (stream.source) manually AND the write side - stream.abort(e); - - if (e instanceof TimeoutError) { - throw new RequestError({code: RequestErrorCode.REQUEST_TIMEOUT}); - } - throw new RequestError({code: RequestErrorCode.REQUEST_ERROR, error: e as Error}); + // Note: libp2p.stop() will close all connections, so not necessary to abort this send on parent stop + await withTimeout( + async (timeoutAndParentSignal) => { + await sendChunks(stream, requestEncode(protocol, requestBody), timeoutAndParentSignal); + await stream.close({signal: timeoutAndParentSignal}); + }, + REQUEST_TIMEOUT, + signal + ).catch((e) => { + stream.abort(e as Error); + + if (e instanceof TimeoutError) { + throw new RequestError({code: RequestErrorCode.REQUEST_TIMEOUT}); } - ); + throw new RequestError({code: RequestErrorCode.REQUEST_ERROR, error: e as Error}); + }); logger.debug("Req request sent", logCtx); @@ -150,67 +200,51 @@ export async function* sendRequest( return; } - // - TTFB_TIMEOUT: The requester MUST wait a maximum of TTFB_TIMEOUT for the first response byte to arrive - // - RESP_TIMEOUT: Requester allows a further RESP_TIMEOUT for each subsequent response_chunk - // - Max total timeout: This timeout is not required by the spec. It may not be necessary, but it's kept as - // safe-guard to close. streams in case of bugs on other timeout mechanisms. - const ttfbTimeoutController = new AbortController(); - const respTimeoutController = new AbortController(); - - let timeoutRESP: NodeJS.Timeout | null = null; + // RESP_TIMEOUT: Maximum time for complete response transfer + const respSignal = createRespSignal(signal, RESP_TIMEOUT); - const timeoutTTFB = setTimeout(() => { - // If we abort on first byte delay, don't need to abort for response delay - if (timeoutRESP) clearTimeout(timeoutRESP); - ttfbTimeoutController.abort(); - }, TTFB_TIMEOUT); - - const restartRespTimeout = (): void => { - if (timeoutRESP) clearTimeout(timeoutRESP); - timeoutRESP = setTimeout(() => respTimeoutController.abort(), RESP_TIMEOUT); - }; + let responseError: Error | null = null; + let responseFullyConsumed = false; try { - // Note: libp2p.stop() will close all connections, so not necessary to abort this pipe on parent stop - yield* pipe( - abortableSource(stream.source as AsyncIterable, [ - { - signal: ttfbTimeoutController.signal, - getError: () => new RequestError({code: RequestErrorCode.TTFB_TIMEOUT}), - }, - { - signal: respTimeoutController.signal, - getError: () => new RequestError({code: RequestErrorCode.RESP_TIMEOUT}), - }, - ]), - - // Transforms `Buffer` chunks to yield `ResponseBody` chunks - responseDecode(protocol, { - onFirstHeader() { - // On first byte, cancel the single use TTFB_TIMEOUT, and start RESP_TIMEOUT - clearTimeout(timeoutTTFB); - timerTTFB?.(); - restartRespTimeout(); - }, - onFirstResponseChunk() { - // On , cancel this chunk's RESP_TIMEOUT and start next's - restartRespTimeout(); - }, - }) - ); + yield* responseDecode(protocol, stream, { + signal: respSignal, + getError: () => + signal?.aborted ? new ErrorAborted("sendRequest") : new RequestError({code: RequestErrorCode.RESP_TIMEOUT}), + }); + responseFullyConsumed = true; // NOTE: Only log once per request to verbose, intermediate steps to debug // NOTE: Do not log the response, logs get extremely cluttered // NOTE: add double space after "Req " to align log with the "Resp " log logger.verbose("Req done", logCtx); + } catch (e) { + responseError = e as Error; + throw e; } finally { - clearTimeout(timeoutTTFB); - if (timeoutRESP !== null) clearTimeout(timeoutRESP); + // On decode/timeout failures abort immediately so mplex can reclaim stream state. + // On normal early consumer exit, close gracefully to avoid stream-id desync with peers. + if (responseError !== null || signal?.aborted) { + stream.abort(responseError ?? new ErrorAborted("sendRequest")); + } else { + await stream.close().catch((e) => { + stream.abort(e as Error); + }); + + if (!responseFullyConsumed) { + // Stop buffering unread inbound data after caller exits early. + // mplex does not support propagating closeRead to the remote, so still + // abort later if the remote never closes write. + await stream.closeRead().catch(() => { + // Ignore closeRead errors - close/abort path below will reclaim stream. + }); - // Necessary to call `stream.close()` since collectResponses() may break out of the source before exhausting it - // `stream.close()` libp2p-mplex will .end() the source (it-pushable instance) - // If collectResponses() exhausts the source, it-pushable.end() can be safely called multiple times - await stream.close(); + if (stream.remoteWriteStatus === "writable") { + scheduleStreamAbortIfNotClosed(stream, RESP_TIMEOUT); + } + } + } + respSignal.clear(); metrics?.outgoingClosedStreams?.inc({method}); logger.verbose("Req stream closed", logCtx); } diff --git a/packages/reqresp/src/response/index.ts b/packages/reqresp/src/response/index.ts index 4226570c2d33..7f032e03c5f3 100644 --- a/packages/reqresp/src/response/index.ts +++ b/packages/reqresp/src/response/index.ts @@ -1,5 +1,4 @@ import {PeerId, Stream} from "@libp2p/interface"; -import {pipe} from "it-pipe"; import {Logger, TimeoutError, withTimeout} from "@lodestar/utils"; import {requestDecode} from "../encoders/requestDecode.js"; import {responseEncodeError, responseEncodeSuccess} from "../encoders/responseEncode.js"; @@ -8,12 +7,12 @@ import {Metrics} from "../metrics.js"; import {ReqRespRateLimiter} from "../rate_limiter/ReqRespRateLimiter.js"; import {RequestError, RequestErrorCode} from "../request/errors.js"; import {Protocol, ReqRespRequest} from "../types.js"; -import {prettyPrintPeerId} from "../utils/index.js"; +import {prettyPrintPeerId, sendChunks} from "../utils/index.js"; import {ResponseError} from "./errors.js"; export {ResponseError}; -// Default spec values from https://github.com/ethereum/consensus-specs/blob/v1.2.0/specs/phase0/p2p-interface.md#configuration +// https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/p2p-interface.md#the-reqresp-domain export const DEFAULT_REQUEST_TIMEOUT = 5 * 1000; // 5 sec export interface HandleRequestOpts { @@ -28,7 +27,7 @@ export interface HandleRequestOpts { requestId?: number; /** Peer client type for logging and metrics: 'prysm' | 'lighthouse' */ peerClient?: string; - /** Non-spec timeout from sending request until write stream closed by responder */ + /** Timeout for reading the incoming request payload */ requestTimeoutMs?: number; } @@ -67,71 +66,66 @@ export async function handleRequest({ metrics?.incomingOpenedStreams.inc({method: protocol.method}); let responseError: Error | null = null; - await pipe( + let streamSendError: Error | null = null; + + try { // Yields success chunks and error chunks in the same generator - // This syntax allows to recycle stream.sink to send success and error chunks without returning + // This syntax allows to recycle stream to send success and error chunks without returning // in case request whose body is a List fails at chunk_i > 0, without breaking out of the for..await..of - (async function* requestHandlerSource() { - try { - // TODO: Does the TTFB timer start on opening stream or after receiving request - const timerTTFB = metrics?.outgoingResponseTTFB.startTimer({method: protocol.method}); - - const requestBody = await withTimeout( - () => pipe(stream.source, requestDecode(protocol)), - REQUEST_TIMEOUT, - signal - ).catch((e: unknown) => { - if (e instanceof TimeoutError) { - throw e; // Let outter catch (_e) {} re-type the error as SERVER_ERROR + await sendChunks( + stream, + (async function* requestHandlerSource() { + try { + const requestBody = await withTimeout( + (timeoutAndParentSignal) => requestDecode(protocol, stream, timeoutAndParentSignal), + REQUEST_TIMEOUT, + signal + ).catch((e: unknown) => { + if (e instanceof TimeoutError) { + throw e; // Let outer catch re-type the error as SERVER_ERROR + } + throw new ResponseError(RespStatus.INVALID_REQUEST, (e as Error).message); + }); + + logger.debug("Req received", logCtx); + + // Max count by request for byRange and byRoot + const requestCount = protocol?.inboundRateLimits?.getRequestCount?.(requestBody) ?? 1; + + if (!rateLimiter.allows(peerId, protocolID, requestCount)) { + throw new RequestError({code: RequestErrorCode.REQUEST_RATE_LIMITED}); } - throw new ResponseError(RespStatus.INVALID_REQUEST, (e as Error).message); - }); - logger.debug("Req received", logCtx); + const requestChunk: ReqRespRequest = { + data: requestBody, + version: protocol.version, + }; - // Max count by request for byRange and byRoot - const requestCount = protocol?.inboundRateLimits?.getRequestCount?.(requestBody) ?? 1; + yield* responseEncodeSuccess(protocol, protocol.handler(requestChunk, peerId, peerClient)); + } catch (e) { + const status = e instanceof ResponseError ? e.status : RespStatus.SERVER_ERROR; + yield* responseEncodeError(protocol, status, (e as Error).message); - if (!rateLimiter.allows(peerId, protocolID, requestCount)) { - throw new RequestError({code: RequestErrorCode.REQUEST_RATE_LIMITED}); + responseError = e as Error; } + })(), + signal + ); + } catch (e) { + streamSendError = e as Error; + throw e; + } finally { + if (streamSendError) { + stream.abort(streamSendError); + } else { + await stream.close().catch((e) => { + stream.abort(e as Error); + }); + } + metrics?.incomingClosedStreams.inc({method: protocol.method}); + } - const requestChunk: ReqRespRequest = { - data: requestBody, - version: protocol.version, - }; - - yield* pipe( - // TODO: Debug the reason for type conversion here - protocol.handler(requestChunk, peerId, peerClient), - // NOTE: Do not log the resp chunk contents, logs get extremely cluttered - // Note: Not logging on each chunk since after 1 year it hasn't add any value when debugging - // onChunk(() => logger.debug("Resp sending chunk", logCtx)), - responseEncodeSuccess(protocol, { - onChunk(chunkIndex) { - if (chunkIndex === 0) timerTTFB?.(); - }, - }) - ); - } catch (e) { - const status = e instanceof ResponseError ? e.status : RespStatus.SERVER_ERROR; - yield* responseEncodeError(protocol, status, (e as Error).message); - - // Should not throw an error here or libp2p-mplex throws with 'AbortError: stream reset' - // throw e; - responseError = e as Error; - } - })(), - stream.sink - ); - - // If streak.sink throws, libp2p-mplex will close stream.source - // If `requestDecode()` throws the stream.source must be closed manually - // To ensure the stream.source it-pushable instance is always closed, stream.close() is called always - await stream.close(); - metrics?.incomingClosedStreams.inc({method: protocol.method}); - - // TODO: It may happen that stream.sink returns before returning stream.source first, + // TODO: It may happen that the response write completes before the request is fully read, // so you never see "Resp received request" in the logs and the response ends without // sending any chunk, triggering EMPTY_RESPONSE error on the requesting side // It has only happened when doing a request too fast upon immediate connection on inbound peer diff --git a/packages/reqresp/src/types.ts b/packages/reqresp/src/types.ts index 4d97e7874a01..09daeff248cc 100644 --- a/packages/reqresp/src/types.ts +++ b/packages/reqresp/src/types.ts @@ -4,11 +4,9 @@ import {ForkName} from "@lodestar/params"; import {LodestarError} from "@lodestar/utils"; import {RateLimiterQuota} from "./rate_limiter/rateLimiterGRCA.js"; -export const protocolPrefix = "/eth2/beacon_chain/req"; - /** * Available request/response encoding strategies: - * https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/p2p-interface.md#encoding-strategies + * https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/p2p-interface.md#encoding-strategies */ export enum Encoding { SSZ_SNAPPY = "ssz_snappy", diff --git a/packages/reqresp/src/utils/abortableSource.ts b/packages/reqresp/src/utils/abortableSource.ts deleted file mode 100644 index 127339bbd597..000000000000 --- a/packages/reqresp/src/utils/abortableSource.ts +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Wraps an AsyncIterable and rejects early if any signal aborts. - * Throws the error returned by `getError()` of each signal options. - * - * Simplified fork of `"abortable-iterator"`. - * Read function's source for reasoning of the fork. - */ -export function abortableSource( - sourceArg: AsyncIterable, - signals: { - signal: AbortSignal; - getError: () => Error; - }[] -): AsyncIterable { - const source = sourceArg as AsyncGenerator; - - async function* abortable(): AsyncIterable { - // Handler that will hold a reference to the `abort()` promise, - // necessary for the signal abort listeners to reject the iterable promise - let nextAbortHandler: ((error: Error) => void) | null = null; - - // For each signal register an abortHandler(), and prepare clean-up with `onDoneCbs` - const onDoneCbs: (() => void)[] = []; - for (const {signal, getError} of signals) { - const abortHandler = (): void => { - if (nextAbortHandler) nextAbortHandler(getError()); - }; - signal.addEventListener("abort", abortHandler); - onDoneCbs.push(() => { - signal.removeEventListener("abort", abortHandler); - }); - } - - try { - while (true) { - // Abort early if any signal is aborted - for (const {signal, getError} of signals) { - if (signal.aborted) { - throw getError(); - } - } - - // Race the iterator and the abort signals - const result = await Promise.race([ - new Promise((_, reject) => { - nextAbortHandler = (error) => reject(error); - }), - source.next(), - ]); - - // source.next() resolved first - nextAbortHandler = null; - - if (result.done) { - return; - } - - yield result.value; - } - } catch (err) { - // End the iterator if it is a generator - if (typeof source.return === "function") { - // This source.return() function may never resolve depending on the source AsyncGenerator implementation. - // This is the main reason to fork "abortable-iterator", which caused our node to get stuck during Sync. - // We choose to call .return() but not await it. In general, source.return should never throw. If it does, - // it a problem of the source implementor, and thus logged as an unhandled rejection. If that happens, - // the source implementor should fix the upstream code. - void source.return(null); - } - - throw err; - } finally { - for (const cb of onDoneCbs) { - cb(); - } - } - } - - return abortable(); -} diff --git a/packages/reqresp/src/utils/bufferedSource.ts b/packages/reqresp/src/utils/bufferedSource.ts deleted file mode 100644 index e5daca1d2f67..000000000000 --- a/packages/reqresp/src/utils/bufferedSource.ts +++ /dev/null @@ -1,46 +0,0 @@ -import {Uint8ArrayList} from "uint8arraylist"; - -/** - * Wraps a buffer chunk stream source with another async iterable - * so it can be reused in multiple for..of statements. - * - * Uses a BufferList internally to make sure all chunks are consumed - * when switching consumers - */ -export class BufferedSource { - isDone = false; - private buffer: Uint8ArrayList; - private source: AsyncGenerator; - - constructor(source: AsyncGenerator) { - this.buffer = new Uint8ArrayList(); - this.source = source; - } - - [Symbol.asyncIterator](): AsyncIterator { - const that = this; - - let firstNext = true; - - return { - async next() { - // Prevent fetching a new chunk if there are pending bytes - // not processed by a previous consumer of this BufferedSource - if (firstNext && that.buffer.length > 0) { - firstNext = false; - return {done: false, value: that.buffer}; - } - - const {done, value: chunk} = await that.source.next(); - if (done === true) { - that.isDone = true; - return {done: true, value: undefined}; - } - - // Concat new chunk and return a reference to its BufferList instance - that.buffer.append(chunk); - return {done: false, value: that.buffer}; - }, - }; - } -} diff --git a/packages/reqresp/src/utils/collectMaxResponse.ts b/packages/reqresp/src/utils/collectMaxResponse.ts index 42ce2c1c8611..9fafa2b00c46 100644 --- a/packages/reqresp/src/utils/collectMaxResponse.ts +++ b/packages/reqresp/src/utils/collectMaxResponse.ts @@ -6,12 +6,11 @@ * Collects a bounded list of responses up to `maxResponses` */ export async function collectMaxResponse(source: AsyncIterable, maxResponses: number): Promise { - // else: zero or more responses const responses: T[] = []; for await (const response of source) { responses.push(response); - if (maxResponses !== undefined && responses.length >= maxResponses) { + if (responses.length >= maxResponses) { break; } } diff --git a/packages/reqresp/src/utils/errorMessage.ts b/packages/reqresp/src/utils/errorMessage.ts index bcfbe2e9742a..be529e010569 100644 --- a/packages/reqresp/src/utils/errorMessage.ts +++ b/packages/reqresp/src/utils/errorMessage.ts @@ -1,8 +1,7 @@ import {decode as varintDecode, encodingLength as varintEncodingLength} from "uint8-varint"; -import {Uint8ArrayList} from "uint8arraylist"; import {writeSszSnappyPayload} from "../encodingStrategies/sszSnappy/encode.js"; import {Encoding} from "../types.js"; -import {SnappyFramesUncompress} from "./snappyIndex.js"; +import {decodeSnappyFrames} from "./snappyIndex.js"; // ErrorMessage schema: // @@ -13,18 +12,21 @@ import {SnappyFramesUncompress} from "./snappyIndex.js"; // By convention, the error_message is a sequence of bytes that MAY be interpreted as a // UTF-8 string (for debugging purposes). Clients MUST treat as valid any byte sequences // -// Spec v1.1.10 https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/p2p-interface.md#responding-side +// https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/p2p-interface.md#responding-side /** * Encodes a UTF-8 string to 256 bytes max */ -export async function* encodeErrorMessage(errorMessage: string, encoding: Encoding): AsyncGenerator { +export function* encodeErrorMessage(errorMessage: string, encoding: Encoding): Generator { const encoder = new TextEncoder(); const bytes = encoder.encode(errorMessage).slice(0, 256); switch (encoding) { case Encoding.SSZ_SNAPPY: yield* writeSszSnappyPayload(bytes); + break; + default: + throw Error("Unsupported encoding"); } } @@ -32,9 +34,9 @@ export async function* encodeErrorMessage(errorMessage: string, encoding: Encodi * Encodes a UTF-8 error message string into a single buffer (max 256 bytes before encoding). * Unlike `encodeErrorMessage`, this collects all encoded chunks into one buffer. */ -export async function encodeErrorMessageToBuffer(errorMessage: string, encoding: Encoding): Promise { +export function encodeErrorMessageToBuffer(errorMessage: string, encoding: Encoding): Buffer { const chunks: Buffer[] = []; - for await (const chunk of encodeErrorMessage(errorMessage, encoding)) { + for (const chunk of encodeErrorMessage(errorMessage, encoding)) { chunks.push(chunk); } return Buffer.concat(chunks); @@ -43,21 +45,20 @@ export async function encodeErrorMessageToBuffer(errorMessage: string, encoding: /** * Decodes error message from network bytes and removes non printable, non ascii characters. */ -export async function decodeErrorMessage(encodedErrorMessage: Uint8Array): Promise { - const encoder = new TextDecoder(); +export function decodeErrorMessage(encodedErrorMessage: Uint8Array): string { + const decoder = new TextDecoder(); let sszDataLength: number; try { sszDataLength = varintDecode(encodedErrorMessage); - const decompressor = new SnappyFramesUncompress(); const varintBytes = varintEncodingLength(sszDataLength); - const errorMessage = decompressor.uncompress(new Uint8ArrayList(encodedErrorMessage.subarray(varintBytes))); - if (errorMessage == null || errorMessage.length !== sszDataLength) { + const errorMessage = decodeSnappyFrames(encodedErrorMessage.subarray(varintBytes)); + if (errorMessage.length !== sszDataLength) { throw new Error("Malformed input: data length mismatch"); } // remove non ascii characters from string - return encoder.decode(errorMessage.subarray(0)).replace(/[^\x20-\x7F]/g, ""); + return decoder.decode(errorMessage.subarray(0)).replace(/[^\x20-\x7F]/g, ""); } catch (_e) { // remove non ascii characters from string - return encoder.decode(encodedErrorMessage.slice(0, 256)).replace(/[^\x20-\x7F]/g, ""); + return decoder.decode(encodedErrorMessage.slice(0, 256)).replace(/[^\x20-\x7F]/g, ""); } } diff --git a/packages/reqresp/src/utils/index.ts b/packages/reqresp/src/utils/index.ts index a0831d4f8647..2655eff6ed90 100644 --- a/packages/reqresp/src/utils/index.ts +++ b/packages/reqresp/src/utils/index.ts @@ -1,9 +1,7 @@ -export * from "./abortableSource.js"; -export * from "./bufferedSource.js"; export * from "./collectExactOne.js"; export * from "./collectMaxResponse.js"; export * from "./errorMessage.js"; -export * from "./onChunk.js"; export * from "./peerId.js"; export * from "./protocolId.js"; export * from "./snappyIndex.js"; +export * from "./stream.js"; diff --git a/packages/reqresp/src/utils/onChunk.ts b/packages/reqresp/src/utils/onChunk.ts deleted file mode 100644 index 4ca4a2ceed94..000000000000 --- a/packages/reqresp/src/utils/onChunk.ts +++ /dev/null @@ -1,12 +0,0 @@ -/** - * Calls `callback` with each `chunk` received from the `source` AsyncIterable - * Useful for logging, or cancelling timeouts - */ -export function onChunk(callback: (chunk: T) => void): (source: AsyncIterable) => AsyncIterable { - return async function* onChunkTransform(source) { - for await (const chunk of source) { - callback(chunk); - yield chunk; - } - }; -} diff --git a/packages/reqresp/src/utils/protocolId.ts b/packages/reqresp/src/utils/protocolId.ts index a86de251f050..9d85eea2e154 100644 --- a/packages/reqresp/src/utils/protocolId.ts +++ b/packages/reqresp/src/utils/protocolId.ts @@ -1,14 +1,14 @@ import {Encoding, ProtocolAttributes} from "../types.js"; /** - * https://github.com/ethereum/consensus-specs/blob/v1.2.0/specs/phase0/p2p-interface.md#protocol-identification + * https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/p2p-interface.md#protocol-identification */ export function formatProtocolID(protocolPrefix: string, method: string, version: number, encoding: Encoding): string { return `${protocolPrefix}/${method}/${version}/${encoding}`; } /** - * https://github.com/ethereum/consensus-specs/blob/v1.2.0/specs/phase0/p2p-interface.md#protocol-identification + * https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/p2p-interface.md#protocol-identification */ export function parseProtocolID(protocolId: string): ProtocolAttributes { const result = protocolId.split("/"); diff --git a/packages/reqresp/src/utils/snappy.ts b/packages/reqresp/src/utils/snappy.ts deleted file mode 100644 index 21bb5a7dfaea..000000000000 --- a/packages/reqresp/src/utils/snappy.ts +++ /dev/null @@ -1,2 +0,0 @@ -export {compressSync as compress} from "snappy"; -export {uncompress} from "snappyjs"; diff --git a/packages/reqresp/src/utils/snappyCompress.ts b/packages/reqresp/src/utils/snappyCompress.ts index e3ac57c62bde..07f7e772e678 100644 --- a/packages/reqresp/src/utils/snappyCompress.ts +++ b/packages/reqresp/src/utils/snappyCompress.ts @@ -3,7 +3,7 @@ import {compressSync} from "snappy"; import {ChunkType, IDENTIFIER_FRAME, UNCOMPRESSED_CHUNK_SIZE, crc} from "./snappyCommon.js"; // The logic in this file is largely copied (in simplified form) from https://github.com/ChainSafe/node-snappy-stream/ -export async function* encodeSnappy(bytes: Buffer): AsyncGenerator { +export function* encodeSnappy(bytes: Buffer): Generator { yield IDENTIFIER_FRAME; for (let i = 0; i < bytes.length; i += UNCOMPRESSED_CHUNK_SIZE) { diff --git a/packages/reqresp/src/utils/snappyIndex.ts b/packages/reqresp/src/utils/snappyIndex.ts index fa79b7ca2a05..c07c566d7886 100644 --- a/packages/reqresp/src/utils/snappyIndex.ts +++ b/packages/reqresp/src/utils/snappyIndex.ts @@ -1,3 +1,3 @@ export * from "./snappyCommon.js"; export {encodeSnappy} from "./snappyCompress.js"; -export {SnappyFramesUncompress} from "./snappyUncompress.js"; +export * from "./snappyUncompress.js"; diff --git a/packages/reqresp/src/utils/snappyUncompress.ts b/packages/reqresp/src/utils/snappyUncompress.ts index af9d5b890e0c..61e8db053fcf 100644 --- a/packages/reqresp/src/utils/snappyUncompress.ts +++ b/packages/reqresp/src/utils/snappyUncompress.ts @@ -2,96 +2,94 @@ import {uncompress} from "snappyjs"; import {Uint8ArrayList} from "uint8arraylist"; import {ChunkType, IDENTIFIER, UNCOMPRESSED_CHUNK_SIZE, crc} from "./snappyCommon.js"; -export class SnappyFramesUncompress { - private buffer = new Uint8ArrayList(); - - private state: UncompressState = { - foundIdentifier: false, - }; - - /** - * Accepts chunk of data containing some part of snappy frames stream - * @param chunk - * @return Buffer if there is one or more whole frames, null if it's partial - */ - uncompress(chunk: Uint8ArrayList): Uint8ArrayList | null { - this.buffer.append(chunk); - const result = new Uint8ArrayList(); - while (this.buffer.length > 0) { - if (this.buffer.length < 4) break; +export function parseSnappyFrameHeader(header: Uint8Array): {type: ChunkType; frameSize: number} { + if (header.length !== 4) { + throw new Error("malformed input: incomplete frame header"); + } - const type = getChunkType(this.buffer.get(0)); + const type = getChunkType(header[0]); + const frameSize = header[1] + (header[2] << 8) + (header[3] << 16); + return {type, frameSize}; +} - if (!this.state.foundIdentifier && type !== ChunkType.IDENTIFIER) { - throw "malformed input: must begin with an identifier"; +export function decodeSnappyFrameData(type: ChunkType, frame: Uint8Array): Uint8Array | null { + switch (type) { + case ChunkType.IDENTIFIER: { + if (!Buffer.prototype.equals.call(frame, IDENTIFIER)) { + throw new Error("malformed input: bad identifier"); + } + return null; + } + case ChunkType.PADDING: + case ChunkType.SKIPPABLE: + return null; + case ChunkType.COMPRESSED: { + if (frame.length < 4) { + throw new Error("malformed input: too short"); } - const frameSize = getFrameSize(this.buffer, 1); + const checksum = frame.subarray(0, 4); + const data = frame.subarray(4); + const uncompressed = uncompress(data, UNCOMPRESSED_CHUNK_SIZE); + if (crc(uncompressed).compare(checksum) !== 0) { + throw new Error("malformed input: bad checksum"); + } + return uncompressed; + } + case ChunkType.UNCOMPRESSED: { + if (frame.length < 4) { + throw new Error("malformed input: too short"); + } - if (this.buffer.length - 4 < frameSize) { - break; + const checksum = frame.subarray(0, 4); + const uncompressed = frame.subarray(4); + if (uncompressed.length > UNCOMPRESSED_CHUNK_SIZE) { + throw new Error("malformed input: too large"); + } + if (crc(uncompressed).compare(checksum) !== 0) { + throw new Error("malformed input: bad checksum"); } + return uncompressed; + } + } +} - const frame = this.buffer.subarray(4, 4 + frameSize); - this.buffer.consume(4 + frameSize); +export function decodeSnappyFrames(data: Uint8Array): Uint8ArrayList { + const out = new Uint8ArrayList(); + let foundIdentifier = false; + let offset = 0; - switch (type) { - case ChunkType.IDENTIFIER: { - if (!Buffer.prototype.equals.call(frame, IDENTIFIER)) { - throw "malformed input: bad identifier"; - } - this.state.foundIdentifier = true; - continue; - } - case ChunkType.PADDING: - case ChunkType.SKIPPABLE: - continue; - case ChunkType.COMPRESSED: { - const checksum = frame.subarray(0, 4); - const data = frame.subarray(4); + while (offset < data.length) { + const remaining = data.length - offset; + if (remaining < 4) { + throw new Error("malformed input: incomplete frame header"); + } - const uncompressed = uncompress(data, UNCOMPRESSED_CHUNK_SIZE); - if (crc(uncompressed).compare(checksum) !== 0) { - throw "malformed input: bad checksum"; - } - result.append(uncompressed); - break; - } - case ChunkType.UNCOMPRESSED: { - const checksum = frame.subarray(0, 4); - const uncompressed = frame.subarray(4); + const {type, frameSize} = parseSnappyFrameHeader(data.subarray(offset, offset + 4)); + if (!foundIdentifier && type !== ChunkType.IDENTIFIER) { + throw new Error("malformed input: must begin with an identifier"); + } - if (uncompressed.length > UNCOMPRESSED_CHUNK_SIZE) { - throw "malformed input: too large"; - } - if (crc(uncompressed).compare(checksum) !== 0) { - throw "malformed input: bad checksum"; - } - result.append(uncompressed); - break; - } - } + offset += 4; + + if (data.length - offset < frameSize) { + throw new Error("malformed input: incomplete frame"); } - if (result.length === 0) { - return null; + + const frame = data.subarray(offset, offset + frameSize); + offset += frameSize; + + if (type === ChunkType.IDENTIFIER) { + foundIdentifier = true; } - return result; - } - reset(): void { - this.buffer = new Uint8ArrayList(); - this.state = { - foundIdentifier: false, - }; + const uncompressed = decodeSnappyFrameData(type, frame); + if (uncompressed !== null) { + out.append(uncompressed); + } } -} - -type UncompressState = { - foundIdentifier: boolean; -}; -function getFrameSize(buffer: Uint8ArrayList, offset: number): number { - return buffer.get(offset) + (buffer.get(offset + 1) << 8) + (buffer.get(offset + 2) << 16); + return out; } function getChunkType(value: number): ChunkType { diff --git a/packages/reqresp/src/utils/stream.ts b/packages/reqresp/src/utils/stream.ts new file mode 100644 index 000000000000..a335059692b6 --- /dev/null +++ b/packages/reqresp/src/utils/stream.ts @@ -0,0 +1,34 @@ +import type {Stream} from "@libp2p/interface"; +import {ByteStream} from "@libp2p/utils"; +import {Uint8ArrayList} from "uint8arraylist"; +import {ErrorAborted} from "@lodestar/utils"; + +export async function sendChunks( + stream: Stream, + source: Iterable | AsyncIterable, + signal?: AbortSignal +): Promise { + for await (const chunk of source) { + if (signal?.aborted) { + throw new ErrorAborted("sendChunks"); + } + + if (!stream.send(chunk)) { + await stream.onDrain({signal}); + } + } +} + +export function drainByteStream(bytes: ByteStream): Uint8Array | undefined { + const readBuffer = ( + bytes as unknown as { + readBuffer?: {byteLength: number; subarray: () => Uint8Array; consume: (bytes: number) => void}; + } + ).readBuffer; + if (readBuffer && readBuffer.byteLength > 0) { + const drained = readBuffer.subarray(); + readBuffer.consume(readBuffer.byteLength); + return drained; + } + return undefined; +} diff --git a/packages/reqresp/test/fixtures/encodingStrategies.ts b/packages/reqresp/test/fixtures/encodingStrategies.ts index bbcb01a6e346..dce2c6c24a6e 100644 --- a/packages/reqresp/test/fixtures/encodingStrategies.ts +++ b/packages/reqresp/test/fixtures/encodingStrategies.ts @@ -71,7 +71,16 @@ export const encodingStrategiesDecodingErrorCases: { id: "if it read more than maxEncodedLen", type: ssz.phase0.Ping, error: SszSnappyErrorCode.TOO_MUCH_BYTES_READ, - chunks: [Buffer.from(varintEncode(ssz.phase0.Ping.minSize)), Buffer.alloc(100)], + chunks: [ + Buffer.from(varintEncode(ssz.phase0.Ping.minSize)), + Buffer.concat([ + // snappy stream identifier frame + Buffer.from([0xff, 0x06, 0x00, 0x00]), + Buffer.from("sNaPpY"), + // many empty skippable frames to force maxEncodedLen overflow + ...new Array(32).fill(Buffer.from([0x80, 0x00, 0x00, 0x00])), + ]), + ], }, { id: "if failed ssz snappy input malformed", diff --git a/packages/reqresp/test/unit/ReqResp.test.ts b/packages/reqresp/test/unit/ReqResp.test.ts index 2cb821b534e1..9e1f9760dd89 100644 --- a/packages/reqresp/test/unit/ReqResp.test.ts +++ b/packages/reqresp/test/unit/ReqResp.test.ts @@ -1,4 +1,4 @@ -import {Libp2p} from "libp2p"; +import type {Libp2p} from "libp2p"; import {afterEach, beforeEach, describe, expect, it, vi} from "vitest"; import {getEmptyLogger} from "@lodestar/logger/empty"; import {Logger} from "@lodestar/utils"; @@ -8,7 +8,7 @@ import {RateLimiterQuota} from "../../src/rate_limiter/rateLimiterGRCA.js"; import {Protocol} from "../../src/types.js"; import {getEmptyHandler, sszSnappyPing} from "../fixtures/messages.js"; import {numberToStringProtocol, numberToStringProtocolDialOnly, pingProtocol} from "../fixtures/protocols.js"; -import {MockLibP2pStream} from "../utils/index.js"; +import {createMockStream} from "../utils/mockStream.js"; import {responseEncode} from "../utils/response.js"; describe("ResResp", () => { @@ -19,19 +19,22 @@ describe("ResResp", () => { beforeEach(() => { libp2p = { - dialProtocol: vi.fn().mockResolvedValue( - new MockLibP2pStream( - responseEncode( - [ - { - status: RespStatus.SUCCESS, - payload: sszSnappyPing.binaryPayload, - }, - ], - ping - ), - ping.method - ) + dialProtocol: vi.fn().mockImplementation( + async () => + ( + await createMockStream({ + protocol: ping.method, + source: responseEncode( + [ + { + status: RespStatus.SUCCESS, + payload: sszSnappyPing.binaryPayload, + }, + ], + ping + ), + }) + ).stream ), handle: vi.fn(), } as unknown as Libp2p; diff --git a/packages/reqresp/test/unit/encoders/reqestEncode.test.ts b/packages/reqresp/test/unit/encoders/reqestEncode.test.ts index 8771e26913cd..0bb7d8edb398 100644 --- a/packages/reqresp/test/unit/encoders/reqestEncode.test.ts +++ b/packages/reqresp/test/unit/encoders/reqestEncode.test.ts @@ -1,5 +1,3 @@ -import all from "it-all"; -import {pipe} from "it-pipe"; import {describe, it} from "vitest"; import {requestEncode} from "../../../src/encoders/requestEncode.js"; import {requestEncodersCases} from "../../fixtures/encoders.js"; @@ -8,9 +6,9 @@ import {expectEqualByteChunks} from "../../utils/index.js"; describe("encoders / requestEncode", () => { describe("valid cases", () => { it.each(requestEncodersCases)("$id", async ({protocol, requestBody, chunks}) => { - const encodedChunks = await pipe(requestEncode(protocol, requestBody), all); + const encodedChunks = await Array.fromAsync(requestEncode(protocol, requestBody)); expectEqualByteChunks( - encodedChunks as Uint8Array[], + encodedChunks, chunks.map((c) => c.subarray()) ); }); diff --git a/packages/reqresp/test/unit/encoders/requestDecode.test.ts b/packages/reqresp/test/unit/encoders/requestDecode.test.ts index cd8c7bc303e6..623f3d498260 100644 --- a/packages/reqresp/test/unit/encoders/requestDecode.test.ts +++ b/packages/reqresp/test/unit/encoders/requestDecode.test.ts @@ -1,21 +1,23 @@ -import {pipe} from "it-pipe"; import {describe, expect, it} from "vitest"; import {requestDecode} from "../../../src/encoders/requestDecode.js"; import {requestEncodersCases, requestEncodersErrorCases} from "../../fixtures/encoders.js"; import {expectRejectedWithLodestarError} from "../../utils/errors.js"; import {arrToSource} from "../../utils/index.js"; +import {createMockStream} from "../../utils/mockStream.js"; describe("encoders / requestDecode", () => { describe("valid cases", () => { it.each(requestEncodersCases)("$id", async ({protocol, requestBody, chunks}) => { - const decodedBody = await pipe(arrToSource(chunks), requestDecode(protocol)); + const {stream} = await createMockStream({source: arrToSource(chunks)}); + const decodedBody = await requestDecode(protocol, stream); expect(decodedBody).toEqual(requestBody); }); }); describe("error cases", () => { it.each(requestEncodersErrorCases.filter((r) => r.errorDecode))("$id", async ({protocol, errorDecode, chunks}) => { - await expectRejectedWithLodestarError(pipe(arrToSource(chunks), requestDecode(protocol)), errorDecode); + const {stream} = await createMockStream({source: arrToSource(chunks)}); + await expectRejectedWithLodestarError(requestDecode(protocol, stream), errorDecode); }); }); }); diff --git a/packages/reqresp/test/unit/encoders/responseDecode.test.ts b/packages/reqresp/test/unit/encoders/responseDecode.test.ts index e8ed906c942f..0a8b13616d46 100644 --- a/packages/reqresp/test/unit/encoders/responseDecode.test.ts +++ b/packages/reqresp/test/unit/encoders/responseDecode.test.ts @@ -1,5 +1,3 @@ -import all from "it-all"; -import {pipe} from "it-pipe"; import {describe, expect, it} from "vitest"; import {LodestarError} from "@lodestar/utils"; import {responseDecode} from "../../../src/encoders/responseDecode.js"; @@ -7,15 +5,13 @@ import {ResponseIncoming} from "../../../src/types.js"; import {responseEncodersErrorTestCases, responseEncodersTestCases} from "../../fixtures/encoders.js"; import {expectRejectedWithLodestarError} from "../../utils/errors.js"; import {arrToSource, onlySuccessResp} from "../../utils/index.js"; +import {createMockStream} from "../../utils/mockStream.js"; describe("encoders / responseDecode", () => { describe("valid cases", () => { it.each(responseEncodersTestCases)("$id", async ({protocol, responseChunks, chunks}) => { - const responses = (await pipe( - arrToSource(chunks), - responseDecode(protocol, {onFirstHeader: () => {}, onFirstResponseChunk: () => {}}), - all - )) as ResponseIncoming[]; + const {stream} = await createMockStream({source: arrToSource(chunks)}); + const responses = (await Array.fromAsync(responseDecode(protocol, stream))) as ResponseIncoming[]; const expectedResponses = responseChunks.filter(onlySuccessResp).map((r) => r.payload); expect(responses.map((r) => ({...r, data: Buffer.from(r.data)}))).toEqual( @@ -28,12 +24,9 @@ describe("encoders / responseDecode", () => { it.each(responseEncodersErrorTestCases.filter((r) => r.decodeError !== undefined))( "$id", async ({protocol, chunks, decodeError}) => { + const {stream} = await createMockStream({source: arrToSource(chunks as Uint8Array[])}); await expectRejectedWithLodestarError( - pipe( - arrToSource(chunks as Uint8Array[]), - responseDecode(protocol, {onFirstHeader: () => {}, onFirstResponseChunk: () => {}}), - all - ), + Array.fromAsync(responseDecode(protocol, stream)), decodeError as LodestarError ); } diff --git a/packages/reqresp/test/unit/encoders/responseEncode.test.ts b/packages/reqresp/test/unit/encoders/responseEncode.test.ts index 9580d0b2fcd8..f43d3bd21949 100644 --- a/packages/reqresp/test/unit/encoders/responseEncode.test.ts +++ b/packages/reqresp/test/unit/encoders/responseEncode.test.ts @@ -1,5 +1,3 @@ -import all from "it-all"; -import {pipe} from "it-pipe"; import {describe, it} from "vitest"; import {Protocol} from "../../../src/types.js"; import {responseEncodersTestCases} from "../../fixtures/encoders.js"; @@ -11,10 +9,9 @@ describe("encoders / responseEncode", () => { it.each(responseEncodersTestCases.filter((f) => !f.skipEncoding))( "$id", async ({protocol, responseChunks, chunks}) => { - const encodedChunks = await pipe(responseEncode(responseChunks, protocol as Protocol), all); - + const encodedChunks = await Array.fromAsync(responseEncode(responseChunks, protocol as Protocol)); expectEqualByteChunks( - encodedChunks as Uint8Array[], + encodedChunks, chunks.map((c) => c.subarray()) ); } diff --git a/packages/reqresp/test/unit/encodingStrategies/sszSnappy/decode.test.ts b/packages/reqresp/test/unit/encodingStrategies/sszSnappy/decode.test.ts index f315cc26316b..6cf4d69f78c7 100644 --- a/packages/reqresp/test/unit/encodingStrategies/sszSnappy/decode.test.ts +++ b/packages/reqresp/test/unit/encodingStrategies/sszSnappy/decode.test.ts @@ -1,19 +1,21 @@ +import {byteStream} from "@libp2p/utils"; import {encode as varintEncode} from "uint8-varint"; import {Uint8ArrayList} from "uint8arraylist"; import {describe, expect, it} from "vitest"; import {readSszSnappyPayload} from "../../../../src/encodingStrategies/sszSnappy/index.js"; -import {BufferedSource} from "../../../../src/utils/index.js"; import { encodingStrategiesDecodingErrorCases, encodingStrategiesMainnetTestCases, encodingStrategiesTestCases, } from "../../../fixtures/index.js"; import {arrToSource} from "../../../utils/index.js"; +import {createMockStream} from "../../../utils/mockStream.js"; describe("encodingStrategies / sszSnappy / decode", () => { it.each(encodingStrategiesTestCases)("$id", async ({type, binaryPayload, chunks}) => { - const bufferedSource = new BufferedSource(arrToSource(chunks)); - const bodyResult = await readSszSnappyPayload(bufferedSource, type); + const {stream} = await createMockStream({source: arrToSource(chunks)}); + const bytes = byteStream(stream); + const bodyResult = await readSszSnappyPayload(bytes, type).finally(() => bytes.unwrap()); expect(bodyResult).toEqual(binaryPayload.data); }); @@ -23,8 +25,9 @@ describe("encodingStrategies / sszSnappy / decode", () => { const streamedBytes = new Uint8ArrayList(Buffer.concat([Buffer.from(varintEncode(bodySize)), streamedBody])); it(id, async () => { - const bufferedSource = new BufferedSource(arrToSource([streamedBytes])); - const bodyResult = await readSszSnappyPayload(bufferedSource, serializer); + const {stream} = await createMockStream({source: arrToSource([streamedBytes])}); + const bytes = byteStream(stream); + const bodyResult = await readSszSnappyPayload(bytes, serializer).finally(() => bytes.unwrap()); expect(bodyResult).toEqual(new Uint8Array(payload.data)); }); @@ -34,8 +37,10 @@ describe("encodingStrategies / sszSnappy / decode", () => { describe("error cases", () => { for (const {id, type, error, chunks} of encodingStrategiesDecodingErrorCases) { it(id, async () => { - const bufferedSource = new BufferedSource(arrToSource([new Uint8ArrayList(...chunks)])); - await expect(readSszSnappyPayload(bufferedSource, type)).rejects.toThrow(error); + const {stream} = await createMockStream({source: arrToSource([new Uint8ArrayList(...chunks)])}); + const bytes = byteStream(stream); + await expect(readSszSnappyPayload(bytes, type)).rejects.toThrow(error); + bytes.unwrap(); }); } }); diff --git a/packages/reqresp/test/unit/encodingStrategies/sszSnappy/encode.test.ts b/packages/reqresp/test/unit/encodingStrategies/sszSnappy/encode.test.ts index 68f5c4e5299d..410cfc581d14 100644 --- a/packages/reqresp/test/unit/encodingStrategies/sszSnappy/encode.test.ts +++ b/packages/reqresp/test/unit/encodingStrategies/sszSnappy/encode.test.ts @@ -1,5 +1,3 @@ -import all from "it-all"; -import {pipe} from "it-pipe"; import {encode as varintEncode} from "uint8-varint"; import {describe, expect, it} from "vitest"; import {writeSszSnappyPayload} from "../../../../src/encodingStrategies/sszSnappy/encode.js"; @@ -8,9 +6,9 @@ import {expectEqualByteChunks} from "../../../utils/index.js"; describe("encodingStrategies / sszSnappy / encode", () => { it.each(encodingStrategiesTestCases)("$id", async ({binaryPayload, chunks}) => { - const encodedChunks = await pipe(writeSszSnappyPayload(Buffer.from(binaryPayload.data)), all); + const encodedChunks = await Array.fromAsync(writeSszSnappyPayload(Buffer.from(binaryPayload.data))); expectEqualByteChunks( - encodedChunks as Uint8Array[], + encodedChunks, chunks.map((c) => c.subarray()) ); }); @@ -19,8 +17,8 @@ describe("encodingStrategies / sszSnappy / encode", () => { it.each(encodingStrategiesMainnetTestCases)("$id", async ({payload, streamedBody}) => { const bodySize = payload.data.length; - const encodedChunks = await pipe(writeSszSnappyPayload(Buffer.from(payload.data)), all); - const encodedStream = Buffer.concat(encodedChunks as Uint8Array[]); + const encodedChunks = await Array.fromAsync(writeSszSnappyPayload(Buffer.from(payload.data))); + const encodedStream = Buffer.concat(encodedChunks); const expectedStreamed = Buffer.concat([Buffer.from(varintEncode(bodySize)), streamedBody]); expect(encodedStream).toEqual(expectedStreamed); }); diff --git a/packages/reqresp/test/unit/encodingStrategies/sszSnappy/snappyFrames/uncompress.test.ts b/packages/reqresp/test/unit/encodingStrategies/sszSnappy/snappyFrames/uncompress.test.ts index 56715a48cb40..8cf7fc307c10 100644 --- a/packages/reqresp/test/unit/encodingStrategies/sszSnappy/snappyFrames/uncompress.test.ts +++ b/packages/reqresp/test/unit/encodingStrategies/sszSnappy/snappyFrames/uncompress.test.ts @@ -1,65 +1,48 @@ -import {pipe} from "it-pipe"; import {Uint8ArrayList} from "uint8arraylist"; import {describe, expect, it} from "vitest"; import { ChunkType, IDENTIFIER_FRAME, - SnappyFramesUncompress, crc, + decodeSnappyFrameData, + decodeSnappyFrames, encodeSnappy, + parseSnappyFrameHeader, } from "../../../../../src/utils/snappyIndex.js"; describe("encodingStrategies / sszSnappy / snappy frames / uncompress", () => { - it("should work with short input", () => - new Promise((done) => { - const testData = "Small test data"; - const compressIterable = encodeSnappy(Buffer.from(testData)); - - const decompress = new SnappyFramesUncompress(); - - void pipe(compressIterable, async (source) => { - for await (const data of source) { - const result = decompress.uncompress(new Uint8ArrayList(data)); - if (result) { - expect(result.subarray().toString()).toBe(testData); - done(); - } - } - }); - })); - - it("should work with huge input", () => - new Promise((done) => { - const testData = Buffer.alloc(100000, 4).toString(); - const compressIterable = encodeSnappy(Buffer.from(testData)); - let result = Buffer.alloc(0); - const decompress = new SnappyFramesUncompress(); - - void pipe(compressIterable, async (source) => { - for await (const data of source) { - // testData will come compressed as two or more chunks - result = Buffer.concat([ - result, - decompress.uncompress(new Uint8ArrayList(data))?.subarray() ?? Buffer.alloc(0), - ]); - if (result.length === testData.length) { - expect(result.toString()).toBe(testData); - done(); - } - } - }); - })); + it("should work with short input", async () => { + const testData = "Small test data"; + const compressIterable = encodeSnappy(Buffer.from(testData)); + const encoded: Uint8Array[] = []; - it("should detect malformed input", () => { - const decompress = new SnappyFramesUncompress(); + for await (const data of compressIterable) { + encoded.push(data); + } - expect(() => decompress.uncompress(new Uint8ArrayList(Buffer.alloc(32, 5)))).toThrow(); + const result = decodeSnappyFrames(Buffer.concat(encoded)); + expect(Buffer.from(result.subarray()).toString()).toBe(testData); }); - it("should return null if not enough data", () => { - const decompress = new SnappyFramesUncompress(); + it("should work with huge input", async () => { + const testData = Buffer.alloc(100000, 4).toString(); + const compressIterable = encodeSnappy(Buffer.from(testData)); + const encoded: Uint8Array[] = []; - expect(decompress.uncompress(new Uint8ArrayList(Buffer.alloc(3, 1)))).toBe(null); + for await (const data of compressIterable) { + encoded.push(data); + } + + const result = decodeSnappyFrames(Buffer.concat(encoded)); + expect(Buffer.from(result.subarray()).toString()).toBe(testData); + }); + + it("should detect malformed input", () => { + expect(() => decodeSnappyFrames(Buffer.alloc(32, 5))).toThrow(); + }); + + it("should return null if not enough data", () => { + expect(() => parseSnappyFrameHeader(Buffer.alloc(3, 1))).toThrow(/incomplete frame header/); }); it("should detect invalid checksum", () => { @@ -71,8 +54,7 @@ describe("encodingStrategies / sszSnappy / snappy frames / uncompress", () => { // 0xffffffff is clearly an invalid checksum chunks.append(Uint8Array.from(Array.from({length: 0x80}, () => 0xff))); - const decompress = new SnappyFramesUncompress(); - expect(() => decompress.uncompress(chunks)).toThrow(/checksum/); + expect(() => decodeSnappyFrames(chunks.subarray())).toThrow(/checksum/); }); it("should detect skippable frames", () => { @@ -82,8 +64,7 @@ describe("encodingStrategies / sszSnappy / snappy frames / uncompress", () => { chunks.append(Uint8Array.from([ChunkType.SKIPPABLE, 0x80, 0x00, 0x00])); chunks.append(Uint8Array.from(Array.from({length: 0x80}, () => 0xff))); - const decompress = new SnappyFramesUncompress(); - expect(decompress.uncompress(chunks)).toBeNull(); + expect(decodeSnappyFrames(chunks.subarray()).length).toBe(0); }); it("should detect large data", () => { @@ -97,7 +78,19 @@ describe("encodingStrategies / sszSnappy / snappy frames / uncompress", () => { chunks.append(checksum); chunks.append(data); - const decompress = new SnappyFramesUncompress(); - expect(() => decompress.uncompress(chunks)).toThrow(/large/); + expect(() => decodeSnappyFrames(chunks.subarray())).toThrow(/large/); + }); + + it("should parse header and decode uncompressed frame", () => { + const payload = Uint8Array.from([1, 2, 3, 4]); + const checksum = crc(payload); + const frame = Buffer.concat([checksum, payload]); + + const header = Uint8Array.from([ChunkType.UNCOMPRESSED, frame.length, 0x00, 0x00]); + const parsed = parseSnappyFrameHeader(header); + const decoded = decodeSnappyFrameData(parsed.type, frame); + + expect(parsed.frameSize).toBe(frame.length); + expect(Buffer.from(decoded ?? [])).toEqual(Buffer.from(payload)); }); }); diff --git a/packages/reqresp/test/unit/request/index.test.ts b/packages/reqresp/test/unit/request/index.test.ts index 578feec130b6..82457f7f24bd 100644 --- a/packages/reqresp/test/unit/request/index.test.ts +++ b/packages/reqresp/test/unit/request/index.test.ts @@ -1,7 +1,5 @@ import {PeerId} from "@libp2p/interface"; -import all from "it-all"; -import {pipe} from "it-pipe"; -import {Libp2p} from "libp2p"; +import type {Libp2p} from "libp2p"; import {afterEach, beforeEach, describe, expect, it, vi} from "vitest"; import {getEmptyLogger} from "@lodestar/logger/empty"; import {LodestarError, sleep} from "@lodestar/utils"; @@ -11,7 +9,7 @@ import {MixedProtocol, Protocol, ResponseIncoming} from "../../../src/types.js"; import {getEmptyHandler, sszSnappyPing} from "../../fixtures/messages.js"; import {pingProtocol} from "../../fixtures/protocols.js"; import {expectRejectedWithLodestarError} from "../../utils/errors.js"; -import {MockLibP2pStream} from "../../utils/index.js"; +import {createMockStream} from "../../utils/mockStream.js"; import {getValidPeerId} from "../../utils/peer.js"; import {responseEncode} from "../../utils/response.js"; @@ -27,7 +25,6 @@ describe("request / sendRequest", () => { id: string; protocols: MixedProtocol[]; requestBody: ResponseIncoming; - maxResponses?: number; expectedReturn: unknown[]; }[] = [ { @@ -36,13 +33,6 @@ describe("request / sendRequest", () => { requestBody: sszSnappyPing.binaryPayload, expectedReturn: [{...sszSnappyPing.binaryPayload, data: Buffer.from(sszSnappyPing.binaryPayload.data)}], }, - // limit to max responses is no longer the responsibility of this package - // { - // id: "Return up to maxResponses for a multi-chunk method", - // protocols: [customProtocol({})], - // requestBody: sszSnappySignedBeaconBlockPhase0.binaryPayload, - // expectedReturn: [sszSnappySignedBeaconBlockPhase0.binaryPayload], - // }, ]; beforeEach(() => { @@ -57,18 +47,25 @@ describe("request / sendRequest", () => { for (const {id, protocols, expectedReturn, requestBody} of testCases) { it(id, async () => { + const encodedResponse = await Array.fromAsync( + responseEncode([{status: RespStatus.SUCCESS, payload: requestBody}], protocols[0] as Protocol) + ); + libp2p = { - dialProtocol: vi - .fn() - .mockResolvedValue( - new MockLibP2pStream( - responseEncode([{status: RespStatus.SUCCESS, payload: requestBody}], protocols[0] as Protocol), - protocols[0].method - ) - ), + dialProtocol: vi.fn().mockImplementation( + async () => + ( + await createMockStream({ + protocol: protocols[0].method, + source: (async function* (): AsyncIterable { + yield Buffer.concat(encodedResponse); + })(), + }) + ).stream + ), } as unknown as Libp2p; - const responses = await pipe( + const responses = await Array.fromAsync( sendRequest( {logger, libp2p, metrics: null}, peerId, @@ -76,13 +73,152 @@ describe("request / sendRequest", () => { protocols.map((p) => p.method), EMPTY_REQUEST, controller.signal - ), - all + ) ); - expect(responses).toEqual(expectedReturn); + expect(responses.map((r) => ({...r, data: Buffer.from(r.data)}))).toEqual(expectedReturn); }); } + it("closes stream gracefully when caller stops consuming responses early", async () => { + const encodedResponse = await Array.fromAsync( + responseEncode( + [ + {status: RespStatus.SUCCESS, payload: sszSnappyPing.binaryPayload}, + {status: RespStatus.SUCCESS, payload: sszSnappyPing.binaryPayload}, + ], + emptyProtocol as Protocol + ) + ); + + let abortCalled = false; + let getStreamState = (): {status: string; readStatus: string; writeStatus: string} => ({ + status: "not-created", + readStatus: "not-created", + writeStatus: "not-created", + }); + libp2p = { + dialProtocol: vi.fn().mockImplementation(async () => { + const streamResult = await createMockStream({ + protocol: emptyProtocol.method, + source: (async function* (): AsyncIterable { + yield Buffer.concat(encodedResponse); + await sleep(100000, controller.signal); + })(), + }); + const reqStream = streamResult.stream as unknown as { + status: string; + readStatus: string; + writeStatus: string; + abort: (error: Error) => void; + }; + const abort = reqStream.abort.bind(reqStream); + reqStream.abort = (error: Error): void => { + abortCalled = true; + abort(error); + }; + getStreamState = () => ({ + status: reqStream.status, + readStatus: reqStream.readStatus, + writeStatus: reqStream.writeStatus, + }); + return streamResult.stream; + }), + } as unknown as Libp2p; + + let responseCount = 0; + for await (const _ of sendRequest( + {logger, libp2p, metrics: null}, + peerId, + [emptyProtocol], + [emptyProtocol.method], + EMPTY_REQUEST, + controller.signal + )) { + responseCount++; + break; + } + + expect(responseCount).toBe(1); + expect(abortCalled).toBe(false); + const streamState = getStreamState(); + expect(streamState.status).not.toBe("aborted"); + expect(streamState.readStatus).toBe("closed"); + expect(streamState.writeStatus).toBe("closed"); + }); + + it("aborts stream if remote never closes after early consumer exit", async () => { + const encodedResponse = await Array.fromAsync( + responseEncode([{status: RespStatus.SUCCESS, payload: sszSnappyPing.binaryPayload}], emptyProtocol as Protocol) + ); + + let getStreamStatus = (): string => "not-created"; + libp2p = { + dialProtocol: vi.fn().mockImplementation(async () => { + const streamResult = await createMockStream({ + protocol: emptyProtocol.method, + source: (async function* (): AsyncIterable { + yield Buffer.concat(encodedResponse); + await sleep(100000, controller.signal); + })(), + }); + const reqStream = streamResult.stream as unknown as {status: string}; + getStreamStatus = () => reqStream.status; + return streamResult.stream; + }), + } as unknown as Libp2p; + + for await (const _ of sendRequest( + {logger, libp2p, metrics: null}, + peerId, + [emptyProtocol], + [emptyProtocol.method], + EMPTY_REQUEST, + controller.signal, + {respTimeoutMs: 20} + )) { + break; + } + + expect(getStreamStatus()).not.toBe("aborted"); + await sleep(50, controller.signal); + expect(getStreamStatus()).toBe("aborted"); + }); + + it("aborts stream on RESP_TIMEOUT", async () => { + const testMethod = "req/test"; + let getStreamStatus = (): string => "not-created"; + libp2p = { + dialProtocol: vi.fn().mockImplementation(async () => { + const streamResult = await createMockStream({ + protocol: testMethod, + source: (async function* (): AsyncIterable { + await sleep(100000, controller.signal); + yield new Uint8Array(); + })(), + }); + const reqStream = streamResult.stream as unknown as {status: string}; + getStreamStatus = () => reqStream.status; + return streamResult.stream; + }), + } as unknown as Libp2p; + + await expectRejectedWithLodestarError( + Array.fromAsync( + sendRequest( + {logger, libp2p, metrics: null}, + peerId, + [emptyProtocol], + [testMethod], + EMPTY_REQUEST, + controller.signal, + {respTimeoutMs: 1} + ) + ), + new RequestError({code: RequestErrorCode.RESP_TIMEOUT}) + ); + expect(getStreamStatus()).toBe("aborted"); + }); + describe("timeout cases", () => { const peerId = getValidPeerId(); const testMethod = "req/test"; @@ -94,13 +230,13 @@ describe("request / sendRequest", () => { error?: LodestarError; }[] = [ { - id: "trigger a TTFB_TIMEOUT", - opts: {ttfbTimeoutMs: 0}, + id: "trigger a RESP_TIMEOUT when first response is delayed", + opts: {respTimeoutMs: 0}, source: async function* () { await sleep(30); // Pause for too long before first byte yield sszSnappyPing.chunks[0]; }, - error: new RequestError({code: RequestErrorCode.TTFB_TIMEOUT}), + error: new RequestError({code: RequestErrorCode.RESP_TIMEOUT}), }, { id: "trigger a RESP_TIMEOUT", @@ -113,18 +249,17 @@ describe("request / sendRequest", () => { error: new RequestError({code: RequestErrorCode.RESP_TIMEOUT}), }, { - // Upstream "abortable-iterator" never throws with an infinite sleep. id: "Infinite sleep on first byte", - opts: {ttfbTimeoutMs: 1, respTimeoutMs: 1}, + opts: {respTimeoutMs: 1}, source: async function* () { await sleep(100000, controller.signal); yield sszSnappyPing.chunks[0]; }, - error: new RequestError({code: RequestErrorCode.TTFB_TIMEOUT}), + error: new RequestError({code: RequestErrorCode.RESP_TIMEOUT}), }, { id: "Infinite sleep on second chunk", - opts: {ttfbTimeoutMs: 1, respTimeoutMs: 1}, + opts: {respTimeoutMs: 1}, source: async function* () { yield sszSnappyPing.chunks[0]; await sleep(100000, controller.signal); @@ -136,11 +271,13 @@ describe("request / sendRequest", () => { for (const {id, source, opts, error} of timeoutTestCases) { it(id, async () => { libp2p = { - dialProtocol: vi.fn().mockResolvedValue(new MockLibP2pStream(source(), testMethod)), + dialProtocol: vi + .fn() + .mockImplementation(async () => (await createMockStream({protocol: testMethod, source: source()})).stream), } as unknown as Libp2p; await expectRejectedWithLodestarError( - pipe( + Array.fromAsync( sendRequest( {logger, libp2p, metrics: null}, peerId, @@ -149,8 +286,7 @@ describe("request / sendRequest", () => { EMPTY_REQUEST, controller.signal, opts - ), - all + ) ), error as LodestarError ); diff --git a/packages/reqresp/test/unit/response/index.test.ts b/packages/reqresp/test/unit/response/index.test.ts index f527961e078f..f808ac0d05a7 100644 --- a/packages/reqresp/test/unit/response/index.test.ts +++ b/packages/reqresp/test/unit/response/index.test.ts @@ -9,7 +9,8 @@ import {handleRequest} from "../../../src/response/index.js"; import {sszSnappyPing} from "../../fixtures/messages.js"; import {pingProtocol} from "../../fixtures/protocols.js"; import {expectRejectedWithLodestarError} from "../../utils/errors.js"; -import {MockLibP2pStream, expectEqualByteChunks} from "../../utils/index.js"; +import {expectEqualByteChunks} from "../../utils/index.js"; +import {createMockStream} from "../../utils/mockStream.js"; import {getValidPeerId} from "../../utils/peer.js"; const testCases: { @@ -59,7 +60,13 @@ describe("response / handleRequest", () => { afterEach(() => controller.abort()); it.each(testCases)("$id", async ({requestChunks, protocol, expectedResponseChunks, expectedError}) => { - const stream = new MockLibP2pStream(requestChunks as any); + const {stream, sentChunks} = await createMockStream({ + source: (async function* (): AsyncIterable { + for (const chunk of requestChunks) { + yield chunk; + } + })(), + }); const rateLimiter = new ReqRespRateLimiter({rateLimitMultiplier: 0}); const resultPromise = handleRequest({ @@ -80,6 +87,9 @@ describe("response / handleRequest", () => { await expect(resultPromise).resolves.toBeUndefined(); } - expectEqualByteChunks(stream.resultChunks, expectedResponseChunks, "Wrong response chunks"); + // Allow stream message events to be delivered before asserting on captured chunks + await new Promise((resolve) => setTimeout(resolve, 0)); + + expectEqualByteChunks(sentChunks, expectedResponseChunks, "Wrong response chunks"); }); }); diff --git a/packages/reqresp/test/unit/utils/errorMessage.test.ts b/packages/reqresp/test/unit/utils/errorMessage.test.ts index 763eaee5526e..4f833d612a0d 100644 --- a/packages/reqresp/test/unit/utils/errorMessage.test.ts +++ b/packages/reqresp/test/unit/utils/errorMessage.test.ts @@ -41,7 +41,7 @@ describe("encode and decode error message", () => { } const encodedMessage = accu.subarray(0); - const decodedErrorMessage = await decodeErrorMessage(encodedMessage); + const decodedErrorMessage = decodeErrorMessage(encodedMessage); expect(decodedErrorMessage).toBe(errorMessage); }); } diff --git a/packages/reqresp/test/utils/index.ts b/packages/reqresp/test/utils/index.ts index 67c5e5875c91..f8a05212c309 100644 --- a/packages/reqresp/test/utils/index.ts +++ b/packages/reqresp/test/utils/index.ts @@ -1,6 +1,3 @@ -import {Direction, ReadStatus, Stream, StreamStatus, WriteStatus} from "@libp2p/interface"; -import {logger} from "@libp2p/logger"; -import {Uint8ArrayList} from "uint8arraylist"; import {expect} from "vitest"; import {toHexString} from "@chainsafe/ssz"; import {fromHex} from "@lodestar/utils"; @@ -8,8 +5,7 @@ import {RespStatus, ResponseIncoming} from "../../src/index.js"; import {ResponseChunk} from "../fixtures/index.js"; /** - * Helper for it-pipe when first argument is an array. - * it-pipe does not convert the chunks array to a generator and BufferedSource breaks + * Converts an array to an async source. */ export async function* arrToSource(arr: T[]): AsyncGenerator { for (const item of arr) { @@ -41,44 +37,6 @@ export function expectInEqualByteChunks(chunks: Uint8Array[], expectedChunks: Ui } } -/** - * Useful to simulate a LibP2P stream source emitting prepared bytes - * and capture the response with a sink accessible via `this.resultChunks` - */ -export class MockLibP2pStream implements Stream { - protocol: string; - id = "mock"; - log = logger("mock"); - direction: Direction = "inbound"; - status: StreamStatus = "open"; - readStatus: ReadStatus = "ready"; - writeStatus: WriteStatus = "ready"; - timeline = { - open: Date.now(), - }; - metadata = {}; - source: Stream["source"]; - resultChunks: Uint8Array[] = []; - - constructor(requestChunks: Uint8ArrayList[] | AsyncIterable | AsyncGenerator, protocol?: string) { - this.source = Array.isArray(requestChunks) - ? arrToSource(requestChunks) - : (requestChunks as AsyncGenerator); - this.protocol = protocol ?? "mock"; - } - - sink: Stream["sink"] = async (source) => { - for await (const chunk of source) { - this.resultChunks.push(chunk.subarray()); - } - }; - - close: Stream["close"] = async () => {}; - closeRead = async (): Promise => {}; - closeWrite = async (): Promise => {}; - abort: Stream["abort"] = () => this.close(); -} - export function fromHexBuf(hex: string): Buffer { return Buffer.from(fromHex(hex)); } diff --git a/packages/reqresp/test/utils/mockStream.ts b/packages/reqresp/test/utils/mockStream.ts new file mode 100644 index 000000000000..f618460b4a65 --- /dev/null +++ b/packages/reqresp/test/utils/mockStream.ts @@ -0,0 +1,48 @@ +import type {Stream} from "@libp2p/interface"; +import {StreamMessageEvent} from "@libp2p/interface"; +import {streamPair} from "@libp2p/utils"; +import {Uint8ArrayList} from "uint8arraylist"; + +type SourceChunk = Uint8Array | Uint8ArrayList; + +/** + * Minimal stream test double for reqresp tests backed by upstream streamPair. + * Pumps source data into one side and returns the other for the code-under-test. + * Captures data sent back through the stream in sentChunks. + */ +export async function createMockStream({ + protocol = "", + source = (async function* (): AsyncIterable {})(), +}: { + protocol?: string; + source?: AsyncIterable; +} = {}): Promise<{stream: Stream; sentChunks: Uint8Array[]}> { + const [writer, reader] = await streamPair({protocol, delay: 0}); + const sentChunks: Uint8Array[] = []; + + // streamPair only sets protocol on the outbound stream; mirror it to the inbound + reader.protocol = protocol; + + // Capture data sent back through the reader stream (received on writer side) + writer.addEventListener("message", (evt: StreamMessageEvent) => { + const data = evt.data; + sentChunks.push(data instanceof Uint8ArrayList ? data.subarray() : data); + }); + + // Pump source data into the writer side so the reader can consume it. + // Use close() to signal EOF — AbstractStream.close() sends closeWrite to the + // remote, which is what byteStream uses to detect end-of-stream. + void (async () => { + try { + for await (const chunk of source) { + const data = chunk instanceof Uint8ArrayList ? chunk : new Uint8ArrayList(chunk); + writer.send(data); + } + } catch { + // Sources may abort on purpose to simulate timeouts + } + await writer.close(); + })(); + + return {stream: reader, sentChunks}; +} diff --git a/packages/reqresp/test/utils/response.ts b/packages/reqresp/test/utils/response.ts index 0452fb6006e4..ce2d636ee470 100644 --- a/packages/reqresp/test/utils/response.ts +++ b/packages/reqresp/test/utils/response.ts @@ -1,4 +1,3 @@ -import {pipe} from "it-pipe"; import {responseEncodeError, responseEncodeSuccess} from "../../src/encoders/responseEncode.js"; import {RespStatus} from "../../src/interface.js"; import {Protocol} from "../../src/types.js"; @@ -6,15 +5,15 @@ import {ResponseChunk} from "../fixtures/encoders.js"; import {beaconConfig} from "../fixtures/messages.js"; import {arrToSource} from "../utils/index.js"; -export async function* responseEncode(responseChunks: ResponseChunk[], protocol: Protocol): AsyncIterable { +export async function* responseEncode(responseChunks: ResponseChunk[], protocol: Protocol): AsyncIterable { for (const chunk of responseChunks) { if (chunk.status === RespStatus.SUCCESS) { const payload = chunk.payload; - yield* pipe( + yield* responseEncodeSuccess( + protocol, arrToSource([ {...payload, boundary: beaconConfig.getForkBoundaryAtEpoch(beaconConfig.forks[payload.fork].epoch)}, - ]), - responseEncodeSuccess(protocol, {onChunk: () => {}}) + ]) ); } else { yield* responseEncodeError(protocol, chunk.status, chunk.errorMessage); diff --git a/packages/reqresp/tsconfig.json b/packages/reqresp/tsconfig.json index 4ee2e1092000..d0d3bd757da9 100644 --- a/packages/reqresp/tsconfig.json +++ b/packages/reqresp/tsconfig.json @@ -1,7 +1,6 @@ { "extends": "../../tsconfig.json", "include": ["src", "test"], - "exclude": ["../../node_modules/it-pipe"], "compilerOptions": { "rootDir": "../..", "outDir": "lib", diff --git a/packages/spec-test-util/package.json b/packages/spec-test-util/package.json index 14575761f351..e5449b76fe38 100644 --- a/packages/spec-test-util/package.json +++ b/packages/spec-test-util/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/spec-test-util", - "version": "1.40.0", + "version": "1.41.0", "description": "Spec test suite generator from yaml test files", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -31,11 +31,11 @@ }, "scripts": { "clean": "rm -rf lib && rm -f *.tsbuildinfo", - "build": "tsc -p tsconfig.build.json", + "build": "tsgo -p tsconfig.build.json", "build:release": "pnpm clean && pnpm build", "build:watch": "pnpm run build --watch", "check-build": "node -e \"(async function() { await import('./lib/downloadTests.js') })()\"", - "check-types": "tsc", + "check-types": "tsgo", "lint": "biome check src/ test/", "lint:fix": "pnpm run lint --write", "test": "pnpm test:unit && pnpm test:e2e", diff --git a/packages/spec-test-util/src/sszGeneric.ts b/packages/spec-test-util/src/sszGeneric.ts index 1898a11e7870..7901bbfe433d 100644 --- a/packages/spec-test-util/src/sszGeneric.ts +++ b/packages/spec-test-util/src/sszGeneric.ts @@ -42,7 +42,7 @@ export function parseSszValidTestcase(dirpath: string, metaFilename: string): Va * | vec_bool_0 * | serialized.ssz_snappy * - * Docs: https://github.com/ethereum/eth2.0-specs/blob/master/tests/formats/ssz_generic/README.md + * Docs: https://github.com/ethereum/consensus-specs/blob/v1.6.1/tests/formats/ssz_generic/README.md */ export function parseSszGenericInvalidTestcase(dirpath: string) { // The serialized value is stored in serialized.ssz_snappy diff --git a/packages/state-transition/package.json b/packages/state-transition/package.json index ebf94eb284fa..8ae51f22506c 100644 --- a/packages/state-transition/package.json +++ b/packages/state-transition/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.40.0", + "version": "1.41.0", "type": "module", "exports": { ".": { @@ -33,6 +33,11 @@ "bun": "./src/slot/index.ts", "types": "./lib/slot/index.d.ts", "import": "./lib/slot/index.js" + }, + "./test-utils": { + "bun": "./src/testUtils/index.ts", + "types": "./lib/testUtils/index.d.ts", + "import": "./lib/testUtils/index.js" } }, "files": [ @@ -42,11 +47,11 @@ ], "scripts": { "clean": "rm -rf lib && rm -f *.tsbuildinfo", - "build": "tsc -p tsconfig.build.json", + "build": "tsgo -p tsconfig.build.json", "build:watch": "pnpm run build --watch", "build:release": "pnpm clean && pnpm build", "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\"", - "check-types": "tsc", + "check-types": "tsgo", "lint": "biome check src/ test/", "lint:fix": "pnpm run lint --write", "test": "pnpm test:unit", @@ -66,7 +71,7 @@ "@lodestar/params": "workspace:^", "@lodestar/types": "workspace:^", "@lodestar/utils": "workspace:^", - "@vekexasia/bigint-buffer2": "^1.1.0" + "@vekexasia/bigint-buffer2": "^1.1.1" }, "devDependencies": { "@lodestar/api": "workspace:^" diff --git a/packages/state-transition/src/block/externalData.ts b/packages/state-transition/src/block/externalData.ts index affe2a3b7fc6..7a02b8a6fa8a 100644 --- a/packages/state-transition/src/block/externalData.ts +++ b/packages/state-transition/src/block/externalData.ts @@ -18,6 +18,8 @@ export enum DataAvailabilityStatus { /* validator activities can't be performed on out of range data */ OutOfRange = "OutOfRange", Available = "Available", + /* Gloas: beacon blocks have no DA requirement, execution payload is separate */ + NotRequired = "NotRequired", } export interface BlockExternalData { diff --git a/packages/state-transition/src/block/index.ts b/packages/state-transition/src/block/index.ts index 1104a52e2fa9..2dc24d48bd5b 100644 --- a/packages/state-transition/src/block/index.ts +++ b/packages/state-transition/src/block/index.ts @@ -13,10 +13,10 @@ import {processBlobKzgCommitments} from "./processBlobKzgCommitments.js"; import {processBlockHeader} from "./processBlockHeader.js"; import {processEth1Data} from "./processEth1Data.js"; import {processExecutionPayload} from "./processExecutionPayload.js"; -import {processExecutionPayloadBid} from "./processExecutionPayloadBid.ts"; -import {processExecutionPayloadEnvelope} from "./processExecutionPayloadEnvelope.ts"; +import {processExecutionPayloadBid} from "./processExecutionPayloadBid.js"; +import {processExecutionPayloadEnvelope} from "./processExecutionPayloadEnvelope.js"; import {processOperations} from "./processOperations.js"; -import {processPayloadAttestation} from "./processPayloadAttestation.ts"; +import {processPayloadAttestation} from "./processPayloadAttestation.js"; import {processRandao} from "./processRandao.js"; import {processSyncAggregate} from "./processSyncCommittee.js"; import {processWithdrawals} from "./processWithdrawals.js"; diff --git a/packages/state-transition/src/block/isValidIndexedAttestation.ts b/packages/state-transition/src/block/isValidIndexedAttestation.ts index d37504ef192b..4015708efe2c 100644 --- a/packages/state-transition/src/block/isValidIndexedAttestation.ts +++ b/packages/state-transition/src/block/isValidIndexedAttestation.ts @@ -1,7 +1,7 @@ import {BeaconConfig} from "@lodestar/config"; import {ForkSeq, MAX_COMMITTEES_PER_SLOT, MAX_VALIDATORS_PER_COMMITTEE} from "@lodestar/params"; import {IndexedAttestation, IndexedAttestationBigint, Slot} from "@lodestar/types"; -import {Index2PubkeyCache} from "../cache/pubkeyCache.js"; +import {PubkeyCache} from "../cache/pubkeyCache.js"; import {getIndexedAttestationBigintSignatureSet, getIndexedAttestationSignatureSet} from "../signatureSets/index.js"; import {verifySignatureSet} from "../util/index.js"; @@ -10,7 +10,7 @@ import {verifySignatureSet} from "../util/index.js"; */ export function isValidIndexedAttestation( config: BeaconConfig, - index2pubkey: Index2PubkeyCache, + pubkeyCache: PubkeyCache, stateSlot: Slot, validatorsLen: number, indexedAttestation: IndexedAttestation, @@ -21,14 +21,14 @@ export function isValidIndexedAttestation( } if (verifySignature) { - return verifySignatureSet(getIndexedAttestationSignatureSet(config, stateSlot, indexedAttestation), index2pubkey); + return verifySignatureSet(getIndexedAttestationSignatureSet(config, stateSlot, indexedAttestation), pubkeyCache); } return true; } export function isValidIndexedAttestationBigint( config: BeaconConfig, - index2pubkey: Index2PubkeyCache, + pubkeyCache: PubkeyCache, stateSlot: Slot, validatorsLen: number, indexedAttestation: IndexedAttestationBigint, @@ -41,7 +41,7 @@ export function isValidIndexedAttestationBigint( if (verifySignature) { return verifySignatureSet( getIndexedAttestationBigintSignatureSet(config, stateSlot, indexedAttestation), - index2pubkey + pubkeyCache ); } return true; @@ -74,9 +74,8 @@ export function isValidIndexedAttestationIndices( prev = index; } - // check if indices are out of bounds, by checking the highest index (since it is sorted) - const lastIndex = indices.at(-1); - if (lastIndex && lastIndex >= validatorsLen) { + // check if indices are out of bounds, `prev` is the highest index since indices are sorted + if (prev >= validatorsLen) { return false; } diff --git a/packages/state-transition/src/block/isValidIndexedPayloadAttestation.ts b/packages/state-transition/src/block/isValidIndexedPayloadAttestation.ts index 5914cf5c5e45..3e733e1922e7 100644 --- a/packages/state-transition/src/block/isValidIndexedPayloadAttestation.ts +++ b/packages/state-transition/src/block/isValidIndexedPayloadAttestation.ts @@ -1,7 +1,7 @@ import {gloas} from "@lodestar/types"; -import {getIndexedPayloadAttestationSignatureSet} from "../signatureSets/index.ts"; +import {getIndexedPayloadAttestationSignatureSet} from "../signatureSets/index.js"; import {CachedBeaconStateGloas} from "../types.js"; -import {verifySignatureSet} from "../util/index.ts"; +import {verifySignatureSet} from "../util/index.js"; export function isValidIndexedPayloadAttestation( state: CachedBeaconStateGloas, @@ -18,7 +18,7 @@ export function isValidIndexedPayloadAttestation( if (verifySignature) { return verifySignatureSet( getIndexedPayloadAttestationSignatureSet(state.config, indexedPayloadAttestation), - state.epochCtx.index2pubkey + state.epochCtx.pubkeyCache ); } diff --git a/packages/state-transition/src/block/processAttestationPhase0.ts b/packages/state-transition/src/block/processAttestationPhase0.ts index 20414784a17b..3d26a1b5d750 100644 --- a/packages/state-transition/src/block/processAttestationPhase0.ts +++ b/packages/state-transition/src/block/processAttestationPhase0.ts @@ -53,7 +53,7 @@ export function processAttestationPhase0( if ( !isValidIndexedAttestation( state.config, - epochCtx.index2pubkey, + epochCtx.pubkeyCache, state.slot, state.validators.length, epochCtx.getIndexedAttestation(ForkSeq.phase0, attestation), diff --git a/packages/state-transition/src/block/processAttestationsAltair.ts b/packages/state-transition/src/block/processAttestationsAltair.ts index e13cfd7d3d38..e6cac307c70f 100644 --- a/packages/state-transition/src/block/processAttestationsAltair.ts +++ b/packages/state-transition/src/block/processAttestationsAltair.ts @@ -18,7 +18,7 @@ import {byteArrayEquals, intSqrt} from "@lodestar/utils"; import {BeaconStateTransitionMetrics} from "../metrics.js"; import {getAttestationWithIndicesSignatureSet} from "../signatureSets/indexedAttestation.js"; import {CachedBeaconStateAltair, CachedBeaconStateGloas} from "../types.js"; -import {isAttestationSameSlot, isAttestationSameSlotRootCache} from "../util/gloas.ts"; +import {isAttestationSameSlot, isAttestationSameSlotRootCache} from "../util/gloas.js"; import {increaseBalance, verifySignatureSet} from "../util/index.js"; import {RootCache} from "../util/rootCache.js"; import {checkpointToStr, isTimelyTarget, validateAttestation} from "./processAttestationPhase0.js"; @@ -64,7 +64,7 @@ export function processAttestationsAltair( // we can verify only that and nothing else. if (verifySignature) { const sigSet = getAttestationWithIndicesSignatureSet(state.config, state.slot, attestation, attestingIndices); - if (!verifySignatureSet(sigSet, state.epochCtx.index2pubkey)) { + if (!verifySignatureSet(sigSet, state.epochCtx.pubkeyCache)) { throw new Error("Attestation signature is not valid"); } } diff --git a/packages/state-transition/src/block/processAttesterSlashing.ts b/packages/state-transition/src/block/processAttesterSlashing.ts index c8739ae506ae..aba93aa3275e 100644 --- a/packages/state-transition/src/block/processAttesterSlashing.ts +++ b/packages/state-transition/src/block/processAttesterSlashing.ts @@ -1,7 +1,7 @@ import {BeaconConfig} from "@lodestar/config"; import {ForkSeq} from "@lodestar/params"; import {AttesterSlashing, Slot} from "@lodestar/types"; -import {Index2PubkeyCache} from "../cache/pubkeyCache.js"; +import {PubkeyCache} from "../cache/pubkeyCache.js"; import {CachedBeaconStateAllForks} from "../types.js"; import {getAttesterSlashableIndices, isSlashableAttestationData, isSlashableValidator} from "../util/index.js"; import {isValidIndexedAttestationBigint} from "./isValidIndexedAttestation.js"; @@ -22,7 +22,7 @@ export function processAttesterSlashing( const {epochCtx} = state; assertValidAttesterSlashing( state.config, - epochCtx.index2pubkey, + epochCtx.pubkeyCache, state.slot, state.validators.length, attesterSlashing, @@ -48,7 +48,7 @@ export function processAttesterSlashing( export function assertValidAttesterSlashing( config: BeaconConfig, - index2pubkey: Index2PubkeyCache, + pubkeyCache: PubkeyCache, stateSlot: Slot, validatorsLen: number, attesterSlashing: AttesterSlashing, @@ -66,7 +66,7 @@ export function assertValidAttesterSlashing( // can be any arbitrary value. Must use bigint variants to hash correctly to all possible values for (const [i, attestation] of [attestation1, attestation2].entries()) { if ( - !isValidIndexedAttestationBigint(config, index2pubkey, stateSlot, validatorsLen, attestation, verifySignatures) + !isValidIndexedAttestationBigint(config, pubkeyCache, stateSlot, validatorsLen, attestation, verifySignatures) ) { throw new Error(`AttesterSlashing attestation${i} is invalid`); } diff --git a/packages/state-transition/src/block/processExecutionPayloadBid.ts b/packages/state-transition/src/block/processExecutionPayloadBid.ts index 4b43fc041040..b7dbac3d5f96 100644 --- a/packages/state-transition/src/block/processExecutionPayloadBid.ts +++ b/packages/state-transition/src/block/processExecutionPayloadBid.ts @@ -2,11 +2,11 @@ import {PublicKey, Signature, verify} from "@chainsafe/blst"; import {BUILDER_INDEX_SELF_BUILD, ForkPostGloas, SLOTS_PER_EPOCH} from "@lodestar/params"; import {BeaconBlock, gloas, ssz} from "@lodestar/types"; import {byteArrayEquals, toHex, toRootHex} from "@lodestar/utils"; -import {G2_POINT_AT_INFINITY} from "../constants/constants.ts"; +import {G2_POINT_AT_INFINITY} from "../constants/constants.js"; import {getExecutionPayloadBidSigningRoot} from "../signatureSets/executionPayloadBid.js"; -import {CachedBeaconStateGloas} from "../types.ts"; -import {canBuilderCoverBid, isActiveBuilder} from "../util/gloas.ts"; -import {getCurrentEpoch, getRandaoMix} from "../util/index.ts"; +import {CachedBeaconStateGloas} from "../types.js"; +import {canBuilderCoverBid, isActiveBuilder} from "../util/gloas.js"; +import {getCurrentEpoch, getRandaoMix} from "../util/index.js"; export function processExecutionPayloadBid(state: CachedBeaconStateGloas, block: BeaconBlock): void { const signedBid = block.body.signedExecutionPayloadBid; diff --git a/packages/state-transition/src/block/processExecutionPayloadEnvelope.ts b/packages/state-transition/src/block/processExecutionPayloadEnvelope.ts index 5f6488bcfa94..6b2a0b8824b0 100644 --- a/packages/state-transition/src/block/processExecutionPayloadEnvelope.ts +++ b/packages/state-transition/src/block/processExecutionPayloadEnvelope.ts @@ -1,68 +1,81 @@ -import {PublicKey, Signature, verify} from "@chainsafe/blst"; -import { - BUILDER_INDEX_SELF_BUILD, - DOMAIN_BEACON_BUILDER, - SLOTS_PER_EPOCH, - SLOTS_PER_HISTORICAL_ROOT, -} from "@lodestar/params"; +import {SLOTS_PER_EPOCH, SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params"; import {gloas, ssz} from "@lodestar/types"; import {byteArrayEquals, toHex, toRootHex} from "@lodestar/utils"; -import {CachedBeaconStateGloas} from "../types.ts"; -import {computeSigningRoot, computeTimeAtSlot} from "../util/index.ts"; -import {processConsolidationRequest} from "./processConsolidationRequest.ts"; -import {processDepositRequest} from "./processDepositRequest.ts"; -import {processWithdrawalRequest} from "./processWithdrawalRequest.ts"; - -// This function does not call execution engine to verify payload. Need to call it from other place +import {getExecutionPayloadEnvelopeSignatureSet} from "../signatureSets/executionPayloadEnvelope.js"; +import {BeaconStateView} from "../stateView/beaconStateView.js"; +import {CachedBeaconStateGloas} from "../types.js"; +import {computeTimeAtSlot} from "../util/index.js"; +import {verifySignatureSet} from "../util/signatureSets.js"; +import {processConsolidationRequest} from "./processConsolidationRequest.js"; +import {processDepositRequest} from "./processDepositRequest.js"; +import {processWithdrawalRequest} from "./processWithdrawalRequest.js"; + +export type ProcessExecutionPayloadEnvelopeOpts = { + verifySignature?: boolean; + verifyStateRoot?: boolean; + dontTransferCache?: boolean; +}; + +// Unlike other block processing functions which mutate state in-place, this function +// clones the state and returns the post-state, similar to stateTransition(). +// This function does not call execution engine to verify payload. Need to call it from other place. export function processExecutionPayloadEnvelope( state: CachedBeaconStateGloas, signedEnvelope: gloas.SignedExecutionPayloadEnvelope, - verify: boolean -): void { + opts?: ProcessExecutionPayloadEnvelopeOpts +): CachedBeaconStateGloas { + const {verifySignature = true, verifyStateRoot = true} = opts ?? {}; const envelope = signedEnvelope.message; const payload = envelope.payload; const fork = state.config.getForkSeq(envelope.slot); - if (verify && !verifyExecutionPayloadEnvelopeSignature(state, signedEnvelope)) { + if (verifySignature && !verifyExecutionPayloadEnvelopeSignature(state, signedEnvelope)) { throw Error(`Execution payload envelope has invalid signature builderIndex=${envelope.builderIndex}`); } - validateExecutionPayloadEnvelope(state, envelope); + // .clone() before mutating state, similar to stateTransition() + const postState = state.clone(opts?.dontTransferCache) as CachedBeaconStateGloas; + + validateExecutionPayloadEnvelope(postState, envelope); const requests = envelope.executionRequests; for (const deposit of requests.deposits) { - processDepositRequest(fork, state, deposit); + processDepositRequest(fork, postState, deposit); } for (const withdrawal of requests.withdrawals) { - processWithdrawalRequest(fork, state, withdrawal); + processWithdrawalRequest(fork, postState, withdrawal); } for (const consolidation of requests.consolidations) { - processConsolidationRequest(state, consolidation); + processConsolidationRequest(postState, consolidation); } // Queue the builder payment - const paymentIndex = SLOTS_PER_EPOCH + (state.slot % SLOTS_PER_EPOCH); - const payment = state.builderPendingPayments.get(paymentIndex).clone(); + const paymentIndex = SLOTS_PER_EPOCH + (postState.slot % SLOTS_PER_EPOCH); + const payment = postState.builderPendingPayments.get(paymentIndex).clone(); const amount = payment.withdrawal.amount; if (amount > 0) { - state.builderPendingWithdrawals.push(payment.withdrawal); + postState.builderPendingWithdrawals.push(payment.withdrawal); } - state.builderPendingPayments.set(paymentIndex, ssz.gloas.BuilderPendingPayment.defaultViewDU()); + postState.builderPendingPayments.set(paymentIndex, ssz.gloas.BuilderPendingPayment.defaultViewDU()); // Cache the execution payload hash - state.executionPayloadAvailability.set(state.slot % SLOTS_PER_HISTORICAL_ROOT, true); - state.latestBlockHash = payload.blockHash; + postState.executionPayloadAvailability.set(postState.slot % SLOTS_PER_HISTORICAL_ROOT, true); + postState.latestBlockHash = payload.blockHash; - if (verify && !byteArrayEquals(envelope.stateRoot, state.hashTreeRoot())) { + postState.commit(); + + if (verifyStateRoot && !byteArrayEquals(envelope.stateRoot, postState.hashTreeRoot())) { throw new Error( - `Envelope's state root does not match state envelope=${toRootHex(envelope.stateRoot)} state=${toRootHex(state.hashTreeRoot())}` + `Envelope's state root does not match state envelope=${toRootHex(envelope.stateRoot)} state=${toRootHex(postState.hashTreeRoot())}` ); } + + return postState; } function validateExecutionPayloadEnvelope( @@ -151,24 +164,12 @@ function verifyExecutionPayloadEnvelopeSignature( state: CachedBeaconStateGloas, signedEnvelope: gloas.SignedExecutionPayloadEnvelope ): boolean { - const builderIndex = signedEnvelope.message.builderIndex; - - const domain = state.config.getDomain(state.slot, DOMAIN_BEACON_BUILDER); - const signingRoot = computeSigningRoot(ssz.gloas.ExecutionPayloadEnvelope, signedEnvelope.message, domain); - - try { - let publicKey: PublicKey; - - if (builderIndex === BUILDER_INDEX_SELF_BUILD) { - const validatorIndex = state.latestBlockHeader.proposerIndex; - publicKey = state.epochCtx.index2pubkey[validatorIndex]; - } else { - publicKey = PublicKey.fromBytes(state.builders.getReadonly(builderIndex).pubkey); - } - const signature = Signature.fromBytes(signedEnvelope.signature, true); - - return verify(signingRoot, publicKey, signature); - } catch (_e) { - return false; // Catch all BLS errors: failed key validation, failed signature validation, invalid signature - } + const signatureSet = getExecutionPayloadEnvelopeSignatureSet( + state.config, + state.epochCtx.pubkeyCache, + new BeaconStateView(state), + signedEnvelope, + state.latestBlockHeader.proposerIndex + ); + return verifySignatureSet(signatureSet); } diff --git a/packages/state-transition/src/block/processOperations.ts b/packages/state-transition/src/block/processOperations.ts index 34b21984c124..b4455f5315d5 100644 --- a/packages/state-transition/src/block/processOperations.ts +++ b/packages/state-transition/src/block/processOperations.ts @@ -14,7 +14,7 @@ import {processBlsToExecutionChange} from "./processBlsToExecutionChange.js"; import {processConsolidationRequest} from "./processConsolidationRequest.js"; import {processDeposit} from "./processDeposit.js"; import {processDepositRequest} from "./processDepositRequest.js"; -import {processPayloadAttestation} from "./processPayloadAttestation.ts"; +import {processPayloadAttestation} from "./processPayloadAttestation.js"; import {processProposerSlashing} from "./processProposerSlashing.js"; import {processVoluntaryExit} from "./processVoluntaryExit.js"; import {processWithdrawalRequest} from "./processWithdrawalRequest.js"; diff --git a/packages/state-transition/src/block/processPayloadAttestation.ts b/packages/state-transition/src/block/processPayloadAttestation.ts index d58cf704dc00..389fbe318017 100644 --- a/packages/state-transition/src/block/processPayloadAttestation.ts +++ b/packages/state-transition/src/block/processPayloadAttestation.ts @@ -1,7 +1,7 @@ import {gloas} from "@lodestar/types"; import {byteArrayEquals} from "@lodestar/utils"; -import {CachedBeaconStateGloas} from "../types.ts"; -import {isValidIndexedPayloadAttestation} from "./isValidIndexedPayloadAttestation.ts"; +import {CachedBeaconStateGloas} from "../types.js"; +import {isValidIndexedPayloadAttestation} from "./isValidIndexedPayloadAttestation.js"; export function processPayloadAttestation( state: CachedBeaconStateGloas, diff --git a/packages/state-transition/src/block/processProposerSlashing.ts b/packages/state-transition/src/block/processProposerSlashing.ts index f66269666678..b136fe8b88c7 100644 --- a/packages/state-transition/src/block/processProposerSlashing.ts +++ b/packages/state-transition/src/block/processProposerSlashing.ts @@ -2,7 +2,7 @@ import {BeaconConfig} from "@lodestar/config"; import {ForkSeq, SLOTS_PER_EPOCH} from "@lodestar/params"; import {Slot, phase0, ssz} from "@lodestar/types"; import {Validator} from "@lodestar/types/phase0"; -import {Index2PubkeyCache} from "../cache/pubkeyCache.js"; +import {PubkeyCache} from "../cache/pubkeyCache.js"; import {getProposerSlashingSignatureSets} from "../signatureSets/index.js"; import {CachedBeaconStateAllForks, CachedBeaconStateGloas} from "../types.js"; import {computeEpochAtSlot, isSlashableValidator} from "../util/index.js"; @@ -24,7 +24,7 @@ export function processProposerSlashing( const proposer = state.validators.getReadonly(proposerSlashing.signedHeader1.message.proposerIndex); assertValidProposerSlashing( state.config, - state.epochCtx.index2pubkey, + state.epochCtx.pubkeyCache, state.slot, proposerSlashing, proposer, @@ -57,7 +57,7 @@ export function processProposerSlashing( export function assertValidProposerSlashing( config: BeaconConfig, - index2pubkey: Index2PubkeyCache, + pubkeyCache: PubkeyCache, stateSlot: Slot, proposerSlashing: phase0.ProposerSlashing, proposer: Validator, @@ -94,7 +94,7 @@ export function assertValidProposerSlashing( if (verifySignatures) { const signatureSets = getProposerSlashingSignatureSets(config, stateSlot, proposerSlashing); for (let i = 0; i < signatureSets.length; i++) { - if (!verifySignatureSet(signatureSets[i], index2pubkey)) { + if (!verifySignatureSet(signatureSets[i], pubkeyCache)) { throw new Error(`ProposerSlashing header${i + 1} signature invalid`); } } diff --git a/packages/state-transition/src/block/processRandao.ts b/packages/state-transition/src/block/processRandao.ts index b43d12bdaac2..0c25293a989d 100644 --- a/packages/state-transition/src/block/processRandao.ts +++ b/packages/state-transition/src/block/processRandao.ts @@ -17,7 +17,7 @@ export function processRandao(state: CachedBeaconStateAllForks, block: BeaconBlo const randaoReveal = block.body.randaoReveal; // verify RANDAO reveal - if (verifySignature && !verifyRandaoSignature(config, epochCtx.index2pubkey, block)) { + if (verifySignature && !verifyRandaoSignature(config, epochCtx.pubkeyCache, block)) { throw new Error("RANDAO reveal is an invalid signature"); } diff --git a/packages/state-transition/src/block/processSyncCommittee.ts b/packages/state-transition/src/block/processSyncCommittee.ts index 243c9935a575..98f3bdb1ec9e 100644 --- a/packages/state-transition/src/block/processSyncCommittee.ts +++ b/packages/state-transition/src/block/processSyncCommittee.ts @@ -32,7 +32,7 @@ export function processSyncAggregate( participantIndices ); // When there's no participation we consider the signature valid and just ignore it - if (signatureSet !== null && !verifySignatureSet(signatureSet, state.epochCtx.index2pubkey)) { + if (signatureSet !== null && !verifySignatureSet(signatureSet, state.epochCtx.pubkeyCache)) { throw Error("Sync committee signature invalid"); } } diff --git a/packages/state-transition/src/block/processVoluntaryExit.ts b/packages/state-transition/src/block/processVoluntaryExit.ts index 1b3a798adbfb..27f823c1dafb 100644 --- a/packages/state-transition/src/block/processVoluntaryExit.ts +++ b/packages/state-transition/src/block/processVoluntaryExit.ts @@ -1,6 +1,7 @@ import {FAR_FUTURE_EPOCH, ForkSeq} from "@lodestar/params"; import {phase0} from "@lodestar/types"; import {verifyVoluntaryExitSignature} from "../signatureSets/index.js"; +import {BeaconStateView} from "../stateView/beaconStateView.js"; import {CachedBeaconStateAllForks, CachedBeaconStateElectra, CachedBeaconStateGloas} from "../types.js"; import { convertValidatorIndexToBuilderIndex, @@ -9,7 +10,7 @@ import { isActiveBuilder, isBuilderIndex, } from "../util/gloas.js"; -import {getPendingBalanceToWithdraw, isActiveValidator, isGloasCachedStateType} from "../util/index.js"; +import {getPendingBalanceToWithdraw, isActiveValidator} from "../util/index.js"; import {initiateValidatorExit} from "./index.js"; export enum VoluntaryExitValidity { @@ -40,8 +41,11 @@ export function processVoluntaryExit( throw Error(`Invalid voluntary exit at forkSeq=${fork} reason=${validity}`); } - if (isGloasCachedStateType(state) && isBuilderIndex(voluntaryExit.validatorIndex)) { - initiateBuilderExit(state, convertValidatorIndexToBuilderIndex(voluntaryExit.validatorIndex)); + if (fork >= ForkSeq.gloas && isBuilderIndex(voluntaryExit.validatorIndex)) { + initiateBuilderExit( + state as CachedBeaconStateGloas, + convertValidatorIndexToBuilderIndex(voluntaryExit.validatorIndex) + ); return; } @@ -64,8 +68,8 @@ export function getVoluntaryExitValidity( } // Check if this is a builder exit - if (isGloasCachedStateType(state) && isBuilderIndex(voluntaryExit.validatorIndex)) { - return getBuilderVoluntaryExitValidity(state, signedVoluntaryExit, verifySignature); + if (fork >= ForkSeq.gloas && isBuilderIndex(voluntaryExit.validatorIndex)) { + return getBuilderVoluntaryExitValidity(state as CachedBeaconStateGloas, signedVoluntaryExit, verifySignature); } return getValidatorVoluntaryExitValidity(fork, state, signedVoluntaryExit, verifySignature); @@ -98,7 +102,10 @@ function getBuilderVoluntaryExitValidity( } // Verify signature - if (verifySignature && !verifyVoluntaryExitSignature(config, epochCtx.index2pubkey, state, signedVoluntaryExit)) { + if ( + verifySignature && + !verifyVoluntaryExitSignature(config, epochCtx.pubkeyCache, new BeaconStateView(state), signedVoluntaryExit) + ) { return VoluntaryExitValidity.invalidSignature; } @@ -145,7 +152,10 @@ function getValidatorVoluntaryExitValidity( } // Verify signature - if (verifySignature && !verifyVoluntaryExitSignature(config, epochCtx.index2pubkey, state, signedVoluntaryExit)) { + if ( + verifySignature && + !verifyVoluntaryExitSignature(config, epochCtx.pubkeyCache, new BeaconStateView(state), signedVoluntaryExit) + ) { return VoluntaryExitValidity.invalidSignature; } diff --git a/packages/state-transition/src/block/processWithdrawalRequest.ts b/packages/state-transition/src/block/processWithdrawalRequest.ts index 6b79bc3aaf2f..a91c745d91a0 100644 --- a/packages/state-transition/src/block/processWithdrawalRequest.ts +++ b/packages/state-transition/src/block/processWithdrawalRequest.ts @@ -21,7 +21,7 @@ export function processWithdrawalRequest( const amount = Number(withdrawalRequest.amount); const {pendingPartialWithdrawals, validators, epochCtx} = state; // no need to use unfinalized pubkey cache from 6110 as validator won't be active anyway - const {pubkey2index, config} = epochCtx; + const {pubkeyCache, config} = epochCtx; const isFullExitRequest = amount === FULL_EXIT_REQUEST_AMOUNT; // If partial withdrawal queue is full, only full exits are processed @@ -31,7 +31,7 @@ export function processWithdrawalRequest( // bail out if validator is not in beacon state // note that we don't need to check for 6110 unfinalized vals as they won't be eligible for withdraw/exit anyway - const validatorIndex = pubkey2index.get(withdrawalRequest.validatorPubkey); + const validatorIndex = pubkeyCache.getIndex(withdrawalRequest.validatorPubkey); if (validatorIndex === null) { return; } diff --git a/packages/state-transition/src/block/processWithdrawals.ts b/packages/state-transition/src/block/processWithdrawals.ts index 768987b76631..e2b54f5e183a 100644 --- a/packages/state-transition/src/block/processWithdrawals.ts +++ b/packages/state-transition/src/block/processWithdrawals.ts @@ -16,7 +16,7 @@ import { convertValidatorIndexToBuilderIndex, isBuilderIndex, isParentBlockFull, -} from "../util/gloas.ts"; +} from "../util/gloas.js"; import { decreaseBalance, getMaxEffectiveBalance, diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index 2f715bbf66c5..d87c998d1e8a 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -1,5 +1,4 @@ import {PublicKey} from "@chainsafe/blst"; -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {BeaconConfig, ChainConfig, createBeaconConfig} from "@lodestar/config"; import { ATTESTATION_SUBNET_COUNT, @@ -56,7 +55,7 @@ import {computeBaseRewardPerIncrement, computeSyncParticipantReward} from "../ut import {sumTargetUnslashedBalanceIncrements} from "../util/targetUnslashedBalance.js"; import {EffectiveBalanceIncrements, getEffectiveBalanceIncrementsWithLen} from "./effectiveBalanceIncrements.js"; import {EpochTransitionCache} from "./epochTransitionCache.js"; -import {Index2PubkeyCache, syncPubkeys} from "./pubkeyCache.js"; +import {PubkeyCache, createPubkeyCache, syncPubkeys} from "./pubkeyCache.js"; import {CachedBeaconStateAllForks, CachedBeaconStateFulu} from "./stateCache.js"; import { SyncCommitteeCache, @@ -71,8 +70,7 @@ export const PROPOSER_WEIGHT_FACTOR = PROPOSER_WEIGHT / (WEIGHT_DENOMINATOR - PR export type EpochCacheImmutableData = { config: BeaconConfig; - pubkey2index: PubkeyIndexMap; - index2pubkey: Index2PubkeyCache; + pubkeyCache: PubkeyCache; }; export type EpochCacheOpts = { @@ -111,15 +109,9 @@ export class EpochCache { /** * Unique globally shared pubkey registry. There should only exist one for the entire application. * - * $VALIDATOR_COUNT x 192 char String -> Number Map + * Couples both index→pubkey and pubkey→index lookups, keeping them in sync atomically. */ - pubkey2index: PubkeyIndexMap; - /** - * Unique globally shared pubkey registry. There should only exist one for the entire application. - * - * $VALIDATOR_COUNT x BLST deserialized pubkey (Jacobian coordinates) - */ - index2pubkey: Index2PubkeyCache; + pubkeyCache: PubkeyCache; /** * Indexes of the block proposers for the current epoch. * For pre-fulu, this is computed and cached from the current shuffling. @@ -251,8 +243,7 @@ export class EpochCache { constructor(data: { config: BeaconConfig; - pubkey2index: PubkeyIndexMap; - index2pubkey: Index2PubkeyCache; + pubkeyCache: PubkeyCache; proposers: number[]; proposersPrevEpoch: number[] | null; proposersNextEpoch: ProposersDeferred; @@ -283,8 +274,7 @@ export class EpochCache { syncPeriod: SyncPeriod; }) { this.config = data.config; - this.pubkey2index = data.pubkey2index; - this.index2pubkey = data.index2pubkey; + this.pubkeyCache = data.pubkeyCache; this.proposers = data.proposers; this.proposersPrevEpoch = data.proposersPrevEpoch; this.proposersNextEpoch = data.proposersNextEpoch; @@ -323,7 +313,7 @@ export class EpochCache { */ static createFromState( state: BeaconStateAllForks, - {config, pubkey2index, index2pubkey}: EpochCacheImmutableData, + {config, pubkeyCache}: EpochCacheImmutableData, opts?: EpochCacheOpts ): EpochCache { const currentEpoch = computeEpochAtSlot(state.slot); @@ -339,9 +329,9 @@ export class EpochCache { const validatorCount = validators.length; // syncPubkeys here to ensure EpochCacheImmutableData is popualted before computing the rest of caches - // - computeSyncCommitteeCache() needs a fully populated pubkey2index cache + // - computeSyncCommitteeCache() needs a fully populated pubkeyCache if (!opts?.skipSyncPubkeys) { - syncPubkeys(validators, pubkey2index, index2pubkey); + syncPubkeys(pubkeyCache, validators); } const effectiveBalanceIncrements = getEffectiveBalanceIncrementsWithLen(validatorCount); @@ -454,8 +444,8 @@ export class EpochCache { // Allow to skip populating sync committee for initializeBeaconStateFromEth1() if (afterAltairFork && !opts?.skipSyncCommitteeCache) { const altairState = state as BeaconStateAltair; - currentSyncCommitteeIndexed = computeSyncCommitteeCache(altairState.currentSyncCommittee, pubkey2index); - nextSyncCommitteeIndexed = computeSyncCommitteeCache(altairState.nextSyncCommittee, pubkey2index); + currentSyncCommitteeIndexed = computeSyncCommitteeCache(altairState.currentSyncCommittee, pubkeyCache); + nextSyncCommitteeIndexed = computeSyncCommitteeCache(altairState.nextSyncCommittee, pubkeyCache); } else { currentSyncCommitteeIndexed = new SyncCommitteeCacheEmpty(); nextSyncCommitteeIndexed = new SyncCommitteeCacheEmpty(); @@ -528,8 +518,7 @@ export class EpochCache { return new EpochCache({ config, - pubkey2index, - index2pubkey, + pubkeyCache, proposers, // On first epoch, set to null to prevent unnecessary work since this is only used for metrics proposersPrevEpoch: null, @@ -572,8 +561,7 @@ export class EpochCache { return new EpochCache({ config: this.config, // Common append-only structures shared with all states, no need to clone - pubkey2index: this.pubkey2index, - index2pubkey: this.index2pubkey, + pubkeyCache: this.pubkeyCache, // Immutable data proposers: this.proposers, proposersPrevEpoch: this.proposersPrevEpoch, @@ -900,16 +888,15 @@ export class EpochCache { * Return pubkey given the validator index. */ getPubkey(index: ValidatorIndex): PublicKey | undefined { - return this.index2pubkey[index]; + return this.pubkeyCache.get(index); } getValidatorIndex(pubkey: Uint8Array): ValidatorIndex | null { - return this.pubkey2index.get(pubkey); + return this.pubkeyCache.getIndex(pubkey); } addPubkey(index: ValidatorIndex, pubkey: Uint8Array): void { - this.pubkey2index.set(pubkey, index); - this.index2pubkey[index] = PublicKey.fromBytes(pubkey); // Optimize for aggregation + this.pubkeyCache.set(index, pubkey); } getShufflingAtSlot(slot: Slot): EpochShuffling { @@ -1121,7 +1108,6 @@ export function createEmptyEpochCacheImmutableData( return { config: createBeaconConfig(chainConfig, state.genesisValidatorsRoot), // This is a test state, there's no need to have a global shared cache of keys - pubkey2index: new PubkeyIndexMap(), - index2pubkey: [], + pubkeyCache: createPubkeyCache(), }; } diff --git a/packages/state-transition/src/cache/pubkeyCache.ts b/packages/state-transition/src/cache/pubkeyCache.ts index 62b7cf1e5c84..a392e5ae643c 100644 --- a/packages/state-transition/src/cache/pubkeyCache.ts +++ b/packages/state-transition/src/cache/pubkeyCache.ts @@ -1,33 +1,74 @@ import {PublicKey} from "@chainsafe/blst"; import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; -import {phase0} from "@lodestar/types"; +import {ValidatorIndex, phase0} from "@lodestar/types"; -export type Index2PubkeyCache = PublicKey[]; +/** + * Unified pubkey cache coupling index→pubkey and pubkey→index lookups. + * Both directions are kept in sync atomically via `set()`. + */ +export interface PubkeyCache { + /** Get deserialized PublicKey by validator index */ + get(index: ValidatorIndex): PublicKey | undefined; + /** Get deserialized PublicKey by validator index or throw if not found */ + getOrThrow(index: ValidatorIndex): PublicKey; + /** Get validator index by pubkey bytes */ + getIndex(pubkey: Uint8Array): ValidatorIndex | null; + /** Set both directions atomically. Takes raw pubkey bytes — deserialization is handled internally. */ + set(index: ValidatorIndex, pubkey: Uint8Array): void; + /** Number of entries */ + readonly size: number; +} + +class StandardPubkeyCache implements PubkeyCache { + private readonly pubkey2index: PubkeyIndexMap; + private readonly index2pubkey: (PublicKey | undefined)[]; + + constructor(pubkey2index?: PubkeyIndexMap, index2pubkey?: (PublicKey | undefined)[]) { + this.pubkey2index = pubkey2index ?? new PubkeyIndexMap(); + this.index2pubkey = index2pubkey ?? []; + } + + get size(): number { + return this.pubkey2index.size; + } + + get(index: ValidatorIndex): PublicKey | undefined { + return this.index2pubkey[index]; + } + + getOrThrow(index: ValidatorIndex): PublicKey { + const pubkey = this.get(index); + if (!pubkey) throw Error(`Missing pubkey for validator index ${index}`); + return pubkey; + } + + getIndex(pubkey: Uint8Array): ValidatorIndex | null { + return this.pubkey2index.get(pubkey); + } + + set(index: ValidatorIndex, pubkey: Uint8Array): void { + this.pubkey2index.set(pubkey, index); + // Pubkeys must be checked for group + inf. This must be done only once when the validator deposit is processed. + // Afterwards any public key in the state is considered validated. + // > Do not do any validation here + this.index2pubkey[index] = PublicKey.fromBytes(pubkey); // Optimize for aggregation + } +} + +export function createPubkeyCache(): PubkeyCache { + return new StandardPubkeyCache(); +} /** * Checks the pubkey indices against a state and adds missing pubkeys * - * Mutates `pubkey2index` and `index2pubkey` + * Mutates `pubkeyCache` * - * If pubkey caches are empty: SLOW CODE - 🐢 + * If pubkey cache is empty: SLOW CODE - 🐢 */ -export function syncPubkeys( - validators: phase0.Validator[], - pubkey2index: PubkeyIndexMap, - index2pubkey: Index2PubkeyCache -): void { - if (pubkey2index.size !== index2pubkey.length) { - throw new Error(`Pubkey indices have fallen out of sync: ${pubkey2index.size} != ${index2pubkey.length}`); - } - +export function syncPubkeys(pubkeyCache: PubkeyCache, validators: phase0.Validator[]): void { const newCount = validators.length; - index2pubkey.length = newCount; - for (let i = pubkey2index.size; i < newCount; i++) { - const pubkey = validators[i].pubkey; - pubkey2index.set(pubkey, i); - // Pubkeys must be checked for group + inf. This must be done only once when the validator deposit is processed. - // Afterwards any public key is the state consider validated. - // > Do not do any validation here - index2pubkey[i] = PublicKey.fromBytes(pubkey); // Optimize for aggregation + for (let i = pubkeyCache.size; i < newCount; i++) { + pubkeyCache.set(i, validators[i].pubkey); } } diff --git a/packages/state-transition/src/cache/stateCache.ts b/packages/state-transition/src/cache/stateCache.ts index f69cd2a44972..6054001a6027 100644 --- a/packages/state-transition/src/cache/stateCache.ts +++ b/packages/state-transition/src/cache/stateCache.ts @@ -1,4 +1,3 @@ -import {PublicKey} from "@chainsafe/blst"; import {BeaconConfig} from "@lodestar/config"; import {loadState} from "../util/loadState/loadState.js"; import {EpochCache, EpochCacheImmutableData, EpochCacheOpts} from "./epochCache.js"; @@ -168,7 +167,7 @@ export function createCachedBeaconState( * Check loadState() api for more details * // TODO: rename to loadUnfinalizedCachedBeaconState() due to ELECTRA */ -export function loadCachedBeaconState( +export function loadCachedBeaconState( cachedSeedState: T, stateBytes: Uint8Array, opts?: EpochCacheOpts, @@ -180,22 +179,19 @@ export function loadCachedBeaconState, - pubkey2index: PubkeyIndexMap + pubkeyCache: {getIndex(pubkey: Uint8Array): number | null} ): SyncCommitteeCache { - const validatorIndices = computeSyncCommitteeValidatorIndices(syncCommittee, pubkey2index); + const validatorIndices = computeSyncCommitteeValidatorIndices(syncCommittee, pubkeyCache); const validatorIndexMap = computeValidatorSyncCommitteeIndexMap(validatorIndices); return { validatorIndices, @@ -79,12 +78,12 @@ export function computeValidatorSyncCommitteeIndexMap( */ function computeSyncCommitteeValidatorIndices( syncCommittee: CompositeViewDU, - pubkey2index: PubkeyIndexMap + pubkeyCache: {getIndex(pubkey: Uint8Array): number | null} ): Uint32Array { const pubkeys = syncCommittee.pubkeys.getAllReadonly(); const validatorIndices = new Uint32Array(pubkeys.length); for (const [i, pubkey] of pubkeys.entries()) { - const validatorIndex = pubkey2index.get(pubkey); + const validatorIndex = pubkeyCache.getIndex(pubkey); if (validatorIndex === null) { throw Error(`SyncCommittee pubkey is unknown ${toPubkeyHex(pubkey)}`); } diff --git a/packages/state-transition/src/epoch/index.ts b/packages/state-transition/src/epoch/index.ts index 562613b44b05..47aa812ef792 100644 --- a/packages/state-transition/src/epoch/index.ts +++ b/packages/state-transition/src/epoch/index.ts @@ -16,7 +16,7 @@ import { CachedBeaconStatePhase0, EpochTransitionCache, } from "../types.js"; -import {processBuilderPendingPayments} from "./processBuilderPendingPayments.ts"; +import {processBuilderPendingPayments} from "./processBuilderPendingPayments.js"; import {processEffectiveBalanceUpdates} from "./processEffectiveBalanceUpdates.js"; import {processEth1DataReset} from "./processEth1DataReset.js"; import {processHistoricalRootsUpdate} from "./processHistoricalRootsUpdate.js"; diff --git a/packages/state-transition/src/epoch/processBuilderPendingPayments.ts b/packages/state-transition/src/epoch/processBuilderPendingPayments.ts index 947b1666be95..a1a95013a63e 100644 --- a/packages/state-transition/src/epoch/processBuilderPendingPayments.ts +++ b/packages/state-transition/src/epoch/processBuilderPendingPayments.ts @@ -1,7 +1,7 @@ import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {ssz} from "@lodestar/types"; -import {CachedBeaconStateGloas} from "../types.ts"; -import {getBuilderPaymentQuorumThreshold} from "../util/gloas.ts"; +import {CachedBeaconStateGloas} from "../types.js"; +import {getBuilderPaymentQuorumThreshold} from "../util/gloas.js"; /** * Processes the builder pending payments from the previous epoch. diff --git a/packages/state-transition/src/index.ts b/packages/state-transition/src/index.ts index c5836ffe9518..d2b23533eb59 100644 --- a/packages/state-transition/src/index.ts +++ b/packages/state-transition/src/index.ts @@ -25,7 +25,7 @@ export { createEmptyEpochCacheImmutableData, } from "./cache/epochCache.js"; export {type EpochTransitionCache, beforeProcessEpoch} from "./cache/epochTransitionCache.js"; -export {type Index2PubkeyCache, syncPubkeys} from "./cache/pubkeyCache.js"; +export {type PubkeyCache, createPubkeyCache, syncPubkeys} from "./cache/pubkeyCache.js"; // Main state caches export { type BeaconStateCache, @@ -35,12 +35,14 @@ export { isStateValidatorsNodesPopulated, loadCachedBeaconState, } from "./cache/stateCache.js"; +export {type SyncCommitteeCache} from "./cache/syncCommitteeCache.js"; export * from "./constants/index.js"; export type {EpochTransitionStep} from "./epoch/index.js"; export {type BeaconStateTransitionMetrics, getMetrics} from "./metrics.js"; export * from "./rewards/index.js"; export * from "./signatureSets/index.js"; export * from "./stateTransition.js"; +export * from "./stateView/index.js"; export type { BeaconStateAllForks, BeaconStateAltair, diff --git a/packages/state-transition/src/lightClient/proofs.ts b/packages/state-transition/src/lightClient/proofs.ts new file mode 100644 index 000000000000..d8b3847d978d --- /dev/null +++ b/packages/state-transition/src/lightClient/proofs.ts @@ -0,0 +1,83 @@ +import {Tree} from "@chainsafe/persistent-merkle-tree"; +import { + BLOCK_BODY_EXECUTION_PAYLOAD_GINDEX, + FINALIZED_ROOT_GINDEX, + FINALIZED_ROOT_GINDEX_ELECTRA, + ForkName, + ForkPostBellatrix, + isForkPostElectra, +} from "@lodestar/params"; +import {BeaconBlockBody, SSZTypesFor, ssz} from "@lodestar/types"; +import {BeaconStateAllForks, CachedBeaconStateAllForks} from "../types.js"; +import {SyncCommitteeWitness} from "./types.js"; + +export function getSyncCommitteesWitness(fork: ForkName, state: BeaconStateAllForks): SyncCommitteeWitness { + const n1 = state.node; + let witness: Uint8Array[]; + let currentSyncCommitteeRoot: Uint8Array; + let nextSyncCommitteeRoot: Uint8Array; + + if (isForkPostElectra(fork)) { + const n2 = n1.left; + const n5 = n2.right; + const n10 = n5.left; + const n21 = n10.right; + const n43 = n21.right; + + currentSyncCommitteeRoot = n43.left.root; // n86 + nextSyncCommitteeRoot = n43.right.root; // n87 + + // Witness branch is sorted by descending gindex + witness = [ + n21.left.root, // 42 + n10.left.root, // 20 + n5.right.root, // 11 + n2.left.root, // 4 + n1.right.root, // 3 + ]; + } else { + const n3 = n1.right; // [1]0110 + const n6 = n3.left; // 1[0]110 + const n13 = n6.right; // 10[1]10 + const n27 = n13.right; // 101[1]0 + currentSyncCommitteeRoot = n27.left.root; // n54 1011[0] + nextSyncCommitteeRoot = n27.right.root; // n55 1011[1] + + // Witness branch is sorted by descending gindex + witness = [ + n13.left.root, // 26 + n6.left.root, // 12 + n3.right.root, // 7 + n1.left.root, // 2 + ]; + } + + return { + witness, + currentSyncCommitteeRoot, + nextSyncCommitteeRoot, + }; +} + +export function getNextSyncCommitteeBranch(syncCommitteesWitness: SyncCommitteeWitness): Uint8Array[] { + // Witness branch is sorted by descending gindex + return [syncCommitteesWitness.currentSyncCommitteeRoot, ...syncCommitteesWitness.witness]; +} + +export function getCurrentSyncCommitteeBranch(syncCommitteesWitness: SyncCommitteeWitness): Uint8Array[] { + // Witness branch is sorted by descending gindex + return [syncCommitteesWitness.nextSyncCommitteeRoot, ...syncCommitteesWitness.witness]; +} + +export function getFinalizedRootProof(state: CachedBeaconStateAllForks): Uint8Array[] { + const finalizedRootGindex = state.epochCtx.isPostElectra() ? FINALIZED_ROOT_GINDEX_ELECTRA : FINALIZED_ROOT_GINDEX; + return new Tree(state.node).getSingleProof(BigInt(finalizedRootGindex)); +} + +export function getBlockBodyExecutionHeaderProof( + fork: ForkPostBellatrix, + body: BeaconBlockBody +): Uint8Array[] { + const bodyView = (ssz[fork].BeaconBlockBody as SSZTypesFor).toView(body); + return new Tree(bodyView.node).getSingleProof(BigInt(BLOCK_BODY_EXECUTION_PAYLOAD_GINDEX)); +} diff --git a/packages/state-transition/src/lightClient/types.ts b/packages/state-transition/src/lightClient/types.ts new file mode 100644 index 000000000000..b9723df501b3 --- /dev/null +++ b/packages/state-transition/src/lightClient/types.ts @@ -0,0 +1,33 @@ +/** + * We aren't creating the sync committee proofs separately because our ssz library automatically adds leaves to composite types, + * so they're already included in the state proof, currently with no way to specify otherwise + * + * remove two offsets so the # of offsets in the state proof will be the # expected + * This is a hack, but properly setting the offsets in the state proof would require either removing witnesses needed for the committees + * or setting the roots of the committees in the state proof + * this will always be 1, syncProofLeavesLength + * + * + * With empty state (minimal) + * - `genesisTime = 0xffffffff` + * - `genesisValidatorsRoot = Buffer.alloc(32, 1)` + * + * Proof: + * ``` + * offsets: [ 5, 4, 3, 2, 1 ] + * leaves: [ + * '0xffffffff00000000000000000000000000000000000000000000000000000000', + * '0x0101010101010101010101010101010101010101010101010101010101010101', + * '0xb11b8bcf59425d6c99019cca1d2c2e47b51a2f74917a67ad132274f43e13ec43', + * '0x74bd1f2437cdf74b0904ee525d8da070a3fa27570942bf42cbab3dc5939600f0', + * '0x7f06739e5a42360c56e519a511675901c95402ea9877edc0d9a87471b1374a6a', + * '0x9f534204ba3c0b69fcb42a11987bfcbc5aea0463e5b0614312ded4b62cf3a380' + * ] + * ``` + */ +export type SyncCommitteeWitness = { + /** Vector[Bytes32, 4] or Vector[Bytes32, 5] depending on the fork */ + witness: Uint8Array[]; + currentSyncCommitteeRoot: Uint8Array; + nextSyncCommitteeRoot: Uint8Array; +}; diff --git a/packages/state-transition/src/rewards/attestationsRewards.ts b/packages/state-transition/src/rewards/attestationsRewards.ts index b3fea34633af..d7d27f58c11a 100644 --- a/packages/state-transition/src/rewards/attestationsRewards.ts +++ b/packages/state-transition/src/rewards/attestationsRewards.ts @@ -1,4 +1,3 @@ -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {BeaconConfig} from "@lodestar/config"; import { EFFECTIVE_BALANCE_INCREMENT, @@ -16,6 +15,7 @@ import { import {ValidatorIndex, rewards} from "@lodestar/types"; import {fromHex} from "@lodestar/utils"; import {EpochTransitionCache, beforeProcessEpoch} from "../cache/epochTransitionCache.js"; +import {PubkeyCache} from "../cache/pubkeyCache.js"; import {CachedBeaconStateAllForks, CachedBeaconStateAltair} from "../types.js"; import { FLAG_ELIGIBLE_ATTESTER, @@ -34,7 +34,7 @@ const defaultAttestationsPenalty = {target: 0, source: 0}; export async function computeAttestationsRewards( config: BeaconConfig, - pubkey2index: PubkeyIndexMap, + pubkeyCache: PubkeyCache, state: CachedBeaconStateAllForks, validatorIds?: (ValidatorIndex | string)[] ): Promise { @@ -53,7 +53,7 @@ export async function computeAttestationsRewards( ); const totalRewards = computeTotalAttestationsRewardsAltair( config, - pubkey2index, + pubkeyCache, stateAltair, transitionCache, idealRewards, @@ -142,7 +142,7 @@ function computeIdealAttestationsRewardsAndPenaltiesAltair( // Same calculation as `getRewardsAndPenaltiesAltair` but returns the breakdown of rewards instead of aggregated function computeTotalAttestationsRewardsAltair( config: BeaconConfig, - pubkey2index: PubkeyIndexMap, + pubkeyCache: PubkeyCache, state: CachedBeaconStateAltair, transitionCache: EpochTransitionCache, idealRewards: rewards.IdealAttestationsReward[], @@ -153,7 +153,7 @@ function computeTotalAttestationsRewardsAltair( const {flags} = transitionCache; const {epochCtx} = state; const validatorIndices = validatorIds - .map((id) => (typeof id === "number" ? id : pubkey2index.get(fromHex(id)))) + .map((id) => (typeof id === "number" ? id : pubkeyCache.getIndex(fromHex(id)))) .filter((index) => index !== undefined); // Validator indices to include in the result const inactivityPenaltyDenominator = config.INACTIVITY_SCORE_BIAS * INACTIVITY_PENALTY_QUOTIENT_ALTAIR; diff --git a/packages/state-transition/src/rewards/syncCommitteeRewards.ts b/packages/state-transition/src/rewards/syncCommitteeRewards.ts index 27e2c24e3dd9..e93d3f61c726 100644 --- a/packages/state-transition/src/rewards/syncCommitteeRewards.ts +++ b/packages/state-transition/src/rewards/syncCommitteeRewards.ts @@ -1,12 +1,12 @@ import {BeaconConfig} from "@lodestar/config"; import {ForkName, SYNC_COMMITTEE_SIZE} from "@lodestar/params"; import {BeaconBlock, ValidatorIndex, altair, rewards} from "@lodestar/types"; -import {Index2PubkeyCache} from "../cache/pubkeyCache.js"; +import {PubkeyCache} from "../cache/pubkeyCache.js"; import {CachedBeaconStateAllForks, CachedBeaconStateAltair} from "../cache/stateCache.js"; export async function computeSyncCommitteeRewards( config: BeaconConfig, - index2pubkey: Index2PubkeyCache, + pubkeyCache: PubkeyCache, block: BeaconBlock, preState: CachedBeaconStateAllForks, validatorIds: (ValidatorIndex | string)[] = [] @@ -47,9 +47,10 @@ export async function computeSyncCommitteeRewards( if (validatorIds.length) { const filtersSet = new Set(validatorIds); - return rewards.filter( - (reward) => filtersSet.has(reward.validatorIndex) || filtersSet.has(index2pubkey[reward.validatorIndex].toHex()) - ); + return rewards.filter((reward) => { + const pubkeyHex = pubkeyCache.get(reward.validatorIndex)?.toHex(); + return filtersSet.has(reward.validatorIndex) || (pubkeyHex !== undefined && filtersSet.has(pubkeyHex)); + }); } return rewards; diff --git a/packages/state-transition/src/signatureSets/executionPayloadEnvelope.ts b/packages/state-transition/src/signatureSets/executionPayloadEnvelope.ts index 40c21f25fd77..6107ef8eceae 100644 --- a/packages/state-transition/src/signatureSets/executionPayloadEnvelope.ts +++ b/packages/state-transition/src/signatureSets/executionPayloadEnvelope.ts @@ -1,7 +1,11 @@ +import {PublicKey} from "@chainsafe/blst"; import {BeaconConfig} from "@lodestar/config"; -import {DOMAIN_BEACON_BUILDER} from "@lodestar/params"; -import {gloas, ssz} from "@lodestar/types"; +import {BUILDER_INDEX_SELF_BUILD, DOMAIN_BEACON_BUILDER} from "@lodestar/params"; +import {ValidatorIndex, gloas, ssz} from "@lodestar/types"; +import {PubkeyCache} from "../cache/pubkeyCache.js"; +import {IBeaconStateView} from "../stateView/interface.js"; import {computeSigningRoot} from "../util/index.js"; +import {type SingleSignatureSet, createSingleSignatureSetFromComponents} from "../util/signatureSets.js"; export function getExecutionPayloadEnvelopeSigningRoot( config: BeaconConfig, @@ -11,3 +15,23 @@ export function getExecutionPayloadEnvelopeSigningRoot( return computeSigningRoot(ssz.gloas.ExecutionPayloadEnvelope, envelope, domain); } + +export function getExecutionPayloadEnvelopeSignatureSet( + config: BeaconConfig, + pubkeyCache: PubkeyCache, + state: IBeaconStateView, + signedEnvelope: gloas.SignedExecutionPayloadEnvelope, + proposerIndex: ValidatorIndex +): SingleSignatureSet { + const envelope = signedEnvelope.message; + const pubkey = + envelope.builderIndex === BUILDER_INDEX_SELF_BUILD + ? pubkeyCache.getOrThrow(proposerIndex) + : PublicKey.fromBytes(state.getBuilder(envelope.builderIndex).pubkey); + + return createSingleSignatureSetFromComponents( + pubkey, + getExecutionPayloadEnvelopeSigningRoot(config, envelope), + signedEnvelope.signature + ); +} diff --git a/packages/state-transition/src/signatureSets/index.ts b/packages/state-transition/src/signatureSets/index.ts index b82e47023dc8..d5bcf44cec85 100644 --- a/packages/state-transition/src/signatureSets/index.ts +++ b/packages/state-transition/src/signatureSets/index.ts @@ -3,6 +3,7 @@ import {ForkSeq} from "@lodestar/params"; import {IndexedAttestation, SignedBeaconBlock, altair, capella} from "@lodestar/types"; import {getSyncCommitteeSignatureSet} from "../block/processSyncCommittee.js"; import {SyncCommitteeCache} from "../cache/syncCommitteeCache.js"; +import {BeaconStateView} from "../stateView/beaconStateView.js"; import {CachedBeaconStateAllForks} from "../types.js"; import {ISignatureSet} from "../util/index.js"; import {getAttesterSlashingsSignatureSets} from "./attesterSlashings.js"; @@ -47,7 +48,7 @@ export function getBlockSignatureSets( ...getProposerSlashingsSignatureSets(config, signedBlock), ...getAttesterSlashingsSignatureSets(config, signedBlock), ...getAttestationsSignatureSets(config, signedBlock, indexedAttestations), - ...getVoluntaryExitsSignatureSets(config, state, signedBlock), + ...getVoluntaryExitsSignatureSets(config, new BeaconStateView(state), signedBlock), ]; if (!opts?.skipProposerSignature) { diff --git a/packages/state-transition/src/signatureSets/proposer.ts b/packages/state-transition/src/signatureSets/proposer.ts index 8505ec22bd3c..f1538d312510 100644 --- a/packages/state-transition/src/signatureSets/proposer.ts +++ b/packages/state-transition/src/signatureSets/proposer.ts @@ -1,17 +1,17 @@ import {BeaconConfig} from "@lodestar/config"; import {DOMAIN_BEACON_PROPOSER} from "@lodestar/params"; import {SignedBeaconBlock, SignedBlindedBeaconBlock, Slot, isBlindedBeaconBlock, phase0, ssz} from "@lodestar/types"; -import {Index2PubkeyCache} from "../cache/pubkeyCache.js"; +import {PubkeyCache} from "../cache/pubkeyCache.js"; import {computeSigningRoot} from "../util/index.js"; import {ISignatureSet, SignatureSetType, verifySignatureSet} from "../util/signatureSets.js"; export function verifyProposerSignature( config: BeaconConfig, - index2pubkey: Index2PubkeyCache, + pubkeyCache: PubkeyCache, signedBlock: SignedBeaconBlock | SignedBlindedBeaconBlock ): boolean { const signatureSet = getBlockProposerSignatureSet(config, signedBlock); - return verifySignatureSet(signatureSet, index2pubkey); + return verifySignatureSet(signatureSet, pubkeyCache); } export function getBlockProposerSignatureSet( diff --git a/packages/state-transition/src/signatureSets/randao.ts b/packages/state-transition/src/signatureSets/randao.ts index d6a132c215b2..0a8372745f23 100644 --- a/packages/state-transition/src/signatureSets/randao.ts +++ b/packages/state-transition/src/signatureSets/randao.ts @@ -1,7 +1,7 @@ import {BeaconConfig} from "@lodestar/config"; import {DOMAIN_RANDAO} from "@lodestar/params"; import {BeaconBlock, ssz} from "@lodestar/types"; -import {Index2PubkeyCache} from "../cache/pubkeyCache.js"; +import {PubkeyCache} from "../cache/pubkeyCache.js"; import { ISignatureSet, SignatureSetType, @@ -10,12 +10,8 @@ import { verifySignatureSet, } from "../util/index.js"; -export function verifyRandaoSignature( - config: BeaconConfig, - index2pubkey: Index2PubkeyCache, - block: BeaconBlock -): boolean { - return verifySignatureSet(getRandaoRevealSignatureSet(config, block), index2pubkey); +export function verifyRandaoSignature(config: BeaconConfig, pubkeyCache: PubkeyCache, block: BeaconBlock): boolean { + return verifySignatureSet(getRandaoRevealSignatureSet(config, block), pubkeyCache); } /** diff --git a/packages/state-transition/src/signatureSets/voluntaryExits.ts b/packages/state-transition/src/signatureSets/voluntaryExits.ts index 2d6037e64875..57a517b14da8 100644 --- a/packages/state-transition/src/signatureSets/voluntaryExits.ts +++ b/packages/state-transition/src/signatureSets/voluntaryExits.ts @@ -1,8 +1,9 @@ import {PublicKey} from "@chainsafe/blst"; import {BeaconConfig} from "@lodestar/config"; -import {SignedBeaconBlock, phase0, ssz} from "@lodestar/types"; -import {Index2PubkeyCache} from "../cache/pubkeyCache.js"; -import {CachedBeaconStateAllForks, CachedBeaconStateGloas} from "../types.js"; +import {ForkSeq} from "@lodestar/params"; +import {SignedBeaconBlock, Slot, phase0, ssz} from "@lodestar/types"; +import {PubkeyCache} from "../cache/pubkeyCache.js"; +import {IBeaconStateView} from "../stateView/interface.js"; import { ISignatureSet, SignatureSetType, @@ -10,17 +11,16 @@ import { computeStartSlotAtEpoch, convertValidatorIndexToBuilderIndex, isBuilderIndex, - isGloasCachedStateType, verifySignatureSet, } from "../util/index.js"; export function verifyVoluntaryExitSignature( config: BeaconConfig, - index2pubkey: Index2PubkeyCache, - state: CachedBeaconStateAllForks, + pubkeyCache: PubkeyCache, + state: IBeaconStateView, signedVoluntaryExit: phase0.SignedVoluntaryExit ): boolean { - return verifySignatureSet(getVoluntaryExitSignatureSet(config, state, signedVoluntaryExit), index2pubkey); + return verifySignatureSet(getVoluntaryExitSignatureSet(config, state, signedVoluntaryExit), pubkeyCache); } /** @@ -28,19 +28,21 @@ export function verifyVoluntaryExitSignature( */ export function getVoluntaryExitSignatureSet( config: BeaconConfig, - state: CachedBeaconStateAllForks, + state: IBeaconStateView, signedVoluntaryExit: phase0.SignedVoluntaryExit ): ISignatureSet { - if (isGloasCachedStateType(state) && isBuilderVoluntaryExit(signedVoluntaryExit)) { + const fork = config.getForkSeq(state.slot); + + if (fork >= ForkSeq.gloas && isBuilderVoluntaryExit(signedVoluntaryExit)) { return getBuilderVoluntaryExitSignatureSet(config, state, signedVoluntaryExit); } - return getValidatorVoluntaryExitSignatureSet(config, state, signedVoluntaryExit); + return getValidatorVoluntaryExitSignatureSet(config, state.slot, signedVoluntaryExit); } export function getVoluntaryExitsSignatureSets( config: BeaconConfig, - state: CachedBeaconStateAllForks, + state: IBeaconStateView, signedBlock: SignedBeaconBlock ): ISignatureSet[] { return signedBlock.message.body.voluntaryExits.map((voluntaryExit) => @@ -50,11 +52,11 @@ export function getVoluntaryExitsSignatureSets( export function getValidatorVoluntaryExitSignatureSet( config: BeaconConfig, - state: CachedBeaconStateAllForks, + stateSlot: Slot, signedVoluntaryExit: phase0.SignedVoluntaryExit ): ISignatureSet { const messageSlot = computeStartSlotAtEpoch(signedVoluntaryExit.message.epoch); - const domain = config.getDomainForVoluntaryExit(state.slot, messageSlot); + const domain = config.getDomainForVoluntaryExit(stateSlot, messageSlot); return { type: SignatureSetType.indexed, @@ -66,13 +68,13 @@ export function getValidatorVoluntaryExitSignatureSet( export function getBuilderVoluntaryExitSignatureSet( config: BeaconConfig, - state: CachedBeaconStateGloas, + state: IBeaconStateView, signedVoluntaryExit: phase0.SignedVoluntaryExit ): ISignatureSet { const messageSlot = computeStartSlotAtEpoch(signedVoluntaryExit.message.epoch); const domain = config.getDomainForVoluntaryExit(state.slot, messageSlot); const builderIndex = convertValidatorIndexToBuilderIndex(signedVoluntaryExit.message.validatorIndex); - const builder = state.builders.getReadonly(builderIndex); + const builder = state.getBuilder(builderIndex); return { type: SignatureSetType.single, diff --git a/packages/state-transition/src/stateTransition.ts b/packages/state-transition/src/stateTransition.ts index 3bbc3478d021..39d742932973 100644 --- a/packages/state-transition/src/stateTransition.ts +++ b/packages/state-transition/src/stateTransition.ts @@ -113,7 +113,7 @@ export function stateTransition( postState = processSlotsWithTransientCache(postState, blockSlot, options, {metrics, validatorMonitor}); // Verify proposer signature only - if (verifyProposer && !verifyProposerSignature(postState.config, postState.epochCtx.index2pubkey, signedBlock)) { + if (verifyProposer && !verifyProposerSignature(postState.config, postState.epochCtx.pubkeyCache, signedBlock)) { throw new Error("Invalid block signature"); } diff --git a/packages/state-transition/src/stateView/beaconStateView.ts b/packages/state-transition/src/stateView/beaconStateView.ts new file mode 100644 index 000000000000..12ce6ce0457f --- /dev/null +++ b/packages/state-transition/src/stateView/beaconStateView.ts @@ -0,0 +1,818 @@ +import {CompactMultiProof, ProofType, Tree, createProof} from "@chainsafe/persistent-merkle-tree"; +import {ByteViews} from "@chainsafe/ssz"; +import {BeaconConfig} from "@lodestar/config"; +import {ForkSeq, SLOTS_PER_HISTORICAL_ROOT, isForkPostGloas} from "@lodestar/params"; +import { + BeaconBlock, + BlindedBeaconBlock, + BuilderIndex, + Bytes32, + Epoch, + ExecutionPayloadBid, + ExecutionPayloadHeader, + Root, + RootHex, + SignedBeaconBlock, + SignedBlindedBeaconBlock, + Slot, + SyncCommittee, + ValidatorIndex, + capella, + electra, + fulu, + getValidatorStatus, + gloas, + mapToGeneralStatus, + phase0, + rewards, +} from "@lodestar/types"; +import {Checkpoint, Fork} from "@lodestar/types/phase0"; +import {processExecutionPayloadEnvelope} from "../block/index.js"; +import {ProcessExecutionPayloadEnvelopeOpts} from "../block/processExecutionPayloadEnvelope.js"; +import {VoluntaryExitValidity, getVoluntaryExitValidity} from "../block/processVoluntaryExit.js"; +import {getExpectedWithdrawals} from "../block/processWithdrawals.js"; +import {EffectiveBalanceIncrements} from "../cache/effectiveBalanceIncrements.js"; +import {EpochTransitionCacheOpts} from "../cache/epochTransitionCache.js"; +import {PubkeyCache, createPubkeyCache} from "../cache/pubkeyCache.js"; +import {RewardCache} from "../cache/rewardCache.js"; +import { + CachedBeaconStateAllForks, + CachedBeaconStateAltair, + CachedBeaconStateCapella, + CachedBeaconStateElectra, + CachedBeaconStateExecutions, + CachedBeaconStateFulu, + CachedBeaconStateGloas, + createCachedBeaconState, + isStateValidatorsNodesPopulated, +} from "../cache/stateCache.js"; +import {SyncCommitteeCache} from "../cache/syncCommitteeCache.js"; +import {BeaconStateAllForks} from "../cache/types.js"; +import {computeUnrealizedCheckpoints} from "../epoch/computeUnrealizedCheckpoints.js"; +import {getFinalizedRootProof, getSyncCommitteesWitness} from "../lightClient/proofs.js"; +import {SyncCommitteeWitness} from "../lightClient/types.js"; +import {computeAttestationsRewards} from "../rewards/attestationsRewards.js"; +import {computeBlockRewards} from "../rewards/blockRewards.js"; +import {computeSyncCommitteeRewards} from "../rewards/syncCommitteeRewards.js"; +import {StateTransitionModules, StateTransitionOpts, processSlots, stateTransition} from "../stateTransition.js"; +import {getEffectiveBalanceIncrementsZeroInactive} from "../util/balance.js"; +import {getBlockRootAtSlot} from "../util/blockRoot.js"; +import {computeAnchorCheckpoint} from "../util/computeAnchorCheckpoint.js"; +import {computeEpochAtSlot, computeStartSlotAtEpoch} from "../util/epoch.js"; +import {EpochShuffling} from "../util/epochShuffling.js"; +import {isExecutionEnabled, isExecutionStateType, isMergeTransitionComplete} from "../util/execution.js"; +import {canBuilderCoverBid} from "../util/gloas.js"; +import {loadState} from "../util/loadState/loadState.js"; +import {getRandaoMix} from "../util/seed.js"; +import {getStateTypeFromBytes} from "../util/sszBytes.js"; +import {getLatestWeakSubjectivityCheckpointEpoch} from "../util/weakSubjectivity.js"; +import {IBeaconStateView} from "./interface.js"; + +export class BeaconStateView implements IBeaconStateView { + private readonly config: BeaconConfig; + // Cached values extracted from the tree + // phase0 + private _fork: Fork | null = null; + private _latestBlockHeader: phase0.BeaconBlockHeader | null = null; + // altair + private _currentSyncCommittee: SyncCommittee | null = null; + private _nextSyncCommittee: SyncCommittee | null = null; + private _previousEpochParticipation: Uint8Array | null = null; + private _currentEpochParticipation: Uint8Array | null = null; + // bellatrix + private _latestExecutionPayloadHeader: ExecutionPayloadHeader | null = null; + // Caches the cross-fork latestBlockHash value + private _latestBlockHash: Bytes32 | null = null; + // capella + private _historicalSummaries: capella.HistoricalSummaries | null = null; + // electra + private _pendingPartialWithdrawals: electra.PendingPartialWithdrawals | null = null; + private _pendingConsolidations: electra.PendingConsolidations | null = null; + private _pendingDeposits: electra.PendingDeposits | null = null; + // fulu + private _proposerLookahead: fulu.ProposerLookahead | null = null; + // gloas + private _executionPayloadAvailability: boolean[] | null = null; + private _latestExecutionPayloadBid: ExecutionPayloadBid | null = null; + + constructor(readonly cachedState: CachedBeaconStateAllForks) { + this.config = cachedState.config; + } + + // phase0 + + get slot(): number { + return this.cachedState.slot; + } + + get fork(): Fork { + if (this._fork === null) { + this._fork = this.cachedState.fork.toValue(); + } + return this._fork; + } + + get epoch(): number { + return computeEpochAtSlot(this.slot); + } + + get genesisTime(): number { + return this.cachedState.genesisTime; + } + + get genesisValidatorsRoot(): Root { + return this.cachedState.genesisValidatorsRoot; + } + + get eth1Data(): phase0.Eth1Data { + return this.cachedState.eth1Data; + } + + get latestBlockHeader(): phase0.BeaconBlockHeader { + if (this._latestBlockHeader === null) { + this._latestBlockHeader = this.cachedState.latestBlockHeader.toValue(); + } + return this._latestBlockHeader; + } + + get previousJustifiedCheckpoint(): Checkpoint { + return this.cachedState.previousJustifiedCheckpoint; + } + + get currentJustifiedCheckpoint(): Checkpoint { + return this.cachedState.currentJustifiedCheckpoint; + } + + get finalizedCheckpoint(): Checkpoint { + return this.cachedState.finalizedCheckpoint; + } + + getBlockRootAtSlot(slot: Slot): Root { + return getBlockRootAtSlot(this.cachedState, slot); + } + + getBlockRootAtEpoch(epoch: Epoch): Root { + return this.getBlockRootAtSlot(computeStartSlotAtEpoch(epoch)); + } + + getStateRootAtSlot(slot: Slot): Root { + return this.cachedState.stateRoots.get(slot % SLOTS_PER_HISTORICAL_ROOT); + } + + getRandaoMix(epoch: Epoch): Bytes32 { + return getRandaoMix(this.cachedState, epoch); + } + + get previousEpochParticipation(): Uint8Array { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.altair) { + throw new Error("previousEpochParticipation is not available before Altair"); + } + + if (this._previousEpochParticipation === null) { + this._previousEpochParticipation = ( + this.cachedState as CachedBeaconStateAltair + ).previousEpochParticipation.serialize(); + } + + return this._previousEpochParticipation; + } + + // altair + + get currentEpochParticipation(): Uint8Array { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.altair) { + throw new Error("currentEpochParticipation is not available before Altair"); + } + + if (this._currentEpochParticipation === null) { + this._currentEpochParticipation = ( + this.cachedState as CachedBeaconStateAltair + ).currentEpochParticipation.serialize(); + } + + return this._currentEpochParticipation; + } + + getPreviousEpochParticipation(validatorIndex: ValidatorIndex): number { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.altair) { + throw new Error("previousEpochParticipation is not available before Altair"); + } + return (this.cachedState as CachedBeaconStateAltair).previousEpochParticipation.get(validatorIndex); + } + + getCurrentEpochParticipation(validatorIndex: ValidatorIndex): number { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.altair) { + throw new Error("currentEpochParticipation is not available before Altair"); + } + return (this.cachedState as CachedBeaconStateAltair).currentEpochParticipation.get(validatorIndex); + } + + // bellatrix + + get latestExecutionPayloadHeader(): ExecutionPayloadHeader { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.bellatrix) { + throw new Error("latestExecutionPayloadHeader is not available before Bellatrix"); + } + + if (this._latestExecutionPayloadHeader === null) { + this._latestExecutionPayloadHeader = ( + this.cachedState as CachedBeaconStateExecutions + ).latestExecutionPayloadHeader.toValue(); + } + + return this._latestExecutionPayloadHeader; + } + + /** + * Cross-fork accessor for the execution block hash of the most recently included payload. + * Pre-gloas: reads from latestExecutionPayloadHeader.blockHash. + * Gloas+: reads the dedicated latestBlockHash field (EIP-7732). + */ + get latestBlockHash(): Bytes32 { + const forkSeq = this.config.getForkSeq(this.cachedState.slot); + if (forkSeq < ForkSeq.bellatrix) { + throw new Error("latestBlockHash is not available before Bellatrix"); + } + + if (this._latestBlockHash === null) { + if (forkSeq >= ForkSeq.gloas) { + this._latestBlockHash = (this.cachedState as CachedBeaconStateGloas).latestBlockHash; + } else { + this._latestBlockHash = ( + this.cachedState as CachedBeaconStateExecutions + ).latestExecutionPayloadHeader.blockHash; + } + } + + return this._latestBlockHash; + } + + /** + * The execution block number of the most recently included payload. + * Named payloadBlockNumber (not latestBlockNumber) to mirror ExecutionPayloadHeader.blockNumber pre-gloas. + * Only available from bellatrix through fulu — not tracked on BeaconState in gloas+ (EIP-7732). + */ + get payloadBlockNumber(): number { + const forkSeq = this.config.getForkSeq(this.cachedState.slot); + if (forkSeq < ForkSeq.bellatrix) { + throw new Error("payloadBlockNumber is not available before Bellatrix"); + } + if (forkSeq >= ForkSeq.gloas) { + throw new Error("payloadBlockNumber is not available post-gloas"); + } + return (this.cachedState as CachedBeaconStateExecutions).latestExecutionPayloadHeader.blockNumber; + } + + // capella + + get historicalSummaries(): capella.HistoricalSummaries { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.capella) { + throw new Error("Historical summaries are not supported before Capella"); + } + + if (this._historicalSummaries === null) { + this._historicalSummaries = (this.cachedState as CachedBeaconStateCapella).historicalSummaries.toValue(); + } + + return this._historicalSummaries; + } + + // electra + + get pendingDeposits(): electra.PendingDeposits { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.electra) { + throw new Error("Pending deposits are not supported before Electra"); + } + + if (this._pendingDeposits === null) { + this._pendingDeposits = (this.cachedState as CachedBeaconStateElectra).pendingDeposits.toValue(); + } + + return this._pendingDeposits; + } + + get pendingDepositsCount(): number { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.electra) { + throw new Error("Pending deposits are not supported before Electra"); + } + + return (this.cachedState as CachedBeaconStateElectra).pendingDeposits.length; + } + + get pendingPartialWithdrawals(): electra.PendingPartialWithdrawals { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.electra) { + throw new Error("Pending partial withdrawals are not supported before Electra"); + } + + if (this._pendingPartialWithdrawals === null) { + this._pendingPartialWithdrawals = ( + this.cachedState as CachedBeaconStateElectra + ).pendingPartialWithdrawals.toValue(); + } + + return this._pendingPartialWithdrawals; + } + + get pendingPartialWithdrawalsCount(): number { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.electra) { + throw new Error("Pending partial withdrawals are not supported before Electra"); + } + + return (this.cachedState as CachedBeaconStateElectra).pendingPartialWithdrawals.length; + } + + get pendingConsolidations(): electra.PendingConsolidations { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.electra) { + throw new Error("Pending consolidations are not supported before Electra"); + } + + if (this._pendingConsolidations === null) { + this._pendingConsolidations = (this.cachedState as CachedBeaconStateElectra).pendingConsolidations.toValue(); + } + + return this._pendingConsolidations; + } + + get pendingConsolidationsCount(): number { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.electra) { + throw new Error("Pending consolidations are not supported before Electra"); + } + + return (this.cachedState as CachedBeaconStateElectra).pendingConsolidations.length; + } + + // fulu + + get proposerLookahead(): fulu.ProposerLookahead { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.fulu) { + throw new Error("Proposer lookahead is not supported before Fulu"); + } + + if (this._proposerLookahead === null) { + this._proposerLookahead = (this.cachedState as CachedBeaconStateFulu).proposerLookahead.toValue(); + } + + return this._proposerLookahead; + } + + // gloas + + get executionPayloadAvailability(): boolean[] { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.gloas) { + throw new Error("executionPayloadAvailability is not available before GLOAS"); + } + + if (this._executionPayloadAvailability === null) { + this._executionPayloadAvailability = (this.cachedState as CachedBeaconStateGloas).executionPayloadAvailability + .toValue() + .toBoolArray(); + } + + return this._executionPayloadAvailability; + } + + get latestExecutionPayloadBid(): ExecutionPayloadBid { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.gloas) { + throw new Error("latestExecutionPayloadBid is not available before GLOAS"); + } + + if (this._latestExecutionPayloadBid === null) { + this._latestExecutionPayloadBid = ( + this.cachedState as CachedBeaconStateGloas + ).latestExecutionPayloadBid.toValue(); + } + return this._latestExecutionPayloadBid; + } + + getBuilder(index: BuilderIndex): gloas.Builder { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.gloas) { + throw new Error("Builders are not supported before GLOAS"); + } + + return (this.cachedState as CachedBeaconStateGloas).builders.getReadonly(index); + } + + canBuilderCoverBid(builderIndex: BuilderIndex, bidAmount: number): boolean { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.gloas) { + throw new Error("Builders are not supported before GLOAS"); + } + + return canBuilderCoverBid(this.cachedState as CachedBeaconStateGloas, builderIndex, bidAmount); + } + + /** + * Return the index of the validator in the PTC committee for the given slot. + * return -1 if validator is not in the PTC committee for the given slot. + */ + validatorPTCCommitteeIndex(validatorIndex: ValidatorIndex, slot: Slot): number { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.gloas) { + throw new Error("PTC committees are not supported before GLOAS"); + } + + const ptcCommittee = (this.cachedState as CachedBeaconStateGloas).epochCtx.getPayloadTimelinessCommittee(slot); + return ptcCommittee.indexOf(validatorIndex); + } + + // Shuffling and committees + + getShufflingAtEpoch(epoch: Epoch): EpochShuffling { + return this.cachedState.epochCtx.getShufflingAtEpoch(epoch); + } + + get previousDecisionRoot(): RootHex { + return this.cachedState.epochCtx.previousDecisionRoot; + } + + get currentDecisionRoot(): RootHex { + return this.cachedState.epochCtx.currentDecisionRoot; + } + + get nextDecisionRoot(): RootHex { + return this.cachedState.epochCtx.nextDecisionRoot; + } + + getShufflingDecisionRoot(epoch: Epoch): RootHex { + return this.cachedState.epochCtx.getShufflingDecisionRoot(epoch); + } + + getPreviousShuffling(): EpochShuffling { + return this.cachedState.epochCtx.previousShuffling; + } + + getCurrentShuffling(): EpochShuffling { + return this.cachedState.epochCtx.currentShuffling; + } + + getNextShuffling(): EpochShuffling { + return this.cachedState.epochCtx.nextShuffling; + } + + // Proposer shuffling + + get previousProposers(): ValidatorIndex[] | null { + return this.cachedState.epochCtx.proposersPrevEpoch; + } + + get currentProposers(): ValidatorIndex[] { + return this.cachedState.epochCtx.getBeaconProposers(); + } + + get nextProposers(): ValidatorIndex[] { + return this.cachedState.epochCtx.getBeaconProposersNextEpoch(); + } + + getBeaconProposer(slot: number): ValidatorIndex { + return this.cachedState.epochCtx.getBeaconProposer(slot); + } + + computeAnchorCheckpoint(): {checkpoint: phase0.Checkpoint; blockHeader: phase0.BeaconBlockHeader} { + return computeAnchorCheckpoint(this.config, this.cachedState); + } + + // Sync committees + + get currentSyncCommittee(): SyncCommittee { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.altair) { + throw new Error("currentSyncCommittee is not available before Altair"); + } + + if (this._currentSyncCommittee === null) { + this._currentSyncCommittee = (this.cachedState as CachedBeaconStateAltair).currentSyncCommittee.toValue(); + } + + return this._currentSyncCommittee; + } + + get nextSyncCommittee(): SyncCommittee { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.altair) { + throw new Error("nextSyncCommittee is not available before Altair"); + } + + if (this._nextSyncCommittee === null) { + this._nextSyncCommittee = (this.cachedState as CachedBeaconStateAltair).nextSyncCommittee.toValue(); + } + + return this._nextSyncCommittee; + } + + get currentSyncCommitteeIndexed(): SyncCommitteeCache { + return this.cachedState.epochCtx.currentSyncCommitteeIndexed; + } + + get syncProposerReward(): number { + return this.cachedState.epochCtx.syncProposerReward; + } + + getIndexedSyncCommitteeAtEpoch(epoch: Epoch): SyncCommitteeCache { + return this.cachedState.epochCtx.getIndexedSyncCommitteeAtEpoch(epoch); + } + + // Validators and balances + + get effectiveBalanceIncrements(): EffectiveBalanceIncrements { + return this.cachedState.epochCtx.effectiveBalanceIncrements; + } + + getEffectiveBalanceIncrementsZeroInactive(): EffectiveBalanceIncrements { + return getEffectiveBalanceIncrementsZeroInactive(this.cachedState); + } + + getBalance(index: number): number { + return this.cachedState.balances.get(index); + } + + getValidator(index: ValidatorIndex): phase0.Validator { + return this.cachedState.validators.getReadonly(index).toValue(); + } + + getValidatorsByStatus(statuses: Set, currentEpoch: Epoch): phase0.Validator[] { + const validators: phase0.Validator[] = []; + const validatorsArr = this.cachedState.validators.getAllReadonlyValues(); + + for (const validator of validatorsArr) { + const validatorStatus = getValidatorStatus(validator, currentEpoch); + if (statuses.has(validatorStatus) || statuses.has(mapToGeneralStatus(validatorStatus))) { + validators.push(validator); + } + } + return validators; + } + + get validatorCount(): number { + return this.cachedState.validators.length; + } + + get activeValidatorCount(): number { + return this.cachedState.epochCtx.currentShuffling.activeIndices.length; + } + + getAllValidators(): phase0.Validator[] { + return this.cachedState.validators.getAllReadonlyValues(); + } + + getAllBalances(): number[] { + return this.cachedState.balances.getAll(); + } + + // Merge + + get isExecutionStateType(): boolean { + return this.config.getForkSeq(this.cachedState.slot) >= ForkSeq.bellatrix; + } + + isExecutionEnabled(block: BeaconBlock | BlindedBeaconBlock): boolean { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.bellatrix) { + return false; + } + + return isExecutionEnabled(this.cachedState as CachedBeaconStateExecutions, block); + } + + get isMergeTransitionComplete(): boolean { + return isExecutionStateType(this.cachedState) && isMergeTransitionComplete(this.cachedState); + } + + // Block production + + getExpectedWithdrawals(): { + expectedWithdrawals: capella.Withdrawal[]; + processedBuilderWithdrawalsCount: number; + processedPartialWithdrawalsCount: number; + processedBuildersSweepCount: number; + processedValidatorSweepCount: number; + } { + const fork = this.config.getForkSeq(this.cachedState.slot); + return getExpectedWithdrawals( + fork, + this.cachedState as CachedBeaconStateCapella | CachedBeaconStateElectra | CachedBeaconStateGloas + ); + } + + // API + + get proposerRewards(): RewardCache { + return this.cachedState.proposerRewards; + } + + async computeBlockRewards(block: BeaconBlock, proposerRewards?: RewardCache): Promise { + return computeBlockRewards(this.cachedState.config, block, this.cachedState, proposerRewards); + } + + async computeAttestationsRewards(validatorIds?: (ValidatorIndex | string)[]): Promise { + return computeAttestationsRewards( + this.cachedState.config, + this.cachedState.epochCtx.pubkeyCache, + this.cachedState, + validatorIds + ); + } + + async computeSyncCommitteeRewards( + block: BeaconBlock, + validatorIds: (ValidatorIndex | string)[] + ): Promise { + return computeSyncCommitteeRewards( + this.cachedState.config, + this.cachedState.epochCtx.pubkeyCache, + block, + this.cachedState, + validatorIds + ); + } + + getLatestWeakSubjectivityCheckpointEpoch(): Epoch { + return getLatestWeakSubjectivityCheckpointEpoch(this.config, this.cachedState); + } + + // Validation + + getVoluntaryExitValidity( + signedVoluntaryExit: phase0.SignedVoluntaryExit, + verifySignature = true + ): VoluntaryExitValidity { + const stateFork = this.config.getForkSeq(this.cachedState.slot); + return getVoluntaryExitValidity(stateFork, this.cachedState, signedVoluntaryExit, verifySignature); + } + + isValidVoluntaryExit(signedVoluntaryExit: phase0.SignedVoluntaryExit, verifySignature: boolean): boolean { + return this.getVoluntaryExitValidity(signedVoluntaryExit, verifySignature) === VoluntaryExitValidity.valid; + } + + // Proofs + + getFinalizedRootProof(): Uint8Array[] { + return getFinalizedRootProof(this.cachedState); + } + + getSyncCommitteesWitness(): SyncCommitteeWitness { + const fork = this.config.getForkName(this.cachedState.slot); + if (ForkSeq[fork] < ForkSeq.altair) { + throw new Error("Sync committees witness is not available before Altair"); + } + + return getSyncCommitteesWitness(fork, this.cachedState); + } + + getSingleProof(gindex: bigint): Uint8Array[] { + return new Tree(this.cachedState.node).getSingleProof(gindex); + } + + createMultiProof(descriptor: Uint8Array): CompactMultiProof { + const stateNode = this.cachedState.node; + return createProof(stateNode, {type: ProofType.compactMulti, descriptor}) as CompactMultiProof; + } + + // Fork choice + + computeUnrealizedCheckpoints(): { + justifiedCheckpoint: phase0.Checkpoint; + finalizedCheckpoint: phase0.Checkpoint; + } { + return computeUnrealizedCheckpoints(this.cachedState); + } + + // this is for backward compatible + + get clonedCount(): number { + return this.cachedState.clonedCount; + } + + get clonedCountWithTransferCache(): number { + return this.cachedState.clonedCountWithTransferCache; + } + + get createdWithTransferCache(): boolean { + return this.cachedState.createdWithTransferCache; + } + + isStateValidatorsNodesPopulated(): boolean { + return isStateValidatorsNodesPopulated(this.cachedState); + } + + // Serialization + + loadOtherState(stateBytes: Uint8Array, seedValidatorsBytes?: Uint8Array): IBeaconStateView { + const {state} = loadState(this.config, this.cachedState, stateBytes, seedValidatorsBytes); + + const cachedState = createCachedBeaconState( + state, + { + config: this.config, + // as of Feb 2026, it's not necessary to sync pubkey cache as it's shared across states in Lodestar + pubkeyCache: this.cachedState.epochCtx.pubkeyCache, + }, + { + skipSyncPubkeys: true, + } + ); + + // load all cache in order for consumers (usually regen.getState()) to process blocks faster + cachedState.validators.getAllReadonlyValues(); + cachedState.balances.getAll(); + + return new BeaconStateView(cachedState); + } + + serialize(): Uint8Array { + return this.cachedState.serialize(); + } + + serializedSize(): number { + return this.cachedState.type.tree_serializedSize(this.cachedState.node); + } + + serializeToBytes(output: ByteViews, offset: number): number { + return this.cachedState.serializeToBytes(output, offset); + } + + serializeValidators(): Uint8Array { + return this.cachedState.validators.serialize(); + } + + serializedValidatorsSize(): number { + const type = this.cachedState.type.fields.validators; + return type.tree_serializedSize(this.cachedState.validators.node); + } + + serializeValidatorsToBytes(output: ByteViews, offset: number): number { + return this.cachedState.validators.serializeToBytes(output, offset); + } + + hashTreeRoot(): Uint8Array { + return this.cachedState.hashTreeRoot(); + } + + // State transition + + stateTransition( + signedBlock: SignedBeaconBlock | SignedBlindedBeaconBlock, + options: StateTransitionOpts, + {metrics, validatorMonitor}: StateTransitionModules + ): IBeaconStateView { + const newState = stateTransition(this.cachedState, signedBlock, options, {metrics, validatorMonitor}); + return new BeaconStateView(newState); + } + + processSlots( + slot: Slot, + epochTransitionCacheOpts?: EpochTransitionCacheOpts & {dontTransferCache?: boolean}, + modules?: StateTransitionModules + ): IBeaconStateView { + const newState = processSlots(this.cachedState, slot, epochTransitionCacheOpts, modules); + return new BeaconStateView(newState); + } + + processExecutionPayloadEnvelope( + signedEnvelope: gloas.SignedExecutionPayloadEnvelope, + opts?: ProcessExecutionPayloadEnvelopeOpts + ): BeaconStateView { + const fork = this.config.getForkName(this.cachedState.slot); + if (!isForkPostGloas(fork)) { + throw Error(`processExecutionPayloadEnvelope is only available for gloas+ forks, got fork=${fork}`); + } + const postPayloadState = processExecutionPayloadEnvelope( + this.cachedState as CachedBeaconStateGloas, + signedEnvelope, + opts + ); + return new BeaconStateView(postPayloadState); + } +} + +/** + * Create BeaconStateView for historical state regen, no need to sync pubkey cache there. + */ +export function createBeaconStateViewForHistoricalRegen( + config: BeaconConfig, + stateBytes: Uint8Array +): IBeaconStateView { + const state = getStateTypeFromBytes(config, stateBytes).deserializeToViewDU(stateBytes); + + const pubkeyCache = createPubkeyCache(); + syncPubkeyCache(state, pubkeyCache); + const cachedState = createCachedBeaconState( + state, + { + config, + pubkeyCache, + }, + { + skipSyncPubkeys: true, + } + ); + + return new BeaconStateView(cachedState); +} + +/** + * Populate a PubkeyIndexMap with any new entries based on a BeaconState + */ +function syncPubkeyCache(state: BeaconStateAllForks, pubkeyCache: PubkeyCache): void { + // Get the validators sub tree once for all the loop + + const newCount = state.validators.length; + for (let i = pubkeyCache.size; i < newCount; i++) { + const pubkey = state.validators.getReadonly(i).pubkey; + pubkeyCache.set(i, pubkey); + } +} diff --git a/packages/state-transition/src/stateView/index.ts b/packages/state-transition/src/stateView/index.ts new file mode 100644 index 000000000000..23854e54232d --- /dev/null +++ b/packages/state-transition/src/stateView/index.ts @@ -0,0 +1,2 @@ +export * from "./beaconStateView.js"; +export * from "./interface.js"; diff --git a/packages/state-transition/src/stateView/interface.ts b/packages/state-transition/src/stateView/interface.ts new file mode 100644 index 000000000000..9ab6b56ff0e1 --- /dev/null +++ b/packages/state-transition/src/stateView/interface.ts @@ -0,0 +1,217 @@ +import {CompactMultiProof} from "@chainsafe/persistent-merkle-tree"; +import {ByteViews} from "@chainsafe/ssz"; +import { + BeaconBlock, + BlindedBeaconBlock, + BuilderIndex, + Bytes32, + Epoch, + ExecutionPayloadBid, + ExecutionPayloadHeader, + Root, + RootHex, + SignedBeaconBlock, + SignedBlindedBeaconBlock, + Slot, + ValidatorIndex, + altair, + capella, + electra, + fulu, + gloas, + phase0, + rewards, +} from "@lodestar/types"; +import {Checkpoint, Fork} from "@lodestar/types/phase0"; +import {ProcessExecutionPayloadEnvelopeOpts} from "../block/processExecutionPayloadEnvelope.js"; +import {VoluntaryExitValidity} from "../block/processVoluntaryExit.js"; +import {EffectiveBalanceIncrements} from "../cache/effectiveBalanceIncrements.js"; +import {EpochTransitionCacheOpts} from "../cache/epochTransitionCache.js"; +import {RewardCache} from "../cache/rewardCache.js"; +import {SyncCommitteeCache} from "../cache/syncCommitteeCache.js"; +import {SyncCommitteeWitness} from "../lightClient/types.js"; +import {StateTransitionModules, StateTransitionOpts} from "../stateTransition.js"; +import {EpochShuffling} from "../util/epochShuffling.js"; + +/** + * A read-only view of the BeaconState. + */ +export interface IBeaconStateView { + // State access + + // phase0 + slot: Slot; + fork: Fork; + epoch: Epoch; + genesisTime: number; + genesisValidatorsRoot: Root; + eth1Data: phase0.Eth1Data; + latestBlockHeader: phase0.BeaconBlockHeader; + previousJustifiedCheckpoint: Checkpoint; + currentJustifiedCheckpoint: Checkpoint; + finalizedCheckpoint: Checkpoint; + getBlockRootAtSlot(slot: Slot): Root; + getBlockRootAtEpoch(epoch: Epoch): Root; + getStateRootAtSlot(slot: Slot): Root; + getRandaoMix(epoch: Epoch): Bytes32; + + // altair + previousEpochParticipation: Uint8Array; + currentEpochParticipation: Uint8Array; + getPreviousEpochParticipation(validatorIndex: ValidatorIndex): number; + getCurrentEpochParticipation(validatorIndex: ValidatorIndex): number; + + // bellatrix + latestExecutionPayloadHeader: ExecutionPayloadHeader; + /** + * Cross-fork accessor for the execution block hash of the most recently included payload. + * Pre-gloas: returns latestExecutionPayloadHeader.blockHash (bellatrix–fulu). + * Gloas+: returns the dedicated latestBlockHash state field (EIP-7732). + * Throws before bellatrix. + */ + latestBlockHash: Bytes32; + /** + * The execution block number of the most recently included payload. + * Named payloadBlockNumber (not latestBlockNumber) to mirror ExecutionPayloadHeader.blockNumber pre-gloas. + * Only available from bellatrix through fulu — not tracked on BeaconState in gloas+ (EIP-7732). + * Throws before bellatrix and from gloas onwards. + */ + payloadBlockNumber: number; + + // capella + historicalSummaries: capella.HistoricalSummaries; + + // electra + pendingDeposits: electra.PendingDeposits; + pendingDepositsCount: number; + pendingPartialWithdrawals: electra.PendingPartialWithdrawals; + pendingPartialWithdrawalsCount: number; + pendingConsolidations: electra.PendingConsolidations; + pendingConsolidationsCount: number; + + // fulu + proposerLookahead: fulu.ProposerLookahead; + + // gloas + executionPayloadAvailability: boolean[]; + latestExecutionPayloadBid: ExecutionPayloadBid; + getBuilder(index: BuilderIndex): gloas.Builder; + canBuilderCoverBid(builderIndex: BuilderIndex, bidAmount: number): boolean; + validatorPTCCommitteeIndex(validatorIndex: ValidatorIndex, slot: Slot): number; + + // Shuffling and committees + getShufflingAtEpoch(epoch: Epoch): EpochShuffling; + // Decision roots + previousDecisionRoot: RootHex; + currentDecisionRoot: RootHex; + nextDecisionRoot: RootHex; + getShufflingDecisionRoot(epoch: Epoch): RootHex; + getPreviousShuffling(): EpochShuffling; + getCurrentShuffling(): EpochShuffling; + getNextShuffling(): EpochShuffling; + + // utils: proposers, anchor checkpoint + previousProposers: ValidatorIndex[] | null; + currentProposers: ValidatorIndex[]; + nextProposers: ValidatorIndex[]; + getBeaconProposer(slot: Slot): ValidatorIndex; + computeAnchorCheckpoint(): {checkpoint: phase0.Checkpoint; blockHeader: phase0.BeaconBlockHeader}; + + // Sync committees + currentSyncCommittee: altair.SyncCommittee; + nextSyncCommittee: altair.SyncCommittee; + currentSyncCommitteeIndexed: SyncCommitteeCache; + syncProposerReward: number; + getIndexedSyncCommitteeAtEpoch(epoch: Epoch): SyncCommitteeCache; + + // Validators and balances + effectiveBalanceIncrements: EffectiveBalanceIncrements; + getEffectiveBalanceIncrementsZeroInactive(): EffectiveBalanceIncrements; + getBalance(index: number): number; + // readonly + getValidator(index: ValidatorIndex): phase0.Validator; + getValidatorsByStatus(statuses: Set, currentEpoch: Epoch): phase0.Validator[]; + validatorCount: number; + // this get number of active validators in the current shuffling + activeValidatorCount: number; + // this is needed for apis only + getAllValidators(): phase0.Validator[]; + getAllBalances(): number[]; + + // Merge + isExecutionStateType: boolean; + isMergeTransitionComplete: boolean; + // TODO this should go away (or rather only need block) + isExecutionEnabled(block: BeaconBlock | BlindedBeaconBlock): boolean; + + // Block production + getExpectedWithdrawals(): { + expectedWithdrawals: capella.Withdrawal[]; + processedBuilderWithdrawalsCount: number; + processedPartialWithdrawalsCount: number; + processedValidatorSweepCount: number; + }; + + // API + proposerRewards: RewardCache; + computeBlockRewards(block: BeaconBlock, proposerRewards?: RewardCache): Promise; + computeAttestationsRewards(validatorIds?: (ValidatorIndex | string)[]): Promise; + computeSyncCommitteeRewards( + block: BeaconBlock, + validatorIds: (ValidatorIndex | string)[] + ): Promise; + getLatestWeakSubjectivityCheckpointEpoch(): Epoch; + + // Validation + getVoluntaryExitValidity( + signedVoluntaryExit: phase0.SignedVoluntaryExit, + verifySignature: boolean + ): VoluntaryExitValidity; + isValidVoluntaryExit(signedVoluntaryExit: phase0.SignedVoluntaryExit, verifySignature: boolean): boolean; + + // Proofs + getFinalizedRootProof(): Uint8Array[]; + getSyncCommitteesWitness(): SyncCommitteeWitness; + getSingleProof(gindex: bigint): Uint8Array[]; + createMultiProof(descriptor: Uint8Array): CompactMultiProof; + + // Fork choice + computeUnrealizedCheckpoints(): { + justifiedCheckpoint: phase0.Checkpoint; + finalizedCheckpoint: phase0.Checkpoint; + }; + + // this is for backward compatible + clonedCount: number; + clonedCountWithTransferCache: number; + createdWithTransferCache: boolean; + // TODO is there a better name that is less implementation specific but still conveys the meaning? + isStateValidatorsNodesPopulated(): boolean; + + // Serialization + loadOtherState(stateBytes: Uint8Array, seedValidatorsBytes?: Uint8Array): IBeaconStateView; + serialize(): Uint8Array; + serializedSize(): number; + serializeToBytes(output: ByteViews, offset: number): number; + serializeValidators(): Uint8Array; + serializedValidatorsSize(): number; + serializeValidatorsToBytes(output: ByteViews, offset: number): number; + + hashTreeRoot(): Uint8Array; + + // State transition + stateTransition( + signedBlock: SignedBeaconBlock | SignedBlindedBeaconBlock, + options: StateTransitionOpts, + modules: StateTransitionModules + ): IBeaconStateView; + processSlots( + slot: Slot, + epochTransitionCacheOpts?: EpochTransitionCacheOpts & {dontTransferCache?: boolean}, + modules?: StateTransitionModules + ): IBeaconStateView; + processExecutionPayloadEnvelope( + signedEnvelope: gloas.SignedExecutionPayloadEnvelope, + opts?: ProcessExecutionPayloadEnvelopeOpts + ): IBeaconStateView; +} diff --git a/packages/state-transition/test/cache.ts b/packages/state-transition/src/testUtils/cache.ts similarity index 80% rename from packages/state-transition/test/cache.ts rename to packages/state-transition/src/testUtils/cache.ts index 59ddfe40b7c1..5d880338aeab 100644 --- a/packages/state-transition/test/cache.ts +++ b/packages/state-transition/src/testUtils/cache.ts @@ -5,4 +5,4 @@ import {fileURLToPath} from "node:url"; // Solutions: https://stackoverflow.com/questions/46745014/alternative-for-dirname-in-node-js-when-using-es6-modules const __dirname = path.dirname(fileURLToPath(import.meta.url)); -export const testCachePath = path.join(__dirname, "../test-cache"); +export const testCachePath = path.join(__dirname, "../../test/test-cache"); diff --git a/packages/state-transition/src/testUtils/index.ts b/packages/state-transition/src/testUtils/index.ts new file mode 100644 index 000000000000..2443c291b0b9 --- /dev/null +++ b/packages/state-transition/src/testUtils/index.ts @@ -0,0 +1,5 @@ +export * from "./cache.js"; +export * from "./interop.js"; +export * from "./params.js"; +export * from "./testFileCache.js"; +export * from "./util.js"; diff --git a/packages/state-transition/test/utils/infura.ts b/packages/state-transition/src/testUtils/infura.ts similarity index 100% rename from packages/state-transition/test/utils/infura.ts rename to packages/state-transition/src/testUtils/infura.ts diff --git a/packages/state-transition/test/utils/interop.ts b/packages/state-transition/src/testUtils/interop.ts similarity index 91% rename from packages/state-transition/test/utils/interop.ts rename to packages/state-transition/src/testUtils/interop.ts index 5a0dca4e0018..16bd63dbe5ac 100644 --- a/packages/state-transition/test/utils/interop.ts +++ b/packages/state-transition/src/testUtils/interop.ts @@ -1,8 +1,8 @@ import fs from "node:fs"; import path from "node:path"; import {fromHexString, toHexString} from "@chainsafe/ssz"; -import {interopSecretKey} from "../../src/index.js"; -import {testCachePath} from "../cache.js"; +import {interopSecretKey} from "../index.js"; +import {testCachePath} from "./cache.js"; const interopPubkeysCachedPath = path.join(testCachePath, "interop-pubkeys.json"); diff --git a/packages/state-transition/test/perf/params.ts b/packages/state-transition/src/testUtils/params.ts similarity index 100% rename from packages/state-transition/test/perf/params.ts rename to packages/state-transition/src/testUtils/params.ts diff --git a/packages/state-transition/src/testUtils/state.ts b/packages/state-transition/src/testUtils/state.ts new file mode 100644 index 000000000000..e6cbac014e73 --- /dev/null +++ b/packages/state-transition/src/testUtils/state.ts @@ -0,0 +1,110 @@ +import {ChainForkConfig, createBeaconConfig} from "@lodestar/config"; +import {config, config as minimalConfig} from "@lodestar/config/default"; +import { + EPOCHS_PER_HISTORICAL_VECTOR, + EPOCHS_PER_SLASHINGS_VECTOR, + GENESIS_EPOCH, + GENESIS_SLOT, + SLOTS_PER_HISTORICAL_ROOT, +} from "@lodestar/params"; +import {phase0, ssz} from "@lodestar/types"; +import {EpochCacheOpts} from "../cache/epochCache.js"; +import {BeaconStateCache} from "../cache/stateCache.js"; +import {ZERO_HASH} from "../constants/index.js"; +import { + BeaconStateAllForks, + BeaconStatePhase0, + CachedBeaconStateAllForks, + createCachedBeaconState, + createPubkeyCache, +} from "../index.js"; +import {newZeroedArray} from "../util/index.js"; + +/** + * Copy of BeaconState, but all fields are marked optional to allow for swapping out variables as needed. + */ +type TestBeaconState = Partial; + +/** + * Generate beaconState, by default it will use the initial state defined when the `ChainStart` log is emitted. + * NOTE: All fields can be overridden through `opts`. + * @param {TestBeaconState} opts + * @returns {BeaconState} + */ +export function generateState(opts?: TestBeaconState): BeaconStatePhase0 { + return ssz.phase0.BeaconState.toViewDU({ + genesisTime: Math.floor(Date.now() / 1000), + genesisValidatorsRoot: ZERO_HASH, + slot: GENESIS_SLOT, + fork: { + previousVersion: config.GENESIS_FORK_VERSION, + currentVersion: config.GENESIS_FORK_VERSION, + epoch: GENESIS_EPOCH, + }, + latestBlockHeader: { + slot: 0, + proposerIndex: 0, + parentRoot: Buffer.alloc(32), + stateRoot: Buffer.alloc(32), + bodyRoot: ssz.phase0.BeaconBlockBody.hashTreeRoot(ssz.phase0.BeaconBlockBody.defaultValue()), + }, + blockRoots: Array.from({length: SLOTS_PER_HISTORICAL_ROOT}, () => ZERO_HASH), + stateRoots: Array.from({length: SLOTS_PER_HISTORICAL_ROOT}, () => ZERO_HASH), + historicalRoots: [], + eth1Data: { + depositRoot: Buffer.alloc(32), + blockHash: Buffer.alloc(32), + depositCount: 0, + }, + eth1DataVotes: [], + eth1DepositIndex: 0, + validators: [], + balances: [], + randaoMixes: Array.from({length: EPOCHS_PER_HISTORICAL_VECTOR}, () => ZERO_HASH), + slashings: newZeroedArray(EPOCHS_PER_SLASHINGS_VECTOR), + previousEpochAttestations: [], + currentEpochAttestations: [], + justificationBits: ssz.phase0.JustificationBits.defaultValue(), + previousJustifiedCheckpoint: { + epoch: GENESIS_EPOCH, + root: ZERO_HASH, + }, + currentJustifiedCheckpoint: { + epoch: GENESIS_EPOCH, + root: ZERO_HASH, + }, + finalizedCheckpoint: { + epoch: GENESIS_EPOCH, + root: ZERO_HASH, + }, + ...opts, + }); +} + +export function generateCachedState( + config: ChainForkConfig = minimalConfig, + opts: TestBeaconState = {} +): CachedBeaconStateAllForks { + const state = generateState(opts); + return createCachedBeaconState(state, { + config: createBeaconConfig(config, state.genesisValidatorsRoot), + // This is a test state, there's no need to have a global shared cache of keys + pubkeyCache: createPubkeyCache(), + }); +} + +export function createCachedBeaconStateTest( + state: T, + configCustom: ChainForkConfig = config, + opts?: EpochCacheOpts +): T & BeaconStateCache { + return createCachedBeaconState( + state, + { + config: createBeaconConfig(configCustom, state.genesisValidatorsRoot), + // This is a test state, there's no need to have a global shared cache of keys + pubkeyCache: createPubkeyCache(), + }, + opts + ); +} diff --git a/packages/state-transition/test/utils/testFileCache.ts b/packages/state-transition/src/testUtils/testFileCache.ts similarity index 94% rename from packages/state-transition/test/utils/testFileCache.ts rename to packages/state-transition/src/testUtils/testFileCache.ts index 1cf381b33beb..e2b98bb64820 100644 --- a/packages/state-transition/test/utils/testFileCache.ts +++ b/packages/state-transition/src/testUtils/testFileCache.ts @@ -5,10 +5,10 @@ import {ChainForkConfig, createChainForkConfig} from "@lodestar/config"; import {NetworkName, networksChainConfig} from "@lodestar/config/networks"; import {SignedBeaconBlock} from "@lodestar/types"; import {fetch} from "@lodestar/utils"; -import {CachedBeaconStateAllForks} from "../../src/index.js"; -import {testCachePath} from "../cache.js"; -import {createCachedBeaconStateTest} from "../utils/state.js"; +import {CachedBeaconStateAllForks} from "../index.js"; +import {testCachePath} from "./cache.js"; import {getInfuraBeaconUrl} from "./infura.js"; +import {createCachedBeaconStateTest} from "./state.js"; /** * Full link example: @@ -96,6 +96,7 @@ export async function getNetworkCachedBlock( async function downloadTestFile(fileId: string): Promise { const fileUrl = `${TEST_FILES_BASE_URL}/${fileId}`; + // biome-ignore lint/suspicious/noConsole: We explicity need to log to console console.log(`Downloading file ${fileUrl}`); try { diff --git a/packages/state-transition/test/perf/util.ts b/packages/state-transition/src/testUtils/util.ts similarity index 90% rename from packages/state-transition/test/perf/util.ts rename to packages/state-transition/src/testUtils/util.ts index 4790be1b1346..2c0daa87117d 100644 --- a/packages/state-transition/test/perf/util.ts +++ b/packages/state-transition/src/testUtils/util.ts @@ -1,5 +1,4 @@ import {PublicKey, SecretKey} from "@chainsafe/blst"; -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {BitArray, fromHexString} from "@chainsafe/ssz"; import {createBeaconConfig, createChainForkConfig} from "@lodestar/config"; import {config} from "@lodestar/config/default"; @@ -14,25 +13,26 @@ import { SLOTS_PER_HISTORICAL_ROOT, } from "@lodestar/params"; import {BeaconState, Slot, phase0, ssz} from "@lodestar/types"; -import {getEffectiveBalanceIncrements} from "../../src/cache/effectiveBalanceIncrements.js"; +import {getEffectiveBalanceIncrements} from "../cache/effectiveBalanceIncrements.js"; import { computeCommitteeCount, computeEpochAtSlot, createCachedBeaconState, + createPubkeyCache, interopSecretKey, newFilledArray, processSlots, -} from "../../src/index.js"; +} from "../index.js"; import { BeaconStateAltair, BeaconStatePhase0, CachedBeaconStateAllForks, CachedBeaconStateAltair, CachedBeaconStatePhase0, -} from "../../src/types.js"; -import {getNextSyncCommittee} from "../../src/util/syncCommittee.js"; -import {getActiveValidatorIndices} from "../../src/util/validator.js"; -import {interopPubkeysCached} from "../utils/interop.js"; +} from "../types.js"; +import {getNextSyncCommittee} from "../util/syncCommittee.js"; +import {getActiveValidatorIndices} from "../util/validator.js"; +import {interopPubkeysCached} from "./interop.js"; let phase0State: BeaconStatePhase0 | null = null; let phase0CachedState23637: CachedBeaconStatePhase0 | null = null; @@ -84,32 +84,21 @@ export function getSecretKeyFromIndexCached(validatorIndex: number): SecretKey { return sk; } -function getPubkeyCaches({pubkeysMod, pubkeysModObj}: ReturnType) { +function getPubkeyCaches({pubkeysMod}: ReturnType) { // Manually sync pubkeys to prevent doing BLS opts 110_000 times - const pubkey2index = new PubkeyIndexMap(); - const index2pubkey = [] as PublicKey[]; + const pubkeyCache = createPubkeyCache(); for (let i = 0; i < numValidators; i++) { const pubkey = pubkeysMod[i % keypairsMod]; - const pubkeyObj = pubkeysModObj[i % keypairsMod]; - pubkey2index.set(pubkey, i); - index2pubkey.push(pubkeyObj); + pubkeyCache.set(i, pubkey); } - // Since most pubkeys are equal the size of pubkey2index is not numValidators. - // Fill with junk up to numValidators - for (let i = pubkey2index.size; i < numValidators; i++) { - const buf = Buffer.alloc(48, 0); - buf.writeInt32LE(i); - pubkey2index.set(buf, i); - } - - return {pubkey2index, index2pubkey}; + return {pubkeyCache}; } export function generatePerfTestCachedStatePhase0(opts?: {goBackOneSlot: boolean}): CachedBeaconStatePhase0 { // Generate only some publicKeys const {pubkeys, pubkeysMod, pubkeysModObj} = getPubkeys(); - const {pubkey2index, index2pubkey} = getPubkeyCaches({pubkeys, pubkeysMod, pubkeysModObj}); + const {pubkeyCache} = getPubkeyCaches({pubkeys, pubkeysMod, pubkeysModObj}); if (!phase0State) { const state = buildPerformanceStatePhase0(); @@ -126,8 +115,7 @@ export function generatePerfTestCachedStatePhase0(opts?: {goBackOneSlot: boolean state.slot -= 1; phase0CachedState23637 = createCachedBeaconState(state, { config: createBeaconConfig(config, state.genesisValidatorsRoot), - pubkey2index, - index2pubkey, + pubkeyCache, }); const currentEpoch = computeEpochAtSlot(state.slot - 1); @@ -219,7 +207,7 @@ export function generatePerfTestCachedStateAltair(opts?: { vc?: number; }): CachedBeaconStateAltair { const {pubkeys, pubkeysMod, pubkeysModObj} = getPubkeys(opts?.vc); - const {pubkey2index, index2pubkey} = getPubkeyCaches({pubkeys, pubkeysMod, pubkeysModObj}); + const {pubkeyCache} = getPubkeyCaches({pubkeys, pubkeysMod, pubkeysModObj}); const altairConfig = createChainForkConfig({ALTAIR_FORK_EPOCH: 0}); @@ -230,8 +218,7 @@ export function generatePerfTestCachedStateAltair(opts?: { state.slot -= 1; altairCachedState23637 = createCachedBeaconState(state, { config: createBeaconConfig(altairConfig, state.genesisValidatorsRoot), - pubkey2index, - index2pubkey, + pubkeyCache, }); } if (!altairCachedState23638) { @@ -389,16 +376,13 @@ export function generateTestCachedBeaconStateOnlyValidators({ slot: Slot; }): CachedBeaconStateAllForks { // Generate only some publicKeys - const {pubkeys, pubkeysMod, pubkeysModObj} = getPubkeys(vc); + const {pubkeys, pubkeysMod} = getPubkeys(vc); // Manually sync pubkeys to prevent doing BLS opts 110_000 times - const pubkey2index = new PubkeyIndexMap(); - const index2pubkey = [] as PublicKey[]; + const pubkeyCache = createPubkeyCache(); for (let i = 0; i < vc; i++) { const pubkey = pubkeysMod[i % keypairsMod]; - const pubkeyObj = pubkeysModObj[i % keypairsMod]; - pubkey2index.set(pubkey, i); - index2pubkey.push(pubkeyObj); + pubkeyCache.set(i, pubkey); } const state = ssz.phase0.BeaconState.defaultViewDU(); @@ -438,8 +422,7 @@ export function generateTestCachedBeaconStateOnlyValidators({ state, { config: createBeaconConfig(config, state.genesisValidatorsRoot), - pubkey2index, - index2pubkey, + pubkeyCache, }, {skipSyncPubkeys: true} ); diff --git a/packages/state-transition/src/util/execution.ts b/packages/state-transition/src/util/execution.ts index 7180abe836f2..d917f8b7c6f6 100644 --- a/packages/state-transition/src/util/execution.ts +++ b/packages/state-transition/src/util/execution.ts @@ -149,7 +149,7 @@ export function executionPayloadToPayloadHeader(fork: ForkSeq, payload: Executio } if (fork >= ForkSeq.deneb) { - // https://github.com/ethereum/consensus-specs/blob/dev/specs/eip4844/beacon-chain.md#process_execution_payload + // https://github.com/ethereum/consensus-specs/blob/v1.3.0-rc.2/specs/eip4844/beacon-chain.md#process_execution_payload (bellatrixPayloadFields as deneb.ExecutionPayloadHeader).blobGasUsed = ( payload as deneb.ExecutionPayloadHeader | deneb.ExecutionPayload ).blobGasUsed; diff --git a/packages/state-transition/src/util/seed.ts b/packages/state-transition/src/util/seed.ts index 978f3e41f557..be01506f6cd4 100644 --- a/packages/state-transition/src/util/seed.ts +++ b/packages/state-transition/src/util/seed.ts @@ -417,8 +417,6 @@ export function naiveComputePayloadTimelinessCommitteeIndices( * https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf * * See the 'generalized domain' algorithm on page 3. - * This is the naive implementation just to make sure lodestar follows the spec, this is not for production. - * The optimized version is in `getComputeShuffledIndexFn`. */ export function computeShuffledIndex(index: number, indexCount: number, seed: Bytes32): number { let permuted = index; @@ -439,75 +437,6 @@ export function computeShuffledIndex(index: number, indexCount: number, seed: By return permuted; } -type ComputeShuffledIndexFn = (index: number) => number; - -/** - * An optimized version of `computeShuffledIndex`, this is for production. - */ -export function getComputeShuffledIndexFn(indexCount: number, seed: Bytes32): ComputeShuffledIndexFn { - // there are possibly SHUFFLE_ROUND_COUNT (90 for mainnet) values for this cache - // this cache will always hit after the 1st call - const pivotByIndex: Map = new Map(); - // given 2M active validators, there are 2 M / 256 = 8k possible positionDiv - // it means there are at most 8k different sources for each round - const sourceByPositionDivByIndex: Map> = new Map(); - // 32 bytes seed + 1 byte i - const pivotBuffer = Buffer.alloc(32 + 1); - pivotBuffer.set(seed, 0); - // 32 bytes seed + 1 byte i + 4 bytes positionDiv - const sourceBuffer = Buffer.alloc(32 + 1 + 4); - sourceBuffer.set(seed, 0); - - return (index): number => { - assert.lt(index, indexCount, "indexCount must be less than index"); - assert.lte(indexCount, 2 ** 40, "indexCount too big"); - let permuted = index; - // const _seed = seed; - for (let i = 0; i < SHUFFLE_ROUND_COUNT; i++) { - // optimized version of the below naive code - // const pivot = Number( - // bytesToBigInt(digest(Buffer.concat([_seed, intToBytes(i, 1)])).slice(0, 8)) % BigInt(indexCount) - // ); - - let pivot = pivotByIndex.get(i); - if (pivot == null) { - // naive version always creates a new buffer, we can reuse the buffer - // pivot = Number( - // bytesToBigInt(digest(Buffer.concat([_seed, intToBytes(i, 1)])).slice(0, 8)) % BigInt(indexCount) - // ); - pivotBuffer[32] = i % 256; - pivot = Number(bytesToBigInt(digest(pivotBuffer).subarray(0, 8)) % BigInt(indexCount)); - pivotByIndex.set(i, pivot); - } - - const flip = (pivot + indexCount - permuted) % indexCount; - const position = Math.max(permuted, flip); - - // optimized version of the below naive code - // const source = digest(Buffer.concat([_seed, intToBytes(i, 1), intToBytes(Math.floor(position / 256), 4)])); - let sourceByPositionDiv = sourceByPositionDivByIndex.get(i); - if (sourceByPositionDiv == null) { - sourceByPositionDiv = new Map(); - sourceByPositionDivByIndex.set(i, sourceByPositionDiv); - } - const positionDiv256 = Math.floor(position / 256); - let source = sourceByPositionDiv.get(positionDiv256); - if (source == null) { - // naive version always creates a new buffer, we can reuse the buffer - // don't want to go through intToBytes() to avoid BigInt - sourceBuffer[32] = i % 256; - sourceBuffer.writeUint32LE(positionDiv256, 33); - source = digest(sourceBuffer); - sourceByPositionDiv.set(positionDiv256, source); - } - const byte = source[Math.floor((position % 256) / 8)]; - const bit = (byte >> (position % 8)) % 2; - permuted = bit ? flip : permuted; - } - return permuted; - }; -} - /** * Return the randao mix at a recent [[epoch]]. */ diff --git a/packages/state-transition/src/util/signatureSets.ts b/packages/state-transition/src/util/signatureSets.ts index eb5a7e41977a..76700d1ea0a5 100644 --- a/packages/state-transition/src/util/signatureSets.ts +++ b/packages/state-transition/src/util/signatureSets.ts @@ -1,6 +1,6 @@ import {PublicKey, Signature, aggregatePublicKeys, fastAggregateVerify, verify} from "@chainsafe/blst"; import {Root} from "@lodestar/types"; -import {Index2PubkeyCache} from "../cache/pubkeyCache.js"; +import {PubkeyCache} from "../cache/pubkeyCache.js"; export enum SignatureSetType { single = "single", @@ -49,18 +49,21 @@ export type ISignatureSet = SingleSignatureSet | IndexedSignatureSet | Aggregate /** * Get the pubkey for a signature set, performing aggregation if necessary. - * Requires index2pubkey cache for indexed and aggregate sets. + * Requires pubkeyCache for indexed and aggregate sets. */ -export function getSignatureSetPubkey(signatureSet: ISignatureSet, index2pubkey: Index2PubkeyCache): PublicKey { +export function getSignatureSetPubkey(signatureSet: ISignatureSet, pubkeyCache: PubkeyCache): PublicKey { switch (signatureSet.type) { case SignatureSetType.single: return signatureSet.pubkey; - case SignatureSetType.indexed: - return index2pubkey[signatureSet.index]; + case SignatureSetType.indexed: { + return pubkeyCache.getOrThrow(signatureSet.index); + } case SignatureSetType.aggregate: { - const pubkeys = signatureSet.indices.map((i) => index2pubkey[i]); + const pubkeys = signatureSet.indices.map((i) => { + return pubkeyCache.getOrThrow(i); + }); return aggregatePublicKeys(pubkeys); } @@ -69,11 +72,11 @@ export function getSignatureSetPubkey(signatureSet: ISignatureSet, index2pubkey: } } -export function verifySignatureSet(signatureSet: SingleSignatureSet, index2pubkey?: Index2PubkeyCache): boolean; -export function verifySignatureSet(signatureSet: IndexedSignatureSet, index2pubkey: Index2PubkeyCache): boolean; -export function verifySignatureSet(signatureSet: AggregatedSignatureSet, index2pubkey: Index2PubkeyCache): boolean; -export function verifySignatureSet(signatureSet: ISignatureSet, index2pubkey: Index2PubkeyCache): boolean; -export function verifySignatureSet(signatureSet: ISignatureSet, index2pubkey?: Index2PubkeyCache): boolean { +export function verifySignatureSet(signatureSet: SingleSignatureSet, pubkeyCache?: PubkeyCache): boolean; +export function verifySignatureSet(signatureSet: IndexedSignatureSet, pubkeyCache: PubkeyCache): boolean; +export function verifySignatureSet(signatureSet: AggregatedSignatureSet, pubkeyCache: PubkeyCache): boolean; +export function verifySignatureSet(signatureSet: ISignatureSet, pubkeyCache: PubkeyCache): boolean; +export function verifySignatureSet(signatureSet: ISignatureSet, pubkeyCache?: PubkeyCache): boolean { // All signatures are not trusted and must be group checked (p2.subgroup_check) const signature = Signature.fromBytes(signatureSet.signature, true); @@ -82,17 +85,20 @@ export function verifySignatureSet(signatureSet: ISignatureSet, index2pubkey?: I return verify(signatureSet.signingRoot, signatureSet.pubkey, signature); case SignatureSetType.indexed: { - if (!index2pubkey) { - throw Error("index2pubkey required for indexed signature set"); + if (!pubkeyCache) { + throw Error("pubkeyCache required for indexed signature set"); } - return verify(signatureSet.signingRoot, index2pubkey[signatureSet.index], signature); + const pubkey = pubkeyCache.getOrThrow(signatureSet.index); + return verify(signatureSet.signingRoot, pubkey, signature); } case SignatureSetType.aggregate: { - if (!index2pubkey) { - throw Error("index2pubkey required for aggregate signature set"); + if (!pubkeyCache) { + throw Error("pubkeyCache required for aggregate signature set"); } - const pubkeys = signatureSet.indices.map((i) => index2pubkey[i]); + const pubkeys = signatureSet.indices.map((i) => { + return pubkeyCache.getOrThrow(i); + }); return fastAggregateVerify(signatureSet.signingRoot, pubkeys, signature); } diff --git a/packages/state-transition/src/util/weakSubjectivity.ts b/packages/state-transition/src/util/weakSubjectivity.ts index ff81f6010b6e..c1dd6b39966b 100644 --- a/packages/state-transition/src/util/weakSubjectivity.ts +++ b/packages/state-transition/src/util/weakSubjectivity.ts @@ -47,7 +47,7 @@ export function computeWeakSubjectivityPeriodCachedState( state: CachedBeaconStateAllForks ): number { const activeValidatorCount = state.epochCtx.currentShuffling.activeIndices.length; - const fork = state.config.getForkName(state.slot); + const fork = config.getForkName(state.slot); return isForkPostElectra(fork) ? computeWeakSubjectivityPeriodFromConstituentsElectra( diff --git a/packages/state-transition/test/perf/analyzeBlocks.ts b/packages/state-transition/test/perf/analyzeBlocks.ts index 1b6b7d592fbd..167ba5378f1f 100644 --- a/packages/state-transition/test/perf/analyzeBlocks.ts +++ b/packages/state-transition/test/perf/analyzeBlocks.ts @@ -1,6 +1,6 @@ import {getClient} from "@lodestar/api"; import {config} from "@lodestar/config/default"; -import {getInfuraBeaconUrl} from "../utils/infura.js"; +import {getInfuraBeaconUrl} from "../../src/testUtils/infura.js"; // Analyze how Ethereum Consensus blocks are in a target network to prepare accurate performance states and blocks diff --git a/packages/state-transition/test/perf/analyzeEpochs.ts b/packages/state-transition/test/perf/analyzeEpochs.ts index c25fa0bcb703..5aed6c010ce9 100644 --- a/packages/state-transition/test/perf/analyzeEpochs.ts +++ b/packages/state-transition/test/perf/analyzeEpochs.ts @@ -11,8 +11,8 @@ import { parseAttesterFlags, processSlots, } from "../../src/index.js"; -import {getInfuraBeaconUrl} from "../utils/infura.js"; -import {createCachedBeaconStateTest} from "../utils/state.js"; +import {getInfuraBeaconUrl} from "../../src/testUtils/infura.js"; +import {createCachedBeaconStateTest} from "../../src/testUtils/state.js"; import {csvAppend, readCsv} from "./csv.js"; // Understand the real network characteristics regarding epoch transitions to accurately produce performance test data. diff --git a/packages/state-transition/test/perf/block/processAttestation.test.ts b/packages/state-transition/test/perf/block/processAttestation.test.ts index 46de84c2566e..733c9891c3af 100644 --- a/packages/state-transition/test/perf/block/processAttestation.test.ts +++ b/packages/state-transition/test/perf/block/processAttestation.test.ts @@ -13,7 +13,7 @@ import { import {phase0} from "@lodestar/types"; import {processAttestationsAltair} from "../../../src/block/processAttestationsAltair.js"; import {CachedBeaconStateAllForks, CachedBeaconStateAltair} from "../../../src/index.js"; -import {generatePerfTestCachedStateAltair, perfStateId} from "../util.js"; +import {generatePerfTestCachedStateAltair, perfStateId} from "../../../src/testUtils/util.js"; import {BlockAltairOpts, getBlockAltair} from "./util.js"; type StateAttestations = { diff --git a/packages/state-transition/test/perf/block/processBlockAltair.test.ts b/packages/state-transition/test/perf/block/processBlockAltair.test.ts index b3d4fea3e3b7..eabceed23874 100644 --- a/packages/state-transition/test/perf/block/processBlockAltair.test.ts +++ b/packages/state-transition/test/perf/block/processBlockAltair.test.ts @@ -16,8 +16,12 @@ import { ExecutionPayloadStatus, stateTransition, } from "../../../src/index.js"; +import { + cachedStateAltairPopulateCaches, + generatePerfTestCachedStateAltair, + perfStateId, +} from "../../../src/testUtils/util.js"; import {StateBlock} from "../types.js"; -import {cachedStateAltairPopulateCaches, generatePerfTestCachedStateAltair, perfStateId} from "../util.js"; import {BlockAltairOpts, getBlockAltair} from "./util.js"; // As of Jun 12 2021 diff --git a/packages/state-transition/test/perf/block/processBlockPhase0.test.ts b/packages/state-transition/test/perf/block/processBlockPhase0.test.ts index 58478a1258fd..b3ea8d3459ec 100644 --- a/packages/state-transition/test/perf/block/processBlockPhase0.test.ts +++ b/packages/state-transition/test/perf/block/processBlockPhase0.test.ts @@ -9,8 +9,8 @@ import { PresetName, } from "@lodestar/params"; import {DataAvailabilityStatus, ExecutionPayloadStatus, stateTransition} from "../../../src/index.js"; +import {generatePerfTestCachedStatePhase0, perfStateId} from "../../../src/testUtils/util.js"; import {StateBlock} from "../types.js"; -import {generatePerfTestCachedStatePhase0, perfStateId} from "../util.js"; import {BlockOpts, getBlockPhase0} from "./util.js"; // As of Jun 12 2021 diff --git a/packages/state-transition/test/perf/block/processEth1Data.test.ts b/packages/state-transition/test/perf/block/processEth1Data.test.ts index b70a63fd8310..f13fd60e0b56 100644 --- a/packages/state-transition/test/perf/block/processEth1Data.test.ts +++ b/packages/state-transition/test/perf/block/processEth1Data.test.ts @@ -3,7 +3,7 @@ import {ACTIVE_PRESET, PresetName, SYNC_COMMITTEE_SIZE} from "@lodestar/params"; import {phase0} from "@lodestar/types"; import {processEth1Data} from "../../../src/block/processEth1Data.js"; import {CachedBeaconStateAllForks} from "../../../src/index.js"; -import {generatePerfTestCachedStateAltair, perfStateId} from "../util.js"; +import {generatePerfTestCachedStateAltair, perfStateId} from "../../../src/testUtils/util.js"; import {getBlockAltair} from "./util.js"; type StateEth1Data = { diff --git a/packages/state-transition/test/perf/block/processWithdrawals.test.ts b/packages/state-transition/test/perf/block/processWithdrawals.test.ts index a95c9f5d7ae2..13cbd8e35dfe 100644 --- a/packages/state-transition/test/perf/block/processWithdrawals.test.ts +++ b/packages/state-transition/test/perf/block/processWithdrawals.test.ts @@ -2,8 +2,8 @@ import {bench, describe} from "@chainsafe/benchmark"; import {ForkSeq} from "@lodestar/params"; import {getExpectedWithdrawals} from "../../../src/block/processWithdrawals.js"; import {CachedBeaconStateCapella} from "../../../src/index.js"; +import {numValidators} from "../../../src/testUtils/util.js"; import {WithdrawalOpts, getExpectedWithdrawalsTestData} from "../../utils/capella.js"; -import {numValidators} from "../util.js"; // PERF: Fixed cost for MAX_WITHDRAWALS_PER_PAYLOAD probes // + cost 'proportional' to $VALIDATOR_COUNT with balances under MAX_EFFECTIVE_BALANCE or diff --git a/packages/state-transition/test/perf/epoch/afterProcessEpoch.test.ts b/packages/state-transition/test/perf/epoch/afterProcessEpoch.test.ts index 9a1f7a645251..2da043c8a51b 100644 --- a/packages/state-transition/test/perf/epoch/afterProcessEpoch.test.ts +++ b/packages/state-transition/test/perf/epoch/afterProcessEpoch.test.ts @@ -1,7 +1,7 @@ import {bench, describe} from "@chainsafe/benchmark"; import {beforeProcessEpoch} from "../../../src/index.js"; +import {generatePerfTestCachedStatePhase0, perfStateId} from "../../../src/testUtils/util.js"; import {StateEpoch} from "../types.js"; -import {generatePerfTestCachedStatePhase0, perfStateId} from "../util.js"; // PERF: Cost = compute attester and proposer shufflings ~ 'proportional' to $VALIDATOR_COUNT, but independent of // network conditions. See also individual benchmarks for shuffling computations. diff --git a/packages/state-transition/test/perf/epoch/beforeProcessEpoch.test.ts b/packages/state-transition/test/perf/epoch/beforeProcessEpoch.test.ts index 3123e7f5a806..abe21f514a32 100644 --- a/packages/state-transition/test/perf/epoch/beforeProcessEpoch.test.ts +++ b/packages/state-transition/test/perf/epoch/beforeProcessEpoch.test.ts @@ -1,7 +1,7 @@ import {bench, describe} from "@chainsafe/benchmark"; import {beforeProcessEpoch} from "../../../src/index.js"; +import {generatePerfTestCachedStatePhase0, perfStateId} from "../../../src/testUtils/util.js"; import {State} from "../types.js"; -import {generatePerfTestCachedStatePhase0, perfStateId} from "../util.js"; // PERF: Two major steps: // 1. Iterate over state.validators. Cost = 'proportional' to $VALIDATOR_COUNT not network conditions diff --git a/packages/state-transition/test/perf/epoch/epochAltair.test.ts b/packages/state-transition/test/perf/epoch/epochAltair.test.ts index 24635acfdb7f..1315dae8a2be 100644 --- a/packages/state-transition/test/perf/epoch/epochAltair.test.ts +++ b/packages/state-transition/test/perf/epoch/epochAltair.test.ts @@ -19,9 +19,9 @@ import { beforeProcessEpoch, computeStartSlotAtEpoch, } from "../../../src/index.js"; +import {altairState} from "../../../src/testUtils/params.js"; +import {getNetworkCachedState} from "../../../src/testUtils/testFileCache.js"; import {LazyValue, beforeValue} from "../../utils/beforeValueBenchmark.js"; -import {getNetworkCachedState} from "../../utils/testFileCache.js"; -import {altairState} from "../params.js"; import {StateEpoch} from "../types.js"; const slot = computeStartSlotAtEpoch(altairState.epoch) - 1; diff --git a/packages/state-transition/test/perf/epoch/epochCapella.test.ts b/packages/state-transition/test/perf/epoch/epochCapella.test.ts index c34d6861aaaf..7d40727f8647 100644 --- a/packages/state-transition/test/perf/epoch/epochCapella.test.ts +++ b/packages/state-transition/test/perf/epoch/epochCapella.test.ts @@ -19,9 +19,9 @@ import { beforeProcessEpoch, computeStartSlotAtEpoch, } from "../../../src/index.js"; +import {capellaState} from "../../../src/testUtils/params.js"; +import {getNetworkCachedState} from "../../../src/testUtils/testFileCache.js"; import {LazyValue, beforeValue} from "../../utils/beforeValueBenchmark.js"; -import {getNetworkCachedState} from "../../utils/testFileCache.js"; -import {capellaState} from "../params.js"; import {StateEpoch} from "../types.js"; const slot = computeStartSlotAtEpoch(capellaState.epoch) - 1; diff --git a/packages/state-transition/test/perf/epoch/epochPhase0.test.ts b/packages/state-transition/test/perf/epoch/epochPhase0.test.ts index 733fb9dd9922..95e18f63dc10 100644 --- a/packages/state-transition/test/perf/epoch/epochPhase0.test.ts +++ b/packages/state-transition/test/perf/epoch/epochPhase0.test.ts @@ -17,9 +17,9 @@ import { beforeProcessEpoch, computeStartSlotAtEpoch, } from "../../../src/index.js"; +import {phase0State} from "../../../src/testUtils/params.js"; +import {getNetworkCachedState} from "../../../src/testUtils/testFileCache.js"; import {LazyValue, beforeValue} from "../../utils/beforeValueBenchmark.js"; -import {getNetworkCachedState} from "../../utils/testFileCache.js"; -import {phase0State} from "../params.js"; import {StateEpoch} from "../types.js"; const slot = computeStartSlotAtEpoch(phase0State.epoch) - 1; diff --git a/packages/state-transition/test/perf/epoch/processEffectiveBalanceUpdates.test.ts b/packages/state-transition/test/perf/epoch/processEffectiveBalanceUpdates.test.ts index 384c1e2cf520..5f624809d82a 100644 --- a/packages/state-transition/test/perf/epoch/processEffectiveBalanceUpdates.test.ts +++ b/packages/state-transition/test/perf/epoch/processEffectiveBalanceUpdates.test.ts @@ -4,9 +4,9 @@ import {ForkSeq} from "@lodestar/params"; import {ssz} from "@lodestar/types"; import {processEffectiveBalanceUpdates} from "../../../src/epoch/processEffectiveBalanceUpdates.js"; import {CachedBeaconStateAllForks, EpochTransitionCache, beforeProcessEpoch} from "../../../src/index.js"; -import {createCachedBeaconStateTest} from "../../utils/state.js"; +import {createCachedBeaconStateTest} from "../../../src/testUtils/state.js"; +import {numValidators} from "../../../src/testUtils/util.js"; import {StateEpoch} from "../types.js"; -import {numValidators} from "../util.js"; // PERF: Cost 'proportional' to $VALIDATOR_COUNT, to iterate over all balances. Then cost is proportional to the amount // of validators whose effectiveBalance changed. Worst case is a massive network leak or a big slashing event which diff --git a/packages/state-transition/test/perf/epoch/processInactivityUpdates.test.ts b/packages/state-transition/test/perf/epoch/processInactivityUpdates.test.ts index 8db7e1d72929..15a09e58fcde 100644 --- a/packages/state-transition/test/perf/epoch/processInactivityUpdates.test.ts +++ b/packages/state-transition/test/perf/epoch/processInactivityUpdates.test.ts @@ -1,7 +1,7 @@ import {bench, describe} from "@chainsafe/benchmark"; import {processInactivityUpdates} from "../../../src/epoch/processInactivityUpdates.js"; +import {generatePerfTestCachedStateAltair, numValidators} from "../../../src/testUtils/util.js"; import {StateAltairEpoch} from "../types.js"; -import {generatePerfTestCachedStateAltair, numValidators} from "../util.js"; import {mutateInactivityScores} from "./util.js"; import {FlagFactors, generateBalanceDeltasEpochTransitionCache} from "./utilPhase0.js"; diff --git a/packages/state-transition/test/perf/epoch/processRegistryUpdates.test.ts b/packages/state-transition/test/perf/epoch/processRegistryUpdates.test.ts index 307fe1de386d..f43fdab6d3a1 100644 --- a/packages/state-transition/test/perf/epoch/processRegistryUpdates.test.ts +++ b/packages/state-transition/test/perf/epoch/processRegistryUpdates.test.ts @@ -2,8 +2,8 @@ import {bench, describe} from "@chainsafe/benchmark"; import {ForkSeq} from "@lodestar/params"; import {processRegistryUpdates} from "../../../src/epoch/processRegistryUpdates.js"; import {CachedBeaconStateAllForks, EpochTransitionCache, beforeProcessEpoch} from "../../../src/index.js"; +import {generatePerfTestCachedStatePhase0, numValidators} from "../../../src/testUtils/util.js"; import {StateEpoch} from "../types.js"; -import {generatePerfTestCachedStatePhase0, numValidators} from "../util.js"; // PERF: Cost 'proportional' to only validators that active + exit. For mainnet conditions: // - indicesEligibleForActivationQueue: Maxing deposits triggers 512 validator mutations diff --git a/packages/state-transition/test/perf/epoch/processRewardsAndPenalties.test.ts b/packages/state-transition/test/perf/epoch/processRewardsAndPenalties.test.ts index 61579256b606..f7596c97508b 100644 --- a/packages/state-transition/test/perf/epoch/processRewardsAndPenalties.test.ts +++ b/packages/state-transition/test/perf/epoch/processRewardsAndPenalties.test.ts @@ -1,7 +1,7 @@ import {bench, describe} from "@chainsafe/benchmark"; import {processRewardsAndPenalties} from "../../../src/epoch/processRewardsAndPenalties.js"; +import {generatePerfTestCachedStateAltair, numValidators} from "../../../src/testUtils/util.js"; import {StateAltairEpoch} from "../types.js"; -import {generatePerfTestCachedStateAltair, numValidators} from "../util.js"; import {mutateInactivityScores} from "./util.js"; import {FlagFactors, generateBalanceDeltasEpochTransitionCache} from "./utilPhase0.js"; diff --git a/packages/state-transition/test/perf/epoch/processRewardsAndPenaltiesPhase0.test.ts b/packages/state-transition/test/perf/epoch/processRewardsAndPenaltiesPhase0.test.ts index 3a7e12f32c8e..cebf95fd2da1 100644 --- a/packages/state-transition/test/perf/epoch/processRewardsAndPenaltiesPhase0.test.ts +++ b/packages/state-transition/test/perf/epoch/processRewardsAndPenaltiesPhase0.test.ts @@ -1,7 +1,7 @@ import {bench, describe} from "@chainsafe/benchmark"; import {getAttestationDeltas} from "../../../src/epoch/getAttestationDeltas.js"; +import {generatePerfTestCachedStatePhase0, numValidators} from "../../../src/testUtils/util.js"; import {StatePhase0Epoch} from "../types.js"; -import {generatePerfTestCachedStatePhase0, numValidators} from "../util.js"; import {FlagFactors, generateBalanceDeltasEpochTransitionCache} from "./utilPhase0.js"; // - On normal mainnet conditions diff --git a/packages/state-transition/test/perf/epoch/processSlashingsAllForks.test.ts b/packages/state-transition/test/perf/epoch/processSlashingsAllForks.test.ts index aa1190f6364a..1dd878abf95c 100644 --- a/packages/state-transition/test/perf/epoch/processSlashingsAllForks.test.ts +++ b/packages/state-transition/test/perf/epoch/processSlashingsAllForks.test.ts @@ -7,8 +7,8 @@ import { EpochTransitionCache, beforeProcessEpoch, } from "../../../src/index.js"; +import {generatePerfTestCachedStatePhase0, numValidators} from "../../../src/testUtils/util.js"; import {StateEpoch} from "../types.js"; -import {generatePerfTestCachedStatePhase0, numValidators} from "../util.js"; // PERF: Cost 'proportional' to only validators that are slashed. For mainnet conditions: // - indicesToSlash: max len is 8704. But it's very unlikely since it would require all validators on the same diff --git a/packages/state-transition/test/perf/epoch/processSyncCommitteeUpdates.test.ts b/packages/state-transition/test/perf/epoch/processSyncCommitteeUpdates.test.ts index 794bcdf510d1..869c04849c29 100644 --- a/packages/state-transition/test/perf/epoch/processSyncCommitteeUpdates.test.ts +++ b/packages/state-transition/test/perf/epoch/processSyncCommitteeUpdates.test.ts @@ -1,8 +1,8 @@ import {bench, describe} from "@chainsafe/benchmark"; import {EPOCHS_PER_SYNC_COMMITTEE_PERIOD, ForkSeq} from "@lodestar/params"; import {processSyncCommitteeUpdates} from "../../../src/epoch/processSyncCommitteeUpdates.js"; +import {generatePerfTestCachedStateAltair, numValidators} from "../../../src/testUtils/util.js"; import {StateAltair} from "../types.js"; -import {generatePerfTestCachedStateAltair, numValidators} from "../util.js"; // PERF: Cost = once per epoch compute committee, proportional to $VALIDATOR_COUNT diff --git a/packages/state-transition/test/perf/hashing.test.ts b/packages/state-transition/test/perf/hashing.test.ts index 21e7c1112d96..ea5aa1a81f34 100644 --- a/packages/state-transition/test/perf/hashing.test.ts +++ b/packages/state-transition/test/perf/hashing.test.ts @@ -2,7 +2,7 @@ import {beforeAll, bench, describe} from "@chainsafe/benchmark"; import {unshuffleList} from "@chainsafe/swap-or-not-shuffle"; import {SHUFFLE_ROUND_COUNT} from "@lodestar/params"; import {ssz} from "@lodestar/types"; -import {generatePerfTestCachedStatePhase0, numValidators} from "./util.js"; +import {generatePerfTestCachedStatePhase0, numValidators} from "../../src/testUtils/util.js"; // Test cost of hashing state after some modifications diff --git a/packages/state-transition/test/perf/misc/byteArrayEquals.test.ts b/packages/state-transition/test/perf/misc/byteArrayEquals.test.ts index 43a62be44ad3..7b904151a51b 100644 --- a/packages/state-transition/test/perf/misc/byteArrayEquals.test.ts +++ b/packages/state-transition/test/perf/misc/byteArrayEquals.test.ts @@ -1,7 +1,7 @@ import crypto from "node:crypto"; import {bench, describe} from "@chainsafe/benchmark"; import {byteArrayEquals} from "@lodestar/utils"; -import {generateState} from "../../utils/state.js"; +import {generateState} from "../../../src/testUtils/state.js"; import {generateValidators} from "../../utils/validator.js"; /** diff --git a/packages/state-transition/test/perf/slot/slots.test.ts b/packages/state-transition/test/perf/slot/slots.test.ts index ee8288e20030..c920ddc622ae 100644 --- a/packages/state-transition/test/perf/slot/slots.test.ts +++ b/packages/state-transition/test/perf/slot/slots.test.ts @@ -1,8 +1,8 @@ import {bench, describe} from "@chainsafe/benchmark"; import {ForkSeq} from "@lodestar/params"; import {processSlot} from "../../../src/slot/index.js"; +import {generatePerfTestCachedStatePhase0} from "../../../src/testUtils/util.js"; import {State} from "../types.js"; -import {generatePerfTestCachedStatePhase0} from "../util.js"; // Test advancing through an empty slot, without any epoch transition diff --git a/packages/state-transition/test/perf/util/balance.test.ts b/packages/state-transition/test/perf/util/balance.test.ts index bca42db717ec..9ed917d63104 100644 --- a/packages/state-transition/test/perf/util/balance.test.ts +++ b/packages/state-transition/test/perf/util/balance.test.ts @@ -1,7 +1,7 @@ import {bench, describe} from "@chainsafe/benchmark"; +import {generatePerfTestCachedStatePhase0, perfStateId} from "../../../src/testUtils/util.js"; import {getEffectiveBalanceIncrementsZeroInactive} from "../../../src/util/index.js"; import {State} from "../types.js"; -import {generatePerfTestCachedStatePhase0, perfStateId} from "../util.js"; describe("getEffectiveBalanceIncrementsZeroInactive", () => { bench({ diff --git a/packages/state-transition/test/perf/util/epochContext.test.ts b/packages/state-transition/test/perf/util/epochContext.test.ts index 520c7987b706..29193045029f 100644 --- a/packages/state-transition/test/perf/util/epochContext.test.ts +++ b/packages/state-transition/test/perf/util/epochContext.test.ts @@ -1,7 +1,7 @@ import {beforeAll, bench, describe} from "@chainsafe/benchmark"; import {Epoch} from "@lodestar/types"; import {CachedBeaconStateAllForks, computeEpochAtSlot} from "../../../src/index.js"; -import {generatePerfTestCachedStatePhase0, numValidators} from "../util.js"; +import {generatePerfTestCachedStatePhase0, numValidators} from "../../../src/testUtils/util.js"; // Current implementation scales very well with number of requested validators // Benchmark data from Wed Jun 30 2021 - Intel(R) Core(TM) i5-8250U CPU @ 1.60GHz diff --git a/packages/state-transition/test/perf/util/loadState/findModifiedValidators.test.ts b/packages/state-transition/test/perf/util/loadState/findModifiedValidators.test.ts index a7071aed52e9..d2df5c335702 100644 --- a/packages/state-transition/test/perf/util/loadState/findModifiedValidators.test.ts +++ b/packages/state-transition/test/perf/util/loadState/findModifiedValidators.test.ts @@ -3,9 +3,9 @@ import {bench, describe} from "@chainsafe/benchmark"; import {CompositeViewDU} from "@chainsafe/ssz"; import {ssz} from "@lodestar/types"; import {byteArrayEquals, bytesToInt} from "@lodestar/utils"; +import {generateState} from "../../../../src/testUtils/state.js"; import {findModifiedValidators} from "../../../../src/util/loadState/findModifiedValidators.js"; import {VALIDATOR_BYTES_SIZE} from "../../../../src/util/sszBytes.js"; -import {generateState} from "../../../utils/state.js"; import {generateValidators} from "../../../utils/validator.js"; /** diff --git a/packages/state-transition/test/perf/util/loadState/loadState.test.ts b/packages/state-transition/test/perf/util/loadState/loadState.test.ts index 526b02bd30e5..5396f03b0c15 100644 --- a/packages/state-transition/test/perf/util/loadState/loadState.test.ts +++ b/packages/state-transition/test/perf/util/loadState/loadState.test.ts @@ -1,10 +1,8 @@ import {bench, describe} from "@chainsafe/benchmark"; -import {PublicKey} from "@chainsafe/blst"; -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; -import {Index2PubkeyCache} from "../../../../src/cache/pubkeyCache.js"; +import {PubkeyCache, createPubkeyCache} from "../../../../src/cache/pubkeyCache.js"; import {createCachedBeaconState} from "../../../../src/cache/stateCache.js"; +import {generatePerfTestCachedStateAltair} from "../../../../src/testUtils/util.js"; import {loadState} from "../../../../src/util/loadState/loadState.js"; -import {generatePerfTestCachedStateAltair} from "../../util.js"; /** * This benchmark shows a stable performance from 2s to 3s on a Mac M1. And it does not really depend on the seed validators, @@ -49,21 +47,18 @@ describe("loadState", () => { migratedState.hashTreeRoot(); // Get the validators sub tree once for all the loop const validators = migratedState.validators; - const pubkey2index = new PubkeyIndexMap(); - const index2pubkey: Index2PubkeyCache = []; + const pubkeyCache: PubkeyCache = createPubkeyCache(); for (const validatorIndex of modifiedValidators) { const validator = validators.getReadonly(validatorIndex); const pubkey = validator.pubkey; - pubkey2index.set(pubkey, validatorIndex); - index2pubkey[validatorIndex] = PublicKey.fromBytes(pubkey); + pubkeyCache.set(validatorIndex, pubkey); } const shufflingGetter = () => seedState.epochCtx.currentShuffling; createCachedBeaconState( migratedState, { config: seedState.config, - pubkey2index, - index2pubkey, + pubkeyCache, }, {skipSyncPubkeys: true, skipSyncCommitteeCache: true, shufflingGetter} ); diff --git a/packages/state-transition/test/perf/util/rootCache.test.ts b/packages/state-transition/test/perf/util/rootCache.test.ts index 015019e3545f..88ab2ef830bb 100644 --- a/packages/state-transition/test/perf/util/rootCache.test.ts +++ b/packages/state-transition/test/perf/util/rootCache.test.ts @@ -1,7 +1,7 @@ import {bench, describe} from "@chainsafe/benchmark"; +import {generatePerfTestCachedStatePhase0, perfStateEpoch, perfStateId} from "../../../src/testUtils/util.js"; import {RootCache, computeStartSlotAtEpoch, getBlockRootAtSlot} from "../../../src/util/index.js"; import {State} from "../types.js"; -import {generatePerfTestCachedStatePhase0, perfStateEpoch, perfStateId} from "../util.js"; const slot = computeStartSlotAtEpoch(perfStateEpoch) - 1; diff --git a/packages/state-transition/test/perf/util/seed.test.ts b/packages/state-transition/test/perf/util/seed.test.ts index 65b196d7c7f1..cd3aa18677a4 100644 --- a/packages/state-transition/test/perf/util/seed.test.ts +++ b/packages/state-transition/test/perf/util/seed.test.ts @@ -1,15 +1,14 @@ import {bench, describe} from "@chainsafe/benchmark"; import {ForkSeq} from "@lodestar/params"; import {fromHex} from "@lodestar/utils"; +import {generatePerfTestCachedStateAltair} from "../../../src/testUtils/util.js"; import { computeProposerIndex, computeShuffledIndex, - getComputeShuffledIndexFn, getNextSyncCommitteeIndices, naiveComputeProposerIndex, naiveGetNextSyncCommitteeIndices, } from "../../../src/util/seed.js"; -import {generatePerfTestCachedStateAltair} from "../util.js"; // I'm not sure how to populate a good test data for this benchmark describe("computeProposerIndex", () => { @@ -88,16 +87,5 @@ describe("computeShuffledIndex", () => { } }, }); - - const shuffledIndexFn = getComputeShuffledIndexFn(vc, seed); - // getComputeShuffledIndexFn() is also not in prod anymore so no need to track it - bench.skip({ - id: `cached computeShuffledIndex ${vc} validators`, - fn: () => { - for (let i = 0; i < vc; i++) { - shuffledIndexFn(i); - } - }, - }); } }); diff --git a/packages/state-transition/test/perf/util/serializeState.test.ts b/packages/state-transition/test/perf/util/serializeState.test.ts index 7d144882d767..d22f76b83e94 100644 --- a/packages/state-transition/test/perf/util/serializeState.test.ts +++ b/packages/state-transition/test/perf/util/serializeState.test.ts @@ -1,6 +1,6 @@ import {bench, describe, setBenchOpts} from "@chainsafe/benchmark"; import {ssz} from "@lodestar/types"; -import {cachedStateAltairPopulateCaches, generatePerfTestCachedStateAltair} from "../util.js"; +import {cachedStateAltairPopulateCaches, generatePerfTestCachedStateAltair} from "../../../src/testUtils/util.js"; /** * This shows different statistics between allocating memory once vs every time. diff --git a/packages/state-transition/test/perf/util/shufflings.test.ts b/packages/state-transition/test/perf/util/shufflings.test.ts index d0b82ac94b77..bb7c6f2fc313 100644 --- a/packages/state-transition/test/perf/util/shufflings.test.ts +++ b/packages/state-transition/test/perf/util/shufflings.test.ts @@ -9,7 +9,7 @@ import { getNextSyncCommittee, getSeed, } from "../../../src/index.js"; -import {generatePerfTestCachedStatePhase0, numValidators} from "../util.js"; +import {generatePerfTestCachedStatePhase0, numValidators} from "../../../src/testUtils/util.js"; describe("epoch shufflings", () => { let state: CachedBeaconStateAllForks; diff --git a/packages/state-transition/test/unit/block/isValidIndexedAttestation.test.ts b/packages/state-transition/test/unit/block/isValidIndexedAttestation.test.ts index 9afa10327b31..0c977dfeadcf 100644 --- a/packages/state-transition/test/unit/block/isValidIndexedAttestation.test.ts +++ b/packages/state-transition/test/unit/block/isValidIndexedAttestation.test.ts @@ -4,7 +4,7 @@ import {FAR_FUTURE_EPOCH, MAX_EFFECTIVE_BALANCE} from "@lodestar/params"; import {IndexedAttestation, ssz} from "@lodestar/types"; import {isValidIndexedAttestation} from "../../../src/block/isValidIndexedAttestation.js"; import {EMPTY_SIGNATURE} from "../../../src/index.js"; -import {generateCachedState} from "../../utils/state.js"; +import {generateCachedState} from "../../../src/testUtils/state.js"; import {generateValidators} from "../../utils/validator.js"; describe("validate indexed attestation", () => { @@ -28,11 +28,21 @@ describe("validate indexed attestation", () => { expectedValue: false, name: "should return invalid indexed attestation - indexes not sorted", }, + { + indices: [0, 0], + expectedValue: false, + name: "should return invalid indexed attestation - duplicate indexes", + }, { indices: [0, 1, 2, 3], expectedValue: true, name: "should return valid indexed attestation", }, + { + indices: [0], + expectedValue: true, + name: "should return valid indexed attestation - single index 0", + }, ]; it.each(testValues)("$name", ({indices, expectedValue}) => { @@ -48,7 +58,7 @@ describe("validate indexed attestation", () => { expect( isValidIndexedAttestation( state.config, - state.epochCtx.index2pubkey, + state.epochCtx.pubkeyCache, state.slot, state.validators.length, indexedAttestation, diff --git a/packages/state-transition/test/unit/block/processConsolidationRequest.test.ts b/packages/state-transition/test/unit/block/processConsolidationRequest.test.ts index 86b9c646b33e..57db5fc6a0ab 100644 --- a/packages/state-transition/test/unit/block/processConsolidationRequest.test.ts +++ b/packages/state-transition/test/unit/block/processConsolidationRequest.test.ts @@ -8,8 +8,8 @@ import { SLOTS_PER_EPOCH, } from "@lodestar/params"; import {ssz} from "@lodestar/types"; -import {generateCachedElectraState} from "../../../../beacon-node/test/utils/state.js"; import {processConsolidationRequest} from "../../../src/block/processConsolidationRequest.js"; +import {generateCachedElectraState} from "../../utils/state.js"; import {generateValidators} from "../../utils/validator.js"; describe("processConsolidationRequest", () => { diff --git a/packages/state-transition/test/unit/block/processWithdrawals.test.ts b/packages/state-transition/test/unit/block/processWithdrawals.test.ts index 939a29809470..ba1ef057c3af 100644 --- a/packages/state-transition/test/unit/block/processWithdrawals.test.ts +++ b/packages/state-transition/test/unit/block/processWithdrawals.test.ts @@ -1,7 +1,7 @@ import {describe, expect, it} from "vitest"; import {ForkSeq} from "@lodestar/params"; import {getExpectedWithdrawals} from "../../../src/block/processWithdrawals.js"; -import {numValidators} from "../../perf/util.js"; +import {numValidators} from "../../../src/testUtils/util.js"; import {beforeValue} from "../../utils/beforeValue.js"; import {WithdrawalOpts, getExpectedWithdrawalsTestData} from "../../utils/capella.js"; diff --git a/packages/state-transition/test/unit/cachedBeaconState.test.ts b/packages/state-transition/test/unit/cachedBeaconState.test.ts index 5da5f698a5a2..9f887634da1d 100644 --- a/packages/state-transition/test/unit/cachedBeaconState.test.ts +++ b/packages/state-transition/test/unit/cachedBeaconState.test.ts @@ -1,15 +1,15 @@ import {describe, expect, it, vi} from "vitest"; -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {fromHexString} from "@chainsafe/ssz"; import {createBeaconConfig} from "@lodestar/config"; import {config as defaultConfig} from "@lodestar/config/default"; import {Epoch, RootHex, ssz} from "@lodestar/types"; import {toHexString} from "@lodestar/utils"; +import {createPubkeyCache} from "../../src/cache/pubkeyCache.js"; import {createCachedBeaconState, loadCachedBeaconState} from "../../src/cache/stateCache.js"; +import {interopPubkeysCached} from "../../src/testUtils/interop.js"; +import {createCachedBeaconStateTest} from "../../src/testUtils/state.js"; import {EpochShuffling, calculateShufflingDecisionRoot} from "../../src/util/epochShuffling.js"; import {modifyStateSameValidator, newStateWithValidators} from "../utils/capella.js"; -import {interopPubkeysCached} from "../utils/interop.js"; -import {createCachedBeaconStateTest} from "../utils/state.js"; describe("CachedBeaconState", () => { vi.setConfig({testTimeout: 30_000, hookTimeout: 30_000}); @@ -92,8 +92,7 @@ describe("CachedBeaconState", () => { stateView, { config, - pubkey2index: new PubkeyIndexMap(), - index2pubkey: [], + pubkeyCache: createPubkeyCache(), }, {skipSyncCommitteeCache: true} ); @@ -195,8 +194,7 @@ describe("CachedBeaconState", () => { state, { config, - pubkey2index: new PubkeyIndexMap(), - index2pubkey: [], + pubkeyCache: createPubkeyCache(), }, {skipSyncCommitteeCache: true, shufflingGetter} ); @@ -207,8 +205,8 @@ describe("CachedBeaconState", () => { // confirm loadCachedBeaconState() result for (let i = 0; i < newCachedState.validators.length; i++) { - expect(newCachedState.epochCtx.pubkey2index.get(newCachedState.validators.get(i).pubkey)).toBe(i); - expect(newCachedState.epochCtx.index2pubkey[i].toBytes()).toEqual(pubkeys[i]); + expect(newCachedState.epochCtx.getValidatorIndex(newCachedState.validators.get(i).pubkey)).toBe(i); + expect(newCachedState.epochCtx.pubkeyCache.get(i)?.toBytes()).toEqual(pubkeys[i]); } }); } diff --git a/packages/state-transition/test/unit/rewards/blockRewards.test.ts b/packages/state-transition/test/unit/rewards/blockRewards.test.ts index dfdf8a4ee506..e3d4347fceb2 100644 --- a/packages/state-transition/test/unit/rewards/blockRewards.test.ts +++ b/packages/state-transition/test/unit/rewards/blockRewards.test.ts @@ -6,9 +6,9 @@ import {ssz} from "@lodestar/types"; import {DataAvailabilityStatus, ExecutionPayloadStatus} from "../../../src/block/externalData.js"; import {computeBlockRewards} from "../../../src/rewards/blockRewards.js"; import {stateTransition} from "../../../src/stateTransition.js"; +import {cachedStateAltairPopulateCaches, generatePerfTestCachedStateAltair} from "../../../src/testUtils/util.js"; import {CachedBeaconStateAllForks} from "../../../src/types.js"; import {BlockAltairOpts, getBlockAltair} from "../../perf/block/util.js"; -import {cachedStateAltairPopulateCaches, generatePerfTestCachedStateAltair} from "../../perf/util.js"; describe("chain / rewards / blockRewards", () => { const config = createBeaconConfig({...chainConfigDef, ALTAIR_FORK_EPOCH: 0}, Buffer.alloc(32, 0xaa)); diff --git a/packages/state-transition/test/unit/sanityCheck.test.ts b/packages/state-transition/test/unit/sanityCheck.test.ts index 9d62cbaa0b12..70fa4809b60d 100644 --- a/packages/state-transition/test/unit/sanityCheck.test.ts +++ b/packages/state-transition/test/unit/sanityCheck.test.ts @@ -1,7 +1,11 @@ import {beforeAll, describe, expect, it, vi} from "vitest"; import {ACTIVE_PRESET, EFFECTIVE_BALANCE_INCREMENT, PresetName} from "@lodestar/params"; import {beforeProcessEpoch} from "../../src/index.js"; -import {generatePerfTestCachedStateAltair, generatePerfTestCachedStatePhase0, perfStateId} from "../perf/util.js"; +import { + generatePerfTestCachedStateAltair, + generatePerfTestCachedStatePhase0, + perfStateId, +} from "../../src/testUtils/util.js"; describe("Perf test sanity check", () => { vi.setConfig({testTimeout: 90 * 1000}); diff --git a/packages/state-transition/test/unit/signatureSets/signatureSets.test.ts b/packages/state-transition/test/unit/signatureSets/signatureSets.test.ts index e754b5551983..fadae9b2d869 100644 --- a/packages/state-transition/test/unit/signatureSets/signatureSets.test.ts +++ b/packages/state-transition/test/unit/signatureSets/signatureSets.test.ts @@ -7,7 +7,7 @@ import {FAR_FUTURE_EPOCH, MAX_EFFECTIVE_BALANCE} from "@lodestar/params"; import {BLSSignature, ValidatorIndex, capella, phase0, ssz} from "@lodestar/types"; import {ZERO_HASH} from "../../../src/constants/index.js"; import {getBlockSignatureSets} from "../../../src/signatureSets/index.js"; -import {generateCachedState} from "../../utils/state.js"; +import {generateCachedState} from "../../../src/testUtils/state.js"; import {generateValidators} from "../../utils/validator.js"; const EMPTY_SIGNATURE = Buffer.alloc(96); diff --git a/packages/state-transition/test/unit/upgradeState.test.ts b/packages/state-transition/test/unit/upgradeState.test.ts index d2f39bdb1a13..573d819e4ab6 100644 --- a/packages/state-transition/test/unit/upgradeState.test.ts +++ b/packages/state-transition/test/unit/upgradeState.test.ts @@ -1,9 +1,9 @@ import {describe, expect, it} from "vitest"; -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {ChainForkConfig, createBeaconConfig, createChainForkConfig} from "@lodestar/config"; import {config as chainConfig} from "@lodestar/config/default"; import {ForkName} from "@lodestar/params"; import {ssz} from "@lodestar/types"; +import {createPubkeyCache} from "../../src/cache/pubkeyCache.js"; import {createCachedBeaconState} from "../../src/cache/stateCache.js"; import {upgradeStateToDeneb} from "../../src/slot/upgradeStateToDeneb.js"; import {upgradeStateToElectra} from "../../src/slot/upgradeStateToElectra.js"; @@ -16,8 +16,7 @@ describe("upgradeState", () => { capellaState, { config: createBeaconConfig(config, capellaState.genesisValidatorsRoot), - pubkey2index: new PubkeyIndexMap(), - index2pubkey: [], + pubkeyCache: createPubkeyCache(), }, {skipSyncCommitteeCache: true} ); @@ -31,8 +30,7 @@ describe("upgradeState", () => { denebState, { config: createBeaconConfig(config, denebState.genesisValidatorsRoot), - pubkey2index: new PubkeyIndexMap(), - index2pubkey: [], + pubkeyCache: createPubkeyCache(), }, {skipSyncCommitteeCache: true} ); diff --git a/packages/state-transition/test/unit/util/balance.test.ts b/packages/state-transition/test/unit/util/balance.test.ts index bd61424452ab..959adc9d52aa 100644 --- a/packages/state-transition/test/unit/util/balance.test.ts +++ b/packages/state-transition/test/unit/util/balance.test.ts @@ -3,8 +3,8 @@ import {config as minimalConfig} from "@lodestar/config/default"; import {EFFECTIVE_BALANCE_INCREMENT} from "@lodestar/params"; import {ValidatorIndex} from "@lodestar/types"; import {getEffectiveBalanceIncrementsZeroInactive, getEffectiveBalanceIncrementsZeroed} from "../../../src/index.js"; +import {generateCachedState, generateState} from "../../../src/testUtils/state.js"; import {decreaseBalance, getTotalBalance, increaseBalance, isActiveValidator} from "../../../src/util/index.js"; -import {generateCachedState, generateState} from "../../utils/state.js"; import {generateValidators} from "../../utils/validator.js"; describe("getTotalBalance", () => { diff --git a/packages/state-transition/test/unit/util/cachedBeaconState.test.ts b/packages/state-transition/test/unit/util/cachedBeaconState.test.ts index c85a8c7a2ffd..a9668f09d6df 100644 --- a/packages/state-transition/test/unit/util/cachedBeaconState.test.ts +++ b/packages/state-transition/test/unit/util/cachedBeaconState.test.ts @@ -1,9 +1,8 @@ import {describe, it} from "vitest"; -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {createBeaconConfig} from "@lodestar/config"; import {config} from "@lodestar/config/default"; import {ssz} from "@lodestar/types"; -import {createCachedBeaconState} from "../../../src/index.js"; +import {createCachedBeaconState, createPubkeyCache} from "../../../src/index.js"; describe("CachedBeaconState", () => { it("Create empty CachedBeaconState", () => { @@ -11,8 +10,7 @@ describe("CachedBeaconState", () => { createCachedBeaconState(emptyState, { config: createBeaconConfig(config, emptyState.genesisValidatorsRoot), - pubkey2index: new PubkeyIndexMap(), - index2pubkey: [], + pubkeyCache: createPubkeyCache(), }); }); }); diff --git a/packages/state-transition/test/unit/util/deposit.test.ts b/packages/state-transition/test/unit/util/deposit.test.ts index 14ae0ef08357..0ed2ab6d4f6e 100644 --- a/packages/state-transition/test/unit/util/deposit.test.ts +++ b/packages/state-transition/test/unit/util/deposit.test.ts @@ -2,8 +2,8 @@ import {describe, expect, it} from "vitest"; import {createChainForkConfig} from "@lodestar/config"; import {MAX_DEPOSITS} from "@lodestar/params"; import {ssz} from "@lodestar/types"; +import {createCachedBeaconStateTest} from "../../../src/testUtils/state.js"; import {getEth1DepositCount} from "../../../src/util/deposit.js"; -import {createCachedBeaconStateTest} from "../../utils/state.js"; describe("getEth1DepositCount", () => { it("Pre Electra", () => { diff --git a/packages/state-transition/test/unit/util/epoch.test.ts b/packages/state-transition/test/unit/util/epoch.test.ts index 54742e4e2ab0..98c60ac0b1ee 100644 --- a/packages/state-transition/test/unit/util/epoch.test.ts +++ b/packages/state-transition/test/unit/util/epoch.test.ts @@ -1,13 +1,13 @@ import {describe, expect, it} from "vitest"; import {GENESIS_SLOT, MAX_SEED_LOOKAHEAD} from "@lodestar/params"; import {Epoch, Slot} from "@lodestar/types"; +import {generateState} from "../../../src/testUtils/state.js"; import { computeActivationExitEpoch, computeEpochAtSlot, computeStartSlotAtEpoch, getPreviousEpoch, } from "../../../src/util/index.js"; -import {generateState} from "../../utils/state.js"; describe("computeEpochAtSlot", () => { it.each([ diff --git a/packages/state-transition/test/unit/util/loadState/findModifiedValidators.test.ts b/packages/state-transition/test/unit/util/loadState/findModifiedValidators.test.ts index a504e5ea15d2..6575e9dd1925 100644 --- a/packages/state-transition/test/unit/util/loadState/findModifiedValidators.test.ts +++ b/packages/state-transition/test/unit/util/loadState/findModifiedValidators.test.ts @@ -1,7 +1,7 @@ import {describe, expect, it} from "vitest"; import {fromHexString} from "@chainsafe/ssz"; +import {generateState} from "../../../../src/testUtils/state.js"; import {findModifiedValidators} from "../../../../src/util/loadState/findModifiedValidators.js"; -import {generateState} from "../../../utils/state.js"; import {generateValidators} from "../../../utils/validator.js"; describe("findModifiedValidators", () => { diff --git a/packages/state-transition/test/unit/util/loadState/loadValidator.test.ts b/packages/state-transition/test/unit/util/loadState/loadValidator.test.ts index cccdec2d5d73..f9b48442385b 100644 --- a/packages/state-transition/test/unit/util/loadState/loadValidator.test.ts +++ b/packages/state-transition/test/unit/util/loadState/loadValidator.test.ts @@ -3,8 +3,8 @@ import {CompositeViewDU} from "@chainsafe/ssz"; import {config} from "@lodestar/config/default"; import {phase0, ssz} from "@lodestar/types"; import {byteArrayEquals} from "@lodestar/utils"; +import {generateState} from "../../../../src/testUtils/state.js"; import {getEffectiveBalancesFromStateBytes, loadValidator} from "../../../../src/util/loadState/loadValidator.js"; -import {generateState} from "../../../utils/state.js"; import {generateValidators} from "../../../utils/validator.js"; describe("loadValidator", () => { diff --git a/packages/state-transition/test/unit/util/misc.test.ts b/packages/state-transition/test/unit/util/misc.test.ts index 9b6bc20c6931..5c8ff44208f8 100644 --- a/packages/state-transition/test/unit/util/misc.test.ts +++ b/packages/state-transition/test/unit/util/misc.test.ts @@ -1,8 +1,8 @@ import {toBigIntLE} from "@vekexasia/bigint-buffer2"; import {describe, expect, it} from "vitest"; import {GENESIS_SLOT, SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params"; +import {generateState} from "../../../src/testUtils/state.js"; import {getBlockRoot} from "../../../src/util/index.js"; -import {generateState} from "../../utils/state.js"; describe("getBlockRoot", () => { it("should return first block root for genesis slot", () => { diff --git a/packages/state-transition/test/unit/util/seed.test.ts b/packages/state-transition/test/unit/util/seed.test.ts index 701ed95c1077..c208e4c73f0e 100644 --- a/packages/state-transition/test/unit/util/seed.test.ts +++ b/packages/state-transition/test/unit/util/seed.test.ts @@ -3,18 +3,16 @@ import {describe, expect, it} from "vitest"; import {toHexString} from "@chainsafe/ssz"; import {ForkSeq, GENESIS_EPOCH, GENESIS_SLOT, SLOTS_PER_EPOCH} from "@lodestar/params"; import {bytesToInt} from "@lodestar/utils"; +import {generateState} from "../../../src/testUtils/state.js"; import { computePayloadTimelinessCommitteeIndices, computeProposerIndex, - computeShuffledIndex, - getComputeShuffledIndexFn, getNextSyncCommitteeIndices, getRandaoMix, naiveComputePayloadTimelinessCommitteeIndices, naiveComputeProposerIndex, naiveGetNextSyncCommitteeIndices, } from "../../../src/util/index.js"; -import {generateState} from "../../utils/state.js"; import {generateValidators} from "../../utils/validator.js"; describe("getRandaoMix", () => { @@ -58,18 +56,6 @@ describe("computeProposerIndex", () => { } }); -describe("computeShuffledIndex", () => { - const seed = crypto.randomBytes(32); - const vc = 1000; - const shuffledIndexFn = getComputeShuffledIndexFn(vc, seed); - it("should be the same to the naive version", () => { - for (let i = 0; i < vc; i++) { - const expectedIndex = computeShuffledIndex(i, vc, seed); - expect(shuffledIndexFn(i)).toBe(expectedIndex); - } - }); -}); - describe("electra getNextSyncCommitteeIndices", () => { const vc = 1000; const validators = generateValidators(vc); diff --git a/packages/state-transition/test/unit/util/shuffling.test.ts b/packages/state-transition/test/unit/util/shuffling.test.ts index 0f37735215e1..9f01deee1c03 100644 --- a/packages/state-transition/test/unit/util/shuffling.test.ts +++ b/packages/state-transition/test/unit/util/shuffling.test.ts @@ -1,8 +1,8 @@ import {describe, expect, it} from "vitest"; import {ssz} from "@lodestar/types"; import {computeEpochAtSlot} from "../../../src/index.js"; +import {generateState} from "../../../src/testUtils/state.js"; import {computeEpochShuffling, computeEpochShufflingAsync} from "../../../src/util/epochShuffling.js"; -import {generateState} from "../../utils/state.js"; describe("EpochShuffling", () => { it("async and sync versions should be identical", async () => { diff --git a/packages/state-transition/test/unit/util/validator.test.ts b/packages/state-transition/test/unit/util/validator.test.ts index 0cb341d3e6ee..e24e79e7c919 100644 --- a/packages/state-transition/test/unit/util/validator.test.ts +++ b/packages/state-transition/test/unit/util/validator.test.ts @@ -1,8 +1,8 @@ import {beforeEach, describe, expect, it} from "vitest"; import {phase0, ssz} from "@lodestar/types"; +import {generateState} from "../../../src/testUtils/state.js"; import {getActiveValidatorIndices, isActiveValidator, isSlashableValidator} from "../../../src/util/index.js"; import {randBetween} from "../../utils/misc.js"; -import {generateState} from "../../utils/state.js"; import {generateValidator} from "../../utils/validator.js"; describe("getActiveValidatorIndices", () => { diff --git a/packages/state-transition/test/utils/capella.ts b/packages/state-transition/test/utils/capella.ts index 249053e2a22a..8a02dc7de11b 100644 --- a/packages/state-transition/test/utils/capella.ts +++ b/packages/state-transition/test/utils/capella.ts @@ -8,9 +8,9 @@ import { } from "@lodestar/params"; import {ssz} from "@lodestar/types"; import {BeaconStateCapella, CachedBeaconStateCapella} from "../../src/index.js"; -import {interopPubkeysCached} from "./interop.js"; +import {interopPubkeysCached} from "../../src/testUtils/interop.js"; +import {createCachedBeaconStateTest} from "../../src/testUtils/state.js"; import {mulberry32} from "./rand.js"; -import {createCachedBeaconStateTest} from "./state.js"; export interface WithdrawalOpts { excessBalance: number; diff --git a/packages/state-transition/test/utils/state.ts b/packages/state-transition/test/utils/state.ts index 520c23669eb7..ef44dd070f01 100644 --- a/packages/state-transition/test/utils/state.ts +++ b/packages/state-transition/test/utils/state.ts @@ -1,112 +1,151 @@ -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {ChainForkConfig, createBeaconConfig} from "@lodestar/config"; -import {config, config as minimalConfig} from "@lodestar/config/default"; -import { - EPOCHS_PER_HISTORICAL_VECTOR, - EPOCHS_PER_SLASHINGS_VECTOR, - GENESIS_EPOCH, - GENESIS_SLOT, - SLOTS_PER_HISTORICAL_ROOT, -} from "@lodestar/params"; -import {phase0, ssz} from "@lodestar/types"; -import {EpochCacheOpts} from "../../src/cache/epochCache.js"; -import {BeaconStateCache} from "../../src/cache/stateCache.js"; -import {ZERO_HASH} from "../../src/constants/index.js"; +import {config as minimalConfig} from "@lodestar/config/default"; +import {getConfig} from "@lodestar/config/test-utils"; +import {FAR_FUTURE_EPOCH, ForkName, ForkSeq, MAX_EFFECTIVE_BALANCE, SYNC_COMMITTEE_SIZE} from "@lodestar/params"; +import {BeaconState, altair, bellatrix, electra, ssz} from "@lodestar/types"; import { BeaconStateAllForks, - BeaconStatePhase0, + BeaconStateBellatrix, + BeaconStateElectra, CachedBeaconStateAllForks, + CachedBeaconStateBellatrix, + CachedBeaconStateElectra, createCachedBeaconState, + createPubkeyCache, } from "../../src/index.js"; -import {newZeroedArray} from "../../src/util/index.js"; +import {generateValidator, generateValidators} from "./validator.js"; /** * Copy of BeaconState, but all fields are marked optional to allow for swapping out variables as needed. */ -type TestBeaconState = Partial; +type TestBeaconState = Partial; /** - * Generate beaconState, by default it will use the initial state defined when the `ChainStart` log is emitted. + * Generate beaconState, by default it will generate a mostly empty state with "just enough" to be valid-ish * NOTE: All fields can be overridden through `opts`. + * should allow 1st test calling generateState more time since creating a new instance is expensive. + * * @param {TestBeaconState} opts + * @param config * @returns {BeaconState} */ -export function generateState(opts?: TestBeaconState): BeaconStatePhase0 { - return ssz.phase0.BeaconState.toViewDU({ - genesisTime: Math.floor(Date.now() / 1000), - genesisValidatorsRoot: ZERO_HASH, - slot: GENESIS_SLOT, - fork: { - previousVersion: config.GENESIS_FORK_VERSION, - currentVersion: config.GENESIS_FORK_VERSION, - epoch: GENESIS_EPOCH, - }, - latestBlockHeader: { - slot: 0, - proposerIndex: 0, - parentRoot: Buffer.alloc(32), - stateRoot: Buffer.alloc(32), - bodyRoot: ssz.phase0.BeaconBlockBody.hashTreeRoot(ssz.phase0.BeaconBlockBody.defaultValue()), - }, - blockRoots: Array.from({length: SLOTS_PER_HISTORICAL_ROOT}, () => ZERO_HASH), - stateRoots: Array.from({length: SLOTS_PER_HISTORICAL_ROOT}, () => ZERO_HASH), - historicalRoots: [], - eth1Data: { - depositRoot: Buffer.alloc(32), - blockHash: Buffer.alloc(32), - depositCount: 0, - }, - eth1DataVotes: [], - eth1DepositIndex: 0, - validators: [], - balances: [], - randaoMixes: Array.from({length: EPOCHS_PER_HISTORICAL_VECTOR}, () => ZERO_HASH), - slashings: newZeroedArray(EPOCHS_PER_SLASHINGS_VECTOR), - previousEpochAttestations: [], - currentEpochAttestations: [], - justificationBits: ssz.phase0.JustificationBits.defaultValue(), - previousJustifiedCheckpoint: { - epoch: GENESIS_EPOCH, - root: ZERO_HASH, - }, - currentJustifiedCheckpoint: { - epoch: GENESIS_EPOCH, - root: ZERO_HASH, - }, - finalizedCheckpoint: { - epoch: GENESIS_EPOCH, - root: ZERO_HASH, - }, - ...opts, +export function generateState( + opts: TestBeaconState = {}, + config: ChainForkConfig = minimalConfig, + withPubkey = false +): BeaconStateAllForks { + const stateSlot = opts.slot ?? 0; + const state = config.getForkTypes(stateSlot).BeaconState.defaultValue(); + const forkSeq = config.getForkSeq(stateSlot); + + // Mutate state adding properties + Object.assign(state, opts); + + const validatorOpts = { + activationEpoch: 0, + effectiveBalance: MAX_EFFECTIVE_BALANCE, + withdrawableEpoch: FAR_FUTURE_EPOCH, + exitEpoch: FAR_FUTURE_EPOCH, + }; + const numValidators = 16; + + const validators = + opts.validators ?? + (withPubkey + ? Array.from({length: numValidators}, (_, _i) => { + return generateValidator({ + ...validatorOpts, + }); + }) + : generateValidators(numValidators, validatorOpts)); + + state.genesisTime = Math.floor(Date.now() / 1000); + state.slot = stateSlot; + state.fork.previousVersion = config.GENESIS_FORK_VERSION; + state.fork.currentVersion = config.GENESIS_FORK_VERSION; + state.latestBlockHeader.bodyRoot = ssz.phase0.BeaconBlockBody.hashTreeRoot(ssz.phase0.BeaconBlockBody.defaultValue()); + state.validators = validators; + state.balances = Array.from({length: validators.length}, () => MAX_EFFECTIVE_BALANCE); + + if (forkSeq >= ForkSeq.altair) { + const stateAltair = state as altair.BeaconState; + stateAltair.previousEpochParticipation = [...[0xff, 0xff], ...Array.from({length: validators.length - 2}, () => 0)]; + stateAltair.currentEpochParticipation = [...[0xff, 0xff], ...Array.from({length: validators.length - 2}, () => 0)]; + stateAltair.currentSyncCommittee = { + pubkeys: Array.from({length: SYNC_COMMITTEE_SIZE}, (_, i) => validators[i % validators.length].pubkey), + aggregatePubkey: ssz.BLSPubkey.defaultValue(), + }; + stateAltair.nextSyncCommittee = { + pubkeys: Array.from({length: SYNC_COMMITTEE_SIZE}, (_, i) => validators[i % validators.length].pubkey), + aggregatePubkey: ssz.BLSPubkey.defaultValue(), + }; + } + + if (forkSeq >= ForkSeq.bellatrix) { + const stateBellatrix = state as bellatrix.BeaconState; + stateBellatrix.latestExecutionPayloadHeader = { + ...ssz.bellatrix.ExecutionPayloadHeader.defaultValue(), + blockNumber: 2022, + }; + } + + if (forkSeq >= ForkSeq.electra) { + const stateElectra = state as electra.BeaconState; + stateElectra.depositRequestsStartIndex = 2023n; + stateElectra.latestExecutionPayloadHeader = ssz.electra.ExecutionPayloadHeader.defaultValue(); + } + + return config.getForkTypes(stateSlot).BeaconState.toViewDU(state); +} + +/** + * This generates state with default pubkey + * TODO: (@matthewkeil) - this is duplicated and exists in state-transition as well + */ +export function generateCachedState(opts?: TestBeaconState): CachedBeaconStateAllForks { + const config = getConfig(ForkName.phase0); + const state = generateState(opts, config); + return createCachedBeaconState(state, { + config: createBeaconConfig(config, state.genesisValidatorsRoot), + // This is a performance test, there's no need to have a global shared cache of keys + pubkeyCache: createPubkeyCache(), }); } -export function generateCachedState( - config: ChainForkConfig = minimalConfig, - opts: TestBeaconState = {} -): CachedBeaconStateAllForks { - const state = generateState(opts); +/** + * This generates state with default pubkey + */ +export function generateCachedAltairState(opts?: TestBeaconState, altairForkEpoch = 0): CachedBeaconStateAllForks { + const config = getConfig(ForkName.altair, altairForkEpoch); + const state = generateState(opts, config); return createCachedBeaconState(state, { config: createBeaconConfig(config, state.genesisValidatorsRoot), - // This is a test state, there's no need to have a global shared cache of keys - pubkey2index: new PubkeyIndexMap(), - index2pubkey: [], + // This is a performance test, there's no need to have a global shared cache of keys + pubkeyCache: createPubkeyCache(), + }); +} + +/** + * This generates state with default pubkey + */ +export function generateCachedBellatrixState(opts?: TestBeaconState): CachedBeaconStateBellatrix { + const config = getConfig(ForkName.bellatrix); + const state = generateState(opts, config); + return createCachedBeaconState(state as BeaconStateBellatrix, { + config: createBeaconConfig(config, state.genesisValidatorsRoot), + // This is a performance test, there's no need to have a global shared cache of keys + pubkeyCache: createPubkeyCache(), }); } -export function createCachedBeaconStateTest( - state: T, - configCustom: ChainForkConfig = config, - opts?: EpochCacheOpts -): T & BeaconStateCache { - return createCachedBeaconState( - state, - { - config: createBeaconConfig(configCustom, state.genesisValidatorsRoot), - // This is a test state, there's no need to have a global shared cache of keys - pubkey2index: new PubkeyIndexMap(), - index2pubkey: [], - }, - opts - ); +/** + * This generates state with default pubkey + */ +export function generateCachedElectraState(opts?: TestBeaconState, electraForkEpoch = 0): CachedBeaconStateElectra { + const config = getConfig(ForkName.electra, electraForkEpoch); + const state = generateState(opts, config); + return createCachedBeaconState(state as BeaconStateElectra, { + config: createBeaconConfig(config, state.genesisValidatorsRoot), + pubkeyCache: createPubkeyCache(), + }); } diff --git a/packages/state-transition/tsconfig.json b/packages/state-transition/tsconfig.json index a0f4f2a31e93..19b9527ba0e3 100644 --- a/packages/state-transition/tsconfig.json +++ b/packages/state-transition/tsconfig.json @@ -1,6 +1,9 @@ { "extends": "../../tsconfig.json", - "include": ["src", "test"], + "include": [ + "src", + "test" + ], "compilerOptions": { "outDir": "lib" } diff --git a/packages/test-utils/package.json b/packages/test-utils/package.json index 1bc6dd34beea..0ddd99ed8318 100644 --- a/packages/test-utils/package.json +++ b/packages/test-utils/package.json @@ -1,7 +1,7 @@ { "name": "@lodestar/test-utils", "private": true, - "version": "1.40.0", + "version": "1.41.0", "description": "Test utilities reused across other packages", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -29,11 +29,11 @@ ], "scripts": { "clean": "rm -rf lib && rm -f *.tsbuildinfo", - "build": "tsc -p tsconfig.build.json", + "build": "tsgo -p tsconfig.build.json", "build:release": "pnpm clean && pnpm build", "build:watch": "pnpm run build --watch", "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\"", - "check-types": "tsc", + "check-types": "tsgo", "lint": "biome check src/", "lint:fix": "pnpm run lint --write", "test:unit": "echo 'No unit tests for @lodestar/test-utils'", diff --git a/packages/test-utils/src/externalSigner.ts b/packages/test-utils/src/externalSigner.ts index b668566b3520..297a0eb53c0a 100644 --- a/packages/test-utils/src/externalSigner.ts +++ b/packages/test-utils/src/externalSigner.ts @@ -3,7 +3,7 @@ import path from "node:path"; import {dirSync as tmpDirSync} from "tmp"; import {ForkSeq} from "@lodestar/params"; import {fetch, retry, withTimeout} from "@lodestar/utils"; -import {runDockerContainer} from "./dockercontainer.ts"; +import {runDockerContainer} from "./dockercontainer.js"; const web3signerVersion = "25.11.0"; const web3signerImage = `consensys/web3signer:${web3signerVersion}`; diff --git a/packages/types/package.json b/packages/types/package.json index 79b052e89058..ba62afc42ba0 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.40.0", + "version": "1.41.0", "type": "module", "exports": { ".": { @@ -62,11 +62,11 @@ ], "scripts": { "clean": "rm -rf lib && rm -f *.tsbuildinfo", - "build": "tsc -p tsconfig.build.json", + "build": "tsgo -p tsconfig.build.json", "build:watch": "pnpm run build --watch", "build:release": "pnpm clean && pnpm build", "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\"", - "check-types": "tsc", + "check-types": "tsgo", "lint": "biome check src/ test/", "lint:fix": "pnpm run lint --write", "test": "pnpm test:unit", diff --git a/packages/types/src/capella/types.ts b/packages/types/src/capella/types.ts index 386f96ecd280..4dda27a73e0b 100644 --- a/packages/types/src/capella/types.ts +++ b/packages/types/src/capella/types.ts @@ -31,3 +31,5 @@ export type LightClientUpdate = ValueOf; export type LightClientFinalityUpdate = ValueOf; export type LightClientOptimisticUpdate = ValueOf; export type LightClientStore = ValueOf; + +export type HistoricalSummaries = ValueOf; diff --git a/packages/types/src/deneb/sszTypes.ts b/packages/types/src/deneb/sszTypes.ts index 9275ac4c4da5..0e18764c7093 100644 --- a/packages/types/src/deneb/sszTypes.ts +++ b/packages/types/src/deneb/sszTypes.ts @@ -18,10 +18,10 @@ const {UintNum64, Slot, Root, BLSSignature, UintBn64, UintBn256, Bytes32, Bytes4 primitiveSsz; // Polynomial commitments -// https://github.com/ethereum/consensus-specs/blob/dev/specs/eip4844/polynomial-commitments.md +// https://github.com/ethereum/consensus-specs/blob/v1.3.0-rc.2/specs/eip4844/polynomial-commitments.md // Custom types -// https://github.com/ethereum/consensus-specs/blob/dev/specs/eip4844/polynomial-commitments.md#custom-types +// https://github.com/ethereum/consensus-specs/blob/v1.3.0-rc.2/specs/eip4844/polynomial-commitments.md#custom-types export const G1Point = Bytes48; export const G2Point = Bytes96; export const BLSFieldElement = Bytes32; @@ -31,7 +31,7 @@ export const KZGProof = Bytes48; // Beacon chain // Custom types -// https://github.com/ethereum/consensus-specs/blob/dev/specs/eip4844/beacon-chain.md#custom-types +// https://github.com/ethereum/consensus-specs/blob/v1.3.0-rc.2/specs/eip4844/beacon-chain.md#custom-types export const Blob = new ByteVectorType(BYTES_PER_FIELD_ELEMENT * FIELD_ELEMENTS_PER_BLOB); export const Blobs = new ListCompositeType(Blob, MAX_BLOB_COMMITMENTS_PER_BLOCK); @@ -62,7 +62,7 @@ export const BlobIdentifier = new ContainerType( ); // Beacon Chain types -// https://github.com/ethereum/consensus-specs/blob/dev/specs/eip4844/beacon-chain.md#containers +// https://github.com/ethereum/consensus-specs/blob/v1.3.0-rc.2/specs/eip4844/beacon-chain.md#containers export const ExecutionPayload = new ContainerType( { diff --git a/packages/types/src/types.ts b/packages/types/src/types.ts index 223269815f6e..8d5562290a0c 100644 --- a/packages/types/src/types.ts +++ b/packages/types/src/types.ts @@ -6,6 +6,7 @@ import { ForkPostDeneb, ForkPostElectra, ForkPostFulu, + ForkPostGloas, } from "@lodestar/params"; import {ts as altair} from "./altair/index.js"; import {ts as bellatrix} from "./bellatrix/index.js"; @@ -321,6 +322,7 @@ type TypesByFork = { AggregateAndProof: electra.AggregateAndProof; SignedAggregateAndProof: electra.SignedAggregateAndProof; ExecutionRequests: electra.ExecutionRequests; + ExecutionPayloadBid: gloas.ExecutionPayloadBid; DataColumnSidecar: gloas.DataColumnSidecar; DataColumnSidecars: gloas.DataColumnSidecars; }; @@ -388,3 +390,4 @@ export type IndexedAttestationBigint = TypesByFork export type AttesterSlashing = TypesByFork[F]["AttesterSlashing"]; export type AggregateAndProof = TypesByFork[F]["AggregateAndProof"]; export type SignedAggregateAndProof = TypesByFork[F]["SignedAggregateAndProof"]; +export type ExecutionPayloadBid = TypesByFork[F]["ExecutionPayloadBid"]; diff --git a/packages/types/src/utils/validatorStatus.ts b/packages/types/src/utils/validatorStatus.ts index 9e72c39d9df2..e5c4692e2f3f 100644 --- a/packages/types/src/utils/validatorStatus.ts +++ b/packages/types/src/utils/validatorStatus.ts @@ -15,6 +15,8 @@ export type ValidatorStatus = | "withdrawal_possible" | "withdrawal_done"; +export type GeneralValidatorStatus = "active" | "pending" | "exited" | "withdrawal"; + /** * Get the status of the validator * based on conditions outlined in https://hackmd.io/ofFJ5gOmQpu1jjHilHbdQQ @@ -50,3 +52,23 @@ export function getValidatorStatus(validator: phase0.Validator, currentEpoch: Ep } throw new Error("ValidatorStatus unknown"); } + +export function mapToGeneralStatus(subStatus: ValidatorStatus): GeneralValidatorStatus { + switch (subStatus) { + case "active_ongoing": + case "active_exiting": + case "active_slashed": + return "active"; + case "pending_initialized": + case "pending_queued": + return "pending"; + case "exited_slashed": + case "exited_unslashed": + return "exited"; + case "withdrawal_possible": + case "withdrawal_done": + return "withdrawal"; + default: + throw new Error(`Unknown substatus: ${subStatus}`); + } +} diff --git a/packages/utils/package.json b/packages/utils/package.json index c468025d36a8..5f832140d0a5 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.40.0", + "version": "1.41.0", "type": "module", "exports": { ".": { @@ -33,11 +33,11 @@ ], "scripts": { "clean": "rm -rf lib && rm -f *.tsbuildinfo", - "build": "tsc -p tsconfig.build.json", + "build": "tsgo -p tsconfig.build.json", "build:watch": "pnpm run build --watch", "build:release": "pnpm clean && pnpm build", "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\"", - "check-types": "tsc && vitest run --project types", + "check-types": "tsgo && vitest run --project types", "lint": "biome check src/ test/", "lint:fix": "pnpm run lint --write", "test:unit": "vitest run --project unit --project unit-minimal", @@ -47,7 +47,7 @@ "types": "lib/index.d.ts", "dependencies": { "@chainsafe/as-sha256": "^1.2.0", - "@vekexasia/bigint-buffer2": "^1.1.0", + "@vekexasia/bigint-buffer2": "^1.1.1", "any-signal": "^4.1.1", "case": "^1.6.3", "js-yaml": "^4.1.0" diff --git a/packages/utils/src/bytes/browser.ts b/packages/utils/src/bytes/browser.ts index 712e99eed25d..12f651c99573 100644 --- a/packages/utils/src/bytes/browser.ts +++ b/packages/utils/src/bytes/browser.ts @@ -187,6 +187,19 @@ export function xor(a: Uint8Array, b: Uint8Array): Uint8Array { return a; } +/** Count set bits in a Uint8Array without allocation (Brian Kernighan's algorithm) */ +export function bitCount(arr: Uint8Array): number { + let count = 0; + for (let i = 0; i < arr.length; i++) { + let byte = arr[i]; + while (byte) { + byte &= byte - 1; + count++; + } + } + return count; +} + /** * Compare two byte arrays for equality. * Note: In Node.js environment, the implementation in nodejs.ts uses Buffer.compare diff --git a/packages/utils/src/bytes/nodejs.ts b/packages/utils/src/bytes/nodejs.ts index c98066c74cbd..816f925ff003 100644 --- a/packages/utils/src/bytes/nodejs.ts +++ b/packages/utils/src/bytes/nodejs.ts @@ -89,4 +89,13 @@ export function byteArrayEquals(a: Uint8Array, b: Uint8Array): boolean { return Buffer.compare(a, b) === 0; } -export {bigIntToBytes, bytesToBigInt, bytesToInt, fromHexInto, intToBytes, toHexString, xor} from "./browser.ts"; +export { + bigIntToBytes, + bitCount, + bytesToBigInt, + bytesToInt, + fromHexInto, + intToBytes, + toHexString, + xor, +} from "./browser.js"; diff --git a/packages/utils/test/perf/bytes.test.ts b/packages/utils/test/perf/bytes.test.ts index 17f7153fef2d..3184158bfb03 100644 --- a/packages/utils/test/perf/bytes.test.ts +++ b/packages/utils/test/perf/bytes.test.ts @@ -1,6 +1,6 @@ import {bench, describe} from "@chainsafe/benchmark"; -import * as browser from "../../src/bytes/browser.ts"; -import * as nodejs from "../../src/bytes/nodejs.ts"; +import * as browser from "../../src/bytes/browser.js"; +import * as nodejs from "../../src/bytes/nodejs.js"; describe("bytes utils", async () => { const runsFactor = 1000; diff --git a/packages/validator/package.json b/packages/validator/package.json index 4711ddac7a52..f25a16adda46 100644 --- a/packages/validator/package.json +++ b/packages/validator/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/validator", - "version": "1.40.0", + "version": "1.41.0", "description": "A Typescript implementation of the validator client", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -23,11 +23,11 @@ ], "scripts": { "clean": "rm -rf lib && rm -f *.tsbuildinfo", - "build": "tsc -p tsconfig.build.json", + "build": "tsgo -p tsconfig.build.json", "build:release": "pnpm clean && pnpm run build", "build:watch": "pnpm run build --watch", "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\"", - "check-types": "tsc", + "check-types": "tsgo", "lint": "biome check src/ test/", "lint:fix": "pnpm run lint --write", "test:unit": "vitest run --project unit --project unit-minimal", @@ -63,7 +63,7 @@ "@lodestar/logger": "workspace:^", "@lodestar/spec-test-util": "workspace:^", "@lodestar/test-utils": "workspace:^", - "@vekexasia/bigint-buffer2": "^1.1.0", + "@vekexasia/bigint-buffer2": "^1.1.1", "rimraf": "^4.4.1" } } diff --git a/packages/validator/src/index.ts b/packages/validator/src/index.ts index 7e46cbb11ad5..06c2ec972198 100644 --- a/packages/validator/src/index.ts +++ b/packages/validator/src/index.ts @@ -1,3 +1,4 @@ +export {Bucket} from "./buckets.js"; export {waitForGenesis} from "./genesis.js"; export {type Metrics, getMetrics} from "./metrics.js"; export * from "./repositories/index.js"; diff --git a/packages/validator/src/validator.ts b/packages/validator/src/validator.ts index 02b5de07c5f7..eca4e47103ab 100644 --- a/packages/validator/src/validator.ts +++ b/packages/validator/src/validator.ts @@ -179,7 +179,11 @@ export class Validator { urls: typeof clientOrUrls === "string" ? [clientOrUrls] : clientOrUrls, // Validator would need the beacon to respond within the slot // See https://github.com/ChainSafe/lodestar/issues/5315 for rationale - globalInit: {timeoutMs: config.SLOT_DURATION_MS, signal: controller.signal, ...globalInit}, + globalInit: { + signal: controller.signal, + ...globalInit, + timeoutMs: globalInit?.timeoutMs ?? config.SLOT_DURATION_MS, + }, }, {config, logger, metrics: metrics?.restApiClient} ); @@ -296,6 +300,7 @@ export class Validator { urls: urls.map(toPrintableUrl).toString(), requestWireFormat: globalInit?.requestWireFormat ?? defaultInit.requestWireFormat, responseWireFormat: globalInit?.responseWireFormat ?? defaultInit.responseWireFormat, + requestTimeoutMs: globalInit?.timeoutMs ?? config.SLOT_DURATION_MS, }); } else { api = clientOrUrls; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d69daae11aaf..0f8017b11771 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -71,6 +71,9 @@ importers: '@types/react': specifier: ^19.1.12 version: 19.1.12 + '@typescript/native-preview': + specifier: 7.0.0-dev.20260303.1 + version: 7.0.0-dev.20260303.1 '@vitest/browser': specifier: 'catalog:' version: 4.0.7(vite@6.1.6(@types/node@24.10.1)(yaml@2.8.2))(vitest@4.0.7) @@ -105,8 +108,8 @@ importers: specifier: ^23.0.1 version: 23.0.1 libp2p: - specifier: 2.9.0 - version: 2.9.0 + specifier: 3.1.6 + version: 3.1.6 node-gyp: specifier: ^9.4.0 version: 9.4.1 @@ -144,17 +147,17 @@ importers: specifier: ^6.3.3 version: 6.3.3 ts-node: - specifier: ^10.9.2 - version: 10.9.2(@swc/core@1.14.0)(@swc/wasm@1.14.0)(@types/node@24.10.1)(typescript@5.9.2) + specifier: ^11.0.0-beta.1 + version: 11.0.0-beta.1(@swc/core@1.14.0)(@swc/wasm@1.14.0)(@types/node@24.10.1)(typescript@6.0.1-rc) typescript: - specifier: ^5.9.2 - version: 5.9.2 + specifier: ^6.0.1-rc + version: 6.0.1-rc vite: specifier: ^6.0.11 version: 6.1.6(@types/node@24.10.1)(yaml@2.8.2) vite-plugin-dts: specifier: ^4.5.4 - version: 4.5.4(@types/node@24.10.1)(rollup@4.52.5)(typescript@5.9.2)(vite@6.1.6(@types/node@24.10.1)(yaml@2.8.2)) + version: 4.5.4(@types/node@24.10.1)(rollup@4.52.5)(typescript@6.0.1-rc)(vite@6.1.6(@types/node@24.10.1)(yaml@2.8.2)) vite-plugin-node-polyfills: specifier: ^0.24.0 version: 0.24.0(rollup@4.52.5)(vite@6.1.6(@types/node@24.10.1)(yaml@2.8.2)) @@ -208,8 +211,8 @@ importers: specifier: ^8.12.0 version: 8.18.0 fastify: - specifier: ^5.7.4 - version: 5.7.4 + specifier: ^5.8.1 + version: 5.8.1 packages/beacon-node: dependencies: @@ -220,17 +223,17 @@ importers: specifier: ^2.2.0 version: 2.2.0 '@chainsafe/discv5': - specifier: ^11.0.4 - version: 11.0.4 + specifier: ^12.0.1 + version: 12.0.1 '@chainsafe/enr': - specifier: ^5.0.1 - version: 5.0.1 - '@chainsafe/libp2p-gossipsub': - specifier: ^14.1.2 - version: 14.1.2 + specifier: ^6.0.1 + version: 6.0.1 '@chainsafe/libp2p-noise': - specifier: ^16.1.5 - version: 16.1.5 + specifier: ^17.0.0 + version: 17.0.0 + '@chainsafe/libp2p-quic': + specifier: ^2.0.0 + version: 2.0.0 '@chainsafe/persistent-merkle-tree': specifier: ^1.2.1 version: 1.2.1 @@ -265,32 +268,35 @@ importers: specifier: ^5.0.1 version: 5.0.1 '@libp2p/bootstrap': - specifier: ^11.0.32 - version: 11.0.46 + specifier: ^12.0.14 + version: 12.0.14 '@libp2p/crypto': - specifier: ^5.0.15 - version: 5.1.7 + specifier: ^5.1.13 + version: 5.1.13 + '@libp2p/gossipsub': + specifier: ^15.0.15 + version: 15.0.15 '@libp2p/identify': - specifier: ^3.0.27 - version: 3.0.38 + specifier: ^4.0.13 + version: 4.0.13 '@libp2p/interface': - specifier: ^2.7.0 - version: 2.10.5 + specifier: ^3.1.0 + version: 3.1.0 '@libp2p/mdns': - specifier: ^11.0.32 - version: 11.0.46 + specifier: ^12.0.14 + version: 12.0.14 '@libp2p/mplex': - specifier: ^11.0.32 - version: 11.0.46 + specifier: ^12.0.14 + version: 12.0.14 '@libp2p/peer-id': - specifier: ^5.1.0 - version: 5.1.8 + specifier: ^6.0.4 + version: 6.0.4 '@libp2p/prometheus-metrics': - specifier: ^4.3.15 - version: 4.3.29 + specifier: ^5.0.14 + version: 5.0.14 '@libp2p/tcp': - specifier: ^10.1.8 - version: 10.1.18 + specifier: ^11.0.13 + version: 11.0.13 '@lodestar/api': specifier: workspace:^ version: link:../api @@ -328,41 +334,35 @@ importers: specifier: workspace:^ version: link:../validator '@multiformats/multiaddr': - specifier: ^12.1.3 - version: 12.5.1 + specifier: ^13.0.1 + version: 13.0.1 datastore-core: - specifier: ^10.0.2 - version: 10.0.2 + specifier: ^11.0.2 + version: 11.0.2 datastore-fs: - specifier: ^10.0.6 - version: 10.0.6 + specifier: ^11.0.2 + version: 11.0.2 datastore-level: - specifier: ^11.0.3 - version: 11.0.4 + specifier: ^12.0.2 + version: 12.0.2 deepmerge: specifier: ^4.3.1 version: 4.3.1 fastify: - specifier: ^5.7.4 - version: 5.7.4 + specifier: ^5.8.1 + version: 5.8.1 interface-datastore: - specifier: ^8.3.0 - version: 8.3.1 - it-all: - specifier: ^3.0.4 - version: 3.0.9 - it-pipe: - specifier: ^3.0.1 - version: 3.0.1 + specifier: ^9.0.2 + version: 9.0.2 jwt-simple: specifier: 0.5.6 version: 0.5.6 libp2p: - specifier: 2.9.0 - version: 2.9.0 + specifier: 3.1.6 + version: 3.1.6 multiformats: - specifier: ^11.0.1 - version: 11.0.2 + specifier: ^13.4.2 + version: 13.4.2 prom-client: specifier: ^15.1.0 version: 15.1.3 @@ -386,11 +386,14 @@ importers: specifier: ^1.2.1 version: 1.2.1 '@libp2p/interface-internal': - specifier: ^2.3.18 - version: 2.3.18 + specifier: ^3.0.13 + version: 3.0.13 '@libp2p/logger': - specifier: ^5.1.21 - version: 5.1.21 + specifier: ^6.2.2 + version: 6.2.2 + '@libp2p/utils': + specifier: ^7.0.13 + version: 7.0.13 '@lodestar/spec-test-util': specifier: workspace:^ version: link:../spec-test-util @@ -403,12 +406,6 @@ importers: '@types/tmp': specifier: ^0.2.3 version: 0.2.3 - it-drain: - specifier: ^3.0.3 - version: 3.0.10 - it-pair: - specifier: ^2.0.6 - version: 2.0.6 js-yaml: specifier: ^4.1.0 version: 4.1.1 @@ -440,11 +437,11 @@ importers: specifier: ^2.2.0 version: 2.2.0 '@chainsafe/discv5': - specifier: ^11.0.4 - version: 11.0.4 + specifier: ^12.0.1 + version: 12.0.1 '@chainsafe/enr': - specifier: ^5.0.1 - version: 5.0.1 + specifier: ^6.0.1 + version: 6.0.1 '@chainsafe/persistent-merkle-tree': specifier: ^1.2.1 version: 1.2.1 @@ -458,14 +455,14 @@ importers: specifier: ^1.11.3 version: 1.11.3 '@libp2p/crypto': - specifier: ^5.0.15 - version: 5.1.7 + specifier: ^5.1.13 + version: 5.1.13 '@libp2p/interface': - specifier: ^2.7.0 - version: 2.10.5 + specifier: ^3.1.0 + version: 3.1.0 '@libp2p/peer-id': - specifier: ^5.1.0 - version: 5.1.8 + specifier: ^6.0.4 + version: 6.0.4 '@lodestar/api': specifier: workspace:^ version: link:../api @@ -500,8 +497,8 @@ importers: specifier: workspace:^ version: link:../validator '@multiformats/multiaddr': - specifier: ^12.1.3 - version: 12.5.1 + specifier: ^13.0.1 + version: 13.0.1 deepmerge: specifier: ^4.3.1 version: 4.3.1 @@ -567,8 +564,8 @@ importers: specifier: ^2.2.1 version: 2.2.1 fastify: - specifier: ^5.7.4 - version: 5.7.4 + specifier: ^5.8.1 + version: 5.8.1 tmp: specifier: ^0.2.4 version: 0.2.4 @@ -590,6 +587,9 @@ importers: '@lodestar/params': specifier: workspace:^ version: link:../params + '@lodestar/spec-test-util': + specifier: workspace:^ + version: link:../spec-test-util '@lodestar/types': specifier: workspace:^ version: link:../types @@ -611,9 +611,6 @@ importers: classic-level: specifier: ^1.4.1 version: 1.4.1 - it-all: - specifier: ^3.0.4 - version: 3.0.9 devDependencies: '@lodestar/logger': specifier: workspace:^ @@ -747,8 +744,8 @@ importers: specifier: ^6.9.7 version: 6.9.7 fastify: - specifier: ^5.7.4 - version: 5.7.4 + specifier: ^5.8.1 + version: 5.8.1 qs: specifier: ^6.11.1 version: 6.14.1 @@ -899,8 +896,11 @@ importers: specifier: ^4.2.0 version: 4.2.0 '@libp2p/interface': - specifier: ^2.7.0 - version: 2.10.5 + specifier: ^3.1.0 + version: 3.1.0 + '@libp2p/utils': + specifier: ^7.0.13 + version: 7.0.13 '@lodestar/config': specifier: workspace:^ version: link:../config @@ -910,12 +910,6 @@ importers: '@lodestar/utils': specifier: workspace:^ version: link:../utils - it-all: - specifier: ^3.0.4 - version: 3.0.9 - it-pipe: - specifier: ^3.0.1 - version: 3.0.1 snappy: specifier: ^7.2.2 version: 7.2.2 @@ -933,26 +927,23 @@ importers: specifier: ^1.2.2 version: 1.2.2 '@libp2p/crypto': - specifier: ^5.1.7 - version: 5.1.7 + specifier: ^5.1.13 + version: 5.1.13 '@libp2p/logger': - specifier: ^5.1.21 - version: 5.1.21 + specifier: ^6.2.2 + version: 6.2.2 '@libp2p/peer-id': - specifier: ^5.1.8 - version: 5.1.8 + specifier: ^6.0.4 + version: 6.0.4 '@lodestar/logger': specifier: workspace:^ version: link:../logger '@lodestar/types': specifier: workspace:^ version: link:../types - it-stream-types: - specifier: ^2.0.2 - version: 2.0.2 libp2p: - specifier: 2.9.0 - version: 2.9.0 + specifier: 3.1.6 + version: 3.1.6 packages/spec-test-util: dependencies: @@ -973,7 +964,7 @@ importers: version: 0.7.0 vitest: specifier: 'catalog:' - version: 4.0.7(@types/node@24.10.1)(@vitest/browser-playwright@4.0.7)(jsdom@23.0.1)(yaml@2.8.2) + version: 4.0.7(@types/node@25.4.0)(@vitest/browser-playwright@4.0.7)(jsdom@23.0.1)(yaml@2.8.2) packages/state-transition: dependencies: @@ -1011,8 +1002,8 @@ importers: specifier: workspace:^ version: link:../utils '@vekexasia/bigint-buffer2': - specifier: ^1.1.0 - version: 1.1.0 + specifier: ^1.1.1 + version: 1.1.1 devDependencies: '@lodestar/api': specifier: workspace:^ @@ -1040,7 +1031,7 @@ importers: version: 0.2.4 vitest: specifier: 'catalog:' - version: 4.0.7(@types/node@24.10.1)(@vitest/browser-playwright@4.0.7)(jsdom@23.0.1)(yaml@2.8.2) + version: 4.0.7(@types/node@25.4.0)(@vitest/browser-playwright@4.0.7)(jsdom@23.0.1)(yaml@2.8.2) devDependencies: '@types/tmp': specifier: ^0.2.3 @@ -1067,8 +1058,8 @@ importers: specifier: ^1.2.0 version: 1.2.0 '@vekexasia/bigint-buffer2': - specifier: ^1.1.0 - version: 1.1.0 + specifier: ^1.1.1 + version: 1.1.1 any-signal: specifier: ^4.1.1 version: 4.1.1 @@ -1135,8 +1126,8 @@ importers: specifier: workspace:^ version: link:../test-utils '@vekexasia/bigint-buffer2': - specifier: ^1.1.0 - version: 1.1.0 + specifier: ^1.1.1 + version: 1.1.1 rimraf: specifier: ^4.4.1 version: 4.4.1 @@ -1396,11 +1387,11 @@ packages: resolution: {integrity: sha512-VBaQoNE2a9d9+skAjQKv3Suk0yGKqp3mZM0YWYJNPj/Ae/f6lAyeVSgKqo2LrsNQBzD/LqrJLKUY8rJT3vDKLA==} engines: {node: '>= 16'} - '@chainsafe/discv5@11.0.4': - resolution: {integrity: sha512-8bUqQXlh3+VBO3f3kzvFL0CLOeShZLjUexlmjCyfXfOgwpP1e46F4G6a9tneEEXaU+AMLPMz8uRCwtP/YsESbw==} + '@chainsafe/discv5@12.0.1': + resolution: {integrity: sha512-isFDtl+DiBOD6R0cV4+4Zch25zlj5mv5nytw2x6c+eR6xDte7xlB6+/UutjfnSScMH/CwysytQCiOi8/pMxrTg==} - '@chainsafe/enr@5.0.1': - resolution: {integrity: sha512-RP5XxJOSYt0f775ne6NqlaG33Qia+IaMjWlP7PlWagz6SOpXlfbm+vaqHx3by6lKPb9aW4UQtO+fiRxz85XwaQ==} + '@chainsafe/enr@6.0.1': + resolution: {integrity: sha512-AoPgLrFAtsnIgyGx6ATzb7dRZkUmHBxA2wXAqf3sLlBeojeoaNprePcYXnU0UxB1E9Y8ZWgF7PI567sF+VrDQw==} '@chainsafe/fast-crc32c@4.2.0': resolution: {integrity: sha512-wbVJ3q1DBuYqs1VGwLe9o5VIDQtxGpSVzr7UY9Hq2SQ283fU6AouII/pwioS9MbiiIK42jDxtY0uLFmTuenb0g==} @@ -1452,12 +1443,58 @@ packages: '@chainsafe/is-ip@2.1.0': resolution: {integrity: sha512-KIjt+6IfysQ4GCv66xihEitBjvhU/bixbbbFxdJ1sqCp4uJ0wuZiYBPhksZoy4lfaF0k9cwNzY5upEW/VWdw3w==} - '@chainsafe/libp2p-gossipsub@14.1.2': - resolution: {integrity: sha512-HuyB1dO+GwqpbvKhq2TdrsvrtNt19iOWLEfjy2N3ItYY1gA2F67KtCOLmZ1NlEcCbjFDg+qRffQMjgkDwHSFOg==} - engines: {npm: '>=8.7.0'} + '@chainsafe/libp2p-noise@17.0.0': + resolution: {integrity: sha512-vwrmY2Y+L1xYhIDiEpl61KHxwrLCZoXzTpwhyk34u+3+6zCAZPL3GxH3i2cs+u5IYNoyLptORdH17RKFXy7upA==} + + '@chainsafe/libp2p-quic-darwin-arm64@2.0.0': + resolution: {integrity: sha512-dY/LEALfn4qRUjY8i3+V5YcmPRzJdeiaE/6eoS92KZr6UoWdVuxUCUz24+3jTl9Ke+EqvqsjZxfIS08CuhWhRA==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + + '@chainsafe/libp2p-quic-darwin-x64@2.0.0': + resolution: {integrity: sha512-XOt+rxMUjJ7W+ainDVUJ76kkmFO/LZYjGI8D9ecVL4P1+SIfxc90awrjikGUDqlozvsk96vjNzvUO6Obw+ftXA==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] - '@chainsafe/libp2p-noise@16.1.5': - resolution: {integrity: sha512-yac0bknwfuYdXXOAFGVL4fYR0de0p1Uk7W0gIkuxlj8JSqhkHr+kEt3958PfnfmX880QJAIogsh/ZupFjg0uJA==} + '@chainsafe/libp2p-quic-linux-arm64-gnu@2.0.0': + resolution: {integrity: sha512-TgRv12gyZMWm8G9RMi7j0Uwjmk272ikqXVvYkDICD5yS13vIkvxueVuvAOtFnkt7qO+pPh2hciPUkdfYeqVjuA==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@chainsafe/libp2p-quic-linux-arm64-musl@2.0.0': + resolution: {integrity: sha512-cax7qIL8fGeN41ydH6yRXcYhbYZbSQOYXa9tSr5lLSDJE24JEOjy8BugeP618kXydNFEbbt3VmyjNc+8XrGd4g==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@chainsafe/libp2p-quic-linux-x64-gnu@2.0.0': + resolution: {integrity: sha512-LiiaS3lmebt4USDQAFQCM+t5n1wuMoGeEDmHco6moSmZ9FJhBaM4CryUS3bXey31tpxEyQIx3q6Ub8SOUyLJ/g==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@chainsafe/libp2p-quic-linux-x64-musl@2.0.0': + resolution: {integrity: sha512-G3SS24AoY/1xkAA9IvL7alq2fp4GJwp+YXM8lOaez3cTCZuhwq4CksQFjy11VzyKrNCs/D9H02l0Aat8UeQaBw==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + libc: [musl] + + '@chainsafe/libp2p-quic-win32-x64-msvc@2.0.0': + resolution: {integrity: sha512-IxLdu8uVcus/KmFTiXHX6xqZE51uazKRJSbEn/9r94UY+uA7AEAeu497Zug7LOfzg3j++vFwd/AOJRSbk4U9Lw==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + + '@chainsafe/libp2p-quic@2.0.0': + resolution: {integrity: sha512-RNxHNdo22XhS5DEFbT0fAaZFfrEmuIVRP5InqgMq8md6r0f61AATZdmYqAuNkcj6VSBareLlCAJ/iuhhNZYt7A==} + engines: {node: '>= 22'} '@chainsafe/netmask@2.0.0': resolution: {integrity: sha512-I3Z+6SWUoaljh3TBzCnCxjlUyN8tA+NAk5L6m9IxvCf1BENQTePzPMis97CoN/iMW1St3WN+AWCCRp+TTBRiDg==} @@ -1666,6 +1703,10 @@ packages: '@dabh/diagnostics@2.0.2': resolution: {integrity: sha512-+A1YivoVDNNVCdfozHSR8v/jyuuLTMXwjWuxPFlFlUapXoGc+Gj9mDlTDDfrwl7rXCl2tNZ0kE8sIBO6YOn96Q==} + '@dnsquery/dns-packet@6.1.1': + resolution: {integrity: sha512-WXTuFvL3G+74SchFAtz3FgIYVOe196ycvGsMgvSH/8Goptb1qpIQtIuM4SOK9G9lhMWYpHxnXyy544ZhluFOew==} + engines: {node: '>=6'} + '@electron/get@2.0.3': resolution: {integrity: sha512-Qkzpg2s9GnVV2I2BjRksUi43U5e6+zaQMcjoJy0C+C5oxaKl+fmckGDQFtRpZpZV0NQekuZZ+tGz7EA9TVnQtQ==} engines: {node: '>=12'} @@ -1848,16 +1889,16 @@ packages: resolution: {integrity: sha512-9PzshkvwO8YBkSD9+vyhJuzM6hxfZlljGnuUbXQlTSGEod7we8BRyzJW53W7nw/WRw5U6wf9Q2fpWypfZFkrbw==} engines: {node: '>=14'} + '@ethereumjs/rlp@10.1.1': + resolution: {integrity: sha512-jbnWTEwcpoY+gE0r+wxfDG9zgiu54DcTcwnc9sX3DsqKR4l5K7x2V8mQL3Et6hURa4DuT9g7z6ukwpBLFchszg==} + engines: {node: '>=20'} + hasBin: true + '@ethereumjs/rlp@4.0.1': resolution: {integrity: sha512-tqsQiBQDQdmPWE1xkkBq4rlSW5QZpLOUJ5RJh2/9fug+q9tnUhuZoVLk7s0scUIKTOzEtR72DFBXI4WiZcMpvw==} engines: {node: '>=14'} hasBin: true - '@ethereumjs/rlp@5.0.2': - resolution: {integrity: sha512-DziebCdg4JpGlEqEdGgXmjqcFoJi+JGulUXwEjsZGAscAQ7MyD/7LE/GVCP29vEQxKc7AAwjT3A2ywHp2xfoCA==} - engines: {node: '>=18'} - hasBin: true - '@ethereumjs/statemanager@1.0.5': resolution: {integrity: sha512-TVkx9Kgc2NtObCzUTTqrpUggNLnftdmxZybzKPd565Bh98FJJB30FrVkWdPwaIV8oB1d9ADtthttfx5Y/kY9gw==} @@ -1991,17 +2032,20 @@ packages: '@fastify/error@4.0.0': resolution: {integrity: sha512-OO/SA8As24JtT1usTUTKgGH7uLvhfwZPwlptRi2Dp5P4KKmJI3gvsZ8MIHnNwDs4sLf/aai5LzTyl66xr7qMxA==} - '@fastify/fast-json-stringify-compiler@5.0.1': - resolution: {integrity: sha512-f2d3JExJgFE3UbdFcpPwqNUEoHWmt8pAKf8f+9YuLESdefA0WgqxeT6DrGL4Yrf/9ihXNSKOqpjEmurV405meA==} + '@fastify/error@4.2.0': + resolution: {integrity: sha512-RSo3sVDXfHskiBZKBPRgnQTtIqpi/7zhJOEmAxCiBcM7d0uwdGdxLlsCaLzGs8v8NnxIRlfG0N51p5yFaOentQ==} + + '@fastify/fast-json-stringify-compiler@5.0.3': + resolution: {integrity: sha512-uik7yYHkLr6fxd8hJSZ8c+xF4WafPK+XzneQDPU+D10r5X19GW8lJcom2YijX2+qtFF1ENJlHXKFM9ouXNJYgQ==} - '@fastify/forwarded@3.0.0': - resolution: {integrity: sha512-kJExsp4JCms7ipzg7SJ3y8DwmePaELHxKYtg+tZow+k0znUTf3cb+npgyqm8+ATZOdmfgfydIebPDWM172wfyA==} + '@fastify/forwarded@3.0.1': + resolution: {integrity: sha512-JqDochHFqXs3C3Ml3gOY58zM7OqO9ENqPo0UqAjAjH8L01fRZqwX9iLeX34//kiJubF7r2ZQHtBRU36vONbLlw==} - '@fastify/merge-json-schemas@0.1.1': - resolution: {integrity: sha512-fERDVz7topgNjtXsJTTW1JKLy0rhuLRcquYqNR9rF7OcVpCa2OVW49ZPDIhaRRCaUuvVxI+N416xUoF76HNSXA==} + '@fastify/merge-json-schemas@0.2.1': + resolution: {integrity: sha512-OA3KGBCy6KtIvLf8DINC5880o5iBlDX4SxzLQS8HorJAbqluzLRn80UXU0bxZn7UOFhFgpRJDasfwn9nG4FG4A==} - '@fastify/proxy-addr@5.0.0': - resolution: {integrity: sha512-37qVVA1qZ5sgH7KpHkkC4z9SK6StIsIcOmpjvMPXNb3vx2GQxhZocogVYbr2PbbeLCQxYIPDok307xEvRZOzGA==} + '@fastify/proxy-addr@5.1.0': + resolution: {integrity: sha512-INS+6gh91cLUjB+PVHfu1UqcB76Sqtpyp7bnL+FYojhjygvOPA9ctiD/JDKsyD9Xgu4hUhCSJBPig/w7duNajw==} '@fastify/send@3.1.1': resolution: {integrity: sha512-LdiV2mle/2tH8vh6GwGl0ubfUAgvY+9yF9oGI1iiwVyNUVOQamvw5n+OFu6iCNNoyuCY80FFURBn4TZCbTe8LA==} @@ -2111,8 +2155,8 @@ packages: '@jridgewell/trace-mapping@0.3.9': resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==} - '@leichtgewicht/ip-codec@2.0.4': - resolution: {integrity: sha512-Hcv+nVC0kZnQ3tD9GVu5xSMR4VVYOteQIr/hwFPVEvPdlXqgGEuRjiheChHgdM+JyqdgNcmzZOX/tnl0JOiI7A==} + '@leichtgewicht/ip-codec@2.0.5': + resolution: {integrity: sha512-Vo+PSpZG2/fmgmiNzYK9qWRh8h/CHrwD0mo1h1DzL4yzHNSfWYujGTYsWGreD000gcgmZ7K4Ys6Tx9TxtsKdDw==} '@lerna-lite/cli@4.10.3': resolution: {integrity: sha512-cRiBwXYuQNjHk0KnbMnfcj9T6ncpkr3iUP1EWwbaXQvDlGO6KzuGqm17LUng1tbR5KzqAex8ih3sZvz9paHFZQ==} @@ -2212,56 +2256,57 @@ packages: resolution: {integrity: sha512-hRFGB47RWHNKZkh9t+ka34kNlv5DQwrIDBvjI+mmzZqZK7UAtPRsJoE+MlQHVlQApDFwUPiSTdxY9UXzJrYFdQ==} engines: {node: ^20.17.0 || >=22.9.0} - '@libp2p/bootstrap@11.0.46': - resolution: {integrity: sha512-VxWHWjryu6YCv5JEylL5L9z2p38DvZPjEhH9jR382Ts19Dg0G+30DAcv7A3Ad1R2Bg8q2acxZGE3W0nwXrpJxQ==} + '@libp2p/bootstrap@12.0.14': + resolution: {integrity: sha512-kVg/t303ac6l7eSo5M1oZs72cs54FRRjYhmXwvHbPEz5v7NgglDpnNgz7ScBJGaS/z4Xn5qeyghVZFc8Qz7BaA==} - '@libp2p/crypto@5.1.7': - resolution: {integrity: sha512-7DO0piidLEKfCuNfS420BlHG0e2tH7W/zugdsPSiC/1Apa/s1B1dBkaIEgfDkGjrRP4S/8Or86Rtq7zXeEu67g==} + '@libp2p/crypto@5.1.13': + resolution: {integrity: sha512-8NN9cQP3jDn+p9+QE9ByiEoZ2lemDFf/unTgiKmS3JF93ph240EUVdbCyyEgOMfykzb0okTM4gzvwfx9osJebQ==} - '@libp2p/identify@3.0.38': - resolution: {integrity: sha512-2eCZU5CSTvyt78jtMLQ88m6Rlrcrt7BVccGHsdjx8hguvFYWqBjq19RQ5Zm47bbHlNzjLwrAUi6vfKx2r7fKIg==} + '@libp2p/gossipsub@15.0.15': + resolution: {integrity: sha512-uZLQj2EH0cDPX8rLgXDjJNL8sY1wYupGfvrY2oPUV/QsyZZQLqJ/a1BrYf1SjWCIqfqqGvyTtpBO8XvK4Ln8OA==} + engines: {npm: '>=8.7.0'} - '@libp2p/interface-internal@2.3.18': - resolution: {integrity: sha512-tnZ20IFASXLbDc2JxeUPZNIXDuN5Ge7be6BU458WLvmquf93NlSqZkWs6xFdi+0yXUrw7GGTgzIP5v+1LnDUmA==} + '@libp2p/identify@4.0.13': + resolution: {integrity: sha512-/zAhl2yMuQeHMyghJZDBRvQ1l3fRwxbsq3zPhT5nDscu9qVDa/CB4xDwquV4jV2Y/pnvefZtngJgj5c+bBIxug==} - '@libp2p/interface@2.10.5': - resolution: {integrity: sha512-Z52n04Mph/myGdwyExbFi5S/HqrmZ9JOmfLc2v4r2Cik3GRdw98vrGH19PFvvwjLwAjaqsweCtlGaBzAz09YDw==} + '@libp2p/interface-internal@3.0.13': + resolution: {integrity: sha512-qZTn1CKOro/1m8Eizb/B1pUvW/eJe5KhP/dvqKETqka26qH89eX5SlTS1OPTINXzJvfbnDFptVJOPxmpa3BfgA==} - '@libp2p/logger@5.1.21': - resolution: {integrity: sha512-V1TWlZM5BuKkiGQ7En4qOnseVP82JwDIpIfNjceUZz1ArL32A5HXJjLQnJchkZ3VW8PVciJzUos/vP6slhPY6Q==} + '@libp2p/interface@3.1.0': + resolution: {integrity: sha512-RE7/XyvC47fQBe1cHxhMvepYKa5bFCUyFrrpj8PuM0E7JtzxU7F+Du5j4VXbg2yLDcToe0+j8mB7jvwE2AThYw==} - '@libp2p/mdns@11.0.46': - resolution: {integrity: sha512-7E3kkdzXe6x+6lBc3ogniJJubaxz6SCAASi70Ne2NWZSMCOYaCjtGtnh5/Ddtel80ZJl92C/0nj9T4Gtgtr+Zg==} + '@libp2p/logger@6.2.2': + resolution: {integrity: sha512-XtanXDT+TuMuZoCK760HGV1AmJsZbwAw5AiRUxWDbsZPwAroYq64nb41AHRu9Gyc0TK9YD+p72+5+FIxbw0hzw==} - '@libp2p/mplex@11.0.46': - resolution: {integrity: sha512-01b077JuqqsDBUcSE68h+6kI1+/9HZM0eog6vM8EOuVttPwDhmyObpuUiBmBTTK8ggCWQrct2KWxR7+PHJDJIw==} + '@libp2p/mdns@12.0.14': + resolution: {integrity: sha512-tGbpZO7veBUQTOsIAHtXhFRLqGP3ek4BN+aE2f9yCVvuX//AmDX5HZLhjuAyd+lLabhaNv4+2MFeNqexC1P6/w==} - '@libp2p/multistream-select@6.0.28': - resolution: {integrity: sha512-ILu65FAX2Hak7x40DXb0gYptF6BmlGGW2kNgGeKIcNeseuvsAkBPO8k0CHwr8MU5mnHamTiweLJh5jD0iVZJ1A==} + '@libp2p/mplex@12.0.14': + resolution: {integrity: sha512-aPa1nEvxkWYAYmIQuE2FdMHrgHWgF3j7E/X82M87cT8irBFQRwWFCjiE/gsxw+Pec09uMubTbSROHHbPxjd6Wg==} - '@libp2p/peer-collections@6.0.34': - resolution: {integrity: sha512-rw8gDGhou4sF6W6i9ntmRARFePX19Dw9MMVpZHr6Kx9q2kvBJq91IXUzsXP06roexEOu1CUlZwxtUAqOBy+Eww==} + '@libp2p/multistream-select@7.0.13': + resolution: {integrity: sha512-nX13GinXiuBFgN+zA/CvIyXZyR/DaftT26agsw6dDfhRvH2RWsoPvf0IGqxk90DsLhpmVxZnTE31rITjmLIKww==} - '@libp2p/peer-id@5.1.8': - resolution: {integrity: sha512-pGaM4BwjnXdGtAtd84L4/wuABpsnFYE+AQ+h3GxNFme0IsTaTVKWd1jBBE5YFeKHBHGUOhF3TlHsdjFfjQA7TA==} + '@libp2p/peer-collections@7.0.13': + resolution: {integrity: sha512-SwNQFT0tfSyfbdUUKZFzHv9DXxsabuT99ch/40as8qC7xgoJJfUmhoa9FSuAuABdpTVHDJmxCI2pIbcb1kBqfg==} - '@libp2p/peer-record@8.0.34': - resolution: {integrity: sha512-GqvRBpvclscoKuF0JUfLyZTv+BwzICBBe50LFiAKio8LijZMBr43b+AcEaSEwFWDwlWmaKU73q8EQLrCb/e67Q==} + '@libp2p/peer-id@6.0.4': + resolution: {integrity: sha512-Z3xK0lwwKn4bPg3ozEpPr1HxsRi2CxZdghOL+MXoFah/8uhJJHxHFA8A/jxtKn4BB8xkk6F8R5vKNIS05yaCYw==} - '@libp2p/peer-store@11.2.6': - resolution: {integrity: sha512-3Lc982/7drqlXa51s9l1/DFHD48zzIjMMYajxFM2KbobyStH+lztYnFc3kNGB9sZijULaW1480PvbTMm9WaJ0g==} + '@libp2p/peer-record@9.0.5': + resolution: {integrity: sha512-disk23OO00yD52O4VmItbDkjJZ/YZJsKbMsqNgVhr+D3PcM+KRpu9VVbiCnN5Tzn9XvFEHhrMJY7BPE+rvT5MQ==} - '@libp2p/prometheus-metrics@4.3.29': - resolution: {integrity: sha512-TPRtOpDZNaeTko8uF7INtumE7gVNwuETPFlLP2C2BVbyp1SrpH8oJSxoioMzmAXRRwfY8esMxx3KUkdHd5BdaA==} + '@libp2p/peer-store@12.0.13': + resolution: {integrity: sha512-hXiIrXEUlXNDJe7i0O32qXRGmLA3ckoRHDjGZcNKMzPnkRDPkGEUQ42v1keA+1QoysMkm95xYyyhF6S3dA6nxg==} - '@libp2p/pubsub@10.1.17': - resolution: {integrity: sha512-hQMld/RiU7HbK81rGO///aNaM5yjBrEaULSo2m1pGB0Qy9SAhXdYJYgFP3ln+Pc18DwEU7wEjDIkE32CCYzpGA==} + '@libp2p/prometheus-metrics@5.0.14': + resolution: {integrity: sha512-v9IkdtvctmY+LVI6tzfjzGpLCEbLyncd4YH54l2ki+5w/U1oXPMTUbO01A94Zlywtt5lTKakQxxOnuliuRBfFA==} - '@libp2p/tcp@10.1.18': - resolution: {integrity: sha512-tB7rjETju5UMiYpyzAEOsX7qGVahJwaoBlsMAfpb35a8lH+KU0AaxBMMK8P4cFbKZZQSuDN86Is0mb5EG7npVA==} + '@libp2p/tcp@11.0.13': + resolution: {integrity: sha512-YTV6rX1NpQVbixYDlsWtIAHALBkW4vYdk4DPmHbFyvvQJdtBofbeHIQakyjPexKrbUtDGaoIXgT5KC2osLRp0g==} - '@libp2p/utils@6.7.1': - resolution: {integrity: sha512-x3WImvw4unmx1ZeAedj8AkRe4UImUlkw0ZItYAiKiekElMNUXwv+Yt48dI/LmB38JIof8sng29XvUeCVU3F6OA==} + '@libp2p/utils@7.0.13': + resolution: {integrity: sha512-BZAn+60HN8EDNwevWhiblopDXFpQ5Z5FnbML73btZKsJ9GGa4yQ2R18CTpWfsbHL8LUo21gTRC0XxUpYWq+UZg==} '@lukeed/ms@2.0.2': resolution: {integrity: sha512-9I2Zn6+NJLfaGoz9jN3lpwDgAYvfGeNYdbAIjJOqzs4Tpc+VU3Jqq4IofSUBKajiDS8k9fZIg18/z13mpk1bsA==} @@ -2280,17 +2325,14 @@ packages: '@microsoft/tsdoc@0.15.1': resolution: {integrity: sha512-4aErSrCR/On/e5G2hDP0wjooqDdauzEbIq8hIkIe5pXV0rtWJZvdCEKL0ykZxex+IxIwBp0eGeV48hQN07dXtw==} - '@multiformats/dns@1.0.6': - resolution: {integrity: sha512-nt/5UqjMPtyvkG9BQYdJ4GfLK3nMqGpFZOzf4hAmIa0sJh2LlS9YKXZ4FgwBDsaHvzZqR/rUFIywIc7pkHNNuw==} - - '@multiformats/mafmt@12.1.6': - resolution: {integrity: sha512-tlJRfL21X+AKn9b5i5VnaTD6bNttpSpcqwKVmDmSHLwxoz97fAHaepqFOk/l1fIu94nImIXneNbhsJx/RQNIww==} + '@multiformats/dns@1.0.13': + resolution: {integrity: sha512-yr4bxtA3MbvJ+2461kYIYMsiiZj/FIqKI64hE4SdvWJUdWF9EtZLar38juf20Sf5tguXKFUruluswAO6JsjS2w==} - '@multiformats/multiaddr-matcher@2.0.1': - resolution: {integrity: sha512-rhEYax74GlL0YPAZo61tK/hu0hwU7Hf9maXpoKNlpekmOPv+eckD+UlSeoTDCpGP8hQqMwLkX8e64MuDpy2Vlg==} + '@multiformats/multiaddr-matcher@3.0.1': + resolution: {integrity: sha512-jvjwzCPysVTQ53F4KqwmcqZw73BqHMk0UUZrMP9P4OtJ/YHrfs122ikTqhVA2upe0P/Qz9l8HVlhEifVYB2q9A==} - '@multiformats/multiaddr@12.5.1': - resolution: {integrity: sha512-+DDlr9LIRUS8KncI1TX/FfUn8F2dl6BIxJgshS/yFQCNB5IAF0OGzcwB39g5NLE22s4qqDePv0Qof6HdpJ/4aQ==} + '@multiformats/multiaddr@13.0.1': + resolution: {integrity: sha512-XToN915cnfr6Lr9EdGWakGJbPT0ghpg/850HvdC+zFX8XvpLZElwa8synCiwa8TuvKNnny6m8j8NVBNCxhIO3g==} '@napi-rs/snappy-android-arm-eabi@7.2.2': resolution: {integrity: sha512-H7DuVkPCK5BlAr1NfSU8bDEN7gYs+R78pSHhDng83QxRnCLmVIZk33ymmIwurmoA1HrdTxbkbuNl+lMvNqnytw==} @@ -2373,17 +2415,25 @@ packages: '@napi-rs/wasm-runtime@0.2.6': resolution: {integrity: sha512-z8YVS3XszxFTO73iwvFDNpQIzdMmSDTP/mB3E/ucR37V3Sx57hSExcXyMoNwaucWxnsWf4xfbZv0iZ30jr0M4Q==} - '@noble/ciphers@1.2.1': - resolution: {integrity: sha512-rONPWMC7PeExE077uLE4oqWrZ1IvAfz3oH9LibVAcVCopJiA9R62uavnbEzdkVmJYI6M6Zgkbeb07+tWjlq2XA==} + '@noble/ciphers@1.3.0': + resolution: {integrity: sha512-2I0gnIVPtfnMw9ee9h1dJG7tp81+8Ob3OJb3Mv37rx5L40/b0i7djjCVvGOVqc9AEIQyvyu1i6ypKdFw8R8gQw==} engines: {node: ^14.21.3 || >=16} + '@noble/ciphers@2.1.1': + resolution: {integrity: sha512-bysYuiVfhxNJuldNXlFEitTVdNnYUc+XNJZd7Qm2a5j1vZHgY+fazadNFWFaMK/2vye0JVlxV3gHmC0WDfAOQw==} + engines: {node: '>= 20.19.0'} + '@noble/curves@1.4.2': resolution: {integrity: sha512-TavHr8qycMChk8UwMld0ZDRvatedkzWfH8IiaeGCfymOP5i0hSCozz9vHOL0nkwk7HRMlFnAiKpS2jrUmSybcw==} - '@noble/curves@1.9.2': - resolution: {integrity: sha512-HxngEd2XUcg9xi20JkwlLCtYwfoFw4JGkuZpT+WlsPD4gB/cxkvTD8fSsoAnphGZhFdZYKeQIPCuFlWPm1uE0g==} + '@noble/curves@1.9.0': + resolution: {integrity: sha512-7YDlXiNMdO1YZeH6t/kvopHHbIZzlxrCV9WLqCY6QhcXOoXiNCMDqJIglZ9Yjx5+w7Dz30TITFrlTjnRg7sKEg==} engines: {node: ^14.21.3 || >=16} + '@noble/curves@2.0.1': + resolution: {integrity: sha512-vs1Az2OOTBiP4q0pwjW5aF0xp9n4MxVrmkFBxc6EKZc6ddYx5gaZiAsZoq0uRRXWbi3AT/sBqn05eRPtn1JCPw==} + engines: {node: '>= 20.19.0'} + '@noble/hashes@1.1.2': resolution: {integrity: sha512-KYRCASVTv6aeUi1tsF8/vpyR7zpfs3FUzy2Jqm+MU+LmUKhQ0y2FpfwqkCcxSg2ua4GALJd8k2R76WxwZGbQpA==} @@ -2395,11 +2445,15 @@ packages: resolution: {integrity: sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==} engines: {node: ^14.21.3 || >=16} + '@noble/hashes@2.0.1': + resolution: {integrity: sha512-XlOlEbQcE9fmuXxrVTXCTlG2nlRXa9Rj3rr5Ue/+tX+nmkgbX720YHh0VR3hBF9xDvwnb8D2shVGOwNx+ulArw==} + engines: {node: '>= 20.19.0'} + '@noble/secp256k1@1.7.1': resolution: {integrity: sha512-hOUk6AyBFmqVrv7k5WAw/LpszxVbj9gGN4JRkIX52fdFAj1UA61KXmZDvqVEm+pOyec3+fIeZB02LYa/pWOArw==} - '@noble/secp256k1@2.2.3': - resolution: {integrity: sha512-l7r5oEQym9Us7EAigzg30/PQAvynhMt2uoYtT3t26eGDVm9Yii5mZ5jWSWmZ/oSIR2Et0xfc6DXrG0bZ787V3w==} + '@noble/secp256k1@3.0.0': + resolution: {integrity: sha512-NJBaR352KyIvj3t6sgT/+7xrNyF9Xk9QlLSIqUGVUYlsnDTAUqY8LOmwpcgEx4AMJXRITQ5XEVHD+mMaPfr3mg==} '@node-rs/crc32-android-arm-eabi@1.10.6': resolution: {integrity: sha512-vZAMuJXm3TpWPOkkhxdrofWDv+Q+I2oO7ucLRbXyAPmXFNDhHtBxbO1rk9Qzz+M3eep8ieS4/+jCL1Q0zacNMQ==} @@ -2886,15 +2940,24 @@ packages: '@scure/base@1.1.8': resolution: {integrity: sha512-6CyAclxj3Nb0XT7GHK6K4zK6k2xJm6E4Ft0Ohjt4WgegiFUHEtFb2CGzmPmGBwoIhrLsqNLYfLr04Y1GePrzZg==} - '@scure/base@1.2.4': - resolution: {integrity: sha512-5Yy9czTO47mqz+/J8GM6GIId4umdCk1wc1q8rKERQulIoc8VP9pzDcghv10Tl2E7R96ZUx/PhND3ESYUQX8NuQ==} + '@scure/base@1.2.6': + resolution: {integrity: sha512-g/nm5FgUa//MCj1gV09zTJTaM6KBAHqLN907YVQqf7zC49+DcO4B1so4ZX07Ef10Twr6nuqYEH9GEggFXA4Fmg==} + + '@scure/base@2.0.0': + resolution: {integrity: sha512-3E1kpuZginKkek01ovG8krQ0Z44E3DHPjc5S2rjJw9lZn3KSQOs8S7wqikF/AH7iRanHypj85uGyxk0XAyC37w==} '@scure/bip32@1.4.0': resolution: {integrity: sha512-sVUpc0Vq3tXCkDGYVWGIZTRfnvu8LoTDaev7vbwh0omSvVORONr960MQWdKqJDCReIEmTj3PAr73O3aoxz7OPg==} + '@scure/bip32@1.7.0': + resolution: {integrity: sha512-E4FFX/N3f4B80AKWp5dP6ow+flD1LQZo/w8UnLGYZO674jS6YnYeepycOOksv+vLPSpgN35wgKgy+ybfTb2SMw==} + '@scure/bip39@1.3.0': resolution: {integrity: sha512-disdg7gHuTDZtY+ZdkmLpPCk7fxZSu3gBiEGuoC1XYxv9cGx3Z6cpTggCgW6odSOOIXCiDjuGejW+aJKCY/pIQ==} + '@scure/bip39@1.6.0': + resolution: {integrity: sha512-+lF0BbLiJNwVlev4eKelw1WWLaiKXw7sSl8T6FvBlWkdX+94aGJ4o8XjUdlyhTCjd8c+B3KT3JfS8P0bLRNU6A==} + '@sec-ant/readable-stream@0.4.1': resolution: {integrity: sha512-831qok9r2t8AlxLko40y2ebgSDhenenCatLVeW/uBtnHPyhHOvG0C7TvfgecV+wHzIm5KUICgzmVpWS+IMEAeg==} @@ -3041,18 +3104,18 @@ packages: resolution: {integrity: sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==} engines: {node: '>= 10'} - '@tsconfig/node10@1.0.8': - resolution: {integrity: sha512-6XFfSQmMgq0CFLY1MslA/CPUfhIL919M1rMsa5lP2P097N2Wd1sSX0tx1u4olM16fLNhtHZpRhedZJphNJqmZg==} - - '@tsconfig/node12@1.0.9': - resolution: {integrity: sha512-/yBMcem+fbvhSREH+s14YJi18sp7J9jpuhYByADT2rypfajMZZN4WQ6zBGgBKp53NKmqI36wFYDb3yaMPurITw==} - '@tsconfig/node14@1.0.1': resolution: {integrity: sha512-509r2+yARFfHHE7T6Puu2jjkoycftovhXRqW328PDXTVGKihlb1P8Z9mMZH04ebyajfRY7dedfGynlrFHJUQCg==} '@tsconfig/node16@1.0.2': resolution: {integrity: sha512-eZxlbI8GZscaGS7kkc/trHTT5xgrjH3/1n2JDwusC9iahPKWMRvRjJSAN5mCXviuTGQ/lHnhvv8Q1YTpnfz9gA==} + '@tsconfig/node18@18.2.6': + resolution: {integrity: sha512-eAWQzAjPj18tKnDzmWstz4OyWewLUNBm9tdoN9LayzoboRktYx3Enk1ZXPmThj55L7c4VWYq/Bzq0A51znZfhw==} + + '@tsconfig/node20@20.1.9': + resolution: {integrity: sha512-IjlTv1RsvnPtUcjTqtVsZExKVq+KQx4g5pCP5tI7rAs6Xesl2qFwSz/tPDBC4JajkL/MlezBu3gPUwqRHl+RIg==} + '@tufjs/canonical-json@2.0.0': resolution: {integrity: sha512-yVtV8zsdo8qFHe+/3kw81dSLyF7D576A5cCFCi4X7B39tWT7SekaEFUnvnWJHz+9qO7qJTah1JbrDjWKqFtdWA==} engines: {node: ^16.14.0 || >=18.0.0} @@ -3121,6 +3184,9 @@ packages: '@types/node@24.10.1': resolution: {integrity: sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ==} + '@types/node@25.4.0': + resolution: {integrity: sha512-9wLpoeWuBlcbBpOY3XmzSTG3oscB6xjBEEtn+pYXTfhyXhIxC5FsBer2KTopBlvKEiW9l13po9fq+SJY/5lkhw==} + '@types/normalize-package-data@2.4.4': resolution: {integrity: sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA==} @@ -3146,11 +3212,11 @@ packages: '@types/retry@0.12.2': resolution: {integrity: sha512-XISRgDJ2Tc5q4TRqvgJtzsRkFYNJzZrhTdtMoGVBttwzzQJkPnS3WWTFc7kuDRoPtPakl+T+OfdEUjYJj7Jbow==} - '@types/sinon@17.0.4': - resolution: {integrity: sha512-RHnIrhfPO3+tJT0s7cFaXGZvsL4bbR3/k7z3P312qMS4JaS2Tk+KiwiLx1S0rQ56ERj00u1/BtdyVd0FY+Pdew==} + '@types/sinon@20.0.0': + resolution: {integrity: sha512-etYGUC6IEevDGSWvR9WrECRA01ucR2/Oi9XMBUAdV0g4bLkNf4HlZWGiGlDOq5lgwXRwcV+PSeKgFcW4QzzYOg==} - '@types/sinonjs__fake-timers@6.0.2': - resolution: {integrity: sha512-dIPoZ3g5gcx9zZEszaxLSVTvMReD3xxyyDnQUjA6IYDG9Ba2AV0otMPs+77sG9ojB4Qr2N2Vk5RnKeuA0X/0bg==} + '@types/sinonjs__fake-timers@15.0.1': + resolution: {integrity: sha512-Ko2tjWJq8oozHzHV+reuvS5KYIRAokHnGbDwGh/J64LntgpbuylF74ipEL24HCyRjf9FOlBiBHWBR1RlVKsI1w==} '@types/through@0.0.30': resolution: {integrity: sha512-FvnCJljyxhPM3gkRgWmxmDZyAQSiBQQWLI0A0VFL0K7W1oRUrPJSqNO0NvTnLkBcotdlp3lKvaT0JrnyRDkzOg==} @@ -3173,8 +3239,47 @@ packages: '@types/yauzl@2.10.0': resolution: {integrity: sha512-Cn6WYCm0tXv8p6k+A8PvbDG763EDpBoTzHdA+Q/MF6H3sapGjCm9NzoaJncJS9tUKSuCoDs9XHxYYsQDgxR6kw==} - '@vekexasia/bigint-buffer2@1.1.0': - resolution: {integrity: sha512-CB19/UHoop2Q+HaRU1lI3fWKUkwju7XtwGTvdDfJgZyPHI+lLXKDYOOkz+NOCLfcBSXpJXpue/vLN0PDZtBT/Q==} + '@typescript/native-preview-darwin-arm64@7.0.0-dev.20260303.1': + resolution: {integrity: sha512-jIIQWmFi0bJY4ML8/7eyz1EGpkI6E0R1E5l4lxJdV/orpMr91vYfAajKICs7DUiMGEJX9HpeiA6TD2piw4DKPQ==} + cpu: [arm64] + os: [darwin] + + '@typescript/native-preview-darwin-x64@7.0.0-dev.20260303.1': + resolution: {integrity: sha512-UkaK+J3f185VXBiAGNG4UKHjGzn4R/nhAz5tArnCKHnIUI7rEnsIm4Xlo5YwmgIATFMU1sVwWUwRshVkMVeFAw==} + cpu: [x64] + os: [darwin] + + '@typescript/native-preview-linux-arm64@7.0.0-dev.20260303.1': + resolution: {integrity: sha512-ELJSV2Q4/mK+ampttssOl4H9s9ZBCc3k7y/u5ivJX8TdlMvZuH/JHqI6cS4Y00flt0R5wc70X+Nlcor4I4+rpw==} + cpu: [arm64] + os: [linux] + + '@typescript/native-preview-linux-arm@7.0.0-dev.20260303.1': + resolution: {integrity: sha512-H84jTRYqUfc/vhuVGQ6VKcBvJoZ4YmomWDx9U4uwYgW6eoUcRpDXqv3S3YqcNJcUmz22d/tTwIYz8ssXNLa/Qw==} + cpu: [arm] + os: [linux] + + '@typescript/native-preview-linux-x64@7.0.0-dev.20260303.1': + resolution: {integrity: sha512-HkddFrPJ0jcrohe+HnCqVTv8PunjqNs7FisRmtIAnc36+ccraDB6MVFEdPyAIL3PUID+TP/ESquqeKNnB7HdrQ==} + cpu: [x64] + os: [linux] + + '@typescript/native-preview-win32-arm64@7.0.0-dev.20260303.1': + resolution: {integrity: sha512-kTTMrBpWuxbHPt9hAFQSWeP//5Oa0KOdAEvceOfXUJhTS8RAA/kZSlFGE/Zw1EtrFLQx2J7uTHUZnYxH1hYXNw==} + cpu: [arm64] + os: [win32] + + '@typescript/native-preview-win32-x64@7.0.0-dev.20260303.1': + resolution: {integrity: sha512-UcVZbf4pra46Yx/eFV6m9F+awvihliPEud4Rq+A8Q3q3zI67VRaNH6R2/qeo4AqqKRahmiEdLM6Tnm+gPtLRQQ==} + cpu: [x64] + os: [win32] + + '@typescript/native-preview@7.0.0-dev.20260303.1': + resolution: {integrity: sha512-BDHJjXlPldInEogbzAc7OCLvT75p3rdkmb5YIA6Je0vjg+5z1UQp3moAvcBGvZQflO/gusOd9a74EfrMVUU/4g==} + hasBin: true + + '@vekexasia/bigint-buffer2@1.1.1': + resolution: {integrity: sha512-uL63NGANn85oKrsrvD0CzgmkuMlWQnGv/J7sz0QQ2CnRaF1nMF0fAoRFH9PFtGRr62ZCA14qkHZATyBZbKYlRQ==} engines: {node: '>= 14.0.0'} peerDependencies: '@vekexasia/bigint-uint8array': '*' @@ -3287,6 +3392,10 @@ packages: resolution: {integrity: sha512-eUP/6pbXBkMbXFdx4IH2fVgvB7M0JvR7/lIL33zcs0IBcwjdzSSl31TOJsaCzmKSSDF9h8QYSOJux4Nd4YJqFg==} engines: {node: '>=12'} + abstract-level@3.1.1: + resolution: {integrity: sha512-CW2gKbJFTuX1feMvOrvsVMmijAOgI9kg2Ie9Dq3gOcMt/dVVoVmqNlLcEUCT13NxHFMEajcUcVBIplbyDroDiw==} + engines: {node: '>=18'} + abstract-logging@2.0.1: resolution: {integrity: sha512-2BjRTZxTPvheOvGbBslFSYOUkr+SjPtOnrLP33f+VIWLzezQpZcqVg7ja3L4dBXmzzgwT+a029jRx5PCi3JuiA==} @@ -3381,6 +3490,10 @@ packages: resolution: {integrity: sha512-iADenERppdC+A2YKbOXXB2WUeABLaM6qnpZ70kZbPZ1cZMMJ7eF+3CaYm+/PhBizgkzlvssC7QuHS30oOiQYWA==} engines: {node: '>=16.0.0', npm: '>=7.0.0'} + any-signal@4.2.0: + resolution: {integrity: sha512-LndMvYuAPf4rC195lk7oSFuHOYFpOszIYrNYv0gHAvz+aEhE9qPZLhmrIz5pXP2BSsPOXvsuHDXEGaiQhIh9wA==} + engines: {node: '>=16.0.0', npm: '>=7.0.0'} + aproba@2.0.0: resolution: {integrity: sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==} @@ -3444,8 +3557,8 @@ packages: resolution: {integrity: sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==} engines: {node: '>= 0.4'} - avvio@9.0.0: - resolution: {integrity: sha512-UbYrOXgE/I+knFG+3kJr9AgC7uNo8DG+FGGODpH9Bj1O1kL/QDjBXnTem9leD3VdQKtaHjV3O85DQ7hHh4IIHw==} + avvio@9.2.0: + resolution: {integrity: sha512-2t/sy01ArdHHE0vRH5Hsay+RtCZt3dLPji7W7/MMOCEgze5b7SNDC4j5H6FnVgPkI1MTNFGzHdHrVXDDl7QSSQ==} aws-sdk@2.934.0: resolution: {integrity: sha512-k7p08ewrKcbs0ikCLFi9OI98Iv9dMND5244xPxUIjK5BLtuT/9Gr6eSCHfN70eCQyM5Y2xG1VJP6zhpkihu9Ew==} @@ -3516,6 +3629,9 @@ packages: browser-level@1.0.1: resolution: {integrity: sha512-XECYKJ+Dbzw0lbydyQuJzwNXtOpbMSq737qxJN11sIRTErOMShvDpbzTlgju7orJKvx4epULolZAuJGLzCmWRQ==} + browser-level@3.0.0: + resolution: {integrity: sha512-kGXtLh29jMwqKaskz5xeDLtCtN1KBz/DbQSqmvH7QdJiyGRC7RAM8PPg6gvUiNMa+wVnaxS9eSmEtP/f5ajOVw==} + browser-resolve@2.0.0: resolution: {integrity: sha512-7sWsQlYL2rGLy2IWm8WL8DCTJvYLc/qlOnsakDac87SOoCd16WLsaAMdCiAqsTNHIe+SXfaqyxyo6THoWqs8WQ==} @@ -3615,6 +3731,10 @@ packages: resolution: {integrity: sha512-K7Qy8O9p76sL3/3m7/zLKbRkyOlSZAgzEaLhyj2mXS8PsCud2Eo4hAb8aLtZqHh0QGqLcb9dlJSu6lHRVENm1w==} engines: {node: '>=6'} + cborg@4.5.8: + resolution: {integrity: sha512-6/viltD51JklRhq4L7jC3zgy6gryuG5xfZ3kzpE+PravtyeQLeQmCYLREhQH7pWENg5pY4Yu/XCd6a7dKScVlw==} + hasBin: true + chai@6.2.0: resolution: {integrity: sha512-aUTnJc/JipRzJrNADXVvpVqi6CO0dn3nx4EVPxijri+fj3LUUDyZQOgVeW54Ob3Y1Xh9Iz8f+CgaCl8v0mn9bA==} engines: {node: '>=18'} @@ -3654,6 +3774,10 @@ packages: resolution: {integrity: sha512-qGx/KJl3bvtOHrGau2WklEZuXhS3zme+jf+fsu6Ej7W7IP/C49v7KNlWIsT1jZu0YnfzSIYDGcEWpCa1wKGWXQ==} engines: {node: '>=12'} + classic-level@3.0.0: + resolution: {integrity: sha512-yGy8j8LjPbN0Bh3+ygmyYvrmskVita92pD/zCoalfcC9XxZj6iDtZTAnz+ot7GG8p9KLTG+MZ84tSA4AhkgVZQ==} + engines: {node: '>=18'} + clean-stack@2.2.0: resolution: {integrity: sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==} engines: {node: '>=6'} @@ -3807,9 +3931,9 @@ packages: engines: {node: '>=18'} hasBin: true - cookie@0.6.0: - resolution: {integrity: sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==} - engines: {node: '>= 0.6'} + cookie@1.1.1: + resolution: {integrity: sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==} + engines: {node: '>=18'} cookiejar@2.1.4: resolution: {integrity: sha512-LDx6oHrK+PhzLKJU9j5S7/Y3jM/mUHvD/DeI1WQmJn652iPC5Y4TBzC9l+5OMOXlyTTA+SmVUPm0HQUwpD5Jqw==} @@ -3878,14 +4002,14 @@ packages: resolution: {integrity: sha512-ZYP5VBHshaDAiVZxjbRVcFJpc+4xGgT0bK3vzy1HLN8jTO975HEbuYzZJcHoQEY5K1a0z8YayJkyVETa08eNTg==} engines: {node: '>=18'} - datastore-core@10.0.2: - resolution: {integrity: sha512-B3WXxI54VxJkpXxnYibiF17si3bLXE1XOjrJB7wM5co9fx2KOEkiePDGiCCEtnapFHTnmAnYCPdA7WZTIpdn/A==} + datastore-core@11.0.2: + resolution: {integrity: sha512-0pN4hMcaCWcnUBo5OL/8j14Lt1l/p1v2VvzryRYeJAKRLqnFrzy2FhAQ7y0yTA63ki760ImQHfm2XlZrfIdFpQ==} - datastore-fs@10.0.6: - resolution: {integrity: sha512-809BIs3wxJMg8/BYDwm734c06oKfrP5svD5CahPPHOIOOk9fSJc3j0OyP1yrb+oUk1OU/V5m1kmeO+eU0tHNCQ==} + datastore-fs@11.0.2: + resolution: {integrity: sha512-5jBSqcc0PzrraaNL1pcoB4nrVc+sNyD7E4axDQoTlkm07wE778rGPHhZNodN8nJEWrvnccOlzqc84w7SLyInKg==} - datastore-level@11.0.4: - resolution: {integrity: sha512-vwiOglxBXMhhEa2r5sh3iQrkrnK3HDYT/7V9dAb+/IvKotqHFsyio+tii9XNuWes1bj3yOCIQ/w8kdV3LZ/Glg==} + datastore-level@12.0.2: + resolution: {integrity: sha512-BMXRvFhDfx7CxlUj7XBAtPt6V2D9CWnKwRGTvZ3KQ4BBTJoXL66UDTxi52rzYLKfd2QppwaO2S+ZhLWhF5PbQQ==} de-indent@1.0.2: resolution: {integrity: sha512-e/1zu3xH5MQryN2zdVaF0OrdNLUbvWxzMbi+iNA6Bky7l1RoP8a2fIbRocyHclXt/arDrrR6lL3TqFD9pMQTsg==} @@ -3949,9 +4073,9 @@ packages: resolution: {integrity: sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==} engines: {node: '>= 0.4'} - delay@6.0.0: - resolution: {integrity: sha512-2NJozoOHQ4NuZuVIr5CWd0iiLVIRSDepakaovIN+9eIDHEhdCAEvSy2cuf1DCrPPQLvHmbqTHODlhHg8UCy4zw==} - engines: {node: '>=16'} + delay@7.0.0: + resolution: {integrity: sha512-C3vaGs818qzZjCvVJ98GQUMVyWeg7dr5w2Nwwb2t5K8G98jOyyVO2ti2bKYk5yoYElqH3F2yA53ykuEnwD6MCg==} + engines: {node: '>=20'} delayed-stream@1.0.0: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} @@ -3975,6 +4099,10 @@ packages: deprecation@2.3.1: resolution: {integrity: sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ==} + dequal@2.0.3: + resolution: {integrity: sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==} + engines: {node: '>=6'} + des.js@1.0.1: resolution: {integrity: sha512-Q0I4pfFrv2VPd34/vfLrFOoRmlYj3OV50i7fskps1jZWK1kApMWWT9G6RRUeYedLcBDIhnSDaUvJMb3AhUlaEA==} @@ -4140,6 +4268,10 @@ packages: ethereum-cryptography@2.2.1: resolution: {integrity: sha512-r/W8lkHSiTLxUxW8Rf3u4HGB0xQweG2RyETjywylKZSzLWoWAijRz8WCuOtJ6wah+avllXBqZuk29HCCvhEIRg==} + ethereum-cryptography@3.2.0: + resolution: {integrity: sha512-Urr5YVsalH+Jo0sYkTkv1MyI9bLYZwW8BENZCeE1QYaTHETEYx0Nv/SVsWkSqpYrzweg6d8KMY1wTjH/1m/BIg==} + engines: {node: ^14.21.3 || >=16, npm: '>=9'} + ethers@5.7.2: resolution: {integrity: sha512-wswUsmWo1aOK8rR7DIKiWSw9DbLWe6x98Jrn8wcTflTVvaXhAMaB5zGAXy0GYQEQp9iO1iSHWVyARQm11zUtyg==} @@ -4157,6 +4289,9 @@ packages: eventemitter3@5.0.1: resolution: {integrity: sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==} + eventemitter3@5.0.4: + resolution: {integrity: sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==} + events@1.1.1: resolution: {integrity: sha512-kEcvvCBByWXGnZy6JUlgAp2gBIUjfCAV6P6TgT1/aaQKcmuAEC4OZTV1I4EWQLz2gxZw76atuVyvHhTxvi0Flw==} engines: {node: '>=0.4.x'} @@ -4208,11 +4343,11 @@ packages: resolution: {integrity: sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==} engines: {node: '>=8.6.0'} - fast-json-stringify@6.0.0: - resolution: {integrity: sha512-FGMKZwniMTgZh7zQp9b6XnBVxUmKVahQLQeRQHqwYmPDqDhcEKZ3BaQsxelFFI5PY7nN71OEeiL47/zUWcYe1A==} + fast-json-stringify@6.3.0: + resolution: {integrity: sha512-oRCntNDY/329HJPlmdNLIdogNtt6Vyjb1WuT01Soss3slIdyUp8kAcDU3saQTOquEK8KFVfwIIF7FebxUAu+yA==} - fast-querystring@1.1.1: - resolution: {integrity: sha512-qR2r+e3HvhEFmpdHMv//U8FnFlnYjaC6QKDuaXALDkw2kvHO8WDjxH+f/rHGR4Me4pnk8p9JAkRNTjYHAKRn2Q==} + fast-querystring@1.1.2: + resolution: {integrity: sha512-g6KuKWmFXc0fID8WWH0jit4g0AGBoJhCkJMb1RmbsSEUNvQ+ZC8D6CUZ+GtF8nMzSPXnhiePyyqqipzNNEnHjg==} fast-safe-stringify@2.1.1: resolution: {integrity: sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==} @@ -4223,9 +4358,6 @@ packages: fast-string-width@3.0.2: resolution: {integrity: sha512-gX8LrtNEI5hq8DVUfRQMbr5lpaS4nMIWV+7XEbXk2b8kiQIizgnlr12B4dA3ZEx3308ze0O4Q1R+cHts8kyUJg==} - fast-uri@2.4.0: - resolution: {integrity: sha512-ypuAmmMKInk5q7XcepxlnUWDLWv4GFtaJqAzWKqn62IpQ3pejtr5dTVbt3vwqVaMKmkNR55sTT+CqUKIaT21BA==} - fast-uri@3.1.0: resolution: {integrity: sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==} @@ -4236,12 +4368,15 @@ packages: fastify-plugin@5.0.1: resolution: {integrity: sha512-HCxs+YnRaWzCl+cWRYFnHmeRFyR5GVnJTAaCJQiYzQSDwK9MgJdyAsuL3nh0EWRCYMgQ5MeziymvmAhUHYHDUQ==} - fastify@5.7.4: - resolution: {integrity: sha512-e6l5NsRdaEP8rdD8VR0ErJASeyaRbzXYpmkrpr2SuvuMq6Si3lvsaVy5C+7gLanEkvjpMDzBXWE5HPeb/hgTxA==} + fastify@5.8.1: + resolution: {integrity: sha512-y0kicFvvn7CYWoPOVLOcvn4YyKQz03DIY7UxmyOy21/J8eXm09R+tmb+tVDBW5h+pja30cHI5dqUcSlvY86V2A==} fastq@1.17.1: resolution: {integrity: sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w==} + fastq@1.20.1: + resolution: {integrity: sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==} + fd-package-json@2.0.0: resolution: {integrity: sha512-jKmm9YtsNXN789RS/0mSzOC1NUq9mkVd65vbSSVsKdjGvYXBuE4oWe2QOEoFeRmJg+lPuZxpmrfFclNhoRMneQ==} @@ -4275,9 +4410,9 @@ packages: resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} engines: {node: '>=8'} - find-my-way@9.0.1: - resolution: {integrity: sha512-/5NN/R0pFWuff16TMajeKt2JyiW+/OE8nOO8vo1DwZTxLaIURb7lcBYPIgRPh61yCNh9l8voeKwcrkUzmB00vw==} - engines: {node: '>=14'} + find-my-way@9.5.0: + resolution: {integrity: sha512-VW2RfnmscZO5KgBY5XVyKREMW5nMZcxDy+buTOsL+zIPnBlbKm+00sgzoQzq1EVh4aALZLfKdwv6atBGcjvjrQ==} + engines: {node: '>=20'} find-up@4.1.0: resolution: {integrity: sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==} @@ -4410,9 +4545,6 @@ packages: resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==} engines: {node: '>= 0.4'} - get-iterator@2.0.1: - resolution: {integrity: sha512-7HuY/hebu4gryTDT7O/XY/fvY9wRByEGdK6QOa4of8npTcv0+NS6frFKABcf6S9EBAsveTuKTsZQQBFMMNILIg==} - get-proto@1.0.1: resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} engines: {node: '>= 0.4'} @@ -4700,11 +4832,11 @@ packages: resolution: {integrity: sha512-3ygAIh8gcZavV9bj6MTdYddG2zPSYswP808fKS46NOwlF0zZljVpnLCHODDqItWJDbDpLb3aouAxGaJbkxoppA==} engines: {node: '>=14.18.0'} - interface-datastore@8.3.1: - resolution: {integrity: sha512-3r0ETmHIi6HmvM5sc09QQiCD3gUfwtEM/AAChOyAd/UAKT69uk8LXfTSUBufbUIO/dU65Vj8nb9O6QjwW8vDSQ==} + interface-datastore@9.0.2: + resolution: {integrity: sha512-jebn+GV/5LTDDoyicNIB4D9O0QszpPqT09Z/MpEWvf3RekjVKpXJCDguM5Au2fwIFxFDAQMZe5bSla0jMamCNg==} - interface-store@6.0.2: - resolution: {integrity: sha512-KSFCXtBlNoG0hzwNa0RmhHtrdhzexp+S+UY2s0rWTBJyfdEIgn6i6Zl9otVqrcFYbYrneBT7hbmHQ8gE0C3umA==} + interface-store@7.0.1: + resolution: {integrity: sha512-OPRRUO3Cs6Jr/t98BrJLQp1jUTPgrRH0PqFfuNoPAqd+J7ABN1tjFVjQdaOBiybYJTS/AyBSZnZVWLPvp3dW3w==} internal-slot@1.0.5: resolution: {integrity: sha512-Y+R5hJrzs52QCG2laLn4udYVnxsfny9CpOhNhUvk/SSSVyF6T27FzRbF0sroPidSu3X8oEAkOn2K804mjpt6UQ==} @@ -4717,8 +4849,8 @@ packages: ip@2.0.1: resolution: {integrity: sha512-lJUL9imLTNi1ZfXT+DU6rBBdbiKGBuay9B6xGSPVjUeQwaH1RIGqef8RZkUtHioLmSNpPR5M4HVKJGm1j8FWVQ==} - ipaddr.js@2.2.0: - resolution: {integrity: sha512-Ag3wB2o37wslZS19hZqorUnrnzSkpOVy+IiiDEiTqNubEYpYuHWIf6K4psgN2ZWKExS4xhVCrRVfb/wfW8fWJA==} + ipaddr.js@2.3.0: + resolution: {integrity: sha512-Zv/pA+ciVFbCSBBjGfaKUya/CcGmUHzTydLMaTwrUUEM2DIEO3iZvueGxmacvmN50fGpGVKeTXpb2LcYQxeVdg==} engines: {node: '>= 10'} is-arguments@1.1.0: @@ -4762,8 +4894,8 @@ packages: engines: {node: '>=8'} hasBin: true - is-electron@2.2.0: - resolution: {integrity: sha512-SpMppC2XR3YdxSzczXReBjqs2zGscWQpBIKqwXYBFic0ERaxNVgwLCHwOLZeESfdJQjX0RDvrJ1lBXX2ij+G1Q==} + is-electron@2.2.2: + resolution: {integrity: sha512-FO/Rhvz5tuw4MCWkpMzHFKWD2LsfHzIb7i6MdPYZ/KW7AlxawyLkqdy+jPZP1WubqEADE3O4FUENlJHDfQASRg==} is-extglob@2.1.1: resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==} @@ -4805,8 +4937,8 @@ packages: resolution: {integrity: sha512-dqJvarLawXsFbNDeJW7zAz8ItJ9cd28YufuuFzh0G8pNHjJMnY08Dv7sYX2uF5UpQOwieAeOExEYAWWfu7ZZUA==} engines: {node: '>= 0.4'} - is-network-error@1.1.0: - resolution: {integrity: sha512-tUdRRAnhT+OtCZR/LxZelH/C7QtjtFrTu5tXCA8pl55eTUElUHT+GPYV8MBMBvea/j+NxQqVt3LbWMRir7Gx9g==} + is-network-error@1.3.1: + resolution: {integrity: sha512-6QCxa49rQbmUWLfk0nuGqzql9U8uaV2H6279bRErPBHe/109hCzsLUBUHfbEtvLIHBd6hyXbgedBSHevm43Edw==} engines: {node: '>=16'} is-number-object@1.0.7: @@ -4924,42 +5056,25 @@ packages: it-batch@3.0.9: resolution: {integrity: sha512-z6p89Q8gm2urBtF3JcpnbJogacijWk3m1uc3xZYI3x0eJUoYLUbgF8IxJ2fnuVObV7yRv3SixfwGCufaZY1NCg==} - it-byte-stream@2.0.3: - resolution: {integrity: sha512-h7FFcn4DWiWsJw1dCJhuPdiY8cGi1z8g4aLAfFspTaJbwQxvEMlEBFG/f8lIVGwM8YK26ClM4/9lxLVhF33b8g==} - it-drain@3.0.10: resolution: {integrity: sha512-0w/bXzudlyKIyD1+rl0xUKTI7k4cshcS43LTlBiGFxI8K1eyLydNPxGcsVLsFVtKh1/ieS8AnVWt6KwmozxyEA==} it-filter@3.1.4: resolution: {integrity: sha512-80kWEKgiFEa4fEYD3mwf2uygo1dTQ5Y5midKtL89iXyjinruA/sNXl6iFkTcdNedydjvIsFhWLiqRPQP4fAwWQ==} - it-foreach@2.1.4: - resolution: {integrity: sha512-gFntBbNLpVK9uDmaHusugICD8/Pp+OCqbF5q1Z8K+B8WaG20YgMePWbMxI1I25+JmNWWr3hk0ecKyiI9pOLgeA==} - it-glob@3.0.4: resolution: {integrity: sha512-73PbGBTK/dHp5PX4l8pkQH1ozCONP0U+PB3qMqltxPonRJQNomINE3Hn9p02m2GOu95VoeVvSZdHI2N+qub0pw==} - it-length-prefixed-stream@2.0.3: - resolution: {integrity: sha512-Ns3jNFy2mcFnV59llCYitJnFHapg8wIcOsWkEaAwOkG9v4HBCk24nze/zGDQjiJdDTyFXTT5GOY3M/uaksot3w==} - it-length-prefixed@10.0.1: resolution: {integrity: sha512-BhyluvGps26u9a7eQIpOI1YN7mFgi8lFwmiPi07whewbBARKAG9LE09Odc8s1Wtbt2MB6rNUrl7j9vvfXTJwdQ==} engines: {node: '>=16.0.0', npm: '>=7.0.0'} - it-length-prefixed@9.1.1: - resolution: {integrity: sha512-O88nBweT6M9ozsmok68/auKH7ik/slNM4pYbM9lrfy2z5QnpokW5SlrepHZDKtN71llhG2sZvd6uY4SAl+lAQg==} - engines: {node: '>=16.0.0', npm: '>=7.0.0'} - it-map@3.1.4: resolution: {integrity: sha512-QB9PYQdE9fUfpVFYfSxBIyvKynUCgblb143c+ktTK6ZuKSKkp7iH58uYFzagqcJ5HcqIfn1xbfaralHWam+3fg==} it-merge@3.0.12: resolution: {integrity: sha512-nnnFSUxKlkZVZD7c0jYw6rDxCcAQYcMsFj27thf7KkDhpj0EA0g9KHPxbFzHuDoc6US2EPS/MtplkNj8sbCx4Q==} - it-pair@2.0.6: - resolution: {integrity: sha512-5M0t5RAcYEQYNG5BV7d7cqbdwbCAp5yLdzvkxsZmkuZsLbTdZzah6MQySYfaAQjNDCq6PUnDt0hqBZ4NwMfW6g==} - engines: {node: '>=16.0.0', npm: '>=7.0.0'} - it-parallel-batch@3.0.9: resolution: {integrity: sha512-TszXWqqLG8IG5DUEnC4cgH9aZI6CsGS7sdkXTiiacMIj913bFy7+ohU3IqsFURCcZkpnXtNLNzrYnXISsKBhbQ==} @@ -4974,17 +5089,14 @@ packages: resolution: {integrity: sha512-sIoNrQl1qSRg2seYSBH/3QxWhJFn9PKYvOf/bHdtCBF0bnghey44VyASsWzn5dAx0DCDDABq1hZIuzKmtBZmKA==} engines: {node: '>=16.0.0', npm: '>=7.0.0'} - it-protobuf-stream@2.0.3: - resolution: {integrity: sha512-Dus9qyylOSnC7l75/3qs6j3Fe9MCM2K5luXi9o175DYijFRne5FPucdOGIYdwaDBDQ4Oy34dNCuFobOpcusvEQ==} - it-pushable@3.2.3: resolution: {integrity: sha512-gzYnXYK8Y5t5b/BnJUr7glfQLO4U5vyb05gPx/TyTw+4Bv1zM9gFk4YsOrnulWefMewlphCjKkakFvj1y99Tcg==} - it-queue@1.1.0: - resolution: {integrity: sha512-aK9unJRIaJc9qiv53LByhF7/I2AuD7Ro4oLfLieVLL9QXNvRx++ANMpv8yCp2UO0KAtBuf70GOxSYb6ElFVRpQ==} + it-queue@1.1.1: + resolution: {integrity: sha512-yeYCV22WF1QDyb3ylw+g3TGEdkmnoHUH2mc12QoGOQuxW4XP1V7Zd3BfsEF1iq2IFBwIK7wCPUcRLTAQVeZ3SQ==} - it-queueless-pushable@2.0.2: - resolution: {integrity: sha512-2BqIt7XvDdgEgudLAdJkdseAwbVSBc0yAd8yPVHrll4eBuJPWIj9+8C3OIxzEKwhswLtd3bi+yLrzgw9gCyxMA==} + it-queueless-pushable@2.0.3: + resolution: {integrity: sha512-USa5EzTvmQswOcVE7+o6qsj2o2G+6KHCxSogPOs23sGYkDWFidhqVO7dAvv6ve/Z+Q+nvxpEa9rrRo6VEK7w4Q==} it-reader@6.0.4: resolution: {integrity: sha512-XCWifEcNFFjjBHtor4Sfaj8rcpt+FkY0L6WdhD578SCDhV4VUm7fCkF3dv5a+fTcfQqvN9BsxBTvWbYO6iCjTg==} @@ -5048,8 +5160,8 @@ packages: resolution: {integrity: sha512-ZF1nxZ28VhQouRWhUcVlUIN3qwSgPuswK05s/HIaoetAoE/9tngVmCHjSxmSQPav1nd+lPtTL0YZ/2AFdR/iYQ==} engines: {node: ^20.17.0 || >=22.9.0} - json-schema-ref-resolver@1.0.1: - resolution: {integrity: sha512-EJAj1pgHc1hxF6vo2Z3s69fMjO1INq6eGHXZ8Z6wCQeldCuwxGK9Sxf4/cScGn3FZubCVUehfWtcDM/PLteCQw==} + json-schema-ref-resolver@3.0.0: + resolution: {integrity: sha512-hOrZIVL5jyYFjzk7+y7n5JDzGlU8rfWDuYyHwGa2WA8/pcmMHezp2xsVwxrebD/Q9t8Nc5DboieySDpCp4WG4A==} json-schema-resolver@2.0.0: resolution: {integrity: sha512-pJ4XLQP4Q9HTxl6RVDLJ8Cyh1uitSs0CzDBAz1uoJ4sRD/Bk7cFSXL1FUXDW3zJ7YnfliJx6eu8Jn283bpZ4Yg==} @@ -5106,10 +5218,18 @@ packages: resolution: {integrity: sha512-PbXpve8rKeNcZ9C1mUicC9auIYFyGpkV9/i6g76tLgANwWhtG2v7I4xNBUlkn3lE2/dZF3Pi0ygYGtLc4RXXdA==} engines: {node: '>=12'} + level-supports@6.2.0: + resolution: {integrity: sha512-QNxVXP0IRnBmMsJIh+sb2kwNCYcKciQZJEt+L1hPCHrKNELllXhvrlClVHXBYZVT+a7aTSM6StgNXdAldoab3w==} + engines: {node: '>=16'} + level-transcoder@1.0.1: resolution: {integrity: sha512-t7bFwFtsQeD8cl8NIoQ2iwxA0CL/9IFw7/9gAjOonH0PWTTiRfY7Hq+Ejbsxh86tXobDQ6IOiddjNYIfOBs06w==} engines: {node: '>=12'} + level@10.0.0: + resolution: {integrity: sha512-aZJvdfRr/f0VBbSRF5C81FHON47ZsC2TkGxbBezXpGGXAUEL/s6+GP73nnhAYRSCIqUNsmJjfeOF4lzRDKbUig==} + engines: {node: '>=18'} + level@8.0.1: resolution: {integrity: sha512-oPBGkheysuw7DmzFQYyFe8NAia5jFLAgEnkgWnK3OXAuJr8qFT+xBQIwokAZPME2bhPFzS8hlYcL16m8UZrtwQ==} engines: {node: '>=12'} @@ -5122,11 +5242,11 @@ packages: resolution: {integrity: sha512-NVPTth/71cfbdYHqypcO9Lt5WFGTzFEcx81lWd7GDJIgZ95ERdYHGUfCtFejHCyqodKsQkNEx2JCkMpreDty/A==} engines: {node: ^20.17.0 || >=22.9.0} - libp2p@2.9.0: - resolution: {integrity: sha512-gzRnhLY+k9KjYifWQCYbdEfmWqCFdM0TZ5Q7qqdY13sAUKXixK0MF5+Z9LMrm5ELGDPWX7pRVLGK8BOSv5/v3Q==} + libp2p@3.1.6: + resolution: {integrity: sha512-p1Tg8htMjQbbyNOQd5GtSsZJXKkJQYQBvRrPGMCa3PZBjGs2pNV4Utr7z0na+WgfJJn+mIbcNvP7NzzcrSD1nw==} - light-my-request@6.0.0: - resolution: {integrity: sha512-kFkFXrmKCL0EEeOmJybMH5amWFd+AFvlvMlvFTRxCUwbhfapZqDmeLMPoWihntnYY6JpoQDE9k+vOzObF1fDqg==} + light-my-request@6.6.0: + resolution: {integrity: sha512-CHYbu8RtboSIoVsHZ6Ye4cj4Aw/yg2oAFimlF7mNvfDV192LR7nDiKtSIfCuLT7KokPSTn/9kfVLm5OGN0A28A==} lilconfig@3.1.3: resolution: {integrity: sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==} @@ -5196,14 +5316,14 @@ packages: lru-cache@10.4.3: resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} - lru-cache@11.0.1: - resolution: {integrity: sha512-CgeuL5uom6j/ZVrg7G/+1IXqRY8JXX4Hghfy5YE0EhoYQWvndP1kufu58cmZLNIDKnRhZrXfdS9urVWx98AipQ==} - engines: {node: 20 || >=22} - lru-cache@11.2.4: resolution: {integrity: sha512-B5Y16Jr9LB9dHVkh6ZevG+vAbOsNOYCX+sXvFWFu7B3Iz5mijW3zdbMyhsh8ANd2mSWBYdJgnqi+mL7/LrOPYg==} engines: {node: 20 || >=22} + lru-cache@11.2.5: + resolution: {integrity: sha512-vFrFJkWtJvJnD5hg+hJvVE8Lh/TcMzKnTgCWmtBipwI5yLX/iX+5UB2tfuyODF5E7k9xEzMdYgGqaSb1c0c5Yw==} + engines: {node: 20 || >=22} + lru-cache@5.1.1: resolution: {integrity: sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==} @@ -5247,6 +5367,10 @@ packages: resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} engines: {node: '>= 0.4'} + maybe-combine-errors@1.0.0: + resolution: {integrity: sha512-eefp6IduNPT6fVdwPp+1NgD0PML1NU5P6j1Mj5nz1nidX8/sWY7119WL8vTAHgqfsY74TzW0w1XPgdYEKkGZ5A==} + engines: {node: '>=10'} + mcl-wasm@0.7.9: resolution: {integrity: sha512-iJIUcQWA88IJB/5L15GnJVnSQJmf/YaxxV6zRavv83HILHaJQb6y0iFyDMdDO0gN8X37tdxmAOrH/P8B6RB8sQ==} engines: {node: '>=8.9.0'} @@ -5430,9 +5554,9 @@ packages: ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} - ms@3.0.0-canary.1: - resolution: {integrity: sha512-kh8ARjh8rMN7Du2igDRO9QJnqCb2xYTJxyQYK7vJJS4TvLLmsbyhiKpSW+t+y26gyOyMd0riphX0GeWKU3ky5g==} - engines: {node: '>=12.13'} + ms@3.0.0-canary.202508261828: + resolution: {integrity: sha512-NotsCoUCIUkojWCzQff4ttdCfIPoA1UGZsyQbi7KmqkNRfKCrvga8JJi2PknHymHOuor0cJSn/ylj52Cbt2IrQ==} + engines: {node: '>=18'} muggle-string@0.4.1: resolution: {integrity: sha512-VNTrAak/KhO2i8dqqnqnAHOa3cYBwXEZe9h+D5h/1ZqFSTEFHdM65lR7RoIqq3tBBYavsOXV84NoHXZ0AkPyqQ==} @@ -5441,12 +5565,8 @@ packages: resolution: {integrity: sha512-2eznPJP8z2BFLX50tf0LuODrpINqP1RVIm/CObbTcBRITQgmC/TjcREF1NeTBzIcR5XO/ukWo+YHOjBbFwIupg==} hasBin: true - multiformats@11.0.2: - resolution: {integrity: sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==} - engines: {node: '>=16.0.0', npm: '>=7.0.0'} - - multiformats@13.3.7: - resolution: {integrity: sha512-meL9DERHj+fFVWoOX9fXqfcYcSpUfSYJPcFvDPKrxitICbwAoWR+Ut4j5NO9zAT917HUHLQmqzQbAsGNHlDcxQ==} + multiformats@13.4.2: + resolution: {integrity: sha512-eh6eHCrRi1+POZ3dA+Dq1C6jhP1GNtr9CRINMb67OKzqW9I5DUuZM/3jLPlzhgpGeiNUlEGEbkCYChXMCc/8DQ==} mute-stream@1.0.0: resolution: {integrity: sha512-avsJQhyd+680gKXyG/sQc0nXaC6rBkPOfyHYcFb9+hdkqQkR9bdnkJ0AMZhke0oesPqIO+mFFJ+IdBc7mst4IA==} @@ -5623,8 +5743,9 @@ packages: observable-fns@0.6.1: resolution: {integrity: sha512-9gRK4+sRWzeN6AOewNBTLXir7Zl/i3GB6Yl26gK4flxz8BXVpD3kt8amREmWNb0mxYOGDotvE5a4N+PtGGKdkg==} - on-exit-leak-free@2.1.0: - resolution: {integrity: sha512-VuCaZZAjReZ3vUwgOB8LxAosIurDiAW0s13rI1YwmaP++jvcxP77AWoQvenZebpCA2m8WC1/EosPYPMjnRAp/w==} + on-exit-leak-free@2.1.2: + resolution: {integrity: sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==} + engines: {node: '>=14.0.0'} once@1.4.0: resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} @@ -5662,9 +5783,9 @@ packages: resolution: {integrity: sha512-Mr5KC5efvAK5VUptYEIopP1bakB85k2IWXaRC0rsh1uwn1L6M0LVml8OIQ4Gudg4oyZakf7FmeRLkMMtZW1i5A==} engines: {node: '>=12'} - p-event@6.0.1: - resolution: {integrity: sha512-Q6Bekk5wpzW5qIyUP4gdMEujObYstZl6DMMOSenwBvV0BlE5LkDwkjs5yHbZmdCEq2o4RJx4tE1vwxFVf2FG1w==} - engines: {node: '>=16.17'} + p-event@7.1.0: + resolution: {integrity: sha512-/lkPs5W1aC3cp6vqZefpdosOn65J571sWodyfOQiF0+tmDCpU+H8Atwpu0vQROCVUlZuToDN5eyTLsMLLc54mg==} + engines: {node: '>=20'} p-limit@2.3.0: resolution: {integrity: sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==} @@ -5706,25 +5827,21 @@ packages: resolution: {integrity: sha512-HkPfFklpZQPUKBFXzKFB6ihLriIHxnmuQdK9WmLDwe4hf2PdhhfWT/FJa+pc3bA1ywvKXtedxIRmd4Y7BTXE4w==} engines: {node: '>=12'} - p-queue@8.1.0: - resolution: {integrity: sha512-mxLDbbGIBEXTJL0zEx8JIylaj3xQ7Z/7eEVjcF9fJX4DBiH9oqe+oahYnlKKxm0Ci9TlWTyhSHgygxMxjIB2jw==} - engines: {node: '>=18'} - p-queue@9.0.1: resolution: {integrity: sha512-RhBdVhSwJb7Ocn3e8ULk4NMwBEuOxe+1zcgphUy9c2e5aR/xbEsdVXxHJ3lynw6Qiqu7OINEyHlZkiblEpaq7w==} engines: {node: '>=20'} + p-queue@9.1.0: + resolution: {integrity: sha512-O/ZPaXuQV29uSLbxWBGGZO1mCQXV2BLIwUr59JUU9SoH76mnYvtms7aafH/isNSNGwuEfP6W/4xD0/TJXxrizw==} + engines: {node: '>=20'} + p-reduce@3.0.0: resolution: {integrity: sha512-xsrIUgI0Kn6iyDYm9StOpOeK29XM1aboGji26+QEortiFST1hGZaUQOLhtEbqHErPpGW/aSz6allwK2qcptp0Q==} engines: {node: '>=12'} - p-retry@6.2.1: - resolution: {integrity: sha512-hEt02O4hUct5wtwg4H4KcWgDdm+l1bOaEy/hWzd8xtXB9BqxTWBBhb+2ImAtH4Cv4rPjV76xN3Zumqk3k3AhhQ==} - engines: {node: '>=16.17'} - - p-timeout@6.1.2: - resolution: {integrity: sha512-UbD77BuZ9Bc9aABo74gfXhNvzC9Tx7SxtHSh1fxvx3jTLLYvmVhiQZZrJzqqU0jKbN32kb5VOKiLEQI/3bIjgQ==} - engines: {node: '>=14.16'} + p-retry@7.1.1: + resolution: {integrity: sha512-J5ApzjyRkkf601HpEeykoiCvzHQjWxPAHhyjFcEUP2SWq0+35NKh8TLhpLw+Dkq5TZBFvUM6UigdE9hIVYTl5w==} + engines: {node: '>=20'} p-timeout@7.0.1: resolution: {integrity: sha512-AxTM2wDGORHGEkPCt8yqxOTMgpfbEHqF51f/5fJCmwFC3C/zNcGT63SymH2ttOAaiIws2zVg4+izQCjrakcwHg==} @@ -5868,8 +5985,8 @@ packages: pino-std-serializers@7.1.0: resolution: {integrity: sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw==} - pino@10.3.0: - resolution: {integrity: sha512-0GNPNzHXBKw6U/InGe79A3Crzyk9bcSyObF9/Gfo9DLEf5qj5RF50RSjsu0W1rZ6ZqRGdzDFCRBQvi9/rSGPtA==} + pino@10.3.1: + resolution: {integrity: sha512-r34yH/GlQpKZbU1BvFFqOjhISRo1MNx1tWYsYvmj6KIRHSPMT2+yHOEb1SG6NMvRoHRF0a07kCOox/9yakl1vg==} hasBin: true pixelmatch@7.1.0: @@ -5895,21 +6012,11 @@ packages: engines: {node: '>=18'} hasBin: true - playwright-core@1.58.2: - resolution: {integrity: sha512-yZkEtftgwS8CsfYo7nm0KE8jsvm6i/PTgVtB8DL726wNf6H2IMsDuxCpJj59KDaxCtSnrWan2AeDqM7JBaultg==} - engines: {node: '>=18'} - hasBin: true - playwright@1.56.1: resolution: {integrity: sha512-aFi5B0WovBHTEvpM3DzXTUaeN6eN0qWnTkKx4NQaH4Wvcmc153PdaY2UBdSYKaGYw+UyWXSVyxDUg5DoPEttjw==} engines: {node: '>=18'} hasBin: true - playwright@1.58.2: - resolution: {integrity: sha512-vA30H8Nvkq/cPBnNw4Q8TWz1EJyqgpuinBcHET0YVJVFldr8JDNiU9LaWAE1KqSkRYazuaBhTpB5ZzShOezQ6A==} - engines: {node: '>=18'} - hasBin: true - pngjs@7.0.0: resolution: {integrity: sha512-LKWqWJRhstyYo9pGvgor/ivk2w94eSjE3RGVuzLGlr3NmD8bf7RcYGze1mNdEHRP6TRP6rMuDHk5t44hnTRyow==} engines: {node: '>=14.19.0'} @@ -5955,8 +6062,8 @@ packages: process-nextick-args@2.0.1: resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==} - process-warning@4.0.0: - resolution: {integrity: sha512-/MyYDxttz7DfGMMHiysAsFE4qF+pQYAA8ziO/3NcRVrQ5fSk+Mns4QZA/oRPFzvcqNoVJXQNWNAsdwBXLUkQKw==} + process-warning@4.0.1: + resolution: {integrity: sha512-3c2LzQ3rY9d0hc1emcsHhfT9Jwz0cChib/QN89oME2R451w5fy3f0afAhERFZAwrbDU43wk12d0ORBpDVME50Q==} process-warning@5.0.0: resolution: {integrity: sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA==} @@ -6007,8 +6114,8 @@ packages: protocols@2.0.1: resolution: {integrity: sha512-/XJ368cyBJ7fzLMwLKv1e4vLxOju2MNAIokcr7meSaNcVbWz/CPcW22cP04mwxOErdA5mwjA8Q6w/cdAQxVn7Q==} - protons-runtime@5.5.0: - resolution: {integrity: sha512-EsALjF9QsrEk6gbCx3lmfHxVN0ah7nG3cY7GySD4xf4g8cr7g543zB88Foh897Sr1RQJ9yDCUsoT1i1H/cVUFA==} + protons-runtime@5.6.0: + resolution: {integrity: sha512-/Kde+sB9DsMFrddJT/UZWe6XqvL7SL5dbag/DBCElFKhkwDj7XKt53S+mzLyaDP5OqS0wXjV5SA572uWDaT0Hg==} proxy-from-env@1.1.0: resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} @@ -6054,15 +6161,15 @@ packages: queue-microtask@1.2.3: resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} - quick-format-unescaped@4.0.3: - resolution: {integrity: sha512-MaL/oqh02mhEo5m5J2rwsVL23Iw2PEaGVHgT2vFt8AAsr0lfvQA5dpXo9TPu0rz7tSBdUPgkbam0j/fj5ZM8yg==} + quick-format-unescaped@4.0.4: + resolution: {integrity: sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==} quick-lru@5.1.1: resolution: {integrity: sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==} engines: {node: '>=10'} - race-event@1.3.0: - resolution: {integrity: sha512-kaLm7axfOnahIqD3jQ4l1e471FIFcEGebXEnhxyLscuUzV8C94xVHtWEqDDXxll7+yu/6lW0w1Ff4HbtvHvOHg==} + race-event@1.6.1: + resolution: {integrity: sha512-vi7WH5g5KoTFpu2mme/HqZiWH14XSOtg5rfp6raBskBHl7wnmy3F/biAIyY5MsK+BHWhoPhxtZ1Y2R7OHHaWyQ==} race-signal@1.1.3: resolution: {integrity: sha512-Mt2NznMgepLfORijhQMncE26IhkmjEphig+/1fKC0OtaKwys/gpvpmswSjoN01SS+VO951mj0L4VIDXdXsjnfA==} @@ -6070,6 +6177,10 @@ packages: race-signal@2.0.0: resolution: {integrity: sha512-P31bLhE4ByBX/70QDXMutxnqgwrF1WUXea1O8DXuviAgkdbQ1iQMQotNgzJIBC9yUSn08u/acZrMUhgw7w6GpA==} + random-int@3.1.0: + resolution: {integrity: sha512-h8CRz8cpvzj0hC/iH/1Gapgcl2TQ6xtnCpyOI5WvWfXf/yrDx2DOU+tD9rX23j36IF11xg1KqB9W11Z18JPMdw==} + engines: {node: '>=12'} + randombytes@2.1.0: resolution: {integrity: sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==} @@ -6151,14 +6262,14 @@ packages: resolution: {integrity: sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==} engines: {node: '>= 4'} - retry@0.13.1: - resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==} - engines: {node: '>= 4'} - reusify@1.0.4: resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==} engines: {iojs: '>=1.0.0', node: '>=0.10.0'} + reusify@1.1.0: + resolution: {integrity: sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==} + engines: {iojs: '>=1.0.0', node: '>=0.10.0'} + rewiremock@3.14.5: resolution: {integrity: sha512-MdPutvaUd+kKVz/lcEz6N6337s4PxRUR5vhphIp2/TJRgfXIckomIkCsIAbwB53MjiSLwi7KBMdQ9lPWE5WpYA==} @@ -6229,13 +6340,17 @@ packages: safe-regex-test@1.0.0: resolution: {integrity: sha512-JBUUzyOgEwXQY1NuPtvcj/qcBDbDmEvWufhlnXZIm75DEHp+afM1r1ujJpJsV/gSM4t59tpDyPi1sd6ZaPFfsA==} - safe-regex2@4.0.0: - resolution: {integrity: sha512-Hvjfv25jPDVr3U+4LDzBuZPPOymELG3PYcSk5hcevooo1yxxamQL/bHs/GrEPGmMoMEwRrHVGiCA1pXi97B8Ew==} + safe-regex2@5.0.0: + resolution: {integrity: sha512-YwJwe5a51WlK7KbOJREPdjNrpViQBI3p4T50lfwPuDhZnE3XGVTlGvi+aolc5+RvxDD6bnUmjVsU9n1eboLUYw==} safe-stable-stringify@2.4.2: resolution: {integrity: sha512-gMxvPJYhP0O9n2pvcfYfIuYgbledAOJFcqRThtPRmjscaipiwcwPPKLytpVzMkG2HAN87Qmo2d4PtGiri1dSLA==} engines: {node: '>=10'} + safe-stable-stringify@2.5.0: + resolution: {integrity: sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==} + engines: {node: '>=10'} + safer-buffer@2.1.2: resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} @@ -6252,8 +6367,8 @@ packages: scrypt-js@3.0.1: resolution: {integrity: sha512-cdwTTnqPu0Hyvf5in5asVdZocVDTNRmR7XEcJuIzMjJeSHybHl7vpB66AzwTaIg6CLSbtjcxc8fqcySfnTkccA==} - secure-json-parse@4.0.0: - resolution: {integrity: sha512-dxtLJO6sc35jWidmLxo7ij+Eg48PM/kleBsxpC8QJE0qJICe+KawkDQmvCMZUr9u7WKVHgMW6vy3fQ7zMiFZMA==} + secure-json-parse@4.1.0: + resolution: {integrity: sha512-l4KnYfEyqYJxDwlNVyRfO2E4NTHfMKAWdUuA8J0yve2Dz/E/PdBepY03RvyJpssIpRFwJoCD55wA+mEDs6ByWA==} semver-compare@1.0.0: resolution: {integrity: sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==} @@ -6276,6 +6391,11 @@ packages: engines: {node: '>=10'} hasBin: true + semver@7.7.4: + resolution: {integrity: sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==} + engines: {node: '>=10'} + hasBin: true + serialize-error@7.0.1: resolution: {integrity: sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==} engines: {node: '>=10'} @@ -6283,8 +6403,8 @@ packages: set-blocking@2.0.0: resolution: {integrity: sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==} - set-cookie-parser@2.7.0: - resolution: {integrity: sha512-lXLOiqpkUumhRdFF3k1osNXCy9akgx/dyPZ5p8qAg9seJzXr5ZrlqZuWIMuY6ejOsVLE6flJ5/h3lsn57fQ/PQ==} + set-cookie-parser@2.7.2: + resolution: {integrity: sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw==} set-function-length@1.2.0: resolution: {integrity: sha512-4DBHDoyHlM1IRPGYcoxexgh67y4ueR53FKV1yyxwFMY7aCqcN/38M1+SwZ/qJQ8iLv7+ck385ot4CcisOAPT9w==} @@ -6391,8 +6511,8 @@ packages: resolution: {integrity: sha512-HLpt+uLy/pxB+bum/9DzAgiKS8CX1EvbWxI4zlmgGCExImLdiad2iCwXT5Z4c9c3Eq8rP2318mPW2c+QbtjK8A==} engines: {node: '>= 10.0.0', npm: '>= 3.0.0'} - sonic-boom@4.2.0: - resolution: {integrity: sha512-INb7TM37/mAcsGmc9hyyI6+QR3rR1zVRu36B0NeGXKnOOLiZOfER5SA+N7X7k3yUYRzLWafduTDvJAfDswwEww==} + sonic-boom@4.2.1: + resolution: {integrity: sha512-w6AxtubXa2wTXAUsZMMWERrsIRAdrK0Sc+FUytWvYAhBJLyuI4llrMIC1DtlNSdI99EI86KZum2MMq3EAZlF9Q==} sort-keys@5.1.0: resolution: {integrity: sha512-aSbHV0DaBcr7u0PVHXzM6NbZNAtrr9sF6+Qfs9UUVG7Ll3jQ6hHi8F/xqIIcn2rvIVbr0v/2zyjSdwSV47AgLQ==} @@ -6480,9 +6600,6 @@ packages: stream-http@3.2.0: resolution: {integrity: sha512-Oq1bLqisTyK3TSCXpPbT4sdeYNdmyZJv1LxpEm2vu1ZhK89kSE5YXwZc3cWk0MagGaKriBh9mCFbVGtO+vY29A==} - stream-to-it@1.0.1: - resolution: {integrity: sha512-AqHYAYPHcmvMrcLNgncE/q0Aj/ajP6A4qGhxP6EVn7K3YTNs0bJpJyk57wc2Heb7MUL64jurvmnmui8D9kjZgA==} - strict-event-emitter-types@2.0.0: resolution: {integrity: sha512-Nk/brWYpD85WlOgzw5h173aci0Teyv8YdIAEtV+N88nDB0dLlazZyJMIsN6eo1/AR61l+p6CJTG1JIyFaoNEEA==} @@ -6559,6 +6676,10 @@ packages: engines: {node: '>=6.4.0'} deprecated: Please upgrade to supertest v7.1.3+, see release notes at https://github.com/forwardemail/supertest/releases/tag/v7.1.3 - maintenance is supported by Forward Email @ https://forwardemail.net + supports-color@10.2.2: + resolution: {integrity: sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==} + engines: {node: '>=18'} + supports-color@5.5.0: resolution: {integrity: sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==} engines: {node: '>=4'} @@ -6571,10 +6692,6 @@ packages: resolution: {integrity: sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==} engines: {node: '>=10'} - supports-color@9.4.0: - resolution: {integrity: sha512-VL+lNrEoIXww1coLPOmiEmK/0sGigko5COxI09KzHc2VJXJsQ37UaQ+8quuxjDeA7+KnLGTWRyOXSLLR2Wb4jw==} - engines: {node: '>=12'} - supports-preserve-symlinks-flag@1.0.0: resolution: {integrity: sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==} engines: {node: '>= 0.4'} @@ -6684,14 +6801,14 @@ packages: triple-beam@1.3.0: resolution: {integrity: sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw==} - ts-node@10.9.2: - resolution: {integrity: sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==} + ts-node@11.0.0-beta.1: + resolution: {integrity: sha512-WMSROP+1pU22Q/Tm40mjfRg130yD8i0g6ROST04ZpocfH8sl1zD75ON4XQMcBEVViXMVemJBH0alflE7xePdRA==} hasBin: true peerDependencies: - '@swc/core': '>=1.2.50' - '@swc/wasm': '>=1.2.50' + '@swc/core': '>=1.3.85' + '@swc/wasm': '>=1.3.85' '@types/node': '*' - typescript: '>=2.7' + typescript: '>=4.4' peerDependenciesMeta: '@swc/core': optional: true @@ -6777,8 +6894,8 @@ packages: engines: {node: '>=14.17'} hasBin: true - typescript@5.9.2: - resolution: {integrity: sha512-CWBzXQrc/qOkhidw1OzBTQuYRbfyxDXJMVJ1XNwUHGROVmuaeiEm3OslpZ1RV96d7SKKjZKrSJu3+t/xlw3R9A==} + typescript@6.0.1-rc: + resolution: {integrity: sha512-7XlzYb+p/7YxX6qSOzwB4mxVFRdAgWWkj1PgAZ+jzldeuFV6Z77vwFbNxHsUXAL/bhlWY2jCT8shLwDJR8337g==} engines: {node: '>=14.17'} hasBin: true @@ -6808,6 +6925,9 @@ packages: undici-types@7.16.0: resolution: {integrity: sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==} + undici-types@7.18.2: + resolution: {integrity: sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==} + undici@5.29.0: resolution: {integrity: sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg==} engines: {node: '>=14.0'} @@ -6852,6 +6972,10 @@ packages: resolution: {integrity: sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==} engines: {node: '>= 10.0.0'} + unlimited-timeout@0.1.0: + resolution: {integrity: sha512-D4g+mxFeQGQHzCfnvij+R35ukJ0658Zzudw7j16p4tBBbNasKkKM4SocYxqhwT5xA7a9JYWDzKkEFyMlRi5sng==} + engines: {node: '>=20'} + upath@2.0.1: resolution: {integrity: sha512-1uEe95xksV1O0CYKXo8vQvN1JEbtJp7lb7C5U9HMsIp6IVwntkH/oNUzyVNQSd4S1sYk2FpSSW44FqMc8qee5w==} engines: {node: '>=4'} @@ -6868,6 +6992,9 @@ packages: url@0.11.0: resolution: {integrity: sha512-kbailJa29QrtXnxgq+DdCEGlbTeYM2eJUxsz6vjZavrCYPMIFHMKQmSKYAIuUK2i7hgPm28a8piX5NTUtM/LKQ==} + utf8-codec@1.0.0: + resolution: {integrity: sha512-S/QSLezp3qvG4ld5PUfXiH7mCFxLKjSVZRFkB3DOjgwHuJPFDkInAXc/anf7BAbHt/D38ozDzL+QMZ6/7gsI6w==} + util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} @@ -7035,8 +7162,8 @@ packages: wcwidth@1.0.1: resolution: {integrity: sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg==} - weald@1.0.4: - resolution: {integrity: sha512-+kYTuHonJBwmFhP1Z4YQK/dGi3jAnJGCYhyODFpHK73rbxnp9lnZQj7a2m+WVgn8fXr5bJaxUpF6l8qZpPeNWQ==} + weald@1.1.1: + resolution: {integrity: sha512-PaEQShzMCz8J/AD2N3dJMc1hTZWkJeLKS2NMeiVkV5KDHwgZe7qXLEzyodsT/SODxWDdXJJqocuwf3kHzcXhSQ==} web3-core@4.0.3: resolution: {integrity: sha512-KJaH1+ajm/gelvhImkXZx8HrBaGZDERqhOCRpikuwReVDTf4X3TlXqF+oKt153qf5HUXWR4CUL6NkNKNQWjhbA==} @@ -7348,10 +7475,6 @@ packages: yauzl@2.10.0: resolution: {integrity: sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==} - yn@3.1.1: - resolution: {integrity: sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==} - engines: {node: '>=6'} - yocto-queue@0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} @@ -7678,31 +7801,30 @@ snapshots: '@chainsafe/blst-linux-x64-musl': 2.2.0 '@chainsafe/blst-win32-x64-msvc': 2.2.0 - '@chainsafe/discv5@11.0.4': - dependencies: - '@chainsafe/enr': 5.0.1 - '@ethereumjs/rlp': 5.0.2 - '@libp2p/crypto': 5.1.7 - '@libp2p/interface': 2.10.5 - '@multiformats/multiaddr': 12.5.1 - '@noble/hashes': 1.8.0 - '@noble/secp256k1': 2.2.3 - debug: 4.4.3 - lru-cache: 10.4.3 + '@chainsafe/discv5@12.0.1': + dependencies: + '@chainsafe/enr': 6.0.1 + '@ethereumjs/rlp': 10.1.1 + '@libp2p/crypto': 5.1.13 + '@libp2p/interface': 3.1.0 + '@libp2p/peer-id': 6.0.4 + '@multiformats/multiaddr': 13.0.1 + '@noble/hashes': 2.0.1 + '@noble/secp256k1': 3.0.0 + ethereum-cryptography: 3.2.0 + lru-cache: 11.2.5 strict-event-emitter-types: 2.0.0 - transitivePeerDependencies: - - supports-color + weald: 1.1.1 - '@chainsafe/enr@5.0.1': + '@chainsafe/enr@6.0.1': dependencies: - '@ethereumjs/rlp': 5.0.2 - '@libp2p/crypto': 5.1.7 - '@libp2p/interface': 2.10.5 - '@libp2p/peer-id': 5.1.8 - '@multiformats/multiaddr': 12.5.1 - '@scure/base': 1.2.4 - ethereum-cryptography: 2.2.1 - uint8-varint: 2.0.4 + '@ethereumjs/rlp': 10.1.1 + '@libp2p/crypto': 5.1.13 + '@libp2p/interface': 3.1.0 + '@libp2p/peer-id': 6.0.4 + '@multiformats/multiaddr': 13.0.1 + '@scure/base': 2.0.0 + ethereum-cryptography: 3.2.0 '@chainsafe/fast-crc32c@4.2.0': optionalDependencies: @@ -7737,43 +7859,62 @@ snapshots: '@chainsafe/is-ip@2.1.0': {} - '@chainsafe/libp2p-gossipsub@14.1.2': - dependencies: - '@libp2p/crypto': 5.1.7 - '@libp2p/interface': 2.10.5 - '@libp2p/interface-internal': 2.3.18 - '@libp2p/peer-id': 5.1.8 - '@libp2p/pubsub': 10.1.17 - '@multiformats/multiaddr': 12.5.1 - denque: 2.1.0 - it-length-prefixed: 9.1.1 - it-pipe: 3.0.1 - it-pushable: 3.2.3 - multiformats: 13.3.7 - protons-runtime: 5.5.0 - uint8arraylist: 2.4.8 - uint8arrays: 5.1.0 - - '@chainsafe/libp2p-noise@16.1.5': + '@chainsafe/libp2p-noise@17.0.0': dependencies: '@chainsafe/as-chacha20poly1305': 0.1.0 '@chainsafe/as-sha256': 1.2.0 - '@libp2p/crypto': 5.1.7 - '@libp2p/interface': 2.10.5 - '@libp2p/peer-id': 5.1.8 - '@noble/ciphers': 1.2.1 - '@noble/curves': 1.9.2 - '@noble/hashes': 1.8.0 - it-length-prefixed: 10.0.1 - it-length-prefixed-stream: 2.0.3 - it-pair: 2.0.6 - it-pipe: 3.0.1 - it-stream-types: 2.0.2 - protons-runtime: 5.5.0 + '@libp2p/crypto': 5.1.13 + '@libp2p/interface': 3.1.0 + '@libp2p/peer-id': 6.0.4 + '@libp2p/utils': 7.0.13 + '@noble/ciphers': 2.1.1 + '@noble/curves': 2.0.1 + '@noble/hashes': 2.0.1 + protons-runtime: 5.6.0 uint8arraylist: 2.4.8 uint8arrays: 5.1.0 wherearewe: 2.0.1 + '@chainsafe/libp2p-quic-darwin-arm64@2.0.0': + optional: true + + '@chainsafe/libp2p-quic-darwin-x64@2.0.0': + optional: true + + '@chainsafe/libp2p-quic-linux-arm64-gnu@2.0.0': + optional: true + + '@chainsafe/libp2p-quic-linux-arm64-musl@2.0.0': + optional: true + + '@chainsafe/libp2p-quic-linux-x64-gnu@2.0.0': + optional: true + + '@chainsafe/libp2p-quic-linux-x64-musl@2.0.0': + optional: true + + '@chainsafe/libp2p-quic-win32-x64-msvc@2.0.0': + optional: true + + '@chainsafe/libp2p-quic@2.0.0': + dependencies: + '@libp2p/crypto': 5.1.13 + '@libp2p/interface': 3.1.0 + '@libp2p/peer-id': 6.0.4 + '@libp2p/utils': 7.0.13 + '@multiformats/multiaddr': 13.0.1 + '@multiformats/multiaddr-matcher': 3.0.1 + race-signal: 1.1.3 + uint8arraylist: 2.4.8 + optionalDependencies: + '@chainsafe/libp2p-quic-darwin-arm64': 2.0.0 + '@chainsafe/libp2p-quic-darwin-x64': 2.0.0 + '@chainsafe/libp2p-quic-linux-arm64-gnu': 2.0.0 + '@chainsafe/libp2p-quic-linux-arm64-musl': 2.0.0 + '@chainsafe/libp2p-quic-linux-x64-gnu': 2.0.0 + '@chainsafe/libp2p-quic-linux-x64-musl': 2.0.0 + '@chainsafe/libp2p-quic-win32-x64-msvc': 2.0.0 + '@chainsafe/netmask@2.0.0': dependencies: '@chainsafe/is-ip': 2.1.0 @@ -7936,6 +8077,11 @@ snapshots: enabled: 2.0.0 kuler: 2.0.0 + '@dnsquery/dns-packet@6.1.1': + dependencies: + '@leichtgewicht/ip-codec': 2.0.5 + utf8-codec: 1.0.0 + '@electron/get@2.0.3': dependencies: debug: 4.4.3 @@ -8103,9 +8249,9 @@ snapshots: - supports-color - utf-8-validate - '@ethereumjs/rlp@4.0.1': {} + '@ethereumjs/rlp@10.1.1': {} - '@ethereumjs/rlp@5.0.2': {} + '@ethereumjs/rlp@4.0.1': {} '@ethereumjs/statemanager@1.0.5': dependencies: @@ -8441,20 +8587,22 @@ snapshots: '@fastify/error@4.0.0': {} - '@fastify/fast-json-stringify-compiler@5.0.1': + '@fastify/error@4.2.0': {} + + '@fastify/fast-json-stringify-compiler@5.0.3': dependencies: - fast-json-stringify: 6.0.0 + fast-json-stringify: 6.3.0 - '@fastify/forwarded@3.0.0': {} + '@fastify/forwarded@3.0.1': {} - '@fastify/merge-json-schemas@0.1.1': + '@fastify/merge-json-schemas@0.2.1': dependencies: - fast-deep-equal: 3.1.3 + dequal: 2.0.3 - '@fastify/proxy-addr@5.0.0': + '@fastify/proxy-addr@5.1.0': dependencies: - '@fastify/forwarded': 3.0.0 - ipaddr.js: 2.2.0 + '@fastify/forwarded': 3.0.1 + ipaddr.js: 2.3.0 '@fastify/send@3.1.1': dependencies: @@ -8582,7 +8730,7 @@ snapshots: '@jridgewell/resolve-uri': 3.1.1 '@jridgewell/sourcemap-codec': 1.5.5 - '@leichtgewicht/ip-codec@2.0.4': {} + '@leichtgewicht/ip-codec@2.0.5': {} '@lerna-lite/cli@4.10.3(@lerna-lite/exec@4.10.3)(@lerna-lite/publish@4.9.4)(@lerna-lite/run@4.9.4)(@lerna-lite/version@4.9.4)(@types/node@24.10.1)': dependencies: @@ -8870,210 +9018,195 @@ snapshots: - conventional-commits-filter - supports-color - '@libp2p/bootstrap@11.0.46': + '@libp2p/bootstrap@12.0.14': dependencies: - '@libp2p/interface': 2.10.5 - '@libp2p/interface-internal': 2.3.18 - '@libp2p/peer-id': 5.1.8 - '@multiformats/mafmt': 12.1.6 - '@multiformats/multiaddr': 12.5.1 + '@libp2p/interface': 3.1.0 + '@libp2p/interface-internal': 3.0.13 + '@libp2p/peer-id': 6.0.4 + '@multiformats/multiaddr': 13.0.1 + '@multiformats/multiaddr-matcher': 3.0.1 main-event: 1.0.1 - '@libp2p/crypto@5.1.7': + '@libp2p/crypto@5.1.13': dependencies: - '@libp2p/interface': 2.10.5 - '@noble/curves': 1.9.2 - '@noble/hashes': 1.8.0 - multiformats: 13.3.7 - protons-runtime: 5.5.0 + '@libp2p/interface': 3.1.0 + '@noble/curves': 2.0.1 + '@noble/hashes': 2.0.1 + multiformats: 13.4.2 + protons-runtime: 5.6.0 uint8arraylist: 2.4.8 uint8arrays: 5.1.0 - '@libp2p/identify@3.0.38': + '@libp2p/gossipsub@15.0.15': dependencies: - '@libp2p/crypto': 5.1.7 - '@libp2p/interface': 2.10.5 - '@libp2p/interface-internal': 2.3.18 - '@libp2p/peer-id': 5.1.8 - '@libp2p/peer-record': 8.0.34 - '@libp2p/utils': 6.7.1 - '@multiformats/multiaddr': 12.5.1 - '@multiformats/multiaddr-matcher': 2.0.1 + '@libp2p/crypto': 5.1.13 + '@libp2p/interface': 3.1.0 + '@libp2p/interface-internal': 3.0.13 + '@libp2p/peer-id': 6.0.4 + '@libp2p/utils': 7.0.13 + '@multiformats/multiaddr': 13.0.1 + denque: 2.1.0 + it-length-prefixed: 10.0.1 + it-pipe: 3.0.1 + it-pushable: 3.2.3 + multiformats: 13.4.2 + protons-runtime: 5.6.0 + uint8arraylist: 2.4.8 + uint8arrays: 5.1.0 + + '@libp2p/identify@4.0.13': + dependencies: + '@libp2p/crypto': 5.1.13 + '@libp2p/interface': 3.1.0 + '@libp2p/interface-internal': 3.0.13 + '@libp2p/peer-id': 6.0.4 + '@libp2p/peer-record': 9.0.5 + '@libp2p/utils': 7.0.13 + '@multiformats/multiaddr': 13.0.1 + '@multiformats/multiaddr-matcher': 3.0.1 it-drain: 3.0.10 it-parallel: 3.0.13 - it-protobuf-stream: 2.0.3 main-event: 1.0.1 - protons-runtime: 5.5.0 + protons-runtime: 5.6.0 uint8arraylist: 2.4.8 uint8arrays: 5.1.0 - '@libp2p/interface-internal@2.3.18': + '@libp2p/interface-internal@3.0.13': dependencies: - '@libp2p/interface': 2.10.5 - '@libp2p/peer-collections': 6.0.34 - '@multiformats/multiaddr': 12.5.1 + '@libp2p/interface': 3.1.0 + '@libp2p/peer-collections': 7.0.13 + '@multiformats/multiaddr': 13.0.1 progress-events: 1.0.1 - '@libp2p/interface@2.10.5': + '@libp2p/interface@3.1.0': dependencies: - '@multiformats/dns': 1.0.6 - '@multiformats/multiaddr': 12.5.1 - it-pushable: 3.2.3 - it-stream-types: 2.0.2 + '@multiformats/dns': 1.0.13 + '@multiformats/multiaddr': 13.0.1 main-event: 1.0.1 - multiformats: 13.3.7 + multiformats: 13.4.2 progress-events: 1.0.1 uint8arraylist: 2.4.8 - '@libp2p/logger@5.1.21': + '@libp2p/logger@6.2.2': dependencies: - '@libp2p/interface': 2.10.5 - '@multiformats/multiaddr': 12.5.1 - interface-datastore: 8.3.1 - multiformats: 13.3.7 - weald: 1.0.4 + '@libp2p/interface': 3.1.0 + '@multiformats/multiaddr': 13.0.1 + interface-datastore: 9.0.2 + multiformats: 13.4.2 + weald: 1.1.1 - '@libp2p/mdns@11.0.46': + '@libp2p/mdns@12.0.14': dependencies: - '@libp2p/interface': 2.10.5 - '@libp2p/interface-internal': 2.3.18 - '@libp2p/peer-id': 5.1.8 - '@libp2p/utils': 6.7.1 - '@multiformats/multiaddr': 12.5.1 + '@libp2p/interface': 3.1.0 + '@libp2p/interface-internal': 3.0.13 + '@libp2p/peer-id': 6.0.4 + '@libp2p/utils': 7.0.13 + '@multiformats/multiaddr': 13.0.1 '@types/multicast-dns': 7.2.4 dns-packet: 5.6.1 main-event: 1.0.1 multicast-dns: 7.2.5 - '@libp2p/mplex@11.0.46': + '@libp2p/mplex@12.0.14': dependencies: - '@libp2p/interface': 2.10.5 - '@libp2p/utils': 6.7.1 - it-pipe: 3.0.1 + '@libp2p/interface': 3.1.0 + '@libp2p/utils': 7.0.13 it-pushable: 3.2.3 - it-stream-types: 2.0.2 uint8-varint: 2.0.4 uint8arraylist: 2.4.8 uint8arrays: 5.1.0 - '@libp2p/multistream-select@6.0.28': + '@libp2p/multistream-select@7.0.13': dependencies: - '@libp2p/interface': 2.10.5 + '@libp2p/interface': 3.1.0 + '@libp2p/utils': 7.0.13 it-length-prefixed: 10.0.1 - it-length-prefixed-stream: 2.0.3 - it-stream-types: 2.0.2 - p-defer: 4.0.1 - race-signal: 1.1.3 - uint8-varint: 2.0.4 uint8arraylist: 2.4.8 uint8arrays: 5.1.0 - '@libp2p/peer-collections@6.0.34': + '@libp2p/peer-collections@7.0.13': dependencies: - '@libp2p/interface': 2.10.5 - '@libp2p/peer-id': 5.1.8 - '@libp2p/utils': 6.7.1 - multiformats: 13.3.7 + '@libp2p/interface': 3.1.0 + '@libp2p/peer-id': 6.0.4 + '@libp2p/utils': 7.0.13 + multiformats: 13.4.2 - '@libp2p/peer-id@5.1.8': + '@libp2p/peer-id@6.0.4': dependencies: - '@libp2p/crypto': 5.1.7 - '@libp2p/interface': 2.10.5 - multiformats: 13.3.7 + '@libp2p/crypto': 5.1.13 + '@libp2p/interface': 3.1.0 + multiformats: 13.4.2 uint8arrays: 5.1.0 - '@libp2p/peer-record@8.0.34': + '@libp2p/peer-record@9.0.5': dependencies: - '@libp2p/crypto': 5.1.7 - '@libp2p/interface': 2.10.5 - '@libp2p/peer-id': 5.1.8 - '@libp2p/utils': 6.7.1 - '@multiformats/multiaddr': 12.5.1 - multiformats: 13.3.7 - protons-runtime: 5.5.0 + '@libp2p/crypto': 5.1.13 + '@libp2p/interface': 3.1.0 + '@libp2p/peer-id': 6.0.4 + '@multiformats/multiaddr': 13.0.1 + multiformats: 13.4.2 + protons-runtime: 5.6.0 uint8-varint: 2.0.4 uint8arraylist: 2.4.8 uint8arrays: 5.1.0 - '@libp2p/peer-store@11.2.6': + '@libp2p/peer-store@12.0.13': dependencies: - '@libp2p/crypto': 5.1.7 - '@libp2p/interface': 2.10.5 - '@libp2p/peer-collections': 6.0.34 - '@libp2p/peer-id': 5.1.8 - '@libp2p/peer-record': 8.0.34 - '@multiformats/multiaddr': 12.5.1 - interface-datastore: 8.3.1 + '@libp2p/crypto': 5.1.13 + '@libp2p/interface': 3.1.0 + '@libp2p/peer-collections': 7.0.13 + '@libp2p/peer-id': 6.0.4 + '@libp2p/peer-record': 9.0.5 + '@multiformats/multiaddr': 13.0.1 + interface-datastore: 9.0.2 it-all: 3.0.9 main-event: 1.0.1 mortice: 3.3.1 - multiformats: 13.3.7 - protons-runtime: 5.5.0 + multiformats: 13.4.2 + protons-runtime: 5.6.0 uint8arraylist: 2.4.8 uint8arrays: 5.1.0 - '@libp2p/prometheus-metrics@4.3.29': + '@libp2p/prometheus-metrics@5.0.14': dependencies: - '@libp2p/interface': 2.10.5 - it-foreach: 2.1.4 - it-stream-types: 2.0.2 + '@libp2p/interface': 3.1.0 prom-client: 15.1.3 - uint8arraylist: 2.4.8 - - '@libp2p/pubsub@10.1.17': - dependencies: - '@libp2p/crypto': 5.1.7 - '@libp2p/interface': 2.10.5 - '@libp2p/interface-internal': 2.3.18 - '@libp2p/peer-collections': 6.0.34 - '@libp2p/peer-id': 5.1.8 - '@libp2p/utils': 6.7.1 - it-length-prefixed: 10.0.1 - it-pipe: 3.0.1 - it-pushable: 3.2.3 - main-event: 1.0.1 - multiformats: 13.3.7 - p-queue: 8.1.0 - uint8arraylist: 2.4.8 - uint8arrays: 5.1.0 - '@libp2p/tcp@10.1.18': + '@libp2p/tcp@11.0.13': dependencies: - '@libp2p/interface': 2.10.5 - '@libp2p/utils': 6.7.1 - '@multiformats/multiaddr': 12.5.1 - '@multiformats/multiaddr-matcher': 2.0.1 - '@types/sinon': 17.0.4 + '@libp2p/interface': 3.1.0 + '@libp2p/utils': 7.0.13 + '@multiformats/multiaddr': 13.0.1 + '@multiformats/multiaddr-matcher': 3.0.1 + '@types/sinon': 20.0.0 main-event: 1.0.1 - p-defer: 4.0.1 - p-event: 6.0.1 + p-event: 7.1.0 progress-events: 1.0.1 - race-event: 1.3.0 - stream-to-it: 1.0.1 + uint8arraylist: 2.4.8 - '@libp2p/utils@6.7.1': + '@libp2p/utils@7.0.13': dependencies: '@chainsafe/is-ip': 2.1.0 '@chainsafe/netmask': 2.0.0 - '@libp2p/crypto': 5.1.7 - '@libp2p/interface': 2.10.5 - '@libp2p/logger': 5.1.21 - '@multiformats/multiaddr': 12.5.1 + '@libp2p/crypto': 5.1.13 + '@libp2p/interface': 3.1.0 + '@libp2p/logger': 6.2.2 + '@multiformats/multiaddr': 13.0.1 '@sindresorhus/fnv1a': 3.1.0 - any-signal: 4.1.1 - delay: 6.0.0 - get-iterator: 2.0.1 + any-signal: 4.2.0 + cborg: 4.5.8 + delay: 7.0.0 is-loopback-addr: 2.0.2 - is-plain-obj: 4.1.0 - it-foreach: 2.1.4 + it-length-prefixed: 10.0.1 it-pipe: 3.0.1 it-pushable: 3.2.3 it-stream-types: 2.0.2 main-event: 1.0.1 netmask: 2.0.2 p-defer: 4.0.1 - race-event: 1.3.0 - race-signal: 1.1.3 + p-event: 7.1.0 + race-signal: 2.0.0 + uint8-varint: 2.0.4 uint8arraylist: 2.4.8 uint8arrays: 5.1.0 @@ -9114,31 +9247,23 @@ snapshots: '@microsoft/tsdoc@0.15.1': {} - '@multiformats/dns@1.0.6': + '@multiformats/dns@1.0.13': dependencies: - '@types/dns-packet': 5.6.5 - buffer: 6.0.3 - dns-packet: 5.6.1 + '@dnsquery/dns-packet': 6.1.1 + '@libp2p/interface': 3.1.0 hashlru: 2.3.0 - p-queue: 8.1.0 + p-queue: 9.1.0 progress-events: 1.0.1 uint8arrays: 5.1.0 - '@multiformats/mafmt@12.1.6': - dependencies: - '@multiformats/multiaddr': 12.5.1 - - '@multiformats/multiaddr-matcher@2.0.1': + '@multiformats/multiaddr-matcher@3.0.1': dependencies: - '@multiformats/multiaddr': 12.5.1 + '@multiformats/multiaddr': 13.0.1 - '@multiformats/multiaddr@12.5.1': + '@multiformats/multiaddr@13.0.1': dependencies: '@chainsafe/is-ip': 2.1.0 - '@chainsafe/netmask': 2.0.0 - '@multiformats/dns': 1.0.6 - abort-error: 1.0.1 - multiformats: 13.3.7 + multiformats: 13.4.2 uint8-varint: 2.0.4 uint8arrays: 5.1.0 @@ -9188,25 +9313,33 @@ snapshots: '@tybys/wasm-util': 0.9.0 optional: true - '@noble/ciphers@1.2.1': {} + '@noble/ciphers@1.3.0': {} + + '@noble/ciphers@2.1.1': {} '@noble/curves@1.4.2': dependencies: '@noble/hashes': 1.4.0 - '@noble/curves@1.9.2': + '@noble/curves@1.9.0': dependencies: '@noble/hashes': 1.8.0 + '@noble/curves@2.0.1': + dependencies: + '@noble/hashes': 2.0.1 + '@noble/hashes@1.1.2': {} '@noble/hashes@1.4.0': {} '@noble/hashes@1.8.0': {} + '@noble/hashes@2.0.1': {} + '@noble/secp256k1@1.7.1': {} - '@noble/secp256k1@2.2.3': {} + '@noble/secp256k1@3.0.0': {} '@node-rs/crc32-android-arm-eabi@1.10.6': optional: true @@ -9287,7 +9420,7 @@ snapshots: agent-base: 7.1.3 http-proxy-agent: 7.0.0 https-proxy-agent: 7.0.6 - lru-cache: 11.2.4 + lru-cache: 11.2.5 socks-proxy-agent: 8.0.5 transitivePeerDependencies: - supports-color @@ -9343,7 +9476,7 @@ snapshots: dependencies: '@npmcli/promise-spawn': 9.0.1 ini: 6.0.0 - lru-cache: 11.2.4 + lru-cache: 11.2.5 npm-pick-manifest: 11.0.3 proc-log: 6.1.0 promise-retry: 2.0.1 @@ -9691,7 +9824,9 @@ snapshots: '@scure/base@1.1.8': {} - '@scure/base@1.2.4': {} + '@scure/base@1.2.6': {} + + '@scure/base@2.0.0': {} '@scure/bip32@1.4.0': dependencies: @@ -9699,11 +9834,22 @@ snapshots: '@noble/hashes': 1.4.0 '@scure/base': 1.1.8 + '@scure/bip32@1.7.0': + dependencies: + '@noble/curves': 1.9.0 + '@noble/hashes': 1.8.0 + '@scure/base': 1.2.6 + '@scure/bip39@1.3.0': dependencies: '@noble/hashes': 1.4.0 '@scure/base': 1.1.8 + '@scure/bip39@1.6.0': + dependencies: + '@noble/hashes': 1.8.0 + '@scure/base': 1.2.6 + '@sec-ant/readable-stream@0.4.1': {} '@sigstore/bundle@4.0.0': @@ -9819,14 +9965,14 @@ snapshots: '@tootallnate/once@2.0.0': {} - '@tsconfig/node10@1.0.8': {} - - '@tsconfig/node12@1.0.9': {} - '@tsconfig/node14@1.0.1': {} '@tsconfig/node16@1.0.2': {} + '@tsconfig/node18@18.2.6': {} + + '@tsconfig/node20@20.1.9': {} + '@tufjs/canonical-json@2.0.0': {} '@tufjs/models@4.0.0': @@ -9861,7 +10007,7 @@ snapshots: '@types/dns-packet@5.6.5': dependencies: - '@types/node': 24.10.1 + '@types/node': 25.4.0 '@types/estree@1.0.8': {} @@ -9889,7 +10035,7 @@ snapshots: '@types/multicast-dns@7.2.4': dependencies: '@types/dns-packet': 5.6.5 - '@types/node': 24.10.1 + '@types/node': 25.4.0 '@types/node@18.15.13': {} @@ -9903,6 +10049,10 @@ snapshots: dependencies: undici-types: 7.16.0 + '@types/node@25.4.0': + dependencies: + undici-types: 7.18.2 + '@types/normalize-package-data@2.4.4': {} '@types/parse-path@7.1.0': @@ -9930,11 +10080,11 @@ snapshots: '@types/retry@0.12.2': {} - '@types/sinon@17.0.4': + '@types/sinon@20.0.0': dependencies: - '@types/sinonjs__fake-timers': 6.0.2 + '@types/sinonjs__fake-timers': 15.0.1 - '@types/sinonjs__fake-timers@6.0.2': {} + '@types/sinonjs__fake-timers@15.0.1': {} '@types/through@0.0.30': dependencies: @@ -9959,7 +10109,38 @@ snapshots: '@types/node': 24.10.1 optional: true - '@vekexasia/bigint-buffer2@1.1.0': {} + '@typescript/native-preview-darwin-arm64@7.0.0-dev.20260303.1': + optional: true + + '@typescript/native-preview-darwin-x64@7.0.0-dev.20260303.1': + optional: true + + '@typescript/native-preview-linux-arm64@7.0.0-dev.20260303.1': + optional: true + + '@typescript/native-preview-linux-arm@7.0.0-dev.20260303.1': + optional: true + + '@typescript/native-preview-linux-x64@7.0.0-dev.20260303.1': + optional: true + + '@typescript/native-preview-win32-arm64@7.0.0-dev.20260303.1': + optional: true + + '@typescript/native-preview-win32-x64@7.0.0-dev.20260303.1': + optional: true + + '@typescript/native-preview@7.0.0-dev.20260303.1': + optionalDependencies: + '@typescript/native-preview-darwin-arm64': 7.0.0-dev.20260303.1 + '@typescript/native-preview-darwin-x64': 7.0.0-dev.20260303.1 + '@typescript/native-preview-linux-arm': 7.0.0-dev.20260303.1 + '@typescript/native-preview-linux-arm64': 7.0.0-dev.20260303.1 + '@typescript/native-preview-linux-x64': 7.0.0-dev.20260303.1 + '@typescript/native-preview-win32-arm64': 7.0.0-dev.20260303.1 + '@typescript/native-preview-win32-x64': 7.0.0-dev.20260303.1 + + '@vekexasia/bigint-buffer2@1.1.1': {} '@vitest/browser-playwright@4.0.7(playwright@1.56.1)(vite@6.1.6(@types/node@24.10.1)(yaml@2.8.2))(vitest@4.0.7)': dependencies: @@ -9974,13 +10155,13 @@ snapshots: - utf-8-validate - vite - '@vitest/browser-playwright@4.0.7(playwright@1.58.2)(vite@6.1.6(@types/node@24.10.1)(yaml@2.8.2))(vitest@4.0.7)': + '@vitest/browser-playwright@4.0.7(playwright@1.56.1)(vite@6.1.6(@types/node@25.4.0)(yaml@2.8.2))(vitest@4.0.7)': dependencies: - '@vitest/browser': 4.0.7(vite@6.1.6(@types/node@24.10.1)(yaml@2.8.2))(vitest@4.0.7) - '@vitest/mocker': 4.0.7(vite@6.1.6(@types/node@24.10.1)(yaml@2.8.2)) - playwright: 1.58.2 + '@vitest/browser': 4.0.7(vite@6.1.6(@types/node@25.4.0)(yaml@2.8.2))(vitest@4.0.7) + '@vitest/mocker': 4.0.7(vite@6.1.6(@types/node@25.4.0)(yaml@2.8.2)) + playwright: 1.56.1 tinyrainbow: 3.0.3 - vitest: 4.0.7(@types/node@24.10.1)(@vitest/browser-playwright@4.0.7)(jsdom@23.0.1)(yaml@2.8.2) + vitest: 4.0.7(@types/node@25.4.0)(@vitest/browser-playwright@4.0.7)(jsdom@23.0.1)(yaml@2.8.2) transitivePeerDependencies: - bufferutil - msw @@ -10005,6 +10186,24 @@ snapshots: - utf-8-validate - vite + '@vitest/browser@4.0.7(vite@6.1.6(@types/node@25.4.0)(yaml@2.8.2))(vitest@4.0.7)': + dependencies: + '@vitest/mocker': 4.0.7(vite@6.1.6(@types/node@25.4.0)(yaml@2.8.2)) + '@vitest/utils': 4.0.7 + magic-string: 0.30.21 + pixelmatch: 7.1.0 + pngjs: 7.0.0 + sirv: 3.0.2 + tinyrainbow: 3.0.3 + vitest: 4.0.7(@types/node@25.4.0)(@vitest/browser-playwright@4.0.7)(jsdom@23.0.1)(yaml@2.8.2) + ws: 8.18.3 + transitivePeerDependencies: + - bufferutil + - msw + - utf-8-validate + - vite + optional: true + '@vitest/coverage-v8@4.0.7(@vitest/browser@4.0.7(vite@6.1.6(@types/node@24.10.1)(yaml@2.8.2))(vitest@4.0.7))(vitest@4.0.7)': dependencies: '@bcoe/v8-coverage': 1.0.2 @@ -10041,6 +10240,14 @@ snapshots: optionalDependencies: vite: 6.1.6(@types/node@24.10.1)(yaml@2.8.2) + '@vitest/mocker@4.0.7(vite@6.1.6(@types/node@25.4.0)(yaml@2.8.2))': + dependencies: + '@vitest/spy': 4.0.7 + estree-walker: 3.0.3 + magic-string: 0.30.21 + optionalDependencies: + vite: 6.1.6(@types/node@25.4.0)(yaml@2.8.2) + '@vitest/pretty-format@2.1.8': dependencies: tinyrainbow: 1.2.0 @@ -10108,7 +10315,7 @@ snapshots: de-indent: 1.0.2 he: 1.2.0 - '@vue/language-core@2.2.0(typescript@5.9.2)': + '@vue/language-core@2.2.0(typescript@6.0.1-rc)': dependencies: '@volar/language-core': 2.4.11 '@vue/compiler-dom': 3.5.13 @@ -10119,7 +10326,7 @@ snapshots: muggle-string: 0.4.1 path-browserify: 1.0.1 optionalDependencies: - typescript: 5.9.2 + typescript: 6.0.1-rc '@vue/shared@3.5.13': {} @@ -10143,6 +10350,15 @@ snapshots: module-error: 1.0.2 queue-microtask: 1.2.3 + abstract-level@3.1.1: + dependencies: + buffer: 6.0.3 + is-buffer: 2.0.5 + level-supports: 6.2.0 + level-transcoder: 1.0.1 + maybe-combine-errors: 1.0.0 + module-error: 1.0.2 + abstract-logging@2.0.1: {} acorn-walk@8.2.0: {} @@ -10231,6 +10447,8 @@ snapshots: any-signal@4.1.1: {} + any-signal@4.2.0: {} + aproba@2.0.0: {} aproba@2.1.0: {} @@ -10302,10 +10520,10 @@ snapshots: available-typed-arrays@1.0.5: {} - avvio@9.0.0: + avvio@9.2.0: dependencies: - '@fastify/error': 4.0.0 - fastq: 1.17.1 + '@fastify/error': 4.2.0 + fastq: 1.20.1 aws-sdk@2.934.0: dependencies: @@ -10391,6 +10609,10 @@ snapshots: module-error: 1.0.2 run-parallel-limit: 1.1.0 + browser-level@3.0.0: + dependencies: + abstract-level: 3.1.1 + browser-resolve@2.0.0: dependencies: resolve: 1.22.10 @@ -10497,7 +10719,7 @@ snapshots: '@npmcli/fs': 5.0.0 fs-minipass: 3.0.1 glob: 13.0.0 - lru-cache: 11.2.4 + lru-cache: 11.2.5 minipass: 7.1.2 minipass-collect: 2.0.1 minipass-flush: 1.0.5 @@ -10545,6 +10767,8 @@ snapshots: catering@2.1.1: {} + cborg@4.5.8: {} + chai@6.2.0: {} chalk@2.4.2: @@ -10581,6 +10805,13 @@ snapshots: napi-macros: 2.2.2 node-gyp-build: 4.5.0 + classic-level@3.0.0: + dependencies: + abstract-level: 3.1.1 + module-error: 1.0.2 + napi-macros: 2.2.2 + node-gyp-build: 4.5.0 + clean-stack@2.2.0: {} cli-cursor@4.0.0: @@ -10733,7 +10964,7 @@ snapshots: conventional-commits-parser: 6.2.1 meow: 13.2.0 - cookie@0.6.0: {} + cookie@1.1.1: {} cookiejar@2.1.4: {} @@ -10824,41 +11055,40 @@ snapshots: whatwg-mimetype: 4.0.0 whatwg-url: 14.0.0 - datastore-core@10.0.2: + datastore-core@11.0.2: dependencies: - '@libp2p/logger': 5.1.21 - interface-datastore: 8.3.1 - interface-store: 6.0.2 + '@libp2p/logger': 6.2.2 + interface-datastore: 9.0.2 + interface-store: 7.0.1 it-drain: 3.0.10 it-filter: 3.1.4 it-map: 3.1.4 it-merge: 3.0.12 it-pipe: 3.0.1 - it-pushable: 3.2.3 it-sort: 3.0.9 it-take: 3.0.9 - datastore-fs@10.0.6: + datastore-fs@11.0.2: dependencies: - datastore-core: 10.0.2 - interface-datastore: 8.3.1 - interface-store: 6.0.2 + datastore-core: 11.0.2 + interface-datastore: 9.0.2 + interface-store: 7.0.1 it-glob: 3.0.4 it-map: 3.1.4 it-parallel-batch: 3.0.9 race-signal: 2.0.0 steno: 4.0.2 - datastore-level@11.0.4: + datastore-level@12.0.2: dependencies: - datastore-core: 10.0.2 - interface-datastore: 8.3.1 - interface-store: 6.0.2 + datastore-core: 11.0.2 + interface-datastore: 9.0.2 + interface-store: 7.0.1 it-filter: 3.1.4 it-map: 3.1.4 it-sort: 3.0.9 it-take: 3.0.9 - level: 8.0.1 + level: 10.0.0 race-signal: 2.0.0 de-indent@1.0.2: {} @@ -10901,7 +11131,10 @@ snapshots: has-property-descriptors: 1.0.1 object-keys: 1.1.1 - delay@6.0.0: {} + delay@7.0.0: + dependencies: + random-int: 3.1.0 + unlimited-timeout: 0.1.0 delayed-stream@1.0.0: {} @@ -10915,6 +11148,8 @@ snapshots: deprecation@2.3.1: {} + dequal@2.0.3: {} + des.js@1.0.1: dependencies: inherits: 2.0.4 @@ -10940,7 +11175,7 @@ snapshots: dns-packet@5.6.1: dependencies: - '@leichtgewicht/ip-codec': 2.0.4 + '@leichtgewicht/ip-codec': 2.0.5 domain-browser@1.2.0: {} @@ -11137,6 +11372,14 @@ snapshots: '@scure/bip32': 1.4.0 '@scure/bip39': 1.3.0 + ethereum-cryptography@3.2.0: + dependencies: + '@noble/ciphers': 1.3.0 + '@noble/curves': 1.9.0 + '@noble/hashes': 1.8.0 + '@scure/bip32': 1.7.0 + '@scure/bip39': 1.6.0 + ethers@5.7.2: dependencies: '@ethersproject/abi': 5.7.0 @@ -11192,6 +11435,8 @@ snapshots: eventemitter3@5.0.1: {} + eventemitter3@5.0.4: {} + events@1.1.1: {} events@3.3.0: {} @@ -11254,17 +11499,16 @@ snapshots: merge2: 1.4.1 micromatch: 4.0.8 - fast-json-stringify@6.0.0: + fast-json-stringify@6.3.0: dependencies: - '@fastify/merge-json-schemas': 0.1.1 + '@fastify/merge-json-schemas': 0.2.1 ajv: 8.18.0 ajv-formats: 3.0.1(ajv@8.18.0) - fast-deep-equal: 3.1.3 - fast-uri: 2.4.0 - json-schema-ref-resolver: 1.0.1 + fast-uri: 3.1.0 + json-schema-ref-resolver: 3.0.0 rfdc: 1.4.1 - fast-querystring@1.1.1: + fast-querystring@1.1.2: dependencies: fast-decode-uri-component: 1.0.1 @@ -11276,8 +11520,6 @@ snapshots: dependencies: fast-string-truncated-width: 3.0.3 - fast-uri@2.4.0: {} - fast-uri@3.1.0: {} fast-xml-parser@4.5.1: @@ -11286,28 +11528,32 @@ snapshots: fastify-plugin@5.0.1: {} - fastify@5.7.4: + fastify@5.8.1: dependencies: '@fastify/ajv-compiler': 4.0.5 - '@fastify/error': 4.0.0 - '@fastify/fast-json-stringify-compiler': 5.0.1 - '@fastify/proxy-addr': 5.0.0 + '@fastify/error': 4.2.0 + '@fastify/fast-json-stringify-compiler': 5.0.3 + '@fastify/proxy-addr': 5.1.0 abstract-logging: 2.0.1 - avvio: 9.0.0 - fast-json-stringify: 6.0.0 - find-my-way: 9.0.1 - light-my-request: 6.0.0 - pino: 10.3.0 + avvio: 9.2.0 + fast-json-stringify: 6.3.0 + find-my-way: 9.5.0 + light-my-request: 6.6.0 + pino: 10.3.1 process-warning: 5.0.0 rfdc: 1.4.1 - secure-json-parse: 4.0.0 - semver: 7.7.3 + secure-json-parse: 4.1.0 + semver: 7.7.4 toad-cache: 3.7.0 fastq@1.17.1: dependencies: reusify: 1.0.4 + fastq@1.20.1: + dependencies: + reusify: 1.1.0 + fd-package-json@2.0.0: dependencies: walk-up-path: 4.0.0 @@ -11339,11 +11585,11 @@ snapshots: dependencies: to-regex-range: 5.0.1 - find-my-way@9.0.1: + find-my-way@9.5.0: dependencies: fast-deep-equal: 3.1.3 - fast-querystring: 1.1.1 - safe-regex2: 4.0.0 + fast-querystring: 1.1.2 + safe-regex2: 5.0.0 find-up@4.1.0: dependencies: @@ -11501,8 +11747,6 @@ snapshots: hasown: 2.0.2 math-intrinsics: 1.1.0 - get-iterator@2.0.1: {} - get-proto@1.0.1: dependencies: dunder-proto: 1.0.1 @@ -11705,7 +11949,7 @@ snapshots: hosted-git-info@9.0.2: dependencies: - lru-cache: 11.2.4 + lru-cache: 11.2.5 html-encoding-sniffer@4.0.0: dependencies: @@ -11837,12 +12081,12 @@ snapshots: through: 2.3.8 wrap-ansi: 8.1.0 - interface-datastore@8.3.1: + interface-datastore@9.0.2: dependencies: - interface-store: 6.0.2 + interface-store: 7.0.1 uint8arrays: 5.1.0 - interface-store@6.0.2: {} + interface-store@7.0.1: {} internal-slot@1.0.5: dependencies: @@ -11854,7 +12098,7 @@ snapshots: ip@2.0.1: {} - ipaddr.js@2.2.0: {} + ipaddr.js@2.3.0: {} is-arguments@1.1.0: dependencies: @@ -11893,7 +12137,7 @@ snapshots: is-docker@2.2.1: {} - is-electron@2.2.0: {} + is-electron@2.2.2: {} is-extglob@2.1.1: {} @@ -11928,7 +12172,7 @@ snapshots: is-negative-zero@2.0.2: {} - is-network-error@1.1.0: {} + is-network-error@1.3.1: {} is-number-object@1.0.7: dependencies: @@ -12026,36 +12270,16 @@ snapshots: it-batch@3.0.9: {} - it-byte-stream@2.0.3: - dependencies: - abort-error: 1.0.1 - it-queueless-pushable: 2.0.2 - it-stream-types: 2.0.2 - race-signal: 1.1.3 - uint8arraylist: 2.4.8 - it-drain@3.0.10: {} it-filter@3.1.4: dependencies: it-peekable: 3.0.1 - it-foreach@2.1.4: - dependencies: - it-peekable: 3.0.1 - it-glob@3.0.4: dependencies: fast-glob: 3.3.3 - it-length-prefixed-stream@2.0.3: - dependencies: - abort-error: 1.0.1 - it-byte-stream: 2.0.3 - it-stream-types: 2.0.2 - uint8-varint: 2.0.4 - uint8arraylist: 2.4.8 - it-length-prefixed@10.0.1: dependencies: it-reader: 6.0.4 @@ -12064,26 +12288,13 @@ snapshots: uint8arraylist: 2.4.8 uint8arrays: 5.1.0 - it-length-prefixed@9.1.1: - dependencies: - it-reader: 6.0.4 - it-stream-types: 2.0.2 - uint8-varint: 2.0.4 - uint8arraylist: 2.4.8 - uint8arrays: 5.1.0 - it-map@3.1.4: dependencies: it-peekable: 3.0.1 it-merge@3.0.12: dependencies: - it-queueless-pushable: 2.0.2 - - it-pair@2.0.6: - dependencies: - it-stream-types: 2.0.2 - p-defer: 4.0.1 + it-queueless-pushable: 2.0.3 it-parallel-batch@3.0.9: dependencies: @@ -12101,30 +12312,23 @@ snapshots: it-pushable: 3.2.3 it-stream-types: 2.0.2 - it-protobuf-stream@2.0.3: - dependencies: - abort-error: 1.0.1 - it-length-prefixed-stream: 2.0.3 - it-stream-types: 2.0.2 - uint8arraylist: 2.4.8 - it-pushable@3.2.3: dependencies: p-defer: 4.0.1 - it-queue@1.1.0: + it-queue@1.1.1: dependencies: abort-error: 1.0.1 it-pushable: 3.2.3 main-event: 1.0.1 - race-event: 1.3.0 - race-signal: 1.1.3 + race-event: 1.6.1 + race-signal: 2.0.0 - it-queueless-pushable@2.0.2: + it-queueless-pushable@2.0.3: dependencies: abort-error: 1.0.1 p-defer: 4.0.1 - race-signal: 1.1.3 + race-signal: 2.0.0 it-reader@6.0.4: dependencies: @@ -12201,9 +12405,9 @@ snapshots: json-parse-even-better-errors@5.0.0: {} - json-schema-ref-resolver@1.0.1: + json-schema-ref-resolver@3.0.0: dependencies: - fast-deep-equal: 3.1.3 + dequal: 2.0.3 json-schema-resolver@2.0.0: dependencies: @@ -12252,11 +12456,19 @@ snapshots: level-supports@4.0.1: {} + level-supports@6.2.0: {} + level-transcoder@1.0.1: dependencies: buffer: 6.0.3 module-error: 1.0.2 + level@10.0.0: + dependencies: + abstract-level: 3.1.1 + browser-level: 3.0.0 + classic-level: 3.0.0 + level@8.0.1: dependencies: abstract-level: 1.0.4 @@ -12283,42 +12495,41 @@ snapshots: transitivePeerDependencies: - supports-color - libp2p@2.9.0: + libp2p@3.1.6: dependencies: '@chainsafe/is-ip': 2.1.0 '@chainsafe/netmask': 2.0.0 - '@libp2p/crypto': 5.1.7 - '@libp2p/interface': 2.10.5 - '@libp2p/interface-internal': 2.3.18 - '@libp2p/logger': 5.1.21 - '@libp2p/multistream-select': 6.0.28 - '@libp2p/peer-collections': 6.0.34 - '@libp2p/peer-id': 5.1.8 - '@libp2p/peer-store': 11.2.6 - '@libp2p/utils': 6.7.1 - '@multiformats/dns': 1.0.6 - '@multiformats/multiaddr': 12.5.1 - '@multiformats/multiaddr-matcher': 2.0.1 - any-signal: 4.1.1 - datastore-core: 10.0.2 - interface-datastore: 8.3.1 - it-byte-stream: 2.0.3 + '@libp2p/crypto': 5.1.13 + '@libp2p/interface': 3.1.0 + '@libp2p/interface-internal': 3.0.13 + '@libp2p/logger': 6.2.2 + '@libp2p/multistream-select': 7.0.13 + '@libp2p/peer-collections': 7.0.13 + '@libp2p/peer-id': 6.0.4 + '@libp2p/peer-store': 12.0.13 + '@libp2p/utils': 7.0.13 + '@multiformats/dns': 1.0.13 + '@multiformats/multiaddr': 13.0.1 + '@multiformats/multiaddr-matcher': 3.0.1 + any-signal: 4.2.0 + datastore-core: 11.0.2 + interface-datastore: 9.0.2 it-merge: 3.0.12 it-parallel: 3.0.13 main-event: 1.0.1 - multiformats: 13.3.7 + multiformats: 13.4.2 p-defer: 4.0.1 - p-retry: 6.2.1 + p-event: 7.1.0 + p-retry: 7.1.1 progress-events: 1.0.1 - race-event: 1.3.0 - race-signal: 1.1.3 + race-signal: 2.0.0 uint8arrays: 5.1.0 - light-my-request@6.0.0: + light-my-request@6.6.0: dependencies: - cookie: 0.6.0 - process-warning: 4.0.0 - set-cookie-parser: 2.7.0 + cookie: 1.1.1 + process-warning: 4.0.1 + set-cookie-parser: 2.7.2 lilconfig@3.1.3: {} @@ -12395,10 +12606,10 @@ snapshots: lru-cache@10.4.3: {} - lru-cache@11.0.1: {} - lru-cache@11.2.4: {} + lru-cache@11.2.5: {} + lru-cache@5.1.1: dependencies: yallist: 3.1.1 @@ -12472,6 +12683,8 @@ snapshots: math-intrinsics@1.1.0: {} + maybe-combine-errors@1.0.0: {} + mcl-wasm@0.7.9: {} md5.js@1.3.5: @@ -12629,14 +12842,14 @@ snapshots: mortice@3.3.1: dependencies: abort-error: 1.0.1 - it-queue: 1.1.0 + it-queue: 1.1.1 main-event: 1.0.1 mrmime@2.0.0: {} ms@2.1.3: {} - ms@3.0.0-canary.1: {} + ms@3.0.0-canary.202508261828: {} muggle-string@0.4.1: {} @@ -12645,9 +12858,7 @@ snapshots: dns-packet: 5.6.1 thunky: 1.1.0 - multiformats@11.0.2: {} - - multiformats@13.3.7: {} + multiformats@13.4.2: {} mute-stream@1.0.0: {} @@ -12895,7 +13106,7 @@ snapshots: observable-fns@0.6.1: {} - on-exit-leak-free@2.1.0: {} + on-exit-leak-free@2.1.2: {} once@1.4.0: dependencies: @@ -12937,9 +13148,9 @@ snapshots: p-defer@4.0.1: {} - p-event@6.0.1: + p-event@7.1.0: dependencies: - p-timeout: 6.1.2 + p-timeout: 7.0.1 p-limit@2.3.0: dependencies: @@ -12977,25 +13188,21 @@ snapshots: p-pipe@4.0.0: {} - p-queue@8.1.0: + p-queue@9.0.1: dependencies: eventemitter3: 5.0.1 - p-timeout: 6.1.2 + p-timeout: 7.0.1 - p-queue@9.0.1: + p-queue@9.1.0: dependencies: - eventemitter3: 5.0.1 + eventemitter3: 5.0.4 p-timeout: 7.0.1 p-reduce@3.0.0: {} - p-retry@6.2.1: + p-retry@7.1.1: dependencies: - '@types/retry': 0.12.2 - is-network-error: 1.1.0 - retry: 0.13.1 - - p-timeout@6.1.2: {} + is-network-error: 1.3.1 p-timeout@7.0.1: {} @@ -13097,7 +13304,7 @@ snapshots: path-scurry@2.0.0: dependencies: - lru-cache: 11.0.1 + lru-cache: 11.2.5 minipass: 7.1.2 path-to-regexp@6.3.0: {} @@ -13138,18 +13345,18 @@ snapshots: pino-std-serializers@7.1.0: {} - pino@10.3.0: + pino@10.3.1: dependencies: '@pinojs/redact': 0.4.0 atomic-sleep: 1.0.0 - on-exit-leak-free: 2.1.0 + on-exit-leak-free: 2.1.2 pino-abstract-transport: 3.0.0 pino-std-serializers: 7.1.0 process-warning: 5.0.0 - quick-format-unescaped: 4.0.3 + quick-format-unescaped: 4.0.4 real-require: 0.2.0 - safe-stable-stringify: 2.4.2 - sonic-boom: 4.2.0 + safe-stable-stringify: 2.5.0 + sonic-boom: 4.2.1 thread-stream: 4.0.0 pixelmatch@7.1.0: @@ -13178,22 +13385,12 @@ snapshots: playwright-core@1.56.1: {} - playwright-core@1.58.2: - optional: true - playwright@1.56.1: dependencies: playwright-core: 1.56.1 optionalDependencies: fsevents: 2.3.2 - playwright@1.58.2: - dependencies: - playwright-core: 1.58.2 - optionalDependencies: - fsevents: 2.3.2 - optional: true - pngjs@7.0.0: {} postcss-selector-parser@7.1.1: @@ -13233,7 +13430,7 @@ snapshots: process-nextick-args@2.0.1: {} - process-warning@4.0.0: {} + process-warning@4.0.1: {} process-warning@5.0.0: {} @@ -13271,7 +13468,7 @@ snapshots: protocols@2.0.1: {} - protons-runtime@5.5.0: + protons-runtime@5.6.0: dependencies: uint8-varint: 2.0.4 uint8arraylist: 2.4.8 @@ -13315,16 +13512,20 @@ snapshots: queue-microtask@1.2.3: {} - quick-format-unescaped@4.0.3: {} + quick-format-unescaped@4.0.4: {} quick-lru@5.1.1: {} - race-event@1.3.0: {} + race-event@1.6.1: + dependencies: + abort-error: 1.0.1 race-signal@1.1.3: {} race-signal@2.0.0: {} + random-int@3.1.0: {} + randombytes@2.1.0: dependencies: safe-buffer: 5.2.1 @@ -13411,10 +13612,10 @@ snapshots: retry@0.12.0: {} - retry@0.13.1: {} - reusify@1.0.4: {} + reusify@1.1.0: {} + rewiremock@3.14.5: dependencies: babel-runtime: 6.26.0 @@ -13523,12 +13724,14 @@ snapshots: get-intrinsic: 1.2.2 is-regex: 1.1.4 - safe-regex2@4.0.0: + safe-regex2@5.0.0: dependencies: ret: 0.5.0 safe-stable-stringify@2.4.2: {} + safe-stable-stringify@2.5.0: {} + safer-buffer@2.1.2: {} sax@1.2.1: {} @@ -13541,7 +13744,7 @@ snapshots: scrypt-js@3.0.1: {} - secure-json-parse@4.0.0: {} + secure-json-parse@4.1.0: {} semver-compare@1.0.0: optional: true @@ -13556,6 +13759,8 @@ snapshots: semver@7.7.3: {} + semver@7.7.4: {} + serialize-error@7.0.1: dependencies: type-fest: 0.13.1 @@ -13563,7 +13768,7 @@ snapshots: set-blocking@2.0.0: {} - set-cookie-parser@2.7.0: {} + set-cookie-parser@2.7.2: {} set-function-length@1.2.0: dependencies: @@ -13705,7 +13910,7 @@ snapshots: ip-address: 10.1.0 smart-buffer: 4.2.0 - sonic-boom@4.2.0: + sonic-boom@4.2.1: dependencies: atomic-sleep: 1.0.0 @@ -13796,10 +14001,6 @@ snapshots: readable-stream: 3.6.2 xtend: 4.0.2 - stream-to-it@1.0.1: - dependencies: - it-stream-types: 2.0.2 - strict-event-emitter-types@2.0.0: {} string-argv@0.3.2: {} @@ -13898,6 +14099,8 @@ snapshots: transitivePeerDependencies: - supports-color + supports-color@10.2.2: {} + supports-color@5.5.0: dependencies: has-flag: 3.0.0 @@ -13910,8 +14113,6 @@ snapshots: dependencies: has-flag: 4.0.0 - supports-color@9.4.0: {} - supports-preserve-symlinks-flag@1.0.0: {} symbol-tree@3.2.4: {} @@ -14006,23 +14207,21 @@ snapshots: triple-beam@1.3.0: {} - ts-node@10.9.2(@swc/core@1.14.0)(@swc/wasm@1.14.0)(@types/node@24.10.1)(typescript@5.9.2): + ts-node@11.0.0-beta.1(@swc/core@1.14.0)(@swc/wasm@1.14.0)(@types/node@24.10.1)(typescript@6.0.1-rc): dependencies: '@cspotcode/source-map-support': 0.8.1 - '@tsconfig/node10': 1.0.8 - '@tsconfig/node12': 1.0.9 '@tsconfig/node14': 1.0.1 '@tsconfig/node16': 1.0.2 + '@tsconfig/node18': 18.2.6 + '@tsconfig/node20': 20.1.9 '@types/node': 24.10.1 acorn: 8.14.0 acorn-walk: 8.2.0 arg: 4.1.3 - create-require: 1.1.1 diff: 4.0.2 make-error: 1.3.6 - typescript: 5.9.2 + typescript: 6.0.1-rc v8-compile-cache-lib: 3.0.1 - yn: 3.1.1 optionalDependencies: '@swc/core': 1.14.0 '@swc/wasm': 1.14.0 @@ -14103,7 +14302,7 @@ snapshots: typescript@5.8.2: {} - typescript@5.9.2: {} + typescript@6.0.1-rc: {} ufo@1.5.4: {} @@ -14121,7 +14320,7 @@ snapshots: uint8arrays@5.1.0: dependencies: - multiformats: 13.3.7 + multiformats: 13.4.2 unbox-primitive@1.0.2: dependencies: @@ -14134,6 +14333,8 @@ snapshots: undici-types@7.16.0: {} + undici-types@7.18.2: {} + undici@5.29.0: dependencies: '@fastify/busboy': 2.1.1 @@ -14168,6 +14369,8 @@ snapshots: universalify@2.0.0: {} + unlimited-timeout@0.1.0: {} + upath@2.0.1: {} uri-js@4.4.1: @@ -14189,6 +14392,8 @@ snapshots: punycode: 1.3.2 querystring: 0.2.0 + utf8-codec@1.0.0: {} + util-deprecate@1.0.2: {} util@0.10.3: @@ -14226,18 +14431,18 @@ snapshots: validate-npm-package-name@7.0.0: {} - vite-plugin-dts@4.5.4(@types/node@24.10.1)(rollup@4.52.5)(typescript@5.9.2)(vite@6.1.6(@types/node@24.10.1)(yaml@2.8.2)): + vite-plugin-dts@4.5.4(@types/node@24.10.1)(rollup@4.52.5)(typescript@6.0.1-rc)(vite@6.1.6(@types/node@24.10.1)(yaml@2.8.2)): dependencies: '@microsoft/api-extractor': 7.53.3(@types/node@24.10.1) '@rollup/pluginutils': 5.1.4(rollup@4.52.5) '@volar/typescript': 2.4.11 - '@vue/language-core': 2.2.0(typescript@5.9.2) + '@vue/language-core': 2.2.0(typescript@6.0.1-rc) compare-versions: 6.1.1 debug: 4.4.3 kolorist: 1.8.0 local-pkg: 1.1.2 magic-string: 0.30.21 - typescript: 5.9.2 + typescript: 6.0.1-rc optionalDependencies: vite: 6.1.6(@types/node@24.10.1)(yaml@2.8.2) transitivePeerDependencies: @@ -14274,6 +14479,16 @@ snapshots: fsevents: 2.3.3 yaml: 2.8.2 + vite@6.1.6(@types/node@25.4.0)(yaml@2.8.2): + dependencies: + esbuild: 0.24.2 + postcss: 8.5.6 + rollup: 4.52.5 + optionalDependencies: + '@types/node': 25.4.0 + fsevents: 2.3.3 + yaml: 2.8.2 + vitest-when@0.9.0(@vitest/expect@4.0.7)(vitest@4.0.7): dependencies: pretty-format: 30.2.0 @@ -14305,7 +14520,47 @@ snapshots: why-is-node-running: 2.3.0 optionalDependencies: '@types/node': 24.10.1 - '@vitest/browser-playwright': 4.0.7(playwright@1.58.2)(vite@6.1.6(@types/node@24.10.1)(yaml@2.8.2))(vitest@4.0.7) + '@vitest/browser-playwright': 4.0.7(playwright@1.56.1)(vite@6.1.6(@types/node@24.10.1)(yaml@2.8.2))(vitest@4.0.7) + jsdom: 23.0.1 + transitivePeerDependencies: + - jiti + - less + - lightningcss + - msw + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + - tsx + - yaml + + vitest@4.0.7(@types/node@25.4.0)(@vitest/browser-playwright@4.0.7)(jsdom@23.0.1)(yaml@2.8.2): + dependencies: + '@vitest/expect': 4.0.7 + '@vitest/mocker': 4.0.7(vite@6.1.6(@types/node@25.4.0)(yaml@2.8.2)) + '@vitest/pretty-format': 4.0.7 + '@vitest/runner': 4.0.7 + '@vitest/snapshot': 4.0.7 + '@vitest/spy': 4.0.7 + '@vitest/utils': 4.0.7 + debug: 4.4.3 + es-module-lexer: 1.7.0 + expect-type: 1.2.2 + magic-string: 0.30.21 + pathe: 2.0.3 + picomatch: 4.0.3 + std-env: 3.10.0 + tinybench: 2.9.0 + tinyexec: 0.3.2 + tinyglobby: 0.2.15 + tinyrainbow: 3.0.3 + vite: 6.1.6(@types/node@25.4.0)(yaml@2.8.2) + why-is-node-running: 2.3.0 + optionalDependencies: + '@types/node': 25.4.0 + '@vitest/browser-playwright': 4.0.7(playwright@1.56.1)(vite@6.1.6(@types/node@25.4.0)(yaml@2.8.2))(vitest@4.0.7) jsdom: 23.0.1 transitivePeerDependencies: - jiti @@ -14343,10 +14598,10 @@ snapshots: dependencies: defaults: 1.0.3 - weald@1.0.4: + weald@1.1.1: dependencies: - ms: 3.0.0-canary.1 - supports-color: 9.4.0 + ms: 3.0.0-canary.202508261828 + supports-color: 10.2.2 web3-core@4.0.3(encoding@0.1.13): dependencies: @@ -14574,7 +14829,7 @@ snapshots: wherearewe@2.0.1: dependencies: - is-electron: 2.2.0 + is-electron: 2.2.2 which-boxed-primitive@1.0.2: dependencies: @@ -14782,8 +15037,6 @@ snapshots: buffer-crc32: 0.2.13 fd-slicer: 1.1.0 - yn@3.1.1: {} - yocto-queue@0.1.0: {} yocto-queue@1.0.0: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 65e7c6f24ca9..74aa36e450d8 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -5,6 +5,12 @@ linkWorkspacePackages: true minimumReleaseAge: 2880 +minimumReleaseAgeExclude: + - "@vekexasia/bigint-buffer2" + - "@chainsafe/*" + - "@libp2p/*" + - "libp2p" + nodeLinker: isolated catalog: diff --git a/scripts/ci/check_native_portability.sh b/scripts/ci/check_native_portability.sh new file mode 100755 index 000000000000..4061b1021f7f --- /dev/null +++ b/scripts/ci/check_native_portability.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# check_native_portability.sh +# +# Scans prebuilt native (.node) binaries for unconditional AVX/AVX2 usage. +# Native modules that use AVX instructions MUST have CPUID-based runtime +# dispatch to fall back on CPUs without AVX support (e.g. Intel Atom/Celeron). +# +# Catches issues like https://github.com/ChainSafe/lodestar/issues/9042 +# where a dependency was compiled with hard-coded -C target-feature=+avx2. + +set -euo pipefail + +if ! command -v objdump > /dev/null 2>&1; then + echo "warning: objdump not found, skipping CPU portability check." + exit 0 +fi + +EXIT_CODE=0 + +echo "Checking native modules for CPU portability..." +echo "" + +found=0 +while IFS= read -r node_file; do + found=1 + # Relative path from node_modules for readable output + name=$(echo "$node_file" | sed 's|.*node_modules/||;s|/[^/]*\.node$||') + + # Disassemble once, then grep for AVX indicators and CPUID calls + dump=$(objdump -d "$node_file" 2>/dev/null || true) + + # Count 256-bit AVX (YMM registers) + ymm_count=$(echo "$dump" | grep -c "ymm" || true) + # Count VEX-encoded instructions (v-prefixed SIMD mnemonics like vmovaps, vzeroupper) + # These also require AVX support and will SIGILL on non-AVX CPUs + vex_count=$(echo "$dump" | grep -cE "\b(vmov|vadd|vsub|vmul|vdiv|vxor|vpxor|vpand|vpor|vpcmp|vshuf|vperm|vbroadcast|vinsert|vextract|vzero|vfmadd|vfmsub|vfnmadd|vfnmsub|vcvt|vpack|vpunpck|vpalign|vblend|vround|vtest|vptest)[a-z]*\b" || true) + avx_count=$((ymm_count + vex_count)) + # Count CPUID calls (runtime CPU feature detection) + cpuid_count=$(echo "$dump" | grep -c "cpuid" || true) + + if [ "$avx_count" -gt 0 ] && [ "$cpuid_count" -eq 0 ]; then + echo "FAIL: $name" + echo " $avx_count AVX instructions ($ymm_count ymm, $vex_count vex-encoded), 0 CPUID dispatch calls" + echo " Binary will crash with SIGILL on CPUs without AVX (e.g. Celeron N5105)" + EXIT_CODE=1 + elif [ "$avx_count" -gt 0 ]; then + echo "OK: $name ($avx_count AVX insns, $cpuid_count CPUID calls)" + else + echo "OK: $name (no AVX)" + fi +done < <(find node_modules -name "*.node" -path "*linux-x64*" ! -path "*/rollup/*" ! -path "*/swc/*" ! -path "*musl*" 2>/dev/null | sort) + +if [ "$found" -eq 0 ]; then + echo "No linux-x64 native modules found (skipped)." + exit 0 +fi + +echo "" +if [ "$EXIT_CODE" -eq 0 ]; then + echo "All native modules have proper CPU feature detection" +else + echo "" + echo "Some native modules use AVX without runtime CPU detection." + echo "These will crash with SIGILL (exit code 132) on CPUs without AVX support." + echo "See: https://github.com/ChainSafe/lodestar/issues/9042" +fi + +exit $EXIT_CODE diff --git a/spec-tests-version.json b/spec-tests-version.json new file mode 100644 index 000000000000..74088e7fb373 --- /dev/null +++ b/spec-tests-version.json @@ -0,0 +1,20 @@ +{ + "ethereumConsensusSpecsTests": { + "specVersion": "v1.7.0-alpha.2", + "specTestsRepoUrl": "https://github.com/ethereum/consensus-specs", + "outputDirBase": "spec-tests", + "testsToDownload": [ + "general", + "mainnet", + "minimal" + ] + }, + "blsSpecTests": { + "specVersion": "v0.1.1", + "specTestsRepoUrl": "https://github.com/ethereum/bls12-381-tests", + "outputDirBase": "spec-tests-bls", + "testsToDownload": [ + "bls_tests_yaml" + ] + } +} diff --git a/specrefs/functions.yml b/specrefs/functions.yml index 7c53864f5fb6..afa88c57aad5 100644 --- a/specrefs/functions.yml +++ b/specrefs/functions.yml @@ -5484,7 +5484,7 @@ - name: is_parent_strong#phase0 sources: - file: packages/fork-choice/src/forkChoice/forkChoice.ts - search: "// https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/fork-choice.md#is_parent_strong" + search: "// https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/fork-choice.md#is_parent_strong" spec: | def is_parent_strong(store: Store, root: Root) -> bool: diff --git a/tsconfig.build.json b/tsconfig.build.json index fc17ee92a43c..75959f0924f9 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -1,44 +1,6 @@ { + "extends": "./tsconfig.common.json", "compilerOptions": { - "target": "es2022", - "lib": ["es2022", "dom"], - - "module": "nodenext", - "moduleResolution": "nodenext", - - "pretty": true, - "strict": true, - "sourceMap": true, - "alwaysStrict": true, - "strictNullChecks": true, - "strictFunctionTypes": true, - "strictBindCallApply": true, - "strictPropertyInitialization": true, - "useUnknownInCatchVariables": true, - "noImplicitAny": true, - "noImplicitThis": true, - "noImplicitReturns": true, - - "allowSyntheticDefaultImports": true, - "esModuleInterop": true, - "declaration": true, - "declarationMap": true, - "incremental": true, - "preserveWatchOutput": true, - "noUncheckedSideEffectImports": true, - "rewriteRelativeImportExtensions": true, - // TODO: Investigate following errors: - // - Cannot find module 'rollup/parseAst' or its corresponding type declarations - "skipLibCheck": true, - - // Required to run benchmark command from root directory - "typeRoots": [ - "node_modules/@types", - "./types", - "${configDir}/node_modules/@types", - "${configDir}/types", - "../../node_modules/@types", - "../../types", - ] + "rootDir": "${configDir}/src", } } diff --git a/tsconfig.common.json b/tsconfig.common.json new file mode 100644 index 000000000000..ab263d04fc6b --- /dev/null +++ b/tsconfig.common.json @@ -0,0 +1,50 @@ +{ + "compilerOptions": { + "target": "es2023", + "lib": [ + "es2023", + "dom" + ], + "module": "nodenext", + "moduleResolution": "nodenext", + "pretty": true, + "strict": true, + "sourceMap": true, + "alwaysStrict": true, + "strictNullChecks": true, + "strictFunctionTypes": true, + "strictBindCallApply": true, + "strictPropertyInitialization": true, + "useUnknownInCatchVariables": true, + "noImplicitAny": true, + "noImplicitThis": true, + "noImplicitReturns": true, + "allowSyntheticDefaultImports": true, + "esModuleInterop": true, + "declaration": true, + "declarationMap": true, + "incremental": true, + "preserveWatchOutput": true, + "noUncheckedSideEffectImports": true, + "rewriteRelativeImportExtensions": true, + // TODO: Investigate following errors: + // - Cannot find module 'rollup/parseAst' or its corresponding type declarations + "skipLibCheck": true, + "types": [ + "node", + "bun", + "snappyjs", + "mitt", + "vitest" + ], + // Required to run benchmark command from root directory + "typeRoots": [ + "node_modules/@types", + "./types", + "${configDir}/node_modules/@types", + "${configDir}/types", + "../../node_modules/@types", + "../../types", + ] + } +} diff --git a/tsconfig.e2e.json b/tsconfig.e2e.json index 14cdc4bf044b..71983027398f 100644 --- a/tsconfig.e2e.json +++ b/tsconfig.e2e.json @@ -1,7 +1,7 @@ { "compilerOptions": { - "target": "es2019", - "lib": ["es2020", "esnext.bigint", "es2020.string", "es2020.symbol.wellknown"], + "target": "es2023", + "lib": ["es2023", "dom"], "module": "esnext", "moduleResolution": "node", diff --git a/tsconfig.json b/tsconfig.json index 50360530b8e9..b7328aca571e 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,17 +1,18 @@ { - "extends": "./tsconfig.build.json", + "extends": "./tsconfig.common.json", "compilerOptions": { + "rootDir": "${configDir}", "emitDeclarationOnly": false, "incremental": false, "noEmit": true, // To be used in the test fixtures "resolveJsonModule": true, // We want to speed up the CI run for all tests, which require us to use the - // `transpileOnly` mode for the `ts-node`. This change requires to treat types for each module + // `transpileOnly` mode for the `ts-node`. This change requires to treat types for each module // independently, which is done by setting the `isolatedModules` flag to `true`. "isolatedModules": true, // Detect unused imports, variables, and functions - "noUnusedLocals": true, + "noUnusedLocals": true }, "ts-node": { "transpileOnly": true diff --git a/types/it-pipe/index.d.ts b/types/it-pipe/index.d.ts deleted file mode 100644 index 0abe013aad78..000000000000 --- a/types/it-pipe/index.d.ts +++ /dev/null @@ -1,120 +0,0 @@ -// Overwrite types until this issue is fixed https://github.com/alanshaw/it-pipe/issues/4 -// Original types contain -// ``` -// export function pipe (f1: any, ...rest: any[]): any -// ``` -// Which act as a @ts-ignore compromising type safety - -declare module "it-pipe" { - type Source = AsyncIterable; - type Sink = (source: TSource) => TReturn; - type Duplex = { - sink: Sink; - source: TSource; - }; - - export function pipe(f1: A | (() => A), f2: (source: A) => B): B; - - export function pipe(f1: A | (() => A), f2: (source: A) => B, f3: (source: B) => C): C; - - export function pipe( - f1: A | (() => A), - f2: (source: A) => B, - f3: (source: B) => C, - f4: (source: C) => D - ): D; - - export function pipe( - f1: A | (() => A), - f2: (source: A) => B, - f3: (source: B) => C, - f4: (source: C) => D, - f5: (source: D) => E - ): E; - - export function pipe( - f1: A | (() => A), - f2: (source: A) => B, - f3: (source: B) => C, - f4: (source: C) => D, - f5: (source: D) => E, - f6: (source: E) => F - ): F; - - export function pipe( - f1: A | (() => A), - f2: (source: A) => B, - f3: (source: B) => C, - f4: (source: C) => D, - f5: (source: D) => E, - f6: (source: E) => F, - f7: (source: F) => G - ): G; - - export function pipe( - f1: A | (() => A), - f2: (source: A) => B, - f3: (source: B) => C, - f4: (source: C) => D, - f5: (source: D) => E, - f6: (source: E) => F, - f7: (source: F) => G, - f8: (source: G) => H - ): H; - - export function pipe( - f1: A | (() => A), - f2: (source: A) => B, - f3: (source: B) => C, - f4: (source: C) => D, - f5: (source: D) => E, - f6: (source: E) => F, - f7: (source: F) => G, - f8: (source: G) => H, - f9: (source: H) => I - ): I; - - // First argument is array - - export function pipe(f1: A[], f2: (source: AsyncIterable) => B): B; - - export function pipe(f1: A[], f2: (source: AsyncIterable) => B, f3: (source: B) => C): C; - - export function pipe( - f1: A[], - f2: (source: AsyncIterable) => B, - f3: (source: B) => C, - f4: (source: C) => D - ): D; - - export function pipe( - f1: A[], - f2: (source: AsyncIterable) => B, - f3: (source: B) => C, - f4: (source: C) => D, - f5: (source: D) => E - ): E; - - // First argument is duplex - - export function pipe(f1: Duplex, f2: (source: AsyncIterable) => B): B; - - export function pipe(f1: Duplex, f2: (source: AsyncIterable) => B, f3: (source: B) => C): C; - - export function pipe( - f1: Duplex, - f2: (source: AsyncIterable) => B, - f3: (source: B) => C, - f4: (source: C) => D - ): D; - - export function pipe( - f1: Duplex, - f2: (source: AsyncIterable) => B, - f3: (source: B) => C, - f4: (source: C) => D, - f5: (source: D) => E - ): E; - - export default pipe; -}