diff --git a/.github/workflows/benchmarkoor-build.yaml b/.github/workflows/benchmarkoor-build.yaml index abb3842..63ce2d8 100644 --- a/.github/workflows/benchmarkoor-build.yaml +++ b/.github/workflows/benchmarkoor-build.yaml @@ -38,6 +38,7 @@ on: - "v1/bal-devnet-7" - "v1/glamsterdam-devnet-6" - "v1/glamsterdam-devnet-7" + - "v1/glamsterdam-devnet-7-full" test-type: description: "Test type to run" required: false diff --git a/.github/workflows/benchmarkoor-run.yaml b/.github/workflows/benchmarkoor-run.yaml index e33153b..851f652 100644 --- a/.github/workflows/benchmarkoor-run.yaml +++ b/.github/workflows/benchmarkoor-run.yaml @@ -32,6 +32,7 @@ on: - "v1/bal-devnet-7" - "v1/glamsterdam-devnet-6" - "v1/glamsterdam-devnet-7" + - "v1/glamsterdam-devnet-7-full" test-type: description: "Test type to run" required: false diff --git a/README.md b/README.md index e28b8b3..2987db4 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,8 @@ The only **context** is `repricing` (gas-repricing / EIP-7870 scenarios). A **su |--------|-------------| | `v1/bal-devnet-7` | BAL devnet-7 variant | | `v1/glamsterdam-devnet-6` | Glamsterdam devnet-6 variant | -| `v1/glamsterdam-devnet-7` | Glamsterdam devnet-7 variant | +| `v1/glamsterdam-devnet-7` | Glamsterdam devnet-7 variant (`-m repricing` subset, 100M–300M gas sweep) | +| `v1/glamsterdam-devnet-7-full` | Glamsterdam devnet-7, full suite: no repricing marker, 200M/300M gas only, frozen `benchmarks/amsterdam` EEST ref | ### Test Types diff --git a/configs/contexts/repricing/v1/glamsterdam-devnet-7-full/clients.yaml b/configs/contexts/repricing/v1/glamsterdam-devnet-7-full/clients.yaml new file mode 100644 index 0000000..77ef30c --- /dev/null +++ b/configs/contexts/repricing/v1/glamsterdam-devnet-7-full/clients.yaml @@ -0,0 +1,110 @@ +runner: + # Full-suite runs: one bal-full instance per client (no sequential/nobatchio/ + # AOT A/B variants — those comparisons live in v1/glamsterdam-devnet-7). + # Instance configs match their v1/glamsterdam-devnet-7 counterparts. + instances: + # Geth + ### NOTE: Geth doesn't have the bal.* flags anymore in ethpandaops/geth:glamsterdam-devnet-7 + - id: geth-bal-full + client: geth + metadata: + labels: + bal-mode: full + image: ethpandaops/geth:glamsterdam-devnet-7 + extra_args: + - --engine.maxreorgdepth=1024 + - --override.amsterdam=1 + - --debug.logslowblock=0 + + # Besu + # + # Pinned to caa5abf rather than the rolling glamsterdam-devnet-7 tag, which + # has moved on to 16c6fae and carries a regression. + - id: besu-bal-full + client: besu + metadata: + labels: + bal-mode: full + image: ethpandaops/besu:glamsterdam-devnet-7-caa5abf + genesis_fork_override: { amsterdam: 1 } + environment: + BESU_OPTS: "-Xms8g -Xmx8g -XX:+AlwaysPreTouch" + extra_args: + - --Xplugin-rocksdb-high-spec-enabled=true + # Bal mode flags + - --Xbal-perfect-parallelization-enabled=true + - --Xbal-state-root-enabled=true + - --Xbal-prefetch-reading-enabled=true + - --Xbal-prefetch-batch-size=8 + + # Reth + - id: reth-bal-full + client: reth + metadata: + labels: + bal-mode: full + image: ethpandaops/reth:glamsterdam-devnet-7 + genesis_fork_override: { amsterdam: 1 } + extra_args: + - --engine.slow-block-threshold=0 + retry_new_payloads_failed_state: + enabled: true + max_retries: 10 + backoff: 1s + + # Nethermind + - id: nethermind-bal-full + client: nethermind + metadata: + labels: + bal-mode: full + image: nethermindeth/nethermind:master + genesis_eip_override: { timestamp: 1, eips: [2780, 7708, 7778, 7843, 7928, 7954, 7976, 7981, 7997, 8024, 8037, 8038, 8246, 8282] } + extra_args: + - --Init.BaseDbPath=/data + - --FlatDb.Enabled=true + - --FlatDb.CompactionOffset=0 + - --FlatDb.MinReorgDepth=1024 + - --FlatDb.MaxReorgDepth=1024 + - --Blocks.ParallelExecution=true # Default is true, but set explicitly for clarity + - --Blocks.ParallelExecutionBatchRead=true # Default is true, but set explicitly for clarity + + # Erigon + - id: erigon-bal-full + client: erigon + metadata: + labels: + bal-mode: full + image: ethpandaops/erigon:glamsterdam-devnet-7 + extra_args: + - --override.amsterdam=1 + - --fcu.background.commit=false + - --exec.no-merge=true + - --exec.no-prune=true + - --exec.no-background-maintenance + - --experimental.parallel-commitment + bootstrap_fcu: + enabled: true + max_retries: 120 + backoff: 30s + + # Ethrex + - id: ethrex-bal-full + client: ethrex + metadata: + labels: + bal-mode: full + image: ethpandaops/ethrex:glamsterdam-devnet-7 + genesis_fork_override: { amsterdam: 1 } + extra_args: + - --skip-genesis-validation + - --no-precompile-cache + + # Nimbus + - id: nimbus-bal-full + client: nimbus + metadata: + labels: + bal-mode: full + image: ethpandaops/nimbus-eth1:glamsterdam-devnet-7 + genesis_fork_override: { amsterdam: 1 } diff --git a/configs/contexts/repricing/v1/glamsterdam-devnet-7-full/global.yaml b/configs/contexts/repricing/v1/glamsterdam-devnet-7-full/global.yaml new file mode 100644 index 0000000..fea4915 --- /dev/null +++ b/configs/contexts/repricing/v1/glamsterdam-devnet-7-full/global.yaml @@ -0,0 +1,3 @@ +global: + env: + EEST_FIXTURES_DIR: /tmp/benchmarkoor/eest-fixtures/glamsterdam-devnet-7-full diff --git a/configs/contexts/repricing/v1/glamsterdam-devnet-7-full/test-source.compute.builder.yaml b/configs/contexts/repricing/v1/glamsterdam-devnet-7-full/test-source.compute.builder.yaml new file mode 100644 index 0000000..284b81d --- /dev/null +++ b/configs/contexts/repricing/v1/glamsterdam-devnet-7-full/test-source.compute.builder.yaml @@ -0,0 +1,73 @@ +builder: + eest_payloads: + pull_policy: always + eest_repo: https://github.com/ethereum/execution-specs.git + # Frozen pre-devnet-8 snapshot of forks/amsterdam (forks/amsterdam now + # merges devnet-8 spec PRs and is no longer usable for benchmark fills). + eest_ref: benchmarks/amsterdam + config: + fork: amsterdam + # Hardhat/Anvil dev key #0 → 0xf39Fd6…2266 (the pre-funded seed above). + rpc_seed_key: "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" + # Full suite: no repricing marker, only the 200M/300M gas points. + gas_benchmark_values: [200,300] + datadir_method: schelk + tests: + - tests/benchmark/compute + # Resolves @pytest.mark.stub_parametrize prefixes against pre-deployed + # state (e.g. bloated_eoa_10GB → 0x87a6…be04). + address_stubs: + bloated_eoa_10GB: + addr: "0x87a6314da5ac8832f6e7a176c8fb133b19f5be04" + pkey: "0x4da32d29f6dcffa26e09dc4e102033f2d105de1444fb893493ae703289275e0e" + # test_blobhash[one_blob] is rejected INVALID by the filler — the fill + # fails without this exclusion (same as the glamsterdam-devnet-7 subdir). + filter: >- + not (test_blobhash and one_blob) + extract_opcode_count: true + targets: + # Nethermind + - name: nethermind + filler_client: nethermind + filler_image: ethpandaops/nethermind:glamsterdam-devnet-7 + source_dir: ${STATE_DIR}/nethermind + filler_extra_args: + - --FlatDb.Enabled=true + - --FlatDb.CompactionOffset=0 + - --FlatDb.MinReorgDepth=1024 + - --FlatDb.MaxReorgDepth=1024 + - --JsonRpc.MaxBatchSize=16384 # Default 1024 + - --Init.AutoDump=None + - --Merge.NewPayloadBlockProcessingTimeout=70000 + - --Blocks.CachePrecompilesOnBlockProcessing=false + # nethermind reads a parity chainspec; amsterdam is activated via the + # per-instance genesis_eip_override (same EIP set as the runner instance). + genesis: ${STATE_DIR}/nethermind/parity-chainspec.json + genesis_eip_override: { timestamp: 1, eips: [2780, 7708, 7778, 7843, 7928, 7954, 7976, 7981, 7997, 8024, 8037, 8038, 8246, 8282] } + output_dir: ${EEST_FIXTURES_DIR}/nethermind + + # Geth + - name: geth + filler_client: geth + filler_image: ethpandaops/geth:glamsterdam-devnet-7 + source_dir: ${STATE_DIR}/geth + # geth boots from the snapshot datadir; activate amsterdam at block 0's + # timestamp (0) + 1 via the override flag — same as the runner geth instance. + filler_extra_args: + - --override.amsterdam=1 + - --engine.maxreorgdepth=1024 # Long-chain benchmarks exceed the 512 default + - --rpc.batch-request-limit=16384 # Default 1000 + output_dir: ${EEST_FIXTURES_DIR}/geth + + # Besu + - name: besu + filler_client: besu + filler_image: ethpandaops/besu:glamsterdam-devnet-7-caa5abf + source_dir: ${STATE_DIR}/besu + # besu reads forks from the geth-format genesis; benchmarkoor patches + # amsterdamTime in at boot (snapshot block 0 ts is 0, so amsterdam at ts 1). + filler_extra_args: + - --rpc-http-max-batch-size=16384 # Default 1024 + genesis: ${STATE_DIR}/besu/besu-chainspec.json + genesis_fork_override: { amsterdam: 1 } + output_dir: ${EEST_FIXTURES_DIR}/besu diff --git a/configs/contexts/repricing/v1/glamsterdam-devnet-7-full/test-source.stateful.builder.yaml b/configs/contexts/repricing/v1/glamsterdam-devnet-7-full/test-source.stateful.builder.yaml new file mode 100644 index 0000000..c350ceb --- /dev/null +++ b/configs/contexts/repricing/v1/glamsterdam-devnet-7-full/test-source.stateful.builder.yaml @@ -0,0 +1,70 @@ +builder: + eest_payloads: + pull_policy: always + eest_repo: https://github.com/ethereum/execution-specs.git + # Frozen pre-devnet-8 snapshot of forks/amsterdam (forks/amsterdam now + # merges devnet-8 spec PRs and is no longer usable for benchmark fills). + eest_ref: benchmarks/amsterdam + config: + fork: amsterdam + # Key #0 → 0xf39Fd6…2266 (the pre-funded seed in the state-actor template) + rpc_seed_key: "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" + # Full suite: no repricing marker, only the 200M/300M gas points. + gas_benchmark_values: [200,300] + datadir_method: schelk + tests: + - tests/benchmark/stateful/bloatnet + # Resolves @pytest.mark.stub_parametrize prefixes against pre-deployed + # state (e.g. bloated_eoa_10GB → 0x87a6…be04). Unmarked tests needing + # stubs the prestate lacks (factory_stub, erc20_stub) warn and skip. + address_stubs: + bloated_eoa_10GB: + addr: "0x87a6314da5ac8832f6e7a176c8fb133b19f5be04" + pkey: "0x4da32d29f6dcffa26e09dc4e102033f2d105de1444fb893493ae703289275e0e" + extract_opcode_count: true + targets: + # Nethermind + - name: nethermind + filler_client: nethermind + filler_image: ethpandaops/nethermind:glamsterdam-devnet-7 + source_dir: ${STATE_DIR}/nethermind + filler_extra_args: + - --FlatDb.Enabled=true + - --FlatDb.CompactionOffset=0 + - --FlatDb.MinReorgDepth=1024 + - --FlatDb.MaxReorgDepth=1024 + - --JsonRpc.MaxBatchSize=16384 # Default 1024 + - --Init.AutoDump=None + - --Merge.NewPayloadBlockProcessingTimeout=70000 + - --Blocks.CachePrecompilesOnBlockProcessing=false + # nethermind reads a parity chainspec; amsterdam is activated via the + # per-instance genesis_eip_override (same EIP set as the runner instance). + genesis: ${STATE_DIR}/nethermind/parity-chainspec.json + genesis_eip_override: { timestamp: 1, eips: [2780, 7708, 7778, 7843, 7928, 7954, 7976, 7981, 7997, 8024, 8037, 8038, 8246, 8282] } + output_dir: ${EEST_FIXTURES_DIR}/nethermind + + # Geth + - name: geth + filler_client: geth + filler_image: ethpandaops/geth:glamsterdam-devnet-7 + source_dir: ${STATE_DIR}/geth + # geth boots from the snapshot datadir; activate amsterdam at block 0's + # timestamp (0) + 1 via the override flag — same as the runner geth instance. + filler_extra_args: + - --override.amsterdam=1 + - --engine.maxreorgdepth=1024 # Long-chain benchmarks exceed the 512 default + - --rpc.batch-request-limit=16384 # Default 1000 + output_dir: ${EEST_FIXTURES_DIR}/geth + + # Besu + - name: besu + filler_client: besu + filler_image: ethpandaops/besu:glamsterdam-devnet-7-caa5abf + source_dir: ${STATE_DIR}/besu + filler_extra_args: + - --rpc-http-max-batch-size=16384 # Default 1024 + # besu reads forks from the geth-format genesis; benchmarkoor patches + # amsterdamTime in at boot (snapshot block 0 ts is 0, so amsterdam at ts 1). + genesis: ${STATE_DIR}/besu/besu-chainspec.json + genesis_fork_override: { amsterdam: 1 } + output_dir: ${EEST_FIXTURES_DIR}/besu