From 1b7b4963e0f4a0f44c864030758ecdf6462b2b22 Mon Sep 17 00:00:00 2001 From: Stefan Date: Thu, 30 Jul 2026 20:54:33 +0200 Subject: [PATCH 1/2] add glamsterdam-devnet-7-full subdir: full suite, no repricing marker, 200M/300M, benchmarks/amsterdam New subdir fills the complete EEST benchmark suite (drop -m repricing) at 200M and 300M gas from the frozen benchmarks/amsterdam branch (forks/amsterdam now merges devnet-8 spec PRs). Test types stay stateful/compute; the suite is distinguished by subdir. Build-only for now: the runner test-source files land with the first release, until then dispatchoor skips the subdir (it discovers test types from runner files). --- .github/workflows/benchmarkoor-build.yaml | 1 + .github/workflows/benchmarkoor-run.yaml | 1 + README.md | 3 +- .../v1/glamsterdam-devnet-7-full/clients.yaml | 325 ++++++++++++++++++ .../v1/glamsterdam-devnet-7-full/global.yaml | 3 + .../test-source.compute.builder.yaml | 72 ++++ .../test-source.stateful.builder.yaml | 65 ++++ 7 files changed, 469 insertions(+), 1 deletion(-) create mode 100644 configs/contexts/repricing/v1/glamsterdam-devnet-7-full/clients.yaml create mode 100644 configs/contexts/repricing/v1/glamsterdam-devnet-7-full/global.yaml create mode 100644 configs/contexts/repricing/v1/glamsterdam-devnet-7-full/test-source.compute.builder.yaml create mode 100644 configs/contexts/repricing/v1/glamsterdam-devnet-7-full/test-source.stateful.builder.yaml diff --git a/.github/workflows/benchmarkoor-build.yaml b/.github/workflows/benchmarkoor-build.yaml index abb3842..63ce2d8 100644 --- a/.github/workflows/benchmarkoor-build.yaml +++ b/.github/workflows/benchmarkoor-build.yaml @@ -38,6 +38,7 @@ on: - "v1/bal-devnet-7" - "v1/glamsterdam-devnet-6" - "v1/glamsterdam-devnet-7" + - "v1/glamsterdam-devnet-7-full" test-type: description: "Test type to run" required: false diff --git a/.github/workflows/benchmarkoor-run.yaml b/.github/workflows/benchmarkoor-run.yaml index e33153b..851f652 100644 --- a/.github/workflows/benchmarkoor-run.yaml +++ b/.github/workflows/benchmarkoor-run.yaml @@ -32,6 +32,7 @@ on: - "v1/bal-devnet-7" - "v1/glamsterdam-devnet-6" - "v1/glamsterdam-devnet-7" + - "v1/glamsterdam-devnet-7-full" test-type: description: "Test type to run" required: false diff --git a/README.md b/README.md index e28b8b3..2987db4 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,8 @@ The only **context** is `repricing` (gas-repricing / EIP-7870 scenarios). A **su |--------|-------------| | `v1/bal-devnet-7` | BAL devnet-7 variant | | `v1/glamsterdam-devnet-6` | Glamsterdam devnet-6 variant | -| `v1/glamsterdam-devnet-7` | Glamsterdam devnet-7 variant | +| `v1/glamsterdam-devnet-7` | Glamsterdam devnet-7 variant (`-m repricing` subset, 100M–300M gas sweep) | +| `v1/glamsterdam-devnet-7-full` | Glamsterdam devnet-7, full suite: no repricing marker, 200M/300M gas only, frozen `benchmarks/amsterdam` EEST ref | ### Test Types diff --git a/configs/contexts/repricing/v1/glamsterdam-devnet-7-full/clients.yaml b/configs/contexts/repricing/v1/glamsterdam-devnet-7-full/clients.yaml new file mode 100644 index 0000000..03b001b --- /dev/null +++ b/configs/contexts/repricing/v1/glamsterdam-devnet-7-full/clients.yaml @@ -0,0 +1,325 @@ +runner: + instances: + # Geth + # - id: geth-bal-sequential + # client: geth + # metadata: + # labels: + # bal-mode: sequential + # image: ethpandaops/geth:glamsterdam-devnet-7 + # extra_args: + # - --engine.maxreorgdepth=1024 + # - --override.amsterdam=1 + # - --debug.logslowblock=0 + # - --bal.executionmode=sequential + # #opcode_extraction: + # # enabled: true + # # timeout: 4m + + ### NOTE: Geth doesn't have the bal.* flags anymore in ethpandaops/geth:glamsterdam-devnet-7 + - id: geth-bal-full + client: geth + metadata: + labels: + bal-mode: full + image: ethpandaops/geth:glamsterdam-devnet-7 + extra_args: + - --engine.maxreorgdepth=1024 + - --override.amsterdam=1 + - --debug.logslowblock=0 + #- --bal.executionmode=full + #- --bal.prefetchworkers=300 + #- --bal.blockingprefetch + + # - id: geth-bal-nobatchio + # client: geth + # metadata: + # labels: + # bal-mode: nobatchio + # image: ethpandaops/geth:glamsterdam-devnet-7 + # extra_args: + # - --engine.maxreorgdepth=1024 + # - --override.amsterdam=1 + # - --debug.logslowblock=0 + # - --bal.executionmode=nobatchio + + # Besu + # + # Pinned to caa5abf rather than the rolling glamsterdam-devnet-7 tag, which + # has moved on to 16c6fae and carries a regression. The AOT cache is only + # valid for the exact jars it was generated against, so the AOT instance + # can't follow the tag anyway; pinning the rest keeps the A/B honest instead + # of comparing AOT@caa5abf against non-AOT@. + # Bump all four together, and only once a matching AOT cache exists. + - id: besu-bal-sequential + client: besu + metadata: + labels: + bal-mode: sequential + image: ethpandaops/besu:glamsterdam-devnet-7-caa5abf + genesis_fork_override: { amsterdam: 1 } + environment: + BESU_OPTS: "-Xms8g -Xmx8g -XX:+AlwaysPreTouch" + extra_args: + - --Xplugin-rocksdb-high-spec-enabled=true + # Bal mode flags + - --Xbal-perfect-parallelization-enabled=false + - --Xbal-state-root-enabled=false + - --Xbal-prefetch-reading-enabled=false + + - id: besu-bal-full + client: besu + metadata: + labels: + bal-mode: full + image: ethpandaops/besu:glamsterdam-devnet-7-caa5abf + genesis_fork_override: { amsterdam: 1 } + environment: + BESU_OPTS: "-Xms8g -Xmx8g -XX:+AlwaysPreTouch" + extra_args: + - --Xplugin-rocksdb-high-spec-enabled=true + # Bal mode flags + - --Xbal-perfect-parallelization-enabled=true + - --Xbal-state-root-enabled=true + - --Xbal-prefetch-reading-enabled=true + - --Xbal-prefetch-batch-size=8 + # The following flags don't exist in the glamsterdam-devnet-7 branch: + # - --Xbal-processing-timeout=-1 + # - --Xbal-state-root-timeout=-1 + + # Same as besu-bal-full, but on the AOT-cache image: the JVM starts + # pre-warmed (Project Leyden) so Besu isn't penalised for JIT/C2 warmup. + # PoC image, built manually by baking a Besu-team-provided AOT cache into + # ethpandaops/besu:glamsterdam-devnet-7-caa5abf (no recompile) and validated to load. This + # is experimental and expected to be reverted once comparison numbers are in. + # BESU_OPTS is a JVM env var, so it must go here, not in extra_args. + # AOTMode=on makes the JVM refuse to start if the cache can't be mapped. + # Without it the JVM silently falls back to a cold run (exit 0), which would + # publish results labelled aot:"true" that got no AOT benefit at all. + - id: besu-bal-full-aot + client: besu + metadata: + labels: + bal-mode: full + aot: "true" + image: ethpandaops/besu:glamsterdam-devnet-7-aot-caa5abf + genesis_fork_override: { amsterdam: 1 } + environment: + BESU_OPTS: "-Xms8g -Xmx8g -XX:+AlwaysPreTouch -XX:AOTCache=/opt/besu/aot/besu.aot -XX:AOTMode=on -Xlog:aot=info" + extra_args: + - --Xplugin-rocksdb-high-spec-enabled=true + # Bal mode flags + - --Xbal-perfect-parallelization-enabled=true + - --Xbal-state-root-enabled=true + - --Xbal-prefetch-reading-enabled=true + - --Xbal-prefetch-batch-size=8 + # The following flags don't exist in the glamsterdam-devnet-7 branch: + # - --Xbal-processing-timeout=-1 + # - --Xbal-state-root-timeout=-1 + + - id: besu-bal-nobatchio + client: besu + metadata: + labels: + bal-mode: nobatchio + image: ethpandaops/besu:glamsterdam-devnet-7-caa5abf + genesis_fork_override: { amsterdam: 1 } + environment: + BESU_OPTS: "-Xms8g -Xmx8g -XX:+AlwaysPreTouch" + extra_args: + - --Xplugin-rocksdb-high-spec-enabled=true + # Bal mode flags + - --Xbal-perfect-parallelization-enabled=true + - --Xbal-state-root-enabled=true + - --Xbal-prefetch-reading-enabled=false + # The following flags don't exist in the glamsterdam-devnet-7 branch: + # - --Xbal-processing-timeout=-1 + # - --Xbal-state-root-timeout=-1 + + # Reth + + - id: reth-bal-sequential + client: reth + metadata: + labels: + bal-mode: sequential + image: ethpandaops/reth:glamsterdam-devnet-7 + genesis_fork_override: { amsterdam: 1 } + extra_args: + - --engine.slow-block-threshold=0 + - --engine.disable-bal-parallel-execution + - --engine.disable-bal-parallel-state-root + - --engine.disable-bal-batch-io + retry_new_payloads_failed_state: + enabled: true + max_retries: 10 + backoff: 1s + + - id: reth-bal-full + client: reth + metadata: + labels: + bal-mode: full + image: ethpandaops/reth:glamsterdam-devnet-7 + genesis_fork_override: { amsterdam: 1 } + extra_args: + - --engine.slow-block-threshold=0 + retry_new_payloads_failed_state: + enabled: true + max_retries: 10 + backoff: 1s + + - id: reth-bal-nobatchio + client: reth + metadata: + labels: + bal-mode: nobatchio + image: ethpandaops/reth:glamsterdam-devnet-7 + genesis_fork_override: { amsterdam: 1 } + extra_args: + - --engine.slow-block-threshold=0 + - --engine.disable-bal-batch-io + retry_new_payloads_failed_state: + enabled: true + max_retries: 10 + backoff: 1s + + # Nethermind + + - id: nethermind-bal-full + client: nethermind + metadata: + labels: + bal-mode: full + image: nethermindeth/nethermind:master + genesis_eip_override: { timestamp: 1, eips: [2780, 7708, 7778, 7843, 7928, 7954, 7976, 7981, 7997, 8024, 8037, 8038, 8246, 8282] } + extra_args: + - --Init.BaseDbPath=/data + - --FlatDb.Enabled=true + - --FlatDb.CompactionOffset=0 + - --FlatDb.MinReorgDepth=1024 + - --FlatDb.MaxReorgDepth=1024 + - --Blocks.ParallelExecution=true # Default is true, but set explicitly for clarity + - --Blocks.ParallelExecutionBatchRead=true # Default is true, but set explicitly for clarity + + - id: nethermind-bal-sequential + client: nethermind + metadata: + labels: + bal-mode: sequential + image: nethermindeth/nethermind:master + genesis_eip_override: { timestamp: 1, eips: [2780, 7708, 7778, 7843, 7928, 7954, 7976, 7981, 7997, 8024, 8037, 8038, 8246, 8282] } + extra_args: + - --Init.BaseDbPath=/data + - --FlatDb.Enabled=true + - --FlatDb.CompactionOffset=0 + - --FlatDb.MinReorgDepth=1024 + - --FlatDb.MaxReorgDepth=1024 + - --Blocks.ParallelExecution=false + - --Blocks.ParallelExecutionBatchRead=false + + - id: nethermind-bal-nobatchio + client: nethermind + metadata: + labels: + bal-mode: nobatchio + image: nethermindeth/nethermind:master + genesis_eip_override: { timestamp: 1, eips: [2780, 7708, 7778, 7843, 7928, 7954, 7976, 7981, 7997, 8024, 8037, 8038, 8246, 8282] } + extra_args: + - --Init.BaseDbPath=/data + - --FlatDb.Enabled=true + - --FlatDb.CompactionOffset=0 + - --FlatDb.MinReorgDepth=1024 + - --FlatDb.MaxReorgDepth=1024 + - --Blocks.ParallelExecution=true + - --Blocks.ParallelExecutionBatchRead=false + + # Erigon + + ## Help for erigon flags: + # --exec.batched-io | true | Turn on/off batched ahead of time db reads + # --exec.state-cache | true | Turn on/off local caching of db reads + # --exec.workers | num cpu/2 | Number of workers used for exec + # --exec.serial | false | Force workers to be 1 + # --exec.no-merge. | false. | Turn off background merging + # --exec.no-prune. | false. | Turn off db removal + ## + - id: erigon-bal-full + client: erigon + metadata: + labels: + bal-mode: full + image: ethpandaops/erigon:glamsterdam-devnet-7 + extra_args: + - --override.amsterdam=1 + - --fcu.background.commit=false + - --exec.no-merge=true + - --exec.no-prune=true + - --exec.no-background-maintenance + - --experimental.parallel-commitment + bootstrap_fcu: + enabled: true + max_retries: 120 + backoff: 30s + + - id: erigon-bal-sequential + client: erigon + metadata: + labels: + bal-mode: sequential + image: ethpandaops/erigon:glamsterdam-devnet-7 + extra_args: + - --override.amsterdam=1 + - --fcu.background.commit=false + - --exec.no-merge=true + - --exec.no-prune=true + - --exec.no-background-maintenance + # Bal mode flag + - --exec.serial + bootstrap_fcu: + enabled: true + max_retries: 120 + backoff: 30s + + + - id: erigon-bal-nobatchio + client: erigon + metadata: + labels: + bal-mode: nobatchio + image: ethpandaops/erigon:glamsterdam-devnet-7 + extra_args: + - --override.amsterdam=1 + - --fcu.background.commit=false + - --exec.no-merge=true + - --exec.no-prune=true + - --exec.no-background-maintenance + # Bal mode flag + - --exec.batched-io=false + bootstrap_fcu: + enabled: true + max_retries: 120 + backoff: 30s + + # Ethrex + + - id: ethrex-bal-full + client: ethrex + metadata: + labels: + bal-mode: full + image: ethpandaops/ethrex:glamsterdam-devnet-7 + genesis_fork_override: { amsterdam: 1 } + extra_args: + - --skip-genesis-validation + - --no-precompile-cache + + # Nimbus + + - id: nimbus-bal-full + client: nimbus + metadata: + labels: + bal-mode: full + image: ethpandaops/nimbus-eth1:glamsterdam-devnet-7 + genesis_fork_override: { amsterdam: 1 } diff --git a/configs/contexts/repricing/v1/glamsterdam-devnet-7-full/global.yaml b/configs/contexts/repricing/v1/glamsterdam-devnet-7-full/global.yaml new file mode 100644 index 0000000..fea4915 --- /dev/null +++ b/configs/contexts/repricing/v1/glamsterdam-devnet-7-full/global.yaml @@ -0,0 +1,3 @@ +global: + env: + EEST_FIXTURES_DIR: /tmp/benchmarkoor/eest-fixtures/glamsterdam-devnet-7-full diff --git a/configs/contexts/repricing/v1/glamsterdam-devnet-7-full/test-source.compute.builder.yaml b/configs/contexts/repricing/v1/glamsterdam-devnet-7-full/test-source.compute.builder.yaml new file mode 100644 index 0000000..e254f2d --- /dev/null +++ b/configs/contexts/repricing/v1/glamsterdam-devnet-7-full/test-source.compute.builder.yaml @@ -0,0 +1,72 @@ +builder: + eest_payloads: + pull_policy: always + eest_repo: https://github.com/ethereum/execution-specs.git + # Frozen pre-devnet-8 snapshot of forks/amsterdam (forks/amsterdam now + # merges devnet-8 spec PRs and is no longer usable for benchmark fills). + eest_ref: benchmarks/amsterdam + config: + fork: amsterdam + # Hardhat/Anvil dev key #0 → 0xf39Fd6…2266 (the pre-funded seed above). + rpc_seed_key: "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" + # Full suite: no repricing marker, only the 200M/300M gas points. + gas_benchmark_values: [200,300] + datadir_method: schelk + tests: + - tests/benchmark/compute + # Resolves @pytest.mark.stub_parametrize prefixes against pre-deployed + # state (e.g. bloated_eoa_10GB → 0x87a6…be04). + address_stubs: + bloated_eoa_10GB: + addr: "0x87a6314da5ac8832f6e7a176c8fb133b19f5be04" + pkey: "0x4da32d29f6dcffa26e09dc4e102033f2d105de1444fb893493ae703289275e0e" + # test_blobhash[one_blob] is rejected INVALID by the filler — the fill + # fails without this exclusion (same as the glamsterdam-devnet-7 subdir). + filter: >- + not (test_blobhash and one_blob) + extract_opcode_count: true + targets: + # Nethermind + - name: nethermind + filler_client: nethermind + filler_image: ethpandaops/nethermind:glamsterdam-devnet-7 + source_dir: ${STATE_DIR}/nethermind + filler_extra_args: + - --FlatDb.Enabled=true + - --FlatDb.CompactionOffset=0 + - --FlatDb.MinReorgDepth=1024 + - --FlatDb.MaxReorgDepth=1024 + - --JsonRpc.MaxBatchSize=16384 # Default 1024 + - --Init.AutoDump=None + - --Merge.NewPayloadBlockProcessingTimeout=70000 + - --Blocks.CachePrecompilesOnBlockProcessing=false + # nethermind reads a parity chainspec; amsterdam is activated via the + # per-instance genesis_eip_override (same EIP set as the runner instance). + genesis: ${STATE_DIR}/nethermind/parity-chainspec.json + genesis_eip_override: { timestamp: 1, eips: [2780, 7708, 7778, 7843, 7928, 7954, 7976, 7981, 7997, 8024, 8037, 8038, 8246, 8282] } + output_dir: ${EEST_FIXTURES_DIR}/nethermind + + # Geth + - name: geth + filler_client: geth + filler_image: ethpandaops/geth:glamsterdam-devnet-7 + source_dir: ${STATE_DIR}/geth + # geth boots from the snapshot datadir; activate amsterdam at block 0's + # timestamp (0) + 1 via the override flag — same as the runner geth instance. + filler_extra_args: + - --override.amsterdam=1 + - --rpc.batch-request-limit=16384 # Default 1000 + output_dir: ${EEST_FIXTURES_DIR}/geth + + # Besu + - name: besu + filler_client: besu + filler_image: ethpandaops/besu:glamsterdam-devnet-7-caa5abf + source_dir: ${STATE_DIR}/besu + # besu reads forks from the geth-format genesis; benchmarkoor patches + # amsterdamTime in at boot (snapshot block 0 ts is 0, so amsterdam at ts 1). + filler_extra_args: + - --rpc-http-max-batch-size=16384 # Default 1024 + genesis: ${STATE_DIR}/besu/besu-chainspec.json + genesis_fork_override: { amsterdam: 1 } + output_dir: ${EEST_FIXTURES_DIR}/besu diff --git a/configs/contexts/repricing/v1/glamsterdam-devnet-7-full/test-source.stateful.builder.yaml b/configs/contexts/repricing/v1/glamsterdam-devnet-7-full/test-source.stateful.builder.yaml new file mode 100644 index 0000000..dd43a1c --- /dev/null +++ b/configs/contexts/repricing/v1/glamsterdam-devnet-7-full/test-source.stateful.builder.yaml @@ -0,0 +1,65 @@ +builder: + eest_payloads: + pull_policy: always + eest_repo: https://github.com/ethereum/execution-specs.git + # Frozen pre-devnet-8 snapshot of forks/amsterdam (forks/amsterdam now + # merges devnet-8 spec PRs and is no longer usable for benchmark fills). + eest_ref: benchmarks/amsterdam + config: + fork: amsterdam + # Key #0 → 0xf39Fd6…2266 (the pre-funded seed in the state-actor template) + rpc_seed_key: "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" + # Full suite: no repricing marker, only the 200M/300M gas points. + gas_benchmark_values: [200,300] + datadir_method: schelk + tests: + - tests/benchmark/stateful/bloatnet + # Resolves @pytest.mark.stub_parametrize prefixes against pre-deployed + # state (e.g. bloated_eoa_10GB → 0x87a6…be04). Unmarked tests needing + # stubs the prestate lacks (factory_stub, erc20_stub) warn and skip. + address_stubs: + bloated_eoa_10GB: + addr: "0x87a6314da5ac8832f6e7a176c8fb133b19f5be04" + pkey: "0x4da32d29f6dcffa26e09dc4e102033f2d105de1444fb893493ae703289275e0e" + extract_opcode_count: true + targets: + # Nethermind + - name: nethermind + filler_client: nethermind + filler_image: ethpandaops/nethermind:glamsterdam-devnet-7 + source_dir: ${STATE_DIR}/nethermind + filler_extra_args: + - --FlatDb.Enabled=true + - --FlatDb.CompactionOffset=0 + - --FlatDb.MinReorgDepth=1024 + - --FlatDb.MaxReorgDepth=1024 + - --Init.AutoDump=None + - --Merge.NewPayloadBlockProcessingTimeout=70000 + - --Blocks.CachePrecompilesOnBlockProcessing=false + # nethermind reads a parity chainspec; amsterdam is activated via the + # per-instance genesis_eip_override (same EIP set as the runner instance). + genesis: ${STATE_DIR}/nethermind/parity-chainspec.json + genesis_eip_override: { timestamp: 1, eips: [2780, 7708, 7778, 7843, 7928, 7954, 7976, 7981, 7997, 8024, 8037, 8038, 8246, 8282] } + output_dir: ${EEST_FIXTURES_DIR}/nethermind + + # Geth + - name: geth + filler_client: geth + filler_image: ethpandaops/geth:glamsterdam-devnet-7 + source_dir: ${STATE_DIR}/geth + # geth boots from the snapshot datadir; activate amsterdam at block 0's + # timestamp (0) + 1 via the override flag — same as the runner geth instance. + filler_extra_args: + - --override.amsterdam=1 + output_dir: ${EEST_FIXTURES_DIR}/geth + + # Besu + - name: besu + filler_client: besu + filler_image: ethpandaops/besu:glamsterdam-devnet-7-caa5abf + source_dir: ${STATE_DIR}/besu + # besu reads forks from the geth-format genesis; benchmarkoor patches + # amsterdamTime in at boot (snapshot block 0 ts is 0, so amsterdam at ts 1). + genesis: ${STATE_DIR}/besu/besu-chainspec.json + genesis_fork_override: { amsterdam: 1 } + output_dir: ${EEST_FIXTURES_DIR}/besu From 5d832979a74fb5a0b60d1fd692acfb221f11269d Mon Sep 17 00:00:00 2001 From: Stefan Date: Fri, 31 Jul 2026 11:58:25 +0200 Subject: [PATCH 2/2] =?UTF-8?q?glamsterdam-devnet-7-full:=20address=20revi?= =?UTF-8?q?ew=20=E2=80=94=20bal-full=20instances=20only,=20align=20filler?= =?UTF-8?q?=20capacity=20flags?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - clients.yaml: drop the sequential/nobatchio/AOT A/B variants; the full suite runs one bal-full instance per client (the comparisons stay in v1/glamsterdam-devnet-7). - fillers: raise RPC batch limits on stateful too (the full suite fills tests the repricing marker excluded, e.g. test_create), and set geth's engine.maxreorgdepth=1024 for the long-chain benchmarks. --- .../v1/glamsterdam-devnet-7-full/clients.yaml | 223 +----------------- .../test-source.compute.builder.yaml | 1 + .../test-source.stateful.builder.yaml | 5 + 3 files changed, 10 insertions(+), 219 deletions(-) diff --git a/configs/contexts/repricing/v1/glamsterdam-devnet-7-full/clients.yaml b/configs/contexts/repricing/v1/glamsterdam-devnet-7-full/clients.yaml index 03b001b..77ef30c 100644 --- a/configs/contexts/repricing/v1/glamsterdam-devnet-7-full/clients.yaml +++ b/configs/contexts/repricing/v1/glamsterdam-devnet-7-full/clients.yaml @@ -1,21 +1,9 @@ runner: + # Full-suite runs: one bal-full instance per client (no sequential/nobatchio/ + # AOT A/B variants — those comparisons live in v1/glamsterdam-devnet-7). + # Instance configs match their v1/glamsterdam-devnet-7 counterparts. instances: # Geth - # - id: geth-bal-sequential - # client: geth - # metadata: - # labels: - # bal-mode: sequential - # image: ethpandaops/geth:glamsterdam-devnet-7 - # extra_args: - # - --engine.maxreorgdepth=1024 - # - --override.amsterdam=1 - # - --debug.logslowblock=0 - # - --bal.executionmode=sequential - # #opcode_extraction: - # # enabled: true - # # timeout: 4m - ### NOTE: Geth doesn't have the bal.* flags anymore in ethpandaops/geth:glamsterdam-devnet-7 - id: geth-bal-full client: geth @@ -27,46 +15,11 @@ runner: - --engine.maxreorgdepth=1024 - --override.amsterdam=1 - --debug.logslowblock=0 - #- --bal.executionmode=full - #- --bal.prefetchworkers=300 - #- --bal.blockingprefetch - - # - id: geth-bal-nobatchio - # client: geth - # metadata: - # labels: - # bal-mode: nobatchio - # image: ethpandaops/geth:glamsterdam-devnet-7 - # extra_args: - # - --engine.maxreorgdepth=1024 - # - --override.amsterdam=1 - # - --debug.logslowblock=0 - # - --bal.executionmode=nobatchio # Besu # # Pinned to caa5abf rather than the rolling glamsterdam-devnet-7 tag, which - # has moved on to 16c6fae and carries a regression. The AOT cache is only - # valid for the exact jars it was generated against, so the AOT instance - # can't follow the tag anyway; pinning the rest keeps the A/B honest instead - # of comparing AOT@caa5abf against non-AOT@. - # Bump all four together, and only once a matching AOT cache exists. - - id: besu-bal-sequential - client: besu - metadata: - labels: - bal-mode: sequential - image: ethpandaops/besu:glamsterdam-devnet-7-caa5abf - genesis_fork_override: { amsterdam: 1 } - environment: - BESU_OPTS: "-Xms8g -Xmx8g -XX:+AlwaysPreTouch" - extra_args: - - --Xplugin-rocksdb-high-spec-enabled=true - # Bal mode flags - - --Xbal-perfect-parallelization-enabled=false - - --Xbal-state-root-enabled=false - - --Xbal-prefetch-reading-enabled=false - + # has moved on to 16c6fae and carries a regression. - id: besu-bal-full client: besu metadata: @@ -83,78 +36,8 @@ runner: - --Xbal-state-root-enabled=true - --Xbal-prefetch-reading-enabled=true - --Xbal-prefetch-batch-size=8 - # The following flags don't exist in the glamsterdam-devnet-7 branch: - # - --Xbal-processing-timeout=-1 - # - --Xbal-state-root-timeout=-1 - - # Same as besu-bal-full, but on the AOT-cache image: the JVM starts - # pre-warmed (Project Leyden) so Besu isn't penalised for JIT/C2 warmup. - # PoC image, built manually by baking a Besu-team-provided AOT cache into - # ethpandaops/besu:glamsterdam-devnet-7-caa5abf (no recompile) and validated to load. This - # is experimental and expected to be reverted once comparison numbers are in. - # BESU_OPTS is a JVM env var, so it must go here, not in extra_args. - # AOTMode=on makes the JVM refuse to start if the cache can't be mapped. - # Without it the JVM silently falls back to a cold run (exit 0), which would - # publish results labelled aot:"true" that got no AOT benefit at all. - - id: besu-bal-full-aot - client: besu - metadata: - labels: - bal-mode: full - aot: "true" - image: ethpandaops/besu:glamsterdam-devnet-7-aot-caa5abf - genesis_fork_override: { amsterdam: 1 } - environment: - BESU_OPTS: "-Xms8g -Xmx8g -XX:+AlwaysPreTouch -XX:AOTCache=/opt/besu/aot/besu.aot -XX:AOTMode=on -Xlog:aot=info" - extra_args: - - --Xplugin-rocksdb-high-spec-enabled=true - # Bal mode flags - - --Xbal-perfect-parallelization-enabled=true - - --Xbal-state-root-enabled=true - - --Xbal-prefetch-reading-enabled=true - - --Xbal-prefetch-batch-size=8 - # The following flags don't exist in the glamsterdam-devnet-7 branch: - # - --Xbal-processing-timeout=-1 - # - --Xbal-state-root-timeout=-1 - - - id: besu-bal-nobatchio - client: besu - metadata: - labels: - bal-mode: nobatchio - image: ethpandaops/besu:glamsterdam-devnet-7-caa5abf - genesis_fork_override: { amsterdam: 1 } - environment: - BESU_OPTS: "-Xms8g -Xmx8g -XX:+AlwaysPreTouch" - extra_args: - - --Xplugin-rocksdb-high-spec-enabled=true - # Bal mode flags - - --Xbal-perfect-parallelization-enabled=true - - --Xbal-state-root-enabled=true - - --Xbal-prefetch-reading-enabled=false - # The following flags don't exist in the glamsterdam-devnet-7 branch: - # - --Xbal-processing-timeout=-1 - # - --Xbal-state-root-timeout=-1 # Reth - - - id: reth-bal-sequential - client: reth - metadata: - labels: - bal-mode: sequential - image: ethpandaops/reth:glamsterdam-devnet-7 - genesis_fork_override: { amsterdam: 1 } - extra_args: - - --engine.slow-block-threshold=0 - - --engine.disable-bal-parallel-execution - - --engine.disable-bal-parallel-state-root - - --engine.disable-bal-batch-io - retry_new_payloads_failed_state: - enabled: true - max_retries: 10 - backoff: 1s - - id: reth-bal-full client: reth metadata: @@ -169,23 +52,7 @@ runner: max_retries: 10 backoff: 1s - - id: reth-bal-nobatchio - client: reth - metadata: - labels: - bal-mode: nobatchio - image: ethpandaops/reth:glamsterdam-devnet-7 - genesis_fork_override: { amsterdam: 1 } - extra_args: - - --engine.slow-block-threshold=0 - - --engine.disable-bal-batch-io - retry_new_payloads_failed_state: - enabled: true - max_retries: 10 - backoff: 1s - # Nethermind - - id: nethermind-bal-full client: nethermind metadata: @@ -202,48 +69,7 @@ runner: - --Blocks.ParallelExecution=true # Default is true, but set explicitly for clarity - --Blocks.ParallelExecutionBatchRead=true # Default is true, but set explicitly for clarity - - id: nethermind-bal-sequential - client: nethermind - metadata: - labels: - bal-mode: sequential - image: nethermindeth/nethermind:master - genesis_eip_override: { timestamp: 1, eips: [2780, 7708, 7778, 7843, 7928, 7954, 7976, 7981, 7997, 8024, 8037, 8038, 8246, 8282] } - extra_args: - - --Init.BaseDbPath=/data - - --FlatDb.Enabled=true - - --FlatDb.CompactionOffset=0 - - --FlatDb.MinReorgDepth=1024 - - --FlatDb.MaxReorgDepth=1024 - - --Blocks.ParallelExecution=false - - --Blocks.ParallelExecutionBatchRead=false - - - id: nethermind-bal-nobatchio - client: nethermind - metadata: - labels: - bal-mode: nobatchio - image: nethermindeth/nethermind:master - genesis_eip_override: { timestamp: 1, eips: [2780, 7708, 7778, 7843, 7928, 7954, 7976, 7981, 7997, 8024, 8037, 8038, 8246, 8282] } - extra_args: - - --Init.BaseDbPath=/data - - --FlatDb.Enabled=true - - --FlatDb.CompactionOffset=0 - - --FlatDb.MinReorgDepth=1024 - - --FlatDb.MaxReorgDepth=1024 - - --Blocks.ParallelExecution=true - - --Blocks.ParallelExecutionBatchRead=false - # Erigon - - ## Help for erigon flags: - # --exec.batched-io | true | Turn on/off batched ahead of time db reads - # --exec.state-cache | true | Turn on/off local caching of db reads - # --exec.workers | num cpu/2 | Number of workers used for exec - # --exec.serial | false | Force workers to be 1 - # --exec.no-merge. | false. | Turn off background merging - # --exec.no-prune. | false. | Turn off db removal - ## - id: erigon-bal-full client: erigon metadata: @@ -262,47 +88,7 @@ runner: max_retries: 120 backoff: 30s - - id: erigon-bal-sequential - client: erigon - metadata: - labels: - bal-mode: sequential - image: ethpandaops/erigon:glamsterdam-devnet-7 - extra_args: - - --override.amsterdam=1 - - --fcu.background.commit=false - - --exec.no-merge=true - - --exec.no-prune=true - - --exec.no-background-maintenance - # Bal mode flag - - --exec.serial - bootstrap_fcu: - enabled: true - max_retries: 120 - backoff: 30s - - - - id: erigon-bal-nobatchio - client: erigon - metadata: - labels: - bal-mode: nobatchio - image: ethpandaops/erigon:glamsterdam-devnet-7 - extra_args: - - --override.amsterdam=1 - - --fcu.background.commit=false - - --exec.no-merge=true - - --exec.no-prune=true - - --exec.no-background-maintenance - # Bal mode flag - - --exec.batched-io=false - bootstrap_fcu: - enabled: true - max_retries: 120 - backoff: 30s - # Ethrex - - id: ethrex-bal-full client: ethrex metadata: @@ -315,7 +101,6 @@ runner: - --no-precompile-cache # Nimbus - - id: nimbus-bal-full client: nimbus metadata: diff --git a/configs/contexts/repricing/v1/glamsterdam-devnet-7-full/test-source.compute.builder.yaml b/configs/contexts/repricing/v1/glamsterdam-devnet-7-full/test-source.compute.builder.yaml index e254f2d..284b81d 100644 --- a/configs/contexts/repricing/v1/glamsterdam-devnet-7-full/test-source.compute.builder.yaml +++ b/configs/contexts/repricing/v1/glamsterdam-devnet-7-full/test-source.compute.builder.yaml @@ -55,6 +55,7 @@ builder: # timestamp (0) + 1 via the override flag — same as the runner geth instance. filler_extra_args: - --override.amsterdam=1 + - --engine.maxreorgdepth=1024 # Long-chain benchmarks exceed the 512 default - --rpc.batch-request-limit=16384 # Default 1000 output_dir: ${EEST_FIXTURES_DIR}/geth diff --git a/configs/contexts/repricing/v1/glamsterdam-devnet-7-full/test-source.stateful.builder.yaml b/configs/contexts/repricing/v1/glamsterdam-devnet-7-full/test-source.stateful.builder.yaml index dd43a1c..c350ceb 100644 --- a/configs/contexts/repricing/v1/glamsterdam-devnet-7-full/test-source.stateful.builder.yaml +++ b/configs/contexts/repricing/v1/glamsterdam-devnet-7-full/test-source.stateful.builder.yaml @@ -33,6 +33,7 @@ builder: - --FlatDb.CompactionOffset=0 - --FlatDb.MinReorgDepth=1024 - --FlatDb.MaxReorgDepth=1024 + - --JsonRpc.MaxBatchSize=16384 # Default 1024 - --Init.AutoDump=None - --Merge.NewPayloadBlockProcessingTimeout=70000 - --Blocks.CachePrecompilesOnBlockProcessing=false @@ -51,6 +52,8 @@ builder: # timestamp (0) + 1 via the override flag — same as the runner geth instance. filler_extra_args: - --override.amsterdam=1 + - --engine.maxreorgdepth=1024 # Long-chain benchmarks exceed the 512 default + - --rpc.batch-request-limit=16384 # Default 1000 output_dir: ${EEST_FIXTURES_DIR}/geth # Besu @@ -58,6 +61,8 @@ builder: filler_client: besu filler_image: ethpandaops/besu:glamsterdam-devnet-7-caa5abf source_dir: ${STATE_DIR}/besu + filler_extra_args: + - --rpc-http-max-batch-size=16384 # Default 1024 # besu reads forks from the geth-format genesis; benchmarkoor patches # amsterdamTime in at boot (snapshot block 0 ts is 0, so amsterdam at ts 1). genesis: ${STATE_DIR}/besu/besu-chainspec.json