diff --git a/.github/workflows/_build-binaries.yaml b/.github/workflows/_build-binaries.yaml index 0a9accc8..89fb73dd 100644 --- a/.github/workflows/_build-binaries.yaml +++ b/.github/workflows/_build-binaries.yaml @@ -80,7 +80,7 @@ jobs: - name: Cache reth binary uses: actions/cache@2f8e54208210a422b2efd51efaa6bd6d7ca8920f # v3.4.3 - id: cache-optimism + id: cache-reth with: path: ~/bin/reth key: ${{ runner.os }}-reth-${{ inputs.optimism_version }} diff --git a/benchmark/flags/flags.go b/benchmark/flags/flags.go index 368ab569..24d562e1 100644 --- a/benchmark/flags/flags.go +++ b/benchmark/flags/flags.go @@ -44,17 +44,17 @@ var ( } RootDirFlag = &cli.StringFlag{ - Name: RootDirFlagName, - Usage: "Root Directory", - EnvVars: prefixEnvVars("ROOT_DIR"), - Required: true, + Name: RootDirFlagName, + Usage: "Root Directory", + EnvVars: prefixEnvVars("ROOT_DIR"), + Value: "./data-dir", } OutputDirFlag = &cli.StringFlag{ - Name: OutputDirFlagName, - Usage: "Output Directory", - EnvVars: prefixEnvVars("OUTPUT_DIR"), - Required: true, + Name: OutputDirFlagName, + Usage: "Output Directory", + Value: "./output", + EnvVars: prefixEnvVars("OUTPUT_DIR"), } TxFuzzBinFlag = &cli.StringFlag{ diff --git a/report/src/components/ChartGrid.tsx b/report/src/components/ChartGrid.tsx index 72174520..bd525329 100644 --- a/report/src/components/ChartGrid.tsx +++ b/report/src/components/ChartGrid.tsx @@ -8,16 +8,31 @@ interface ProvidedProps { role: "sequencer" | "validator" | null; } +function resolveMetricKey( + data: DataSeries[], + primaryKey: string, + aliases: string[] = [], +): string { + const keys = [primaryKey, ...aliases]; + const chartData = data.flatMap((s) => s.data); + for (const key of keys) { + if (chartData.some((d) => d.ExecutionMetrics[key] !== undefined)) { + return key; + } + } + return primaryKey; +} + const ChartGrid: React.FC = ({ data, role }: ProvidedProps) => { return (
{SORTED_CHART_CONFIG.map(([metricKey, config]) => { - // sequencer and validator have different thresholds + const resolvedKey = resolveMetricKey(data, metricKey, config.aliases); const thresholdKey = role ? `${role}/${metricKey}` : null; const chartData = data.flatMap((s) => s.data); const thresholds = data[0]?.thresholds; const executionMetrics = chartData - .map((d) => d.ExecutionMetrics[metricKey]) + .map((d) => d.ExecutionMetrics[resolvedKey]) .filter((v) => v !== undefined); if (executionMetrics.length === 0) { @@ -26,7 +41,7 @@ const ChartGrid: React.FC = ({ data, role }: ProvidedProps) => { const chartProps = { series: data, - metricKey, + metricKey: resolvedKey, title: config.title, description: config.description, unit: config.unit, diff --git a/report/src/metricDefinitions.ts b/report/src/metricDefinitions.ts index f9c6cdd9..58085ebd 100644 --- a/report/src/metricDefinitions.ts +++ b/report/src/metricDefinitions.ts @@ -136,112 +136,114 @@ export const CHART_CONFIG = { description: "Shows the median gas per block", unit: "gas", }, - reth_sync_execution_execution_duration: { + reth_sync_execution_execution_duration_avg: { type: "line", title: "Reth Sync Execution Duration", - description: "Shows the time taken for execution during reth sync", + description: "Shows the average time taken for execution during reth sync", unit: "s", + aliases: ["reth_sync_execution_execution_duration"], }, - reth_sync_block_validation_state_root_duration: { + reth_sync_block_validation_state_root_duration_avg: { type: "line", title: "Reth Sync Block Validation State Root Duration", description: - "Shows the time taken for state root validation during reth sync", + "Shows the average time taken for state root validation during reth sync", unit: "s", + aliases: ["reth_sync_block_validation_state_root_duration"], }, - reth_op_rbuilder_block_built_success: { + reth_base_builder_block_built_success: { type: "line", - title: "Reth OP Builder Block Built Success", - description: "Indicates whether the Builder successfully built a block", + title: "Builder Block Built Success", + description: "Number of blocks successfully built per block interval", unit: "count", + aliases: ["reth_op_rbuilder_block_built_success"], }, - reth_op_rbuilder_flashblock_count: { + reth_base_builder_flashblock_count: { type: "line", - title: "Reth OP Builder Flashblock Count", - description: "Shows the number of flashblocks built by Builder", + title: "Builder Flashblock Count", + description: "Number of flashblock bundles sent per block", unit: "count", + aliases: ["reth_op_rbuilder_flashblock_count"], }, - reth_op_rbuilder_total_block_built_duration: { + reth_base_builder_flashblock_count_avg: { type: "line", - title: "Reth OP Builder Total Block Built Duration", - description: "Shows the total time taken to build a block by Builder", - unit: "s", - }, - reth_op_rbuilder_flashblock_build_duration: { - type: "line", - title: "Reth OP Builder Flashblock Build Duration", - description: "Shows the time taken to build a flashblock by Builder", - unit: "s", + title: "Builder Flashblocks per Block (avg)", + description: "Average number of flashblocks included per block", + unit: "count", }, - reth_op_rbuilder_state_root_calculation_duration: { + reth_base_builder_total_block_built_duration_avg: { type: "line", - title: "Reth OP Builder State Root Calculation Duration", - description: "Shows the time taken to calculate the state root by Builder", + title: "Builder Total Block Built Duration", + description: "Average total time taken to build a block", unit: "s", + aliases: ["reth_op_rbuilder_total_block_built_duration"], }, - reth_op_rbuilder_sequencer_tx_duration: { + reth_base_builder_flashblock_build_duration_avg: { type: "line", - title: "Reth OP Builder Sequencer Tx Duration", - description: "Shows the time taken for sequencer transactions in Builder", + title: "Builder Flashblock Build Duration", + description: "Average time taken to build a single flashblock", unit: "s", + aliases: ["reth_op_rbuilder_flashblock_build_duration"], }, - reth_op_rbuilder_payload_tx_simulation_duration: { + reth_base_builder_state_root_calculation_duration_avg: { type: "line", - title: "Reth OP Builder Payload Tx Simulation Duration", - description: - "Shows the time taken for payload transaction simulation in Builder", + title: "Builder State Root Calculation Duration", + description: "Average time taken to calculate the state root", unit: "s", + aliases: ["reth_op_rbuilder_state_root_calculation_duration"], }, - reth_sync_state_provider_total_storage_fetch_latency: { + reth_base_builder_sequencer_tx_duration_avg: { type: "line", - title: "Validator Storage Load Latency", - description: "Shows the 90th percentile latency for storage slot loads", + title: "Builder Sequencer Tx Duration", + description: "Average time taken to process sequencer transactions", unit: "s", + aliases: ["reth_op_rbuilder_sequencer_tx_duration"], }, - reth_sync_state_provider_total_code_fetch_latency: { + reth_base_builder_payload_transaction_simulation_duration_avg: { type: "line", - title: "Validator Code Load Latency", - description: "Shows the 90th percentile latency for code loads", + title: "Builder Payload Tx Simulation Duration", + description: "Average time taken for payload transaction simulation", unit: "s", + aliases: ["reth_op_rbuilder_payload_tx_simulation_duration"], }, - reth_sync_state_provider_total_account_fetch_latency: { + reth_base_builder_tx_simulation_duration_avg: { type: "line", - title: "Validator Account Load Latency", - description: "Shows the 90th percentile latency for account loads", + title: "Builder Tx Simulation Duration", + description: "Average per-transaction simulation duration", unit: "s", }, - reth_reth_flashblocks_block_processing_duration: { + reth_base_builder_payload_num_tx_gauge: { type: "line", - title: "Flashblock Processing Duration", - description: "Shows the time taken to process flashblocks on the validator", - unit: "s", + title: "Builder Payload Tx Count", + description: "Number of transactions included in the most recent payload", + unit: "count", }, - reth_reth_flashblocks_flashblocks_in_block: { + reth_base_builder_flashblock_gas_headroom_pct_avg: { type: "line", - title: "Flashblocks per Block", - description: "Shows the number of flashblocks received per block", + title: "Builder Flashblock Gas Headroom %", + description: "Average gas headroom percentage across flashblocks", unit: "count", }, - reth_reth_flashblocks_sender_recovery_duration: { + reth_sync_state_provider_total_storage_fetch_latency_avg: { type: "line", - title: "Flashblock Sender Recovery Duration", - description: - "Shows the time taken to recover senders from flashblock transactions", + title: "Validator Storage Load Latency", + description: "Average latency for storage slot loads during validation", unit: "s", + aliases: ["reth_sync_state_provider_total_storage_fetch_latency"], }, - reth_reth_flashblocks_upstream_messages: { + reth_sync_state_provider_total_code_fetch_latency_avg: { type: "line", - title: "Flashblock Upstream Messages", - description: - "Shows the number of upstream messages received from the flashblock stream", - unit: "count", + title: "Validator Code Load Latency", + description: "Average latency for bytecode loads during validation", + unit: "s", + aliases: ["reth_sync_state_provider_total_code_fetch_latency"], }, - reth_reth_flashblocks_bundle_state_clone_duration: { + reth_sync_state_provider_total_account_fetch_latency_avg: { type: "line", - title: "Flashblock Bundle State Clone Duration", - description: - "Shows the time taken to clone the bundle state for flashblock processing", + title: "Validator Account Load Latency", + description: "Average latency for account loads during validation", unit: "s", + aliases: ["reth_sync_state_provider_total_account_fetch_latency"], }, } satisfies Record; diff --git a/report/src/types.ts b/report/src/types.ts index 0049f821..862c4369 100644 --- a/report/src/types.ts +++ b/report/src/types.ts @@ -49,7 +49,8 @@ export interface ChartConfig { | "gas" | "count" | "gas/s" - | "blocks"; // Add 'gas/s', ensure 's' is present + | "blocks"; + aliases?: string[]; } export interface MachineInfo { diff --git a/runner/benchmark/benchmark.go b/runner/benchmark/benchmark.go index 135e5463..f1614af3 100644 --- a/runner/benchmark/benchmark.go +++ b/runner/benchmark/benchmark.go @@ -7,7 +7,6 @@ import ( "os" "strings" "sync/atomic" - "time" "github.com/base/base-bench/runner/network/types" "github.com/ethereum/go-ethereum/core" @@ -29,9 +28,8 @@ const ( ) var DefaultParams = &types.RunParams{ - NodeType: "geth", - GasLimit: 50e9, - BlockTime: 1 * time.Second, + NodeType: "geth", + GasLimit: 50e9, } // NewParamsFromValues constructs a new benchmark params given a config and a set of transaction payloads to run. diff --git a/runner/benchmark/definition.go b/runner/benchmark/definition.go index 0b869747..e9e96ff5 100644 --- a/runner/benchmark/definition.go +++ b/runner/benchmark/definition.go @@ -8,6 +8,7 @@ import ( "path" "path/filepath" "strings" + "time" "github.com/base/base-bench/runner/payload" ) @@ -87,13 +88,40 @@ func (s SnapshotDefinition) CreateSnapshot(nodeType string, outputDir string) er return cmd.Run() } +// FlashblocksConfig holds top-level flashblocks configuration. +type FlashblocksConfig struct { + BlockTime string `yaml:"block_time"` +} + +const DefaultFlashblocksBlockTime = "250" +const DefaultBlockTime = "1s" + type BenchmarkConfig struct { Name string `yaml:"name"` Description *string `yaml:"description"` + BlockTime *string `yaml:"block_time"` + Flashblocks *FlashblocksConfig `yaml:"flashblocks"` Benchmarks []TestDefinition `yaml:"benchmarks"` TransactionPayloads []payload.Definition `yaml:"payloads"` } +// GetBlockTime returns the configured block time as a duration, or the default (1s). +func (bc *BenchmarkConfig) GetBlockTime() (time.Duration, error) { + raw := DefaultBlockTime + if bc.BlockTime != nil && *bc.BlockTime != "" { + raw = *bc.BlockTime + } + return time.ParseDuration(raw) +} + +// FlashblocksBlockTime returns the configured flashblocks block time, or the default. +func (bc *BenchmarkConfig) FlashblocksBlockTime() string { + if bc.Flashblocks != nil && bc.Flashblocks.BlockTime != "" { + return bc.Flashblocks.BlockTime + } + return DefaultFlashblocksBlockTime +} + type DatadirConfig struct { Sequencer *string `yaml:"sequencer"` Validator *string `yaml:"validator"` diff --git a/runner/benchmark/matrix.go b/runner/benchmark/matrix.go index bfdc8d6c..430aa892 100644 --- a/runner/benchmark/matrix.go +++ b/runner/benchmark/matrix.go @@ -47,6 +47,11 @@ func NewTestPlanFromConfig(c TestDefinition, testFileName string, config *Benchm // ResolveTestRunsFromMatrix constructs a new ParamsMatrix from a config. func ResolveTestRunsFromMatrix(c TestDefinition, testFileName string, config *BenchmarkConfig) ([]TestRun, error) { + blockTime, err := config.GetBlockTime() + if err != nil { + return nil, fmt.Errorf("invalid block_time: %w", err) + } + seenParams := make(map[string]bool) // Multiple payloads can run in a single benchmark. @@ -107,6 +112,7 @@ func ResolveTestRunsFromMatrix(c TestDefinition, testFileName string, config *Be return nil, err } + params.BlockTime = blockTime params.Name = config.Name if config.Description != nil { params.Description = *config.Description diff --git a/runner/benchmark/matrix_test.go b/runner/benchmark/matrix_test.go index d4d822aa..5440dccf 100644 --- a/runner/benchmark/matrix_test.go +++ b/runner/benchmark/matrix_test.go @@ -2,6 +2,7 @@ package benchmark_test import ( "testing" + "time" "github.com/base/base-bench/runner/benchmark" "github.com/base/base-bench/runner/network/types" @@ -31,7 +32,7 @@ func TestResolveTestRunsFromMatrix(t *testing.T) { NodeType: "geth", PayloadID: "simple", GasLimit: benchmark.DefaultParams.GasLimit, - BlockTime: benchmark.DefaultParams.BlockTime, + BlockTime: 1 * time.Second, }, }, }, @@ -57,7 +58,7 @@ func TestResolveTestRunsFromMatrix(t *testing.T) { NodeType: "geth", GasLimit: benchmark.DefaultParams.GasLimit, PayloadID: "simple", - BlockTime: benchmark.DefaultParams.BlockTime, + BlockTime: 1 * time.Second, }, }, { @@ -65,7 +66,7 @@ func TestResolveTestRunsFromMatrix(t *testing.T) { NodeType: "erigon", GasLimit: benchmark.DefaultParams.GasLimit, PayloadID: "simple", - BlockTime: benchmark.DefaultParams.BlockTime, + BlockTime: 1 * time.Second, }, }, { @@ -73,7 +74,7 @@ func TestResolveTestRunsFromMatrix(t *testing.T) { NodeType: "geth", GasLimit: benchmark.DefaultParams.GasLimit, PayloadID: "complex", - BlockTime: benchmark.DefaultParams.BlockTime, + BlockTime: 1 * time.Second, }, }, { @@ -81,7 +82,7 @@ func TestResolveTestRunsFromMatrix(t *testing.T) { NodeType: "erigon", GasLimit: benchmark.DefaultParams.GasLimit, PayloadID: "complex", - BlockTime: benchmark.DefaultParams.BlockTime, + BlockTime: 1 * time.Second, }, }, }, diff --git a/runner/clients/builder/client.go b/runner/clients/builder/client.go index 660b41a1..b229905c 100644 --- a/runner/clients/builder/client.go +++ b/runner/clients/builder/client.go @@ -49,6 +49,12 @@ func (r *BuilderClient) Run(ctx context.Context, cfg *types.RuntimeConfig) error cfg2 := *cfg cfg2.Args = append(cfg2.Args, "--flashblocks.port", fmt.Sprintf("%d", r.websocketPort)) + if cfg.FlashblocksBlockTime != "" { + cfg2.Args = append(cfg2.Args, "--flashblocks.block-time", cfg.FlashblocksBlockTime) + } + if cfg.BlockTimeMs > 0 { + cfg2.Args = append(cfg2.Args, "--rollup.chain-block-time", fmt.Sprintf("%d", cfg.BlockTimeMs)) + } err := r.elClient.Run(ctx, &cfg2) if err != nil { return err diff --git a/runner/clients/types/types.go b/runner/clients/types/types.go index f7f7e882..a0be0080 100644 --- a/runner/clients/types/types.go +++ b/runner/clients/types/types.go @@ -10,10 +10,12 @@ import ( ) type RuntimeConfig struct { - Stdout io.WriteCloser - Stderr io.WriteCloser - Args []string - FlashblocksURL *string // Optional URL for flashblocks websocket server (only used by clients that support it) + Stdout io.WriteCloser + Stderr io.WriteCloser + Args []string + FlashblocksURL *string // Optional URL for flashblocks websocket server (only used by clients that support it) + FlashblocksBlockTime string // Block time for flashblocks in milliseconds (e.g. "250") + BlockTimeMs uint64 // L2 block time in milliseconds } // ExecutionClient is an abstraction over the different clients that can be used to run the chain like diff --git a/runner/network/configutil/rollup_config.go b/runner/network/configutil/rollup_config.go index ea55c767..150f172e 100644 --- a/runner/network/configutil/rollup_config.go +++ b/runner/network/configutil/rollup_config.go @@ -12,8 +12,8 @@ import ( "github.com/ethereum/go-ethereum/params" ) -// GetRollupConfig creates a rollup configuration for the given genesis and chain -func GetRollupConfig(genesis *core.Genesis, chain fakel1.L1Chain, batcherAddr common.Address) *rollup.Config { +// GetRollupConfig creates a rollup configuration for the given genesis, chain, and block time (in seconds). +func GetRollupConfig(genesis *core.Genesis, chain fakel1.L1Chain, batcherAddr common.Address, blockTimeSec uint64) *rollup.Config { var eipParams eth.Bytes8 copy(eipParams[:], eip1559.EncodeHolocene1559Params(50, 1)) @@ -50,7 +50,7 @@ func GetRollupConfig(genesis *core.Genesis, chain fakel1.L1Chain, batcherAddr co }), }, }, - BlockTime: 1, + BlockTime: blockTimeSec, MaxSequencerDrift: 20, SeqWindowSize: 24, L1ChainID: big.NewInt(1), diff --git a/runner/network/consensus/sequencer_consensus.go b/runner/network/consensus/sequencer_consensus.go index 6b62f967..661bb5fa 100644 --- a/runner/network/consensus/sequencer_consensus.go +++ b/runner/network/consensus/sequencer_consensus.go @@ -289,6 +289,7 @@ func (f *SequencerConsensusClient) Propose(ctx context.Context, blockMetrics *me blockMetrics.AddExecutionMetric(networktypes.UpdateForkChoiceLatencyMetric, duration) f.currentPayloadID = payloadID + f.log.Info("Waiting for block time", "block_time", f.options.BlockTime) // wait block time time.Sleep(f.options.BlockTime) diff --git a/runner/network/fault_proof_benchmark.go b/runner/network/fault_proof_benchmark.go index b762ad99..7f1625ac 100644 --- a/runner/network/fault_proof_benchmark.go +++ b/runner/network/fault_proof_benchmark.go @@ -43,8 +43,8 @@ type opProgramBenchmark struct { rollupCfg *rollup.Config } -func NewOPProgramBenchmark(genesis *core.Genesis, log log.Logger, opProgramBin string, l2RPCURL string, l1Chain fakel1.L1Chain, batcherKey *ecdsa.PrivateKey) ProofProgramBenchmark { - rollupCfg := configutil.GetRollupConfig(genesis, l1Chain, crypto.PubkeyToAddress(batcherKey.PublicKey)) +func NewOPProgramBenchmark(genesis *core.Genesis, log log.Logger, opProgramBin string, l2RPCURL string, l1Chain fakel1.L1Chain, batcherKey *ecdsa.PrivateKey, blockTimeSec uint64) ProofProgramBenchmark { + rollupCfg := configutil.GetRollupConfig(genesis, l1Chain, crypto.PubkeyToAddress(batcherKey.PublicKey), blockTimeSec) batcher := proofprogram.NewBatcher(rollupCfg, batcherKey, l1Chain) return &opProgramBenchmark{ diff --git a/runner/network/network_benchmark.go b/runner/network/network_benchmark.go index 53246b3f..2bf13775 100644 --- a/runner/network/network_benchmark.go +++ b/runner/network/network_benchmark.go @@ -44,20 +44,22 @@ type NetworkBenchmark struct { testConfig *benchtypes.TestConfig proofConfig *benchmark.ProofProgramOptions - transactionPayload payload.Definition - ports portmanager.PortManager + transactionPayload payload.Definition + ports portmanager.PortManager + flashblocksBlockTime string } // NewNetworkBenchmark creates a new network benchmark and initializes the payload worker and consensus client -func NewNetworkBenchmark(config *benchtypes.TestConfig, log log.Logger, sequencerOptions *config.InternalClientOptions, validatorOptions *config.InternalClientOptions, proofConfig *benchmark.ProofProgramOptions, transactionPayload payload.Definition, ports portmanager.PortManager) (*NetworkBenchmark, error) { +func NewNetworkBenchmark(config *benchtypes.TestConfig, log log.Logger, sequencerOptions *config.InternalClientOptions, validatorOptions *config.InternalClientOptions, proofConfig *benchmark.ProofProgramOptions, transactionPayload payload.Definition, ports portmanager.PortManager, flashblocksBlockTime string) (*NetworkBenchmark, error) { return &NetworkBenchmark{ - log: log, - sequencerOptions: sequencerOptions, - validatorOptions: validatorOptions, - testConfig: config, - proofConfig: proofConfig, - transactionPayload: transactionPayload, - ports: ports, + log: log, + sequencerOptions: sequencerOptions, + validatorOptions: validatorOptions, + testConfig: config, + proofConfig: proofConfig, + transactionPayload: transactionPayload, + ports: ports, + flashblocksBlockTime: flashblocksBlockTime, }, nil } @@ -88,7 +90,7 @@ func (nb *NetworkBenchmark) Run(ctx context.Context) error { } func (nb *NetworkBenchmark) benchmarkSequencer(ctx context.Context, l1Chain *l1Chain) (*benchtypes.PayloadResult, uint64, types.ExecutionClient, error) { - sequencerClient, err := setupNode(ctx, nb.log, nb.testConfig.Params.NodeType, nb.testConfig.Params, nb.sequencerOptions, nb.ports, "") + sequencerClient, err := setupNode(ctx, nb.log, nb.testConfig.Params.NodeType, nb.testConfig.Params, nb.sequencerOptions, nb.ports, "", nb.flashblocksBlockTime) if err != nil { return nil, 0, nil, fmt.Errorf("failed to setup sequencer node: %w", err) } @@ -158,7 +160,7 @@ func (nb *NetworkBenchmark) benchmarkValidator(ctx context.Context, payloadResul validatorNodeType = nb.testConfig.Params.NodeType } - validatorClient, err := setupNode(ctx, nb.log, validatorNodeType, nb.testConfig.Params, nb.validatorOptions, nb.ports, flashblockServerURL) + validatorClient, err := setupNode(ctx, nb.log, validatorNodeType, nb.testConfig.Params, nb.validatorOptions, nb.ports, flashblockServerURL, nb.flashblocksBlockTime) if err != nil { sequencerClient.Stop() return fmt.Errorf("failed to setup validator node: %w", err) @@ -253,7 +255,7 @@ func (nb *NetworkBenchmark) GetResult() (*benchmark.RunResult, error) { }, nil } -func setupNode(ctx context.Context, l log.Logger, nodeTypeStr string, params benchtypes.RunParams, options *config.InternalClientOptions, portManager portmanager.PortManager, flashblockServerURL string) (types.ExecutionClient, error) { +func setupNode(ctx context.Context, l log.Logger, nodeTypeStr string, params benchtypes.RunParams, options *config.InternalClientOptions, portManager portmanager.PortManager, flashblockServerURL string, flashblocksBlockTime string) (types.ExecutionClient, error) { if options == nil { return nil, errors.New("client options cannot be nil") } @@ -293,10 +295,12 @@ func setupNode(ctx context.Context, l log.Logger, nodeTypeStr string, params ben } runtimeConfig := &types.RuntimeConfig{ - Stdout: stdoutLogger, - Stderr: stderrLogger, - Args: args, - FlashblocksURL: flashblocksURLPtr, + Stdout: stdoutLogger, + Stderr: stderrLogger, + Args: args, + FlashblocksURL: flashblocksURLPtr, + FlashblocksBlockTime: flashblocksBlockTime, + BlockTimeMs: uint64(params.BlockTime.Milliseconds()), } if err := client.Run(ctx, runtimeConfig); err != nil { diff --git a/runner/network/sequencer_benchmark.go b/runner/network/sequencer_benchmark.go index 05a15fb7..29aa8c28 100644 --- a/runner/network/sequencer_benchmark.go +++ b/runner/network/sequencer_benchmark.go @@ -272,7 +272,8 @@ func (nb *sequencerBenchmark) Run(ctx context.Context, metricsCollector metrics. pendingTxs = 0 } - time.Sleep(1000 * time.Millisecond) + log.Info("Sleeping for block time", "block_time", params.BlockTime) + time.Sleep(params.BlockTime) err = metricsCollector.Collect(benchmarkCtx, blockMetrics) if err != nil { diff --git a/runner/network/validator_benchmark.go b/runner/network/validator_benchmark.go index cf5e5880..b3e397fb 100644 --- a/runner/network/validator_benchmark.go +++ b/runner/network/validator_benchmark.go @@ -53,7 +53,8 @@ func (vb *validatorBenchmark) benchmarkFaultProofProgram(ctx context.Context, pa return fmt.Errorf("proof program binary does not exist at %s", binaryPath) } - opProgramBenchmark := NewOPProgramBenchmark(&vb.config.Genesis, vb.log, binaryPath, vb.validatorClient.ClientURL(), l1Chain, batcherKey) + blockTimeSec := uint64(vb.config.Params.BlockTime.Seconds()) + opProgramBenchmark := NewOPProgramBenchmark(&vb.config.Genesis, vb.log, binaryPath, vb.validatorClient.ClientURL(), l1Chain, batcherKey, blockTimeSec) return opProgramBenchmark.Run(ctx, payloads, lastSetupBlock) } diff --git a/runner/service.go b/runner/service.go index 163736ed..2e1c0309 100644 --- a/runner/service.go +++ b/runner/service.go @@ -162,6 +162,54 @@ type TestRunMetadata struct { Error *string `json:"error,omitempty"` } +const maxLogDumpSize = 1 << 20 // 1 MB + +// dumpLogFile reads the last maxLogDumpSize bytes of the EL log file and prints them to stdout. +// This is called on benchmark failure so CI logs contain useful subprocess output. +func (s *service) dumpLogFile(testDirs *config.InternalClientOptions, nodeType string) { + logsPath := path.Join(testDirs.TestDirPath, network.ExecutionLayerLogFileName) + + f, err := os.Open(logsPath) + if err != nil { + s.log.Warn("could not open log file for dump", "path", logsPath, "err", err) + return + } + defer func() { _ = f.Close() }() + + stat, err := f.Stat() + if err != nil { + s.log.Warn("could not stat log file for dump", "path", logsPath, "err", err) + return + } + + size := stat.Size() + truncated := false + readSize := size + if readSize > maxLogDumpSize { + truncated = true + readSize = maxLogDumpSize + if _, err := f.Seek(-readSize, io.SeekEnd); err != nil { + s.log.Warn("could not seek log file for dump", "path", logsPath, "err", err) + return + } + } + + buf := make([]byte, readSize) + n, err := io.ReadFull(f, buf) + if err != nil && err != io.ErrUnexpectedEOF { + s.log.Warn("could not read log file for dump", "path", logsPath, "err", err) + return + } + buf = buf[:n] + + fmt.Printf("\n=== %s EL logs ===\n", nodeType) + if truncated { + fmt.Printf("[truncated: showing last %d bytes of %d total]\n", readSize, size) + } + _, _ = os.Stdout.Write(buf) + fmt.Printf("\n=== end %s EL logs ===\n\n", nodeType) +} + func (s *service) exportOutput(testName string, returnedError error, testDirs *config.InternalClientOptions, testOutputDir string, nodeType string) error { // package up logs from the EL client and write them to the output dir // outputDir/ @@ -320,7 +368,7 @@ func (s *service) setupBlobsDir(workingDir string) error { return nil } -func (s *service) runTest(ctx context.Context, params types.RunParams, workingDir string, outputDir string, snapshotConfig *benchmark.SnapshotDefinition, proofConfig *benchmark.ProofProgramOptions, transactionPayload payload.Definition, datadirsConfig *benchmark.DatadirConfig) (*benchmark.RunResult, error) { +func (s *service) runTest(ctx context.Context, params types.RunParams, workingDir string, outputDir string, snapshotConfig *benchmark.SnapshotDefinition, proofConfig *benchmark.ProofProgramOptions, transactionPayload payload.Definition, datadirsConfig *benchmark.DatadirConfig, flashblocksBlockTime string) (*benchmark.RunResult, error) { s.log.Info(fmt.Sprintf("Running benchmark with params: %+v", params)) @@ -384,7 +432,7 @@ func (s *service) runTest(ctx context.Context, params types.RunParams, workingDi } // Run benchmark - benchmark, err := network.NewNetworkBenchmark(config, s.log, sequencerOptions, validatorOptions, proofConfig, transactionPayload, s.portState) + benchmark, err := network.NewNetworkBenchmark(config, s.log, sequencerOptions, validatorOptions, proofConfig, transactionPayload, s.portState, flashblocksBlockTime) if err != nil { return nil, errors.Wrap(err, "failed to create network benchmark") } @@ -401,6 +449,8 @@ func (s *service) runTest(ctx context.Context, params types.RunParams, workingDi } if runErr != nil { + s.dumpLogFile(sequencerOptions, "sequencer") + s.dumpLogFile(validatorOptions, "validator") return nil, errors.Wrap(runErr, "failed to run benchmark") } @@ -567,7 +617,7 @@ outerLoop: return errors.Wrap(err, "failed to create output directory") } - metricSummary, err := s.runTest(ctx, c.Params, s.config.DataDir(), outputDir, testPlan.Snapshot, testPlan.ProofProgram, transactionPayloads[c.Params.PayloadID], testPlan.Datadir) + metricSummary, err := s.runTest(ctx, c.Params, s.config.DataDir(), outputDir, testPlan.Snapshot, testPlan.ProofProgram, transactionPayloads[c.Params.PayloadID], testPlan.Datadir, config.FlashblocksBlockTime()) if err != nil { log.Error("Failed to run test", "err", err) metricSummary = &benchmark.RunResult{