From 0b3690a9ad53b8e7f86634b2b940cc08dd841378 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Mon, 23 Feb 2026 10:09:10 +0000 Subject: [PATCH 01/63] fix: log payload attestations for gloas block bodies (#8942) --- .../src/chain/produceBlock/produceBlockBody.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts b/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts index 76deccf25be0..c3eee555e734 100644 --- a/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts +++ b/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts @@ -4,6 +4,7 @@ import { BUILDER_INDEX_SELF_BUILD, ForkName, ForkPostBellatrix, + ForkPostCapella, ForkPostDeneb, ForkPostFulu, ForkPostGloas, @@ -567,8 +568,14 @@ export async function produceBlockBody( }); } - if (ForkSeq[fork] >= ForkSeq.capella) { - const {blsToExecutionChanges, executionPayload} = blockBody as capella.BeaconBlockBody; + if (ForkSeq[fork] >= ForkSeq.gloas) { + const {blsToExecutionChanges, payloadAttestations} = blockBody as BeaconBlockBody; + Object.assign(logMeta, { + blsToExecutionChanges: blsToExecutionChanges.length, + payloadAttestations: payloadAttestations.length, + }); + } else if (ForkSeq[fork] >= ForkSeq.capella) { + const {blsToExecutionChanges, executionPayload} = blockBody as BeaconBlockBody; Object.assign(logMeta, { blsToExecutionChanges: blsToExecutionChanges.length, }); From 49f680d9f0a1efd88f807f9a88f5441d90b22041 Mon Sep 17 00:00:00 2001 From: Nazar Hussain Date: Mon, 23 Feb 2026 11:42:34 +0100 Subject: [PATCH 02/63] feat: create standalone fork-choice dashboard (#8928) **Motivation** Need to add more `metrics` to the fork choice which makes it easier to have it's own dashboard. **Description** - Create new dashboard for the forkchoice - Remove fork choice visualizations from the block processing dashboard --- dashboards/lodestar_block_processor.json | 1146 +------------------ dashboards/lodestar_fork_choice.json | 1282 ++++++++++++++++++++++ 2 files changed, 1320 insertions(+), 1108 deletions(-) create mode 100644 dashboards/lodestar_fork_choice.json diff --git a/dashboards/lodestar_block_processor.json b/dashboards/lodestar_block_processor.json index 4107450b4c86..f46257193f5e 100644 --- a/dashboards/lodestar_block_processor.json +++ b/dashboards/lodestar_block_processor.json @@ -3898,1078 +3898,13 @@ "title": "Prepare Next Epoch hash_tree_root", "type": "heatmap" }, - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 173 - }, - "id": 136, - "panels": [], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "refId": "A" - } - ], - "title": "Fork-Choice Stats", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 28, - "gradientMode": "opacity", - "hideFrom": { - "graph": false, - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "s" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "prepare_next_slot" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 174 - }, - "id": 130, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - }, - "tooltipOptions": { - "mode": "single" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "rate(beacon_fork_choice_find_head_seconds_sum[$rate_interval])/rate(beacon_fork_choice_find_head_seconds_count[$rate_interval])", - "interval": "", - "legendFormat": "{{caller}}", - "range": true, - "refId": "A" - } - ], - "title": "updateHead() avg time", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "graph": false, - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [] - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "beacon_fork_choice_errors_total{group=\"beta\", instance=\"contabo-13\", job=~\"$beacon_job|beacon\"}" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 174 - }, - "id": 140, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - }, - "tooltipOptions": { - "mode": "single" - } - }, - "pluginVersion": "8.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "beacon_fork_choice_errors_total", - "interval": "", - "legendFormat": "", - "range": true, - "refId": "A" - } - ], - "title": "updateHead() errors", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "graph": false, - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [] - }, - "overrides": [ - { - "matcher": { - "id": "byFrameRefID", - "options": "C" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "percentunit" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 182 - }, - "id": 132, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - }, - "tooltipOptions": { - "mode": "single" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "12 * rate(beacon_fork_choice_find_head_seconds_count[$rate_interval])", - "interval": "", - "legendFormat": "{{caller}}_updateHead_calls", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "", - "hide": false, - "interval": "", - "legendFormat": "updateHead calls", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "rate(beacon_fork_choice_find_head_seconds_sum[$rate_interval])", - "hide": false, - "interval": "", - "legendFormat": "{{caller}}_usage_rate", - "range": true, - "refId": "C" - } - ], - "title": "updateHead() calls / slot", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "left", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "opacity", - "hideFrom": { - "graph": false, - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "percentunit" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "{job=~\"$beacon_job|beacon\"}" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "purple", - "mode": "palette-classic" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 182 - }, - "id": 138, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - }, - "tooltipOptions": { - "mode": "single" - } - }, - "pluginVersion": "8.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "rate(beacon_fork_choice_changed_head_total[6m])/rate(beacon_fork_choice_requests_total[6m])", - "hide": false, - "interval": "", - "legendFormat": "Changed Heads", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "", - "hide": false, - "interval": "", - "legendFormat": "Total Requests", - "refId": "B" - } - ], - "title": "ratio of updateHead() calls that change head", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [] - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "queued_attestations" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "validated_attestation_datas" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 190 - }, - "id": 531, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "beacon_fork_choice_votes_count", - "legendFormat": "votes", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "beacon_fork_choice_queued_attestations_count", - "hide": false, - "legendFormat": "queued_attestations", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "beacon_fork_choice_validated_attestation_datas_count", - "hide": false, - "legendFormat": "validated_attestation_datas", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "beacon_fork_choice_balances_length", - "hide": false, - "legendFormat": "balances_length", - "range": true, - "refId": "D" - } - ], - "title": "Forkchoice Internal Data", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [] - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 190 - }, - "id": 533, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "beacon_fork_choice_nodes_count", - "legendFormat": "nodes", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "beacon_fork_choice_indices_count", - "hide": false, - "legendFormat": "indices", - "range": true, - "refId": "B" - } - ], - "title": "Proto Array", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 198 - }, - "id": 587, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "rate(beacon_fork_choice_compute_deltas_seconds_sum[$rate_interval])\n/\nrate(beacon_fork_choice_compute_deltas_seconds_count[$rate_interval])", - "instant": false, - "legendFormat": "duration", - "range": true, - "refId": "A" - } - ], - "title": "computeDeltas() avg time", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [] - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "deltas" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 198 - }, - "id": 588, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "beacon_fork_choice_compute_deltas_deltas_count", - "instant": false, - "legendFormat": "deltas", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "beacon_fork_choice_compute_deltas_zero_deltas_count", - "hide": false, - "instant": false, - "legendFormat": "zero_deltas", - "range": true, - "refId": "B" - } - ], - "title": "computeDeltas() - deltas result", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [] - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 206 - }, - "id": 589, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "beacon_fork_choice_compute_deltas_equivocating_validators_count", - "instant": false, - "legendFormat": "equivocating_validators", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "beacon_fork_choice_compute_deltas_old_inactive_validators_count", - "hide": false, - "instant": false, - "legendFormat": "old_inactive_validators", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "beacon_fork_choice_compute_deltas_new_inactive_validators_count", - "hide": false, - "instant": false, - "legendFormat": "new_inactive_validators", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "beacon_fork_choice_compute_deltas_unchanged_vote_validators_count", - "hide": false, - "instant": false, - "legendFormat": "unchanged_vote_validators", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "beacon_fork_choice_compute_deltas_new_vote_validators_count", - "hide": false, - "instant": false, - "legendFormat": "new_vote_validators", - "range": true, - "refId": "E" - } - ], - "title": "computeDeltas() - validators result", - "type": "timeseries" - }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 214 + "y": 173 }, "id": 569, "panels": [], @@ -5027,7 +3962,7 @@ "h": 8, "w": 12, "x": 0, - "y": 215 + "y": 174 }, "id": 580, "options": { @@ -5241,7 +4176,7 @@ "h": 8, "w": 12, "x": 12, - "y": 215 + "y": 174 }, "id": 578, "options": { @@ -5439,7 +4374,7 @@ "h": 8, "w": 12, "x": 0, - "y": 223 + "y": 182 }, "id": 577, "options": { @@ -5572,7 +4507,7 @@ "h": 8, "w": 12, "x": 12, - "y": 223 + "y": 182 }, "id": 586, "options": { @@ -5655,7 +4590,7 @@ "h": 8, "w": 12, "x": 0, - "y": 231 + "y": 190 }, "id": 584, "options": { @@ -5772,7 +4707,7 @@ "h": 8, "w": 12, "x": 12, - "y": 231 + "y": 190 }, "id": 582, "options": { @@ -5873,7 +4808,7 @@ "h": 8, "w": 12, "x": 12, - "y": 239 + "y": 198 }, "id": 585, "options": { @@ -5931,7 +4866,7 @@ "h": 1, "w": 24, "x": 0, - "y": 247 + "y": 206 }, "id": 538, "panels": [], @@ -5963,7 +4898,7 @@ "h": 8, "w": 12, "x": 0, - "y": 248 + "y": 207 }, "id": 539, "options": { @@ -6045,7 +4980,7 @@ "h": 8, "w": 12, "x": 12, - "y": 248 + "y": 207 }, "id": 540, "options": { @@ -6154,7 +5089,7 @@ "h": 8, "w": 12, "x": 0, - "y": 256 + "y": 215 }, "id": 553, "options": { @@ -6241,7 +5176,7 @@ "h": 8, "w": 12, "x": 12, - "y": 256 + "y": 215 }, "id": 554, "options": { @@ -6300,7 +5235,7 @@ "h": 8, "w": 12, "x": 0, - "y": 264 + "y": 223 }, "id": 556, "options": { @@ -6388,7 +5323,7 @@ "h": 8, "w": 12, "x": 12, - "y": 264 + "y": 223 }, "id": 542, "options": { @@ -6497,7 +5432,7 @@ "h": 8, "w": 12, "x": 0, - "y": 272 + "y": 231 }, "id": 541, "options": { @@ -6583,7 +5518,7 @@ "h": 8, "w": 12, "x": 12, - "y": 272 + "y": 231 }, "id": 555, "options": { @@ -6642,7 +5577,7 @@ "h": 8, "w": 12, "x": 0, - "y": 280 + "y": 239 }, "id": 557, "options": { @@ -6731,7 +5666,7 @@ "h": 8, "w": 12, "x": 12, - "y": 280 + "y": 239 }, "id": 544, "options": { @@ -6847,7 +5782,7 @@ "h": 8, "w": 12, "x": 0, - "y": 288 + "y": 247 }, "id": 543, "options": { @@ -6933,7 +5868,7 @@ "h": 8, "w": 12, "x": 12, - "y": 288 + "y": 247 }, "id": 558, "options": { @@ -6992,7 +5927,7 @@ "h": 8, "w": 12, "x": 0, - "y": 296 + "y": 255 }, "id": 559, "options": { @@ -7081,7 +6016,7 @@ "h": 8, "w": 12, "x": 12, - "y": 296 + "y": 255 }, "id": 546, "options": { @@ -7197,7 +6132,7 @@ "h": 8, "w": 12, "x": 0, - "y": 304 + "y": 263 }, "id": 545, "options": { @@ -7283,7 +6218,7 @@ "h": 8, "w": 12, "x": 12, - "y": 304 + "y": 263 }, "id": 560, "options": { @@ -7342,7 +6277,7 @@ "h": 8, "w": 12, "x": 0, - "y": 312 + "y": 271 }, "id": 561, "options": { @@ -7431,7 +6366,7 @@ "h": 8, "w": 12, "x": 12, - "y": 312 + "y": 271 }, "id": 548, "options": { @@ -7547,7 +6482,7 @@ "h": 8, "w": 12, "x": 0, - "y": 320 + "y": 279 }, "id": 547, "options": { @@ -7633,7 +6568,7 @@ "h": 8, "w": 12, "x": 12, - "y": 320 + "y": 279 }, "id": 562, "options": { @@ -7692,7 +6627,7 @@ "h": 8, "w": 12, "x": 0, - "y": 328 + "y": 287 }, "id": 563, "options": { @@ -7781,7 +6716,7 @@ "h": 8, "w": 12, "x": 12, - "y": 328 + "y": 287 }, "id": 550, "options": { @@ -7897,7 +6832,7 @@ "h": 8, "w": 12, "x": 0, - "y": 336 + "y": 295 }, "id": 549, "options": { @@ -7983,7 +6918,7 @@ "h": 8, "w": 12, "x": 12, - "y": 336 + "y": 295 }, "id": 564, "options": { @@ -8042,7 +6977,7 @@ "h": 8, "w": 12, "x": 0, - "y": 344 + "y": 303 }, "id": 565, "options": { @@ -8124,7 +7059,7 @@ "h": 8, "w": 12, "x": 12, - "y": 344 + "y": 303 }, "id": 552, "options": { @@ -8233,7 +7168,7 @@ "h": 8, "w": 12, "x": 0, - "y": 352 + "y": 311 }, "id": 551, "options": { @@ -8319,7 +7254,7 @@ "h": 8, "w": 12, "x": 12, - "y": 352 + "y": 311 }, "id": 566, "options": { @@ -8474,11 +7409,6 @@ "type": "adhoc" }, { - "current": { - "selected": false, - "text": "${VAR_BEACON_JOB}", - "value": "${VAR_BEACON_JOB}" - }, "description": "Job name used in Prometheus config to scrape Beacon node", "hide": 2, "label": "Beacon node job name", @@ -8510,6 +7440,6 @@ "timezone": "utc", "title": "Lodestar - block processor", "uid": "lodestar_block_processor", - "version": 16, + "version": 24, "weekStart": "monday" } diff --git a/dashboards/lodestar_fork_choice.json b/dashboards/lodestar_fork_choice.json new file mode 100644 index 000000000000..4f8605a01d18 --- /dev/null +++ b/dashboards/lodestar_fork_choice.json @@ -0,0 +1,1282 @@ +{ + "__inputs": [ + { + "description": "", + "label": "Prometheus", + "name": "DS_PROMETHEUS", + "pluginId": "prometheus", + "pluginName": "Prometheus", + "type": "datasource" + }, + { + "description": "", + "label": "Beacon node job name", + "name": "VAR_BEACON_JOB", + "type": "constant", + "value": "beacon" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "lodestar" + ], + "targetBlank": false, + "title": "Lodestar dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 136, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "refId": "A" + } + ], + "title": "Fork-Choice Stats", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 28, + "gradientMode": "opacity", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "s" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "prepare_next_slot" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 130, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "rate(beacon_fork_choice_find_head_seconds_sum[$rate_interval])/rate(beacon_fork_choice_find_head_seconds_count[$rate_interval])", + "interval": "", + "legendFormat": "{{caller}}", + "range": true, + "refId": "A" + } + ], + "title": "updateHead() avg time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "beacon_fork_choice_errors_total{group=\"beta\", instance=\"contabo-13\", job=~\"$beacon_job|beacon\"}" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 140, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "8.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "beacon_fork_choice_errors_total", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "updateHead() errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "percentunit" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 132, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "12 * rate(beacon_fork_choice_find_head_seconds_count[$rate_interval])", + "interval": "", + "legendFormat": "{{caller}}_updateHead_calls", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": false, + "expr": "", + "hide": false, + "interval": "", + "legendFormat": "updateHead calls", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "rate(beacon_fork_choice_find_head_seconds_sum[$rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "{{caller}}_usage_rate", + "range": true, + "refId": "C" + } + ], + "title": "updateHead() calls / slot", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "left", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "opacity", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "{job=~\"$beacon_job|beacon\"}" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "palette-classic" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 138, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "8.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": false, + "expr": "rate(beacon_fork_choice_changed_head_total[6m])/rate(beacon_fork_choice_requests_total[6m])", + "hide": false, + "interval": "", + "legendFormat": "Changed Heads", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": false, + "expr": "", + "hide": false, + "interval": "", + "legendFormat": "Total Requests", + "refId": "B" + } + ], + "title": "ratio of updateHead() calls that change head", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "queued_attestations" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "validated_attestation_datas" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 531, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "beacon_fork_choice_votes_count", + "legendFormat": "votes", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "beacon_fork_choice_queued_attestations_count", + "hide": false, + "legendFormat": "queued_attestations", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "beacon_fork_choice_validated_attestation_datas_count", + "hide": false, + "legendFormat": "validated_attestation_datas", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "beacon_fork_choice_balances_length", + "hide": false, + "legendFormat": "balances_length", + "range": true, + "refId": "D" + } + ], + "title": "Forkchoice Internal Data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 533, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "beacon_fork_choice_nodes_count", + "legendFormat": "nodes", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "beacon_fork_choice_indices_count", + "hide": false, + "legendFormat": "indices", + "range": true, + "refId": "B" + } + ], + "title": "Proto Array", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 587, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(beacon_fork_choice_compute_deltas_seconds_sum[$rate_interval])\n/\nrate(beacon_fork_choice_compute_deltas_seconds_count[$rate_interval])", + "instant": false, + "legendFormat": "duration", + "range": true, + "refId": "A" + } + ], + "title": "computeDeltas() avg time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [] + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "deltas" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 588, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "beacon_fork_choice_compute_deltas_deltas_count", + "instant": false, + "legendFormat": "deltas", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "beacon_fork_choice_compute_deltas_zero_deltas_count", + "hide": false, + "instant": false, + "legendFormat": "zero_deltas", + "range": true, + "refId": "B" + } + ], + "title": "computeDeltas() - deltas result", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 33 + }, + "id": 589, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "beacon_fork_choice_compute_deltas_equivocating_validators_count", + "instant": false, + "legendFormat": "equivocating_validators", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "beacon_fork_choice_compute_deltas_old_inactive_validators_count", + "hide": false, + "instant": false, + "legendFormat": "old_inactive_validators", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "beacon_fork_choice_compute_deltas_new_inactive_validators_count", + "hide": false, + "instant": false, + "legendFormat": "new_inactive_validators", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "beacon_fork_choice_compute_deltas_unchanged_vote_validators_count", + "hide": false, + "instant": false, + "legendFormat": "unchanged_vote_validators", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "beacon_fork_choice_compute_deltas_new_vote_validators_count", + "hide": false, + "instant": false, + "legendFormat": "new_vote_validators", + "range": true, + "refId": "E" + } + ], + "title": "computeDeltas() - validators result", + "type": "timeseries" + } + ], + "refresh": "10s", + "schemaVersion": 39, + "tags": [ + "lodestar" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "auto": true, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": false, + "text": "1h", + "value": "1h" + }, + "hide": 0, + "label": "rate() interval", + "name": "rate_interval", + "options": [ + { + "selected": false, + "text": "auto", + "value": "$__auto_interval_rate_interval" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": true, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "queryValue": "", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_local" + }, + "filters": [ + { + "condition": "", + "key": "instance", + "operator": "=", + "value": "unstable-super" + } + ], + "hide": 0, + "name": "Filters", + "skipUrlSync": false, + "type": "adhoc" + }, + { + "description": "Job name used in Prometheus config to scrape Beacon node", + "hide": 2, + "label": "Beacon node job name", + "name": "beacon_job", + "query": "${VAR_BEACON_JOB}", + "skipUrlSync": false, + "type": "constant" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "utc", + "title": "Lodestar - fork choice", + "uid": "lodestar_fork_choice", + "version": 2, + "weekStart": "monday" +} From aeb5a213ee7d739ebe8ead1d27f742e5042a8b25 Mon Sep 17 00:00:00 2001 From: twoeths <10568965+twoeths@users.noreply.github.com> Date: Mon, 23 Feb 2026 21:03:27 +0700 Subject: [PATCH 03/63] feat: model BeaconStateView (#8773) **Motivation** - see #8650 **Description** - implement `IBeaconStateView` interface - implement the `ts` `BeaconStateView` blocked by #8728 to build successfully --------- Co-authored-by: Tuyen Nguyen Co-authored-by: Cayman --- .../src/api/impl/beacon/state/utils.ts | 33 +- .../state-transition/src/cache/stateCache.ts | 2 +- packages/state-transition/src/index.ts | 2 + .../src/lightClient/proofs.ts | 83 ++ .../state-transition/src/lightClient/types.ts | 33 + .../src/stateView/beaconStateView.ts | 746 ++++++++++++++++++ .../state-transition/src/stateView/index.ts | 2 + .../src/stateView/interface.ts | 196 +++++ .../src/util/weakSubjectivity.ts | 2 +- packages/types/src/capella/types.ts | 2 + packages/types/src/types.ts | 3 + packages/types/src/utils/validatorStatus.ts | 22 + 12 files changed, 1101 insertions(+), 25 deletions(-) create mode 100644 packages/state-transition/src/lightClient/proofs.ts create mode 100644 packages/state-transition/src/lightClient/types.ts create mode 100644 packages/state-transition/src/stateView/beaconStateView.ts create mode 100644 packages/state-transition/src/stateView/index.ts create mode 100644 packages/state-transition/src/stateView/interface.ts diff --git a/packages/beacon-node/src/api/impl/beacon/state/utils.ts b/packages/beacon-node/src/api/impl/beacon/state/utils.ts index 8353f0823ed2..2d63c28e569d 100644 --- a/packages/beacon-node/src/api/impl/beacon/state/utils.ts +++ b/packages/beacon-node/src/api/impl/beacon/state/utils.ts @@ -3,7 +3,16 @@ import {routes} from "@lodestar/api"; import {CheckpointWithHex, IForkChoice} from "@lodestar/fork-choice"; import {GENESIS_SLOT} from "@lodestar/params"; import {BeaconStateAllForks, CachedBeaconStateAllForks} from "@lodestar/state-transition"; -import {BLSPubkey, Epoch, RootHex, Slot, ValidatorIndex, getValidatorStatus, phase0} from "@lodestar/types"; +import { + BLSPubkey, + Epoch, + RootHex, + Slot, + ValidatorIndex, + getValidatorStatus, + mapToGeneralStatus, + phase0, +} from "@lodestar/types"; import {fromHex} from "@lodestar/utils"; import {IBeaconChain} from "../../../../chain/index.js"; import {ApiError, ValidationError} from "../../errors.js"; @@ -65,28 +74,6 @@ export async function getStateResponseWithRegen( return res; } -type GeneralValidatorStatus = "active" | "pending" | "exited" | "withdrawal"; - -function mapToGeneralStatus(subStatus: routes.beacon.ValidatorStatus): GeneralValidatorStatus { - switch (subStatus) { - case "active_ongoing": - case "active_exiting": - case "active_slashed": - return "active"; - case "pending_initialized": - case "pending_queued": - return "pending"; - case "exited_slashed": - case "exited_unslashed": - return "exited"; - case "withdrawal_possible": - case "withdrawal_done": - return "withdrawal"; - default: - throw new Error(`Unknown substatus: ${subStatus}`); - } -} - export function toValidatorResponse( index: ValidatorIndex, validator: phase0.Validator, diff --git a/packages/state-transition/src/cache/stateCache.ts b/packages/state-transition/src/cache/stateCache.ts index f69cd2a44972..a3c0d2325201 100644 --- a/packages/state-transition/src/cache/stateCache.ts +++ b/packages/state-transition/src/cache/stateCache.ts @@ -168,7 +168,7 @@ export function createCachedBeaconState( * Check loadState() api for more details * // TODO: rename to loadUnfinalizedCachedBeaconState() due to ELECTRA */ -export function loadCachedBeaconState( +export function loadCachedBeaconState( cachedSeedState: T, stateBytes: Uint8Array, opts?: EpochCacheOpts, diff --git a/packages/state-transition/src/index.ts b/packages/state-transition/src/index.ts index c5836ffe9518..23d2d6fd6e61 100644 --- a/packages/state-transition/src/index.ts +++ b/packages/state-transition/src/index.ts @@ -35,12 +35,14 @@ export { isStateValidatorsNodesPopulated, loadCachedBeaconState, } from "./cache/stateCache.js"; +export {type SyncCommitteeCache} from "./cache/syncCommitteeCache.js"; export * from "./constants/index.js"; export type {EpochTransitionStep} from "./epoch/index.js"; export {type BeaconStateTransitionMetrics, getMetrics} from "./metrics.js"; export * from "./rewards/index.js"; export * from "./signatureSets/index.js"; export * from "./stateTransition.js"; +export * from "./stateView/index.js"; export type { BeaconStateAllForks, BeaconStateAltair, diff --git a/packages/state-transition/src/lightClient/proofs.ts b/packages/state-transition/src/lightClient/proofs.ts new file mode 100644 index 000000000000..d8b3847d978d --- /dev/null +++ b/packages/state-transition/src/lightClient/proofs.ts @@ -0,0 +1,83 @@ +import {Tree} from "@chainsafe/persistent-merkle-tree"; +import { + BLOCK_BODY_EXECUTION_PAYLOAD_GINDEX, + FINALIZED_ROOT_GINDEX, + FINALIZED_ROOT_GINDEX_ELECTRA, + ForkName, + ForkPostBellatrix, + isForkPostElectra, +} from "@lodestar/params"; +import {BeaconBlockBody, SSZTypesFor, ssz} from "@lodestar/types"; +import {BeaconStateAllForks, CachedBeaconStateAllForks} from "../types.js"; +import {SyncCommitteeWitness} from "./types.js"; + +export function getSyncCommitteesWitness(fork: ForkName, state: BeaconStateAllForks): SyncCommitteeWitness { + const n1 = state.node; + let witness: Uint8Array[]; + let currentSyncCommitteeRoot: Uint8Array; + let nextSyncCommitteeRoot: Uint8Array; + + if (isForkPostElectra(fork)) { + const n2 = n1.left; + const n5 = n2.right; + const n10 = n5.left; + const n21 = n10.right; + const n43 = n21.right; + + currentSyncCommitteeRoot = n43.left.root; // n86 + nextSyncCommitteeRoot = n43.right.root; // n87 + + // Witness branch is sorted by descending gindex + witness = [ + n21.left.root, // 42 + n10.left.root, // 20 + n5.right.root, // 11 + n2.left.root, // 4 + n1.right.root, // 3 + ]; + } else { + const n3 = n1.right; // [1]0110 + const n6 = n3.left; // 1[0]110 + const n13 = n6.right; // 10[1]10 + const n27 = n13.right; // 101[1]0 + currentSyncCommitteeRoot = n27.left.root; // n54 1011[0] + nextSyncCommitteeRoot = n27.right.root; // n55 1011[1] + + // Witness branch is sorted by descending gindex + witness = [ + n13.left.root, // 26 + n6.left.root, // 12 + n3.right.root, // 7 + n1.left.root, // 2 + ]; + } + + return { + witness, + currentSyncCommitteeRoot, + nextSyncCommitteeRoot, + }; +} + +export function getNextSyncCommitteeBranch(syncCommitteesWitness: SyncCommitteeWitness): Uint8Array[] { + // Witness branch is sorted by descending gindex + return [syncCommitteesWitness.currentSyncCommitteeRoot, ...syncCommitteesWitness.witness]; +} + +export function getCurrentSyncCommitteeBranch(syncCommitteesWitness: SyncCommitteeWitness): Uint8Array[] { + // Witness branch is sorted by descending gindex + return [syncCommitteesWitness.nextSyncCommitteeRoot, ...syncCommitteesWitness.witness]; +} + +export function getFinalizedRootProof(state: CachedBeaconStateAllForks): Uint8Array[] { + const finalizedRootGindex = state.epochCtx.isPostElectra() ? FINALIZED_ROOT_GINDEX_ELECTRA : FINALIZED_ROOT_GINDEX; + return new Tree(state.node).getSingleProof(BigInt(finalizedRootGindex)); +} + +export function getBlockBodyExecutionHeaderProof( + fork: ForkPostBellatrix, + body: BeaconBlockBody +): Uint8Array[] { + const bodyView = (ssz[fork].BeaconBlockBody as SSZTypesFor).toView(body); + return new Tree(bodyView.node).getSingleProof(BigInt(BLOCK_BODY_EXECUTION_PAYLOAD_GINDEX)); +} diff --git a/packages/state-transition/src/lightClient/types.ts b/packages/state-transition/src/lightClient/types.ts new file mode 100644 index 000000000000..b9723df501b3 --- /dev/null +++ b/packages/state-transition/src/lightClient/types.ts @@ -0,0 +1,33 @@ +/** + * We aren't creating the sync committee proofs separately because our ssz library automatically adds leaves to composite types, + * so they're already included in the state proof, currently with no way to specify otherwise + * + * remove two offsets so the # of offsets in the state proof will be the # expected + * This is a hack, but properly setting the offsets in the state proof would require either removing witnesses needed for the committees + * or setting the roots of the committees in the state proof + * this will always be 1, syncProofLeavesLength + * + * + * With empty state (minimal) + * - `genesisTime = 0xffffffff` + * - `genesisValidatorsRoot = Buffer.alloc(32, 1)` + * + * Proof: + * ``` + * offsets: [ 5, 4, 3, 2, 1 ] + * leaves: [ + * '0xffffffff00000000000000000000000000000000000000000000000000000000', + * '0x0101010101010101010101010101010101010101010101010101010101010101', + * '0xb11b8bcf59425d6c99019cca1d2c2e47b51a2f74917a67ad132274f43e13ec43', + * '0x74bd1f2437cdf74b0904ee525d8da070a3fa27570942bf42cbab3dc5939600f0', + * '0x7f06739e5a42360c56e519a511675901c95402ea9877edc0d9a87471b1374a6a', + * '0x9f534204ba3c0b69fcb42a11987bfcbc5aea0463e5b0614312ded4b62cf3a380' + * ] + * ``` + */ +export type SyncCommitteeWitness = { + /** Vector[Bytes32, 4] or Vector[Bytes32, 5] depending on the fork */ + witness: Uint8Array[]; + currentSyncCommitteeRoot: Uint8Array; + nextSyncCommitteeRoot: Uint8Array; +}; diff --git a/packages/state-transition/src/stateView/beaconStateView.ts b/packages/state-transition/src/stateView/beaconStateView.ts new file mode 100644 index 000000000000..b1c7151da227 --- /dev/null +++ b/packages/state-transition/src/stateView/beaconStateView.ts @@ -0,0 +1,746 @@ +import {CompactMultiProof, ProofType, Tree, createProof} from "@chainsafe/persistent-merkle-tree"; +import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; +import {ByteViews} from "@chainsafe/ssz"; +import {BeaconConfig} from "@lodestar/config"; +import {ForkSeq, SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params"; +import { + BeaconBlock, + BlindedBeaconBlock, + BuilderIndex, + Bytes32, + Epoch, + ExecutionPayloadBid, + ExecutionPayloadHeader, + Root, + RootHex, + SignedBeaconBlock, + SignedBlindedBeaconBlock, + Slot, + SyncCommittee, + ValidatorIndex, + capella, + electra, + fulu, + getValidatorStatus, + gloas, + mapToGeneralStatus, + phase0, + rewards, +} from "@lodestar/types"; +import {Checkpoint, Fork} from "@lodestar/types/phase0"; +import {VoluntaryExitValidity, getVoluntaryExitValidity} from "../block/processVoluntaryExit.js"; +import {getExpectedWithdrawals} from "../block/processWithdrawals.js"; +import {EffectiveBalanceIncrements} from "../cache/effectiveBalanceIncrements.js"; +import {EpochTransitionCacheOpts} from "../cache/epochTransitionCache.js"; +import {RewardCache} from "../cache/rewardCache.js"; +import { + CachedBeaconStateAllForks, + CachedBeaconStateAltair, + CachedBeaconStateCapella, + CachedBeaconStateElectra, + CachedBeaconStateExecutions, + CachedBeaconStateFulu, + CachedBeaconStateGloas, + createCachedBeaconState, + isStateValidatorsNodesPopulated, +} from "../cache/stateCache.js"; +import {SyncCommitteeCache} from "../cache/syncCommitteeCache.js"; +import {BeaconStateAllForks} from "../cache/types.js"; +import {computeUnrealizedCheckpoints} from "../epoch/computeUnrealizedCheckpoints.js"; +import {getFinalizedRootProof, getSyncCommitteesWitness} from "../lightClient/proofs.js"; +import {SyncCommitteeWitness} from "../lightClient/types.js"; +import {computeAttestationsRewards} from "../rewards/attestationsRewards.js"; +import {computeBlockRewards} from "../rewards/blockRewards.js"; +import {computeSyncCommitteeRewards} from "../rewards/syncCommitteeRewards.js"; +import {StateTransitionModules, StateTransitionOpts, processSlots, stateTransition} from "../stateTransition.js"; +import {getEffectiveBalanceIncrementsZeroInactive} from "../util/balance.js"; +import {getBlockRootAtSlot} from "../util/blockRoot.js"; +import {computeAnchorCheckpoint} from "../util/computeAnchorCheckpoint.js"; +import {computeEpochAtSlot, computeStartSlotAtEpoch} from "../util/epoch.js"; +import {EpochShuffling} from "../util/epochShuffling.js"; +import {isExecutionEnabled, isExecutionStateType, isMergeTransitionComplete} from "../util/execution.js"; +import {canBuilderCoverBid} from "../util/gloas.js"; +import {loadState} from "../util/loadState/loadState.js"; +import {getRandaoMix} from "../util/seed.js"; +import {getStateTypeFromBytes} from "../util/sszBytes.js"; +import {getLatestWeakSubjectivityCheckpointEpoch} from "../util/weakSubjectivity.js"; +import {IBeaconStateView} from "./interface.js"; + +export class BeaconStateView implements IBeaconStateView { + private readonly config: BeaconConfig; + // Cached values extracted from the tree + // phase0 + private _fork: Fork | null = null; + private _latestBlockHeader: phase0.BeaconBlockHeader | null = null; + // altair + private _currentSyncCommittee: SyncCommittee | null = null; + private _nextSyncCommittee: SyncCommittee | null = null; + private _previousEpochParticipation: number[] | null = null; + private _currentEpochParticipation: number[] | null = null; + // bellatrix + private _latestExecutionPayloadHeader: ExecutionPayloadHeader | null = null; + // capella + private _historicalSummaries: capella.HistoricalSummaries | null = null; + // electra + private _pendingPartialWithdrawals: electra.PendingPartialWithdrawals | null = null; + private _pendingConsolidations: electra.PendingConsolidations | null = null; + private _pendingDeposits: electra.PendingDeposits | null = null; + // fulu + private _proposerLookahead: fulu.ProposerLookahead | null = null; + // gloas + private _executionPayloadAvailability: boolean[] | null = null; + private _latestExecutionPayloadBid: ExecutionPayloadBid | null = null; + + constructor(readonly cachedState: CachedBeaconStateAllForks) { + this.config = cachedState.config; + } + + // phase0 + + get slot(): number { + return this.cachedState.slot; + } + + get fork(): Fork { + if (this._fork === null) { + this._fork = this.cachedState.fork.toValue(); + } + return this._fork; + } + + get epoch(): number { + return computeEpochAtSlot(this.slot); + } + + get genesisTime(): number { + return this.cachedState.genesisTime; + } + + get genesisValidatorsRoot(): Root { + return this.cachedState.genesisValidatorsRoot; + } + + get eth1Data(): phase0.Eth1Data { + return this.cachedState.eth1Data; + } + + get latestBlockHeader(): phase0.BeaconBlockHeader { + if (this._latestBlockHeader === null) { + this._latestBlockHeader = this.cachedState.latestBlockHeader.toValue(); + } + return this._latestBlockHeader; + } + + get previousJustifiedCheckpoint(): Checkpoint { + return this.cachedState.previousJustifiedCheckpoint; + } + + get currentJustifiedCheckpoint(): Checkpoint { + return this.cachedState.currentJustifiedCheckpoint; + } + + get finalizedCheckpoint(): Checkpoint { + return this.cachedState.finalizedCheckpoint; + } + + getBlockRootAtSlot(slot: Slot): Root { + return getBlockRootAtSlot(this.cachedState, slot); + } + + getBlockRootAtEpoch(epoch: Epoch): Root { + return this.getBlockRootAtSlot(computeStartSlotAtEpoch(epoch)); + } + + getStateRootAtSlot(slot: Slot): Root { + return this.cachedState.stateRoots.get(slot % SLOTS_PER_HISTORICAL_ROOT); + } + + getRandaoMix(epoch: Epoch): Bytes32 { + return getRandaoMix(this.cachedState, epoch); + } + + get previousEpochParticipation(): number[] { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.altair) { + throw new Error("previousEpochParticipation is not available before Altair"); + } + + if (this._previousEpochParticipation === null) { + this._previousEpochParticipation = ( + this.cachedState as CachedBeaconStateAltair + ).previousEpochParticipation.toValue(); + } + + return this._previousEpochParticipation; + } + + // altair + + get currentEpochParticipation(): number[] { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.altair) { + throw new Error("currentEpochParticipation is not available before Altair"); + } + + if (this._currentEpochParticipation === null) { + this._currentEpochParticipation = ( + this.cachedState as CachedBeaconStateAltair + ).currentEpochParticipation.toValue(); + } + + return this._currentEpochParticipation; + } + + // bellatrix + + get latestExecutionPayloadHeader(): ExecutionPayloadHeader { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.bellatrix) { + throw new Error("latestExecutionPayloadHeader is not available before Bellatrix"); + } + + if (this._latestExecutionPayloadHeader === null) { + this._latestExecutionPayloadHeader = ( + this.cachedState as CachedBeaconStateExecutions + ).latestExecutionPayloadHeader.toValue(); + } + + return this._latestExecutionPayloadHeader; + } + + // capella + + get historicalSummaries(): capella.HistoricalSummaries { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.capella) { + throw new Error("Historical summaries are not supported before Capella"); + } + + if (this._historicalSummaries === null) { + this._historicalSummaries = (this.cachedState as CachedBeaconStateCapella).historicalSummaries.toValue(); + } + + return this._historicalSummaries; + } + + // electra + + get pendingDeposits(): electra.PendingDeposits { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.electra) { + throw new Error("Pending deposits are not supported before Electra"); + } + + if (this._pendingDeposits === null) { + this._pendingDeposits = (this.cachedState as CachedBeaconStateElectra).pendingDeposits.toValue(); + } + + return this._pendingDeposits; + } + + get pendingDepositsCount(): number { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.electra) { + throw new Error("Pending deposits are not supported before Electra"); + } + + return (this.cachedState as CachedBeaconStateElectra).pendingDeposits.length; + } + + get pendingPartialWithdrawals(): electra.PendingPartialWithdrawals { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.electra) { + throw new Error("Pending partial withdrawals are not supported before Electra"); + } + + if (this._pendingPartialWithdrawals === null) { + this._pendingPartialWithdrawals = ( + this.cachedState as CachedBeaconStateElectra + ).pendingPartialWithdrawals.toValue(); + } + + return this._pendingPartialWithdrawals; + } + + get pendingPartialWithdrawalsCount(): number { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.electra) { + throw new Error("Pending partial withdrawals are not supported before Electra"); + } + + return (this.cachedState as CachedBeaconStateElectra).pendingPartialWithdrawals.length; + } + + get pendingConsolidations(): electra.PendingConsolidations { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.electra) { + throw new Error("Pending consolidations are not supported before Electra"); + } + + if (this._pendingConsolidations === null) { + this._pendingConsolidations = (this.cachedState as CachedBeaconStateElectra).pendingConsolidations.toValue(); + } + + return this._pendingConsolidations; + } + + get pendingConsolidationsCount(): number { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.electra) { + throw new Error("Pending consolidations are not supported before Electra"); + } + + return (this.cachedState as CachedBeaconStateElectra).pendingConsolidations.length; + } + + // fulu + + get proposerLookahead(): fulu.ProposerLookahead { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.fulu) { + throw new Error("Proposer lookahead is not supported before Fulu"); + } + + if (this._proposerLookahead === null) { + this._proposerLookahead = (this.cachedState as CachedBeaconStateFulu).proposerLookahead.toValue(); + } + + return this._proposerLookahead; + } + + // gloas + + get executionPayloadAvailability(): boolean[] { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.gloas) { + throw new Error("executionPayloadAvailability is not available before GLOAS"); + } + + if (this._executionPayloadAvailability === null) { + this._executionPayloadAvailability = (this.cachedState as CachedBeaconStateGloas).executionPayloadAvailability + .toValue() + .toBoolArray(); + } + + return this._executionPayloadAvailability; + } + + get latestExecutionPayloadBid(): ExecutionPayloadBid { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.gloas) { + throw new Error("latestExecutionPayloadBid is not available before GLOAS"); + } + + if (this._latestExecutionPayloadBid === null) { + this._latestExecutionPayloadBid = ( + this.cachedState as CachedBeaconStateGloas + ).latestExecutionPayloadBid.toValue(); + } + return this._latestExecutionPayloadBid; + } + + getBuilder(index: BuilderIndex): gloas.Builder { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.gloas) { + throw new Error("Builders are not supported before GLOAS"); + } + + return (this.cachedState as CachedBeaconStateGloas).builders.getReadonly(index); + } + + canBuilderCoverBid(builderIndex: BuilderIndex, bidAmount: number): boolean { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.gloas) { + throw new Error("Builders are not supported before GLOAS"); + } + + return canBuilderCoverBid(this.cachedState as CachedBeaconStateGloas, builderIndex, bidAmount); + } + + /** + * Return the index of the validator in the PTC committee for the given slot. + * return -1 if validator is not in the PTC committee for the given slot. + */ + validatorPTCCommitteeIndex(validatorIndex: ValidatorIndex, slot: Slot): number { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.gloas) { + throw new Error("PTC committees are not supported before GLOAS"); + } + + const ptcCommittee = (this.cachedState as CachedBeaconStateGloas).epochCtx.getPayloadTimelinessCommittee(slot); + return ptcCommittee.indexOf(validatorIndex); + } + + // Shuffling and committees + + getShufflingAtEpoch(epoch: Epoch): EpochShuffling { + return this.cachedState.epochCtx.getShufflingAtEpoch(epoch); + } + + get previousDecisionRoot(): RootHex { + return this.cachedState.epochCtx.previousDecisionRoot; + } + + get currentDecisionRoot(): RootHex { + return this.cachedState.epochCtx.currentDecisionRoot; + } + + get nextDecisionRoot(): RootHex { + return this.cachedState.epochCtx.nextDecisionRoot; + } + + getShufflingDecisionRoot(epoch: Epoch): RootHex { + return this.cachedState.epochCtx.getShufflingDecisionRoot(epoch); + } + + getPreviousShuffling(): EpochShuffling { + return this.cachedState.epochCtx.previousShuffling; + } + + getCurrentShuffling(): EpochShuffling { + return this.cachedState.epochCtx.currentShuffling; + } + + getNextShuffling(): EpochShuffling { + return this.cachedState.epochCtx.nextShuffling; + } + + // Proposer shuffling + + get previousProposers(): ValidatorIndex[] | null { + return this.cachedState.epochCtx.proposersPrevEpoch; + } + + get currentProposers(): ValidatorIndex[] { + return this.cachedState.epochCtx.getBeaconProposers(); + } + + get nextProposers(): ValidatorIndex[] { + return this.cachedState.epochCtx.getBeaconProposersNextEpoch(); + } + + getBeaconProposer(slot: number): ValidatorIndex { + return this.cachedState.epochCtx.getBeaconProposer(slot); + } + + computeAnchorCheckpoint(): {checkpoint: phase0.Checkpoint; blockHeader: phase0.BeaconBlockHeader} { + return computeAnchorCheckpoint(this.config, this.cachedState); + } + + // Sync committees + + get currentSyncCommittee(): SyncCommittee { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.altair) { + throw new Error("currentSyncCommittee is not available before Altair"); + } + + if (this._currentSyncCommittee === null) { + this._currentSyncCommittee = (this.cachedState as CachedBeaconStateAltair).currentSyncCommittee.toValue(); + } + + return this._currentSyncCommittee; + } + + get nextSyncCommittee(): SyncCommittee { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.altair) { + throw new Error("nextSyncCommittee is not available before Altair"); + } + + if (this._nextSyncCommittee === null) { + this._nextSyncCommittee = (this.cachedState as CachedBeaconStateAltair).nextSyncCommittee.toValue(); + } + + return this._nextSyncCommittee; + } + + get currentSyncCommitteeIndexed(): SyncCommitteeCache { + return this.cachedState.epochCtx.currentSyncCommitteeIndexed; + } + + get syncProposerReward(): number { + return this.cachedState.epochCtx.syncProposerReward; + } + + getIndexedSyncCommitteeAtEpoch(epoch: Epoch): SyncCommitteeCache { + return this.cachedState.epochCtx.getIndexedSyncCommitteeAtEpoch(epoch); + } + + // Validators and balances + + get effectiveBalanceIncrements(): EffectiveBalanceIncrements { + return this.cachedState.epochCtx.effectiveBalanceIncrements; + } + + getEffectiveBalanceIncrementsZeroInactive(): EffectiveBalanceIncrements { + return getEffectiveBalanceIncrementsZeroInactive(this.cachedState); + } + + getBalance(index: number): number { + return this.cachedState.balances.get(index); + } + + getValidator(index: ValidatorIndex): phase0.Validator { + return this.cachedState.validators.getReadonly(index).toValue(); + } + + getValidatorsByStatus(statuses: Set, currentEpoch: Epoch): phase0.Validator[] { + const validators: phase0.Validator[] = []; + const validatorsArr = this.cachedState.validators.getAllReadonlyValues(); + + for (const validator of validatorsArr) { + const validatorStatus = getValidatorStatus(validator, currentEpoch); + if (statuses.has(validatorStatus) || statuses.has(mapToGeneralStatus(validatorStatus))) { + validators.push(validator); + } + } + return validators; + } + + get validatorCount(): number { + return this.cachedState.validators.length; + } + + get activeValidatorCount(): number { + return this.cachedState.epochCtx.currentShuffling.activeIndices.length; + } + + getAllValidators(): phase0.Validator[] { + return this.cachedState.validators.getAllReadonlyValues(); + } + + getAllBalances(): number[] { + return this.cachedState.balances.getAll(); + } + + // Merge + + get isExecutionStateType(): boolean { + return this.config.getForkSeq(this.cachedState.slot) >= ForkSeq.bellatrix; + } + + isExecutionEnabled(block: BeaconBlock | BlindedBeaconBlock): boolean { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.bellatrix) { + return false; + } + + return isExecutionEnabled(this.cachedState as CachedBeaconStateExecutions, block); + } + + get isMergeTransitionComplete(): boolean { + return isExecutionStateType(this.cachedState) && isMergeTransitionComplete(this.cachedState); + } + + // Block production + + getExpectedWithdrawals(): { + expectedWithdrawals: capella.Withdrawal[]; + processedBuilderWithdrawalsCount: number; + processedPartialWithdrawalsCount: number; + processedBuildersSweepCount: number; + processedValidatorSweepCount: number; + } { + const fork = this.config.getForkSeq(this.cachedState.slot); + return getExpectedWithdrawals( + fork, + this.cachedState as CachedBeaconStateCapella | CachedBeaconStateElectra | CachedBeaconStateGloas + ); + } + + // API + + get proposerRewards(): RewardCache { + return this.cachedState.proposerRewards; + } + + async computeBlockRewards(block: BeaconBlock, proposerRewards?: RewardCache): Promise { + return computeBlockRewards(this.cachedState.config, block, this.cachedState, proposerRewards); + } + + async computeAttestationsRewards(validatorIds?: (ValidatorIndex | string)[]): Promise { + return computeAttestationsRewards( + this.cachedState.config, + this.cachedState.epochCtx.pubkey2index, + this.cachedState, + validatorIds + ); + } + + async computeSyncCommitteeRewards( + block: BeaconBlock, + validatorIds: (ValidatorIndex | string)[] + ): Promise { + return computeSyncCommitteeRewards( + this.cachedState.config, + this.cachedState.epochCtx.index2pubkey, + block, + this.cachedState, + validatorIds + ); + } + + getLatestWeakSubjectivityCheckpointEpoch(): Epoch { + return getLatestWeakSubjectivityCheckpointEpoch(this.config, this.cachedState); + } + + // Validation + + getVoluntaryExitValidity( + signedVoluntaryExit: phase0.SignedVoluntaryExit, + verifySignature = true + ): VoluntaryExitValidity { + const stateFork = this.config.getForkSeq(this.cachedState.slot); + return getVoluntaryExitValidity(stateFork, this.cachedState, signedVoluntaryExit, verifySignature); + } + + isValidVoluntaryExit(signedVoluntaryExit: phase0.SignedVoluntaryExit, verifySignature: boolean): boolean { + return this.getVoluntaryExitValidity(signedVoluntaryExit, verifySignature) === VoluntaryExitValidity.valid; + } + + // Proofs + + getFinalizedRootProof(): Uint8Array[] { + return getFinalizedRootProof(this.cachedState); + } + + getSyncCommitteesWitness(): SyncCommitteeWitness { + const fork = this.config.getForkName(this.cachedState.slot); + if (ForkSeq[fork] < ForkSeq.altair) { + throw new Error("Sync committees witness is not available before Altair"); + } + + return getSyncCommitteesWitness(fork, this.cachedState); + } + + getSingleProof(gindex: bigint): Uint8Array[] { + return new Tree(this.cachedState.node).getSingleProof(gindex); + } + + createMultiProof(descriptor: Uint8Array): CompactMultiProof { + const stateNode = this.cachedState.node; + return createProof(stateNode, {type: ProofType.compactMulti, descriptor}) as CompactMultiProof; + } + + // Fork choice + + computeUnrealizedCheckpoints(): { + justifiedCheckpoint: phase0.Checkpoint; + finalizedCheckpoint: phase0.Checkpoint; + } { + return computeUnrealizedCheckpoints(this.cachedState); + } + + // this is for backward compatible + + get clonedCount(): number { + return this.cachedState.clonedCount; + } + + get clonedCountWithTransferCache(): number { + return this.cachedState.clonedCountWithTransferCache; + } + + get createdWithTransferCache(): boolean { + return this.cachedState.createdWithTransferCache; + } + + isStateValidatorsNodesPopulated(): boolean { + return isStateValidatorsNodesPopulated(this.cachedState); + } + + // Serialization + + loadOtherState(stateBytes: Uint8Array, seedValidatorsBytes?: Uint8Array): IBeaconStateView { + const {state} = loadState(this.config, this.cachedState, stateBytes, seedValidatorsBytes); + + const cachedState = createCachedBeaconState( + state, + { + config: this.config, + // as of Feb 2026, it's not necessary to sync pubkey cache as it's shared across states in Lodestar + pubkey2index: this.cachedState.epochCtx.pubkey2index, + index2pubkey: this.cachedState.epochCtx.index2pubkey, + }, + { + skipSyncPubkeys: true, + } + ); + + // load all cache in order for consumers (usually regen.getState()) to process blocks faster + cachedState.validators.getAllReadonlyValues(); + cachedState.balances.getAll(); + + return new BeaconStateView(cachedState); + } + + serialize(): Uint8Array { + return this.cachedState.serialize(); + } + + serializedSize(): number { + return this.cachedState.type.tree_serializedSize(this.cachedState.node); + } + + serializeToBytes(output: ByteViews, offset: number): number { + return this.cachedState.serializeToBytes(output, offset); + } + + serializeValidators(): Uint8Array { + return this.cachedState.validators.serialize(); + } + + serializedValidatorsSize(): number { + const type = this.cachedState.type.fields.validators; + return type.tree_serializedSize(this.cachedState.validators.node); + } + + serializeValidatorsToBytes(output: ByteViews, offset: number): number { + return this.cachedState.validators.serializeToBytes(output, offset); + } + + hashTreeRoot(): Uint8Array { + return this.cachedState.hashTreeRoot(); + } + + // State transition + + stateTransition( + signedBlock: SignedBeaconBlock | SignedBlindedBeaconBlock, + options: StateTransitionOpts, + {metrics, validatorMonitor}: StateTransitionModules + ): IBeaconStateView { + const newState = stateTransition(this.cachedState, signedBlock, options, {metrics, validatorMonitor}); + return new BeaconStateView(newState); + } + + processSlots( + slot: Slot, + epochTransitionCacheOpts?: EpochTransitionCacheOpts & {dontTransferCache?: boolean}, + modules?: StateTransitionModules + ): IBeaconStateView { + const newState = processSlots(this.cachedState, slot, epochTransitionCacheOpts, modules); + return new BeaconStateView(newState); + } +} + +/** + * Create BeaconStateView for historical state regen, no need to sync pubkey cache there. + */ +export function createBeaconStateViewForHistoricalRegen( + config: BeaconConfig, + stateBytes: Uint8Array +): IBeaconStateView { + const state = getStateTypeFromBytes(config, stateBytes).deserializeToViewDU(stateBytes); + + const pubkey2index = new PubkeyIndexMap(); + syncPubkeyCache(state, pubkey2index); + const cachedState = createCachedBeaconState( + state, + { + config, + pubkey2index, + index2pubkey: [], + }, + { + skipSyncPubkeys: true, + } + ); + + return new BeaconStateView(cachedState); +} + +/** + * Populate a PubkeyIndexMap with any new entries based on a BeaconState + */ +function syncPubkeyCache(state: BeaconStateAllForks, pubkey2index: PubkeyIndexMap): void { + // Get the validators sub tree once for all the loop + + const newCount = state.validators.length; + for (let i = pubkey2index.size; i < newCount; i++) { + const pubkey = state.validators.getReadonly(i).pubkey; + pubkey2index.set(pubkey, i); + } +} diff --git a/packages/state-transition/src/stateView/index.ts b/packages/state-transition/src/stateView/index.ts new file mode 100644 index 000000000000..23854e54232d --- /dev/null +++ b/packages/state-transition/src/stateView/index.ts @@ -0,0 +1,2 @@ +export * from "./beaconStateView.js"; +export * from "./interface.js"; diff --git a/packages/state-transition/src/stateView/interface.ts b/packages/state-transition/src/stateView/interface.ts new file mode 100644 index 000000000000..1059187f3b48 --- /dev/null +++ b/packages/state-transition/src/stateView/interface.ts @@ -0,0 +1,196 @@ +import {CompactMultiProof} from "@chainsafe/persistent-merkle-tree"; +import {ByteViews} from "@chainsafe/ssz"; +import { + BeaconBlock, + BlindedBeaconBlock, + BuilderIndex, + Bytes32, + Epoch, + ExecutionPayloadBid, + ExecutionPayloadHeader, + Root, + RootHex, + SignedBeaconBlock, + SignedBlindedBeaconBlock, + Slot, + ValidatorIndex, + altair, + capella, + electra, + fulu, + gloas, + phase0, + rewards, +} from "@lodestar/types"; +import {Checkpoint, Fork} from "@lodestar/types/phase0"; +import {VoluntaryExitValidity} from "../block/processVoluntaryExit.js"; +import {EffectiveBalanceIncrements} from "../cache/effectiveBalanceIncrements.js"; +import {EpochTransitionCacheOpts} from "../cache/epochTransitionCache.js"; +import {RewardCache} from "../cache/rewardCache.js"; +import {SyncCommitteeCache} from "../cache/syncCommitteeCache.js"; +import {SyncCommitteeWitness} from "../lightClient/types.js"; +import {StateTransitionModules, StateTransitionOpts} from "../stateTransition.js"; +import {EpochShuffling} from "../util/epochShuffling.js"; + +/** + * A read-only view of the BeaconState. + */ +export interface IBeaconStateView { + // State access + + // phase0 + slot: Slot; + fork: Fork; + epoch: Epoch; + genesisTime: number; + genesisValidatorsRoot: Root; + eth1Data: phase0.Eth1Data; + latestBlockHeader: phase0.BeaconBlockHeader; + previousJustifiedCheckpoint: Checkpoint; + currentJustifiedCheckpoint: Checkpoint; + finalizedCheckpoint: Checkpoint; + getBlockRootAtSlot(slot: Slot): Root; + getBlockRootAtEpoch(epoch: Epoch): Root; + getStateRootAtSlot(slot: Slot): Root; + getRandaoMix(epoch: Epoch): Bytes32; + + // altair + previousEpochParticipation: number[]; + currentEpochParticipation: number[]; + + // bellatrix + latestExecutionPayloadHeader: ExecutionPayloadHeader; + + // capella + historicalSummaries: capella.HistoricalSummaries; + + // electra + pendingDeposits: electra.PendingDeposits; + pendingDepositsCount: number; + pendingPartialWithdrawals: electra.PendingPartialWithdrawals; + pendingPartialWithdrawalsCount: number; + pendingConsolidations: electra.PendingConsolidations; + pendingConsolidationsCount: number; + + // fulu + proposerLookahead: fulu.ProposerLookahead; + + // gloas + executionPayloadAvailability: boolean[]; + latestExecutionPayloadBid: ExecutionPayloadBid; + getBuilder(index: BuilderIndex): gloas.Builder; + canBuilderCoverBid(builderIndex: BuilderIndex, bidAmount: number): boolean; + validatorPTCCommitteeIndex(validatorIndex: ValidatorIndex, slot: Slot): number; + + // Shuffling and committees + getShufflingAtEpoch(epoch: Epoch): EpochShuffling; + // Decision roots + previousDecisionRoot: RootHex; + currentDecisionRoot: RootHex; + nextDecisionRoot: RootHex; + getShufflingDecisionRoot(epoch: Epoch): RootHex; + getPreviousShuffling(): EpochShuffling; + getCurrentShuffling(): EpochShuffling; + getNextShuffling(): EpochShuffling; + + // utils: proposers, anchor checkpoint + previousProposers: ValidatorIndex[] | null; + currentProposers: ValidatorIndex[]; + nextProposers: ValidatorIndex[]; + getBeaconProposer(slot: Slot): ValidatorIndex; + computeAnchorCheckpoint(): {checkpoint: phase0.Checkpoint; blockHeader: phase0.BeaconBlockHeader}; + + // Sync committees + currentSyncCommittee: altair.SyncCommittee; + nextSyncCommittee: altair.SyncCommittee; + currentSyncCommitteeIndexed: SyncCommitteeCache; + syncProposerReward: number; + getIndexedSyncCommitteeAtEpoch(epoch: Epoch): SyncCommitteeCache; + + // Validators and balances + effectiveBalanceIncrements: EffectiveBalanceIncrements; + getEffectiveBalanceIncrementsZeroInactive(): EffectiveBalanceIncrements; + getBalance(index: number): number; + // readonly + getValidator(index: ValidatorIndex): phase0.Validator; + getValidatorsByStatus(statuses: Set, currentEpoch: Epoch): phase0.Validator[]; + validatorCount: number; + // this get number of active validators in the current shuffling + activeValidatorCount: number; + // this is needed for apis only + getAllValidators(): phase0.Validator[]; + getAllBalances(): number[]; + + // Merge + isExecutionStateType: boolean; + isMergeTransitionComplete: boolean; + // TODO this should go away (or rather only need block) + isExecutionEnabled(block: BeaconBlock | BlindedBeaconBlock): boolean; + + // Block production + getExpectedWithdrawals(): { + expectedWithdrawals: capella.Withdrawal[]; + processedBuilderWithdrawalsCount: number; + processedPartialWithdrawalsCount: number; + processedValidatorSweepCount: number; + }; + + // API + proposerRewards: RewardCache; + computeBlockRewards(block: BeaconBlock, proposerRewards?: RewardCache): Promise; + computeAttestationsRewards(validatorIds?: (ValidatorIndex | string)[]): Promise; + computeSyncCommitteeRewards( + block: BeaconBlock, + validatorIds: (ValidatorIndex | string)[] + ): Promise; + getLatestWeakSubjectivityCheckpointEpoch(): Epoch; + + // Validation + getVoluntaryExitValidity( + signedVoluntaryExit: phase0.SignedVoluntaryExit, + verifySignature: boolean + ): VoluntaryExitValidity; + isValidVoluntaryExit(signedVoluntaryExit: phase0.SignedVoluntaryExit, verifySignature: boolean): boolean; + + // Proofs + getFinalizedRootProof(): Uint8Array[]; + getSyncCommitteesWitness(): SyncCommitteeWitness; + getSingleProof(gindex: bigint): Uint8Array[]; + createMultiProof(descriptor: Uint8Array): CompactMultiProof; + + // Fork choice + computeUnrealizedCheckpoints(): { + justifiedCheckpoint: phase0.Checkpoint; + finalizedCheckpoint: phase0.Checkpoint; + }; + + // this is for backward compatible + clonedCount: number; + clonedCountWithTransferCache: number; + createdWithTransferCache: boolean; + // TODO is there a better name that is less implementation specific but still conveys the meaning? + isStateValidatorsNodesPopulated(): boolean; + + // Serialization + loadOtherState(stateBytes: Uint8Array, seedValidatorsBytes?: Uint8Array): IBeaconStateView; + serialize(): Uint8Array; + serializedSize(): number; + serializeToBytes(output: ByteViews, offset: number): number; + serializeValidators(): Uint8Array; + serializedValidatorsSize(): number; + serializeValidatorsToBytes(output: ByteViews, offset: number): number; + + hashTreeRoot(): Uint8Array; + + // State transition + stateTransition( + signedBlock: SignedBeaconBlock | SignedBlindedBeaconBlock, + options: StateTransitionOpts, + modules: StateTransitionModules + ): IBeaconStateView; + processSlots( + slot: Slot, + epochTransitionCacheOpts?: EpochTransitionCacheOpts & {dontTransferCache?: boolean}, + modules?: StateTransitionModules + ): IBeaconStateView; +} diff --git a/packages/state-transition/src/util/weakSubjectivity.ts b/packages/state-transition/src/util/weakSubjectivity.ts index ff81f6010b6e..c1dd6b39966b 100644 --- a/packages/state-transition/src/util/weakSubjectivity.ts +++ b/packages/state-transition/src/util/weakSubjectivity.ts @@ -47,7 +47,7 @@ export function computeWeakSubjectivityPeriodCachedState( state: CachedBeaconStateAllForks ): number { const activeValidatorCount = state.epochCtx.currentShuffling.activeIndices.length; - const fork = state.config.getForkName(state.slot); + const fork = config.getForkName(state.slot); return isForkPostElectra(fork) ? computeWeakSubjectivityPeriodFromConstituentsElectra( diff --git a/packages/types/src/capella/types.ts b/packages/types/src/capella/types.ts index 386f96ecd280..4dda27a73e0b 100644 --- a/packages/types/src/capella/types.ts +++ b/packages/types/src/capella/types.ts @@ -31,3 +31,5 @@ export type LightClientUpdate = ValueOf; export type LightClientFinalityUpdate = ValueOf; export type LightClientOptimisticUpdate = ValueOf; export type LightClientStore = ValueOf; + +export type HistoricalSummaries = ValueOf; diff --git a/packages/types/src/types.ts b/packages/types/src/types.ts index 223269815f6e..8d5562290a0c 100644 --- a/packages/types/src/types.ts +++ b/packages/types/src/types.ts @@ -6,6 +6,7 @@ import { ForkPostDeneb, ForkPostElectra, ForkPostFulu, + ForkPostGloas, } from "@lodestar/params"; import {ts as altair} from "./altair/index.js"; import {ts as bellatrix} from "./bellatrix/index.js"; @@ -321,6 +322,7 @@ type TypesByFork = { AggregateAndProof: electra.AggregateAndProof; SignedAggregateAndProof: electra.SignedAggregateAndProof; ExecutionRequests: electra.ExecutionRequests; + ExecutionPayloadBid: gloas.ExecutionPayloadBid; DataColumnSidecar: gloas.DataColumnSidecar; DataColumnSidecars: gloas.DataColumnSidecars; }; @@ -388,3 +390,4 @@ export type IndexedAttestationBigint = TypesByFork export type AttesterSlashing = TypesByFork[F]["AttesterSlashing"]; export type AggregateAndProof = TypesByFork[F]["AggregateAndProof"]; export type SignedAggregateAndProof = TypesByFork[F]["SignedAggregateAndProof"]; +export type ExecutionPayloadBid = TypesByFork[F]["ExecutionPayloadBid"]; diff --git a/packages/types/src/utils/validatorStatus.ts b/packages/types/src/utils/validatorStatus.ts index 9e72c39d9df2..e5c4692e2f3f 100644 --- a/packages/types/src/utils/validatorStatus.ts +++ b/packages/types/src/utils/validatorStatus.ts @@ -15,6 +15,8 @@ export type ValidatorStatus = | "withdrawal_possible" | "withdrawal_done"; +export type GeneralValidatorStatus = "active" | "pending" | "exited" | "withdrawal"; + /** * Get the status of the validator * based on conditions outlined in https://hackmd.io/ofFJ5gOmQpu1jjHilHbdQQ @@ -50,3 +52,23 @@ export function getValidatorStatus(validator: phase0.Validator, currentEpoch: Ep } throw new Error("ValidatorStatus unknown"); } + +export function mapToGeneralStatus(subStatus: ValidatorStatus): GeneralValidatorStatus { + switch (subStatus) { + case "active_ongoing": + case "active_exiting": + case "active_slashed": + return "active"; + case "pending_initialized": + case "pending_queued": + return "pending"; + case "exited_slashed": + case "exited_unslashed": + return "exited"; + case "withdrawal_possible": + case "withdrawal_done": + return "withdrawal"; + default: + throw new Error(`Unknown substatus: ${subStatus}`); + } +} From f2252230009da1fb2e680997806b4b58a4112523 Mon Sep 17 00:00:00 2001 From: Cayman Date: Mon, 23 Feb 2026 16:39:47 -0500 Subject: [PATCH 04/63] refactor: unified pubkey cache (#8945) - helpful towards lodestar-z integration - combining index2pubkey and pubkey2index allows them to stay synchronized by construction --- .../src/api/impl/beacon/state/index.ts | 16 ++-- .../src/api/impl/beacon/state/utils.ts | 11 ++- .../src/api/impl/validator/index.ts | 7 +- .../historicalState/getHistoricalState.ts | 21 +++-- .../archiveStore/historicalState/worker.ts | 6 +- .../src/chain/bls/multithread/index.ts | 14 ++-- .../src/chain/bls/multithread/jobItem.ts | 6 +- .../beacon-node/src/chain/bls/singleThread.ts | 10 +-- packages/beacon-node/src/chain/bls/utils.ts | 13 +-- packages/beacon-node/src/chain/chain.ts | 25 +++--- packages/beacon-node/src/chain/interface.ts | 11 +-- .../src/chain/validation/attestation.ts | 5 +- .../src/chain/validation/attesterSlashing.ts | 2 +- .../validation/payloadAttestationMessage.ts | 10 ++- .../src/chain/validation/proposerSlashing.ts | 2 +- .../syncCommitteeContributionAndProof.ts | 2 +- packages/beacon-node/src/node/nodejs.ts | 12 +-- .../test/e2e/chain/bls/multithread.test.ts | 8 +- packages/beacon-node/test/fixtures/capella.ts | 8 +- .../test/mocks/mockedBeaconChain.ts | 5 +- .../perf/api/impl/validator/attester.test.ts | 2 +- .../test/perf/chain/opPools/opPool.test.ts | 14 ++-- .../produceBlock/produceBlockBody.test.ts | 5 +- .../perf/chain/verifyImportBlocks.test.ts | 3 +- .../test/sim/electra-interop.test.ts | 4 +- .../test/spec/presets/fork_choice.test.ts | 14 ++-- .../unit/api/impl/beacon/state/utils.test.ts | 18 ++-- .../test/unit/chain/bls/bls.test.ts | 13 +-- ...idateGossipAttestationsSameAttData.test.ts | 14 ++-- .../test/utils/networkWithMockDb.ts | 14 ++-- .../beacon-node/test/utils/node/beacon.ts | 14 ++-- packages/beacon-node/test/utils/state.ts | 14 ++-- .../test/utils/validationData/attestation.ts | 6 +- packages/cli/src/cmds/beacon/handler.ts | 14 ++-- .../src/block/isValidIndexedAttestation.ts | 10 +-- .../block/isValidIndexedPayloadAttestation.ts | 2 +- .../src/block/processAttestationPhase0.ts | 2 +- .../src/block/processAttestationsAltair.ts | 2 +- .../src/block/processAttesterSlashing.ts | 8 +- .../block/processExecutionPayloadEnvelope.ts | 6 +- .../src/block/processProposerSlashing.ts | 8 +- .../src/block/processRandao.ts | 2 +- .../src/block/processSyncCommittee.ts | 2 +- .../src/block/processVoluntaryExit.ts | 2 +- .../src/block/processWithdrawalRequest.ts | 4 +- .../state-transition/src/cache/epochCache.ts | 48 ++++------- .../state-transition/src/cache/pubkeyCache.ts | 83 ++++++++++++++----- .../state-transition/src/cache/stateCache.ts | 10 +-- .../src/cache/syncCommitteeCache.ts | 9 +- packages/state-transition/src/index.ts | 2 +- .../src/rewards/attestationsRewards.ts | 10 +-- .../src/rewards/syncCommitteeRewards.ts | 11 +-- .../src/signatureSets/proposer.ts | 6 +- .../src/signatureSets/randao.ts | 10 +-- .../src/signatureSets/voluntaryExits.ts | 6 +- .../state-transition/src/stateTransition.ts | 2 +- .../src/stateView/beaconStateView.ts | 22 +++-- .../src/util/signatureSets.ts | 40 +++++---- packages/state-transition/test/perf/util.ts | 43 +++------- .../perf/util/loadState/loadState.test.ts | 13 +-- .../block/isValidIndexedAttestation.test.ts | 2 +- .../test/unit/cachedBeaconState.test.ts | 12 ++- .../test/unit/upgradeState.test.ts | 8 +- .../test/unit/util/cachedBeaconState.test.ts | 6 +- packages/state-transition/test/utils/state.ts | 8 +- 65 files changed, 358 insertions(+), 384 deletions(-) diff --git a/packages/beacon-node/src/api/impl/beacon/state/index.ts b/packages/beacon-node/src/api/impl/beacon/state/index.ts index ddbf9e73907d..e8b3b92a5155 100644 --- a/packages/beacon-node/src/api/impl/beacon/state/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/state/index.ts @@ -95,14 +95,14 @@ export function getBeaconStateApi({ const {state, executionOptimistic, finalized} = await getState(stateId); const currentEpoch = getCurrentEpoch(state); const {validators, balances} = state; // Get the validators sub tree once for all the loop - const {pubkey2index} = chain; + const {pubkeyCache} = chain; const validatorResponses: routes.beacon.ValidatorResponse[] = []; if (validatorIds.length) { assertUniqueItems(validatorIds, "Duplicate validator IDs provided"); for (const id of validatorIds) { - const resp = getStateValidatorIndex(id, state, pubkey2index); + const resp = getStateValidatorIndex(id, state, pubkeyCache); if (resp.valid) { const validatorIndex = resp.validatorIndex; const validator = validators.getReadonly(validatorIndex); @@ -127,7 +127,7 @@ export function getBeaconStateApi({ if (statuses.length) { assertUniqueItems(statuses, "Duplicate statuses provided"); - const validatorsByStatus = filterStateValidatorsByStatus(statuses, state, pubkey2index, currentEpoch); + const validatorsByStatus = filterStateValidatorsByStatus(statuses, state, pubkeyCache, currentEpoch); return { data: validatorsByStatus, meta: {executionOptimistic, finalized}, @@ -154,7 +154,7 @@ export function getBeaconStateApi({ async postStateValidatorIdentities({stateId, validatorIds = []}) { const {state, executionOptimistic, finalized} = await getState(stateId); - const {pubkey2index} = chain; + const {pubkeyCache} = chain; let validatorIdentities: routes.beacon.ValidatorIdentities; @@ -163,7 +163,7 @@ export function getBeaconStateApi({ validatorIdentities = []; for (const id of validatorIds) { - const resp = getStateValidatorIndex(id, state, pubkey2index); + const resp = getStateValidatorIndex(id, state, pubkeyCache); if (resp.valid) { const index = resp.validatorIndex; const {pubkey, activationEpoch} = state.validators.getReadonly(index); @@ -187,9 +187,9 @@ export function getBeaconStateApi({ async getStateValidator({stateId, validatorId}) { const {state, executionOptimistic, finalized} = await getState(stateId); - const {pubkey2index} = chain; + const {pubkeyCache} = chain; - const resp = getStateValidatorIndex(validatorId, state, pubkey2index); + const resp = getStateValidatorIndex(validatorId, state, pubkeyCache); if (!resp.valid) { throw new ApiError(resp.code, resp.reason); } @@ -214,7 +214,7 @@ export function getBeaconStateApi({ const balances: routes.beacon.ValidatorBalance[] = []; for (const id of validatorIds) { - const resp = getStateValidatorIndex(id, state, chain.pubkey2index); + const resp = getStateValidatorIndex(id, state, chain.pubkeyCache); if (resp.valid) { balances.push({ diff --git a/packages/beacon-node/src/api/impl/beacon/state/utils.ts b/packages/beacon-node/src/api/impl/beacon/state/utils.ts index 2d63c28e569d..c231f5da8964 100644 --- a/packages/beacon-node/src/api/impl/beacon/state/utils.ts +++ b/packages/beacon-node/src/api/impl/beacon/state/utils.ts @@ -1,8 +1,7 @@ -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {routes} from "@lodestar/api"; import {CheckpointWithHex, IForkChoice} from "@lodestar/fork-choice"; import {GENESIS_SLOT} from "@lodestar/params"; -import {BeaconStateAllForks, CachedBeaconStateAllForks} from "@lodestar/state-transition"; +import {BeaconStateAllForks, CachedBeaconStateAllForks, PubkeyCache} from "@lodestar/state-transition"; import { BLSPubkey, Epoch, @@ -91,7 +90,7 @@ export function toValidatorResponse( export function filterStateValidatorsByStatus( statuses: string[], state: BeaconStateAllForks, - pubkey2index: PubkeyIndexMap, + pubkeyCache: PubkeyCache, currentEpoch: Epoch ): routes.beacon.ValidatorResponse[] { const responses: routes.beacon.ValidatorResponse[] = []; @@ -102,7 +101,7 @@ export function filterStateValidatorsByStatus( const validatorStatus = getValidatorStatus(validator, currentEpoch); const generalStatus = mapToGeneralStatus(validatorStatus); - const resp = getStateValidatorIndex(validator.pubkey, state, pubkey2index); + const resp = getStateValidatorIndex(validator.pubkey, state, pubkeyCache); if (resp.valid && (statusSet.has(validatorStatus) || statusSet.has(generalStatus))) { responses.push( toValidatorResponse(resp.validatorIndex, validator, state.balances.get(resp.validatorIndex), currentEpoch) @@ -119,7 +118,7 @@ type StateValidatorIndexResponse = export function getStateValidatorIndex( id: routes.beacon.ValidatorId | BLSPubkey, state: BeaconStateAllForks, - pubkey2index: PubkeyIndexMap + pubkeyCache: PubkeyCache ): StateValidatorIndexResponse { if (typeof id === "string") { // mutate `id` and fallthrough to below @@ -147,7 +146,7 @@ export function getStateValidatorIndex( } // typeof id === Uint8Array - const validatorIndex = pubkey2index.get(id); + const validatorIndex = pubkeyCache.getIndex(id); if (validatorIndex === null) { return {valid: false, code: 404, reason: "Validator pubkey not found in state"}; } diff --git a/packages/beacon-node/src/api/impl/validator/index.ts b/packages/beacon-node/src/api/impl/validator/index.ts index c2d759c73d78..023ca07fd46e 100644 --- a/packages/beacon-node/src/api/impl/validator/index.ts +++ b/packages/beacon-node/src/api/impl/validator/index.ts @@ -1,4 +1,3 @@ -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {routes} from "@lodestar/api"; import {ApplicationMethods} from "@lodestar/api/server"; import {ExecutionStatus, ProtoBlock} from "@lodestar/fork-choice"; @@ -27,6 +26,7 @@ import { computeStartSlotAtEpoch, computeTimeAtSlot, createCachedBeaconState, + createPubkeyCache, getBlockRootAtSlot, getCurrentSlot, loadState, @@ -1123,8 +1123,7 @@ export function getValidatorApi( { config: chain.config, // Not required to compute proposers - pubkey2index: new PubkeyIndexMap(), - index2pubkey: [], + pubkeyCache: createPubkeyCache(), }, {skipSyncPubkeys: true, skipSyncCommitteeCache: true} ); @@ -1585,7 +1584,7 @@ export function getValidatorApi( const filteredRegistrations = registrations.filter((registration) => { const {pubkey} = registration.message; - const validatorIndex = chain.pubkey2index.get(pubkey); + const validatorIndex = chain.pubkeyCache.getIndex(pubkey); if (validatorIndex === null) return false; const validator = headState.validators.getReadonly(validatorIndex); diff --git a/packages/beacon-node/src/chain/archiveStore/historicalState/getHistoricalState.ts b/packages/beacon-node/src/chain/archiveStore/historicalState/getHistoricalState.ts index e6381575e2e0..c110e9d78d5a 100644 --- a/packages/beacon-node/src/chain/archiveStore/historicalState/getHistoricalState.ts +++ b/packages/beacon-node/src/chain/archiveStore/historicalState/getHistoricalState.ts @@ -1,10 +1,10 @@ -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {BeaconConfig} from "@lodestar/config"; import { BeaconStateAllForks, CachedBeaconStateAllForks, DataAvailabilityStatus, ExecutionPayloadStatus, + PubkeyCache, createCachedBeaconState, stateTransition, } from "@lodestar/state-transition"; @@ -15,16 +15,16 @@ import {HistoricalStateRegenMetrics} from "./metrics.js"; import {RegenErrorType} from "./types.js"; /** - * Populate a PubkeyIndexMap with any new entries based on a BeaconState + * Populate a PubkeyCache with any new entries based on a BeaconState */ -export function syncPubkeyCache(state: BeaconStateAllForks, pubkey2index: PubkeyIndexMap): void { +export function syncPubkeyCache(state: BeaconStateAllForks, pubkeyCache: PubkeyCache): void { // Get the validators sub tree once for all the loop const validators = state.validators; const newCount = state.validators.length; - for (let i = pubkey2index.size; i < newCount; i++) { + for (let i = pubkeyCache.size; i < newCount; i++) { const pubkey = validators.getReadonly(i).pubkey; - pubkey2index.set(pubkey, i); + pubkeyCache.set(i, pubkey); } } @@ -35,7 +35,7 @@ export async function getNearestState( slot: number, config: BeaconConfig, db: IBeaconDb, - pubkey2index: PubkeyIndexMap + pubkeyCache: PubkeyCache ): Promise { const stateBytesArr = await db.stateArchive.binaries({limit: 1, lte: slot, reverse: true}); if (!stateBytesArr.length) { @@ -44,14 +44,13 @@ export async function getNearestState( const stateBytes = stateBytesArr[0]; const state = getStateTypeFromBytes(config, stateBytes).deserializeToViewDU(stateBytes); - syncPubkeyCache(state, pubkey2index); + syncPubkeyCache(state, pubkeyCache); return createCachedBeaconState( state, { config, - pubkey2index, - index2pubkey: [], + pubkeyCache, }, { skipSyncPubkeys: true, @@ -66,13 +65,13 @@ export async function getHistoricalState( slot: number, config: BeaconConfig, db: IBeaconDb, - pubkey2index: PubkeyIndexMap, + pubkeyCache: PubkeyCache, metrics?: HistoricalStateRegenMetrics ): Promise { const regenTimer = metrics?.regenTime.startTimer(); const loadStateTimer = metrics?.loadStateTime.startTimer(); - let state = await getNearestState(slot, config, db, pubkey2index).catch((e) => { + let state = await getNearestState(slot, config, db, pubkeyCache).catch((e) => { metrics?.regenErrorCount.inc({reason: RegenErrorType.loadState}); throw e; }); diff --git a/packages/beacon-node/src/chain/archiveStore/historicalState/worker.ts b/packages/beacon-node/src/chain/archiveStore/historicalState/worker.ts index 2b8ede8b8e81..62925444b2d8 100644 --- a/packages/beacon-node/src/chain/archiveStore/historicalState/worker.ts +++ b/packages/beacon-node/src/chain/archiveStore/historicalState/worker.ts @@ -1,9 +1,9 @@ import worker from "node:worker_threads"; -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {Transfer, expose} from "@chainsafe/threads/worker"; import {chainConfigFromJson, createBeaconConfig} from "@lodestar/config"; import {LevelDbController} from "@lodestar/db/controller/level"; import {getNodeLogger} from "@lodestar/logger/node"; +import {createPubkeyCache} from "@lodestar/state-transition"; import {BeaconDb} from "../../../db/index.js"; import {RegistryMetricCreator, collectNodeJSMetrics} from "../../../metrics/index.js"; import {JobFnQueue} from "../../../util/queue/fnQueue.js"; @@ -52,7 +52,7 @@ const queue = new JobFnQueue( queueMetrics ); -const pubkey2index = new PubkeyIndexMap(); +const pubkeyCache = createPubkeyCache(); const api: HistoricalStateWorkerApi = { async close() { @@ -65,7 +65,7 @@ const api: HistoricalStateWorkerApi = { historicalStateRegenMetrics?.regenRequestCount.inc(); const stateBytes = await queue.push(() => - getHistoricalState(slot, config, db, pubkey2index, historicalStateRegenMetrics) + getHistoricalState(slot, config, db, pubkeyCache, historicalStateRegenMetrics) ); const result = Transfer(stateBytes, [stateBytes.buffer]) as unknown as Uint8Array; diff --git a/packages/beacon-node/src/chain/bls/multithread/index.ts b/packages/beacon-node/src/chain/bls/multithread/index.ts index a3ca9ca803b6..d5b95609afe2 100644 --- a/packages/beacon-node/src/chain/bls/multithread/index.ts +++ b/packages/beacon-node/src/chain/bls/multithread/index.ts @@ -7,7 +7,7 @@ import {Worker, spawn} from "@chainsafe/threads"; self = undefined; import {PublicKey} from "@chainsafe/blst"; -import {ISignatureSet, Index2PubkeyCache} from "@lodestar/state-transition"; +import {ISignatureSet, PubkeyCache} from "@lodestar/state-transition"; import {Logger} from "@lodestar/utils"; import {Metrics} from "../../../metrics/index.js"; import {LinkedList} from "../../../util/array.js"; @@ -34,7 +34,7 @@ const workerDir = process.env.NODE_ENV === "test" ? "../../../../lib/chain/bls/m export type BlsMultiThreadWorkerPoolModules = { logger: Logger; metrics: Metrics | null; - index2pubkey: Index2PubkeyCache; + pubkeyCache: PubkeyCache; }; export type BlsMultiThreadWorkerPoolOptions = { @@ -114,7 +114,7 @@ type WorkerDescriptor = { export class BlsMultiThreadWorkerPool implements IBlsVerifier { private readonly logger: Logger; private readonly metrics: Metrics | null; - private readonly index2pubkey: Index2PubkeyCache; + private readonly pubkeyCache: PubkeyCache; private readonly workers: WorkerDescriptor[]; private readonly jobs = new LinkedList(); @@ -130,10 +130,10 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier { private workersBusy = 0; constructor(options: BlsMultiThreadWorkerPoolOptions, modules: BlsMultiThreadWorkerPoolModules) { - const {logger, metrics, index2pubkey} = modules; + const {logger, metrics, pubkeyCache} = modules; this.logger = logger; this.metrics = metrics; - this.index2pubkey = index2pubkey; + this.pubkeyCache = pubkeyCache; this.blsVerifyAllMultiThread = options.blsVerifyAllMultiThread ?? false; // Use compressed for herumi for now. @@ -173,7 +173,7 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier { try { return verifySignatureSetsMaybeBatch( sets.map((set) => ({ - publicKey: getAggregatedPubkey(set, this.index2pubkey), + publicKey: getAggregatedPubkey(set, this.pubkeyCache), message: set.signingRoot.valueOf(), signature: set.signature, })) @@ -398,7 +398,7 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier { try { // Note: This can throw, must be handled per-job. // Pubkey and signature aggregation is defered here - workReq = await jobItemWorkReq(job, this.index2pubkey, this.metrics); + workReq = await jobItemWorkReq(job, this.pubkeyCache, this.metrics); } catch (e) { this.metrics?.blsThreadPool.errorAggregateSignatureSetsCount.inc({type: job.type}); diff --git a/packages/beacon-node/src/chain/bls/multithread/jobItem.ts b/packages/beacon-node/src/chain/bls/multithread/jobItem.ts index a5d28a524490..1f4b6a7e2f84 100644 --- a/packages/beacon-node/src/chain/bls/multithread/jobItem.ts +++ b/packages/beacon-node/src/chain/bls/multithread/jobItem.ts @@ -1,5 +1,5 @@ import {PublicKey, asyncAggregateWithRandomness} from "@chainsafe/blst"; -import {ISignatureSet, Index2PubkeyCache, SignatureSetType} from "@lodestar/state-transition"; +import {ISignatureSet, PubkeyCache, SignatureSetType} from "@lodestar/state-transition"; import {Metrics} from "../../../metrics/metrics.js"; import {LinkedList} from "../../../util/array.js"; import {VerifySignatureOpts} from "../interface.js"; @@ -50,7 +50,7 @@ export function jobItemSigSets(job: JobQueueItem): number { */ export async function jobItemWorkReq( job: JobQueueItem, - index2pubkey: Index2PubkeyCache, + pubkeyCache: PubkeyCache, metrics: Metrics | null ): Promise { switch (job.type) { @@ -59,7 +59,7 @@ export async function jobItemWorkReq( opts: job.opts, sets: job.sets.map((set) => ({ // this can throw, handled in the consumer code - publicKey: getAggregatedPubkey(set, index2pubkey, metrics).toBytes(), + publicKey: getAggregatedPubkey(set, pubkeyCache, metrics).toBytes(), signature: set.signature, message: set.signingRoot, })), diff --git a/packages/beacon-node/src/chain/bls/singleThread.ts b/packages/beacon-node/src/chain/bls/singleThread.ts index 673035e204c7..1e2f29f791ab 100644 --- a/packages/beacon-node/src/chain/bls/singleThread.ts +++ b/packages/beacon-node/src/chain/bls/singleThread.ts @@ -1,5 +1,5 @@ import {PublicKey, Signature, aggregatePublicKeys, aggregateSignatures, verify} from "@chainsafe/blst"; -import {ISignatureSet, Index2PubkeyCache} from "@lodestar/state-transition"; +import {ISignatureSet, PubkeyCache} from "@lodestar/state-transition"; import {Metrics} from "../../metrics/index.js"; import {IBlsVerifier} from "./interface.js"; import {verifySignatureSetsMaybeBatch} from "./maybeBatch.js"; @@ -7,18 +7,18 @@ import {getAggregatedPubkey, getAggregatedPubkeysCount} from "./utils.js"; export class BlsSingleThreadVerifier implements IBlsVerifier { private readonly metrics: Metrics | null; - private readonly index2pubkey: Index2PubkeyCache; + private readonly pubkeyCache: PubkeyCache; - constructor({metrics = null, index2pubkey}: {metrics: Metrics | null; index2pubkey: Index2PubkeyCache}) { + constructor({metrics = null, pubkeyCache}: {metrics: Metrics | null; pubkeyCache: PubkeyCache}) { this.metrics = metrics; - this.index2pubkey = index2pubkey; + this.pubkeyCache = pubkeyCache; } async verifySignatureSets(sets: ISignatureSet[]): Promise { this.metrics?.bls.aggregatedPubkeys.inc(getAggregatedPubkeysCount(sets)); const setsAggregated = sets.map((set) => ({ - publicKey: getAggregatedPubkey(set, this.index2pubkey, this.metrics), + publicKey: getAggregatedPubkey(set, this.pubkeyCache, this.metrics), message: set.signingRoot, signature: set.signature, })); diff --git a/packages/beacon-node/src/chain/bls/utils.ts b/packages/beacon-node/src/chain/bls/utils.ts index e01ab788026f..d73528dad4ef 100644 --- a/packages/beacon-node/src/chain/bls/utils.ts +++ b/packages/beacon-node/src/chain/bls/utils.ts @@ -1,22 +1,25 @@ import {PublicKey, aggregatePublicKeys} from "@chainsafe/blst"; -import {ISignatureSet, Index2PubkeyCache, SignatureSetType} from "@lodestar/state-transition"; +import {ISignatureSet, PubkeyCache, SignatureSetType} from "@lodestar/state-transition"; import {Metrics} from "../../metrics/metrics.js"; export function getAggregatedPubkey( signatureSet: ISignatureSet, - index2pubkey: Index2PubkeyCache, + pubkeyCache: PubkeyCache, metrics: Metrics | null = null ): PublicKey { switch (signatureSet.type) { case SignatureSetType.single: return signatureSet.pubkey; - case SignatureSetType.indexed: - return index2pubkey[signatureSet.index]; + case SignatureSetType.indexed: { + return pubkeyCache.getOrThrow(signatureSet.index); + } case SignatureSetType.aggregate: { const timer = metrics?.blsThreadPool.pubkeysAggregationMainThreadDuration.startTimer(); - const pubkeys = signatureSet.indices.map((i) => index2pubkey[i]); + const pubkeys = signatureSet.indices.map((i) => { + return pubkeyCache.getOrThrow(i); + }); const aggregated = aggregatePublicKeys(pubkeys); timer?.(); return aggregated; diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index d579bd34c58e..a0a29bcc0c0e 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -1,6 +1,5 @@ import path from "node:path"; import {PrivateKey} from "@libp2p/interface"; -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {CompositeTypeAny, TreeView, Type} from "@chainsafe/ssz"; import {BeaconConfig} from "@lodestar/config"; import {CheckpointWithHex, IForkChoice, ProtoBlock, UpdateHeadOpt} from "@lodestar/fork-choice"; @@ -21,7 +20,7 @@ import { CachedBeaconStateGloas, EffectiveBalanceIncrements, EpochShuffling, - Index2PubkeyCache, + PubkeyCache, computeAnchorCheckpoint, computeAttestationsRewards, computeBlockRewards, @@ -192,8 +191,7 @@ export class BeaconChain implements IBeaconChain { readonly seenBlockAttesters = new SeenBlockAttesters(); // Global state caches - readonly pubkey2index: PubkeyIndexMap; - readonly index2pubkey: Index2PubkeyCache; + readonly pubkeyCache: PubkeyCache; readonly beaconProposerCache: BeaconProposerCache; readonly checkpointBalancesCache: CheckpointBalancesCache; @@ -239,8 +237,7 @@ export class BeaconChain implements IBeaconChain { { privateKey, config, - pubkey2index, - index2pubkey, + pubkeyCache, db, dbName, dataDir, @@ -256,8 +253,7 @@ export class BeaconChain implements IBeaconChain { }: { privateKey: PrivateKey; config: BeaconConfig; - pubkey2index: PubkeyIndexMap; - index2pubkey: Index2PubkeyCache; + pubkeyCache: PubkeyCache; db: IBeaconDb; dbName: string; dataDir: string; @@ -289,8 +285,8 @@ export class BeaconChain implements IBeaconChain { const emitter = new ChainEventEmitter(); // by default, verify signatures on both main threads and worker threads const bls = opts.blsVerifyAllMainThread - ? new BlsSingleThreadVerifier({metrics, index2pubkey}) - : new BlsMultiThreadWorkerPool(opts, {logger, metrics, index2pubkey}); + ? new BlsSingleThreadVerifier({metrics, pubkeyCache}) + : new BlsMultiThreadWorkerPool(opts, {logger, metrics, pubkeyCache}); if (!clock) clock = new Clock({config, genesisTime: this.genesisTime, signal}); @@ -346,8 +342,7 @@ export class BeaconChain implements IBeaconChain { ]); // Global cache of validators pubkey/index mapping - this.pubkey2index = pubkey2index; - this.index2pubkey = index2pubkey; + this.pubkeyCache = pubkeyCache; const fileDataStore = opts.nHistoricalStatesFileDataStore ?? true; const blockStateCache = new FIFOBlockStateCache(this.opts, {metrics}); @@ -920,7 +915,7 @@ export class BeaconChain implements IBeaconChain { RegenCaller.produceBlock ); const proposerIndex = state.epochCtx.getBeaconProposer(slot); - const proposerPubKey = this.index2pubkey[proposerIndex].toBytes(); + const proposerPubKey = this.pubkeyCache.getOrThrow(proposerIndex).toBytes(); const {body, produceResult, executionPayloadValue, shouldOverrideBuilder} = await produceBlockBody.call( this, @@ -1536,7 +1531,7 @@ export class BeaconChain implements IBeaconChain { throw Error(`State is not in cache for slot ${slot}`); } - const rewards = await computeAttestationsRewards(this.config, this.pubkey2index, cachedState, validatorIds); + const rewards = await computeAttestationsRewards(this.config, this.pubkeyCache, cachedState, validatorIds); return {rewards, executionOptimistic, finalized}; } @@ -1553,6 +1548,6 @@ export class BeaconChain implements IBeaconChain { preState = processSlots(preState, block.slot); // Dial preState's slot to block.slot - return computeSyncCommitteeRewards(this.config, this.index2pubkey, block, preState, validatorIds); + return computeSyncCommitteeRewards(this.config, this.pubkeyCache, block, preState, validatorIds); } } diff --git a/packages/beacon-node/src/chain/interface.ts b/packages/beacon-node/src/chain/interface.ts index 7a480d9e2610..cc300e9a5dd7 100644 --- a/packages/beacon-node/src/chain/interface.ts +++ b/packages/beacon-node/src/chain/interface.ts @@ -1,13 +1,7 @@ -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {CompositeTypeAny, TreeView, Type} from "@chainsafe/ssz"; import {BeaconConfig} from "@lodestar/config"; import {CheckpointWithHex, IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; -import { - BeaconStateAllForks, - CachedBeaconStateAllForks, - EpochShuffling, - Index2PubkeyCache, -} from "@lodestar/state-transition"; +import {BeaconStateAllForks, CachedBeaconStateAllForks, EpochShuffling, PubkeyCache} from "@lodestar/state-transition"; import { BeaconBlock, BlindedBeaconBlock, @@ -117,8 +111,7 @@ export interface IBeaconChain { readonly regen: IStateRegenerator; readonly lightClientServer?: LightClientServer; readonly reprocessController: ReprocessController; - readonly pubkey2index: PubkeyIndexMap; - readonly index2pubkey: Index2PubkeyCache; + readonly pubkeyCache: PubkeyCache; readonly archiveStore: IArchiveStore; // Ops pool diff --git a/packages/beacon-node/src/chain/validation/attestation.ts b/packages/beacon-node/src/chain/validation/attestation.ts index fd49a8fb494d..9a2a62a11432 100644 --- a/packages/beacon-node/src/chain/validation/attestation.ts +++ b/packages/beacon-node/src/chain/validation/attestation.ts @@ -143,7 +143,10 @@ export async function validateGossipAttestationsSameAttData( if (batchableBls) { // all signature sets should have same signing root since we filtered in network processor signatureValids = await chain.bls.verifySignatureSetsSameMessage( - signatureSets.map((set) => ({publicKey: chain.index2pubkey[set.index], signature: set.signature})), + signatureSets.map((set) => { + const publicKey = chain.pubkeyCache.getOrThrow(set.index); + return {publicKey, signature: set.signature}; + }), signatureSets[0].signingRoot ); } else { diff --git a/packages/beacon-node/src/chain/validation/attesterSlashing.ts b/packages/beacon-node/src/chain/validation/attesterSlashing.ts index df6774f26b22..3ddfac6166de 100644 --- a/packages/beacon-node/src/chain/validation/attesterSlashing.ts +++ b/packages/beacon-node/src/chain/validation/attesterSlashing.ts @@ -45,7 +45,7 @@ export async function validateAttesterSlashing( // verifySignature = false, verified in batch below assertValidAttesterSlashing( chain.config, - chain.index2pubkey, + chain.pubkeyCache, state.slot, state.validators.length, attesterSlashing, diff --git a/packages/beacon-node/src/chain/validation/payloadAttestationMessage.ts b/packages/beacon-node/src/chain/validation/payloadAttestationMessage.ts index b8ac7fb98f1a..09f81c2d45d7 100644 --- a/packages/beacon-node/src/chain/validation/payloadAttestationMessage.ts +++ b/packages/beacon-node/src/chain/validation/payloadAttestationMessage.ts @@ -87,8 +87,16 @@ async function validatePayloadAttestationMessage( } // [REJECT] `payload_attestation_message.signature` is valid with respect to the validator's public key. + const validatorPubkey = chain.pubkeyCache.get(validatorIndex); + if (!validatorPubkey) { + throw new PayloadAttestationError(GossipAction.REJECT, { + code: PayloadAttestationErrorCode.INVALID_ATTESTER, + attesterIndex: validatorIndex, + }); + } + const signatureSet = createSingleSignatureSetFromComponents( - chain.index2pubkey[validatorIndex], + validatorPubkey, getPayloadAttestationDataSigningRoot(chain.config, data), payloadAttestationMessage.signature ); diff --git a/packages/beacon-node/src/chain/validation/proposerSlashing.ts b/packages/beacon-node/src/chain/validation/proposerSlashing.ts index 350f2e701668..bf8a2f02359e 100644 --- a/packages/beacon-node/src/chain/validation/proposerSlashing.ts +++ b/packages/beacon-node/src/chain/validation/proposerSlashing.ts @@ -37,7 +37,7 @@ async function validateProposerSlashing( try { const proposer = state.validators.getReadonly(proposerSlashing.signedHeader1.message.proposerIndex); // verifySignature = false, verified in batch below - assertValidProposerSlashing(chain.config, chain.index2pubkey, state.slot, proposerSlashing, proposer, false); + assertValidProposerSlashing(chain.config, chain.pubkeyCache, state.slot, proposerSlashing, proposer, false); } catch (e) { throw new ProposerSlashingError(GossipAction.REJECT, { code: ProposerSlashingErrorCode.INVALID, diff --git a/packages/beacon-node/src/chain/validation/syncCommitteeContributionAndProof.ts b/packages/beacon-node/src/chain/validation/syncCommitteeContributionAndProof.ts index 7d94b6bf8a53..f70351269c41 100644 --- a/packages/beacon-node/src/chain/validation/syncCommitteeContributionAndProof.ts +++ b/packages/beacon-node/src/chain/validation/syncCommitteeContributionAndProof.ts @@ -106,7 +106,7 @@ export async function validateSyncCommitteeGossipContributionAndProof( /** * Retrieve pubkeys in contribution aggregate using epochCtx: * - currSyncCommitteeIndexes cache - * - index2pubkey cache + * - pubkeyCache */ function getContributionIndices( state: CachedBeaconStateAltair, diff --git a/packages/beacon-node/src/node/nodejs.ts b/packages/beacon-node/src/node/nodejs.ts index 18e7961c7604..0a3c321d16c7 100644 --- a/packages/beacon-node/src/node/nodejs.ts +++ b/packages/beacon-node/src/node/nodejs.ts @@ -2,12 +2,11 @@ import {setMaxListeners} from "node:events"; import {PrivateKey} from "@libp2p/interface"; import {Registry} from "prom-client"; import {hasher} from "@chainsafe/persistent-merkle-tree"; -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {BeaconApiMethods} from "@lodestar/api/beacon/server"; import {BeaconConfig} from "@lodestar/config"; import type {LoggerNode} from "@lodestar/logger/node"; import {ZERO_HASH_HEX} from "@lodestar/params"; -import {CachedBeaconStateAllForks, Index2PubkeyCache, isExecutionCachedStateType} from "@lodestar/state-transition"; +import {CachedBeaconStateAllForks, PubkeyCache, isExecutionCachedStateType} from "@lodestar/state-transition"; import {phase0} from "@lodestar/types"; import {sleep, toRootHex} from "@lodestar/utils"; import {ProcessShutdownCallback} from "@lodestar/validator"; @@ -47,8 +46,7 @@ export type BeaconNodeModules = { export type BeaconNodeInitModules = { opts: IBeaconNodeOptions; config: BeaconConfig; - pubkey2index: PubkeyIndexMap; - index2pubkey: Index2PubkeyCache; + pubkeyCache: PubkeyCache; db: IBeaconDb; logger: LoggerNode; processShutdownCallback: ProcessShutdownCallback; @@ -150,8 +148,7 @@ export class BeaconNode { static async init({ opts, config, - pubkey2index, - index2pubkey, + pubkeyCache, db, logger, processShutdownCallback, @@ -240,8 +237,7 @@ export class BeaconNode { privateKey, config, clock, - pubkey2index, - index2pubkey, + pubkeyCache, dataDir, db, dbName: opts.db.name, diff --git a/packages/beacon-node/test/e2e/chain/bls/multithread.test.ts b/packages/beacon-node/test/e2e/chain/bls/multithread.test.ts index 12e5130c0208..0428389a3df1 100644 --- a/packages/beacon-node/test/e2e/chain/bls/multithread.test.ts +++ b/packages/beacon-node/test/e2e/chain/bls/multithread.test.ts @@ -1,6 +1,6 @@ import {afterEach, beforeAll, beforeEach, describe, expect, it} from "vitest"; import {PublicKey, SecretKey} from "@chainsafe/blst"; -import {ISignatureSet, SignatureSetType} from "@lodestar/state-transition"; +import {ISignatureSet, SignatureSetType, createPubkeyCache} from "@lodestar/state-transition"; import {VerifySignatureOpts} from "../../../../src/chain/bls/interface.js"; import {BlsMultiThreadWorkerPool} from "../../../../src/chain/bls/multithread/index.js"; import {testLogger} from "../../../utils/logger.js"; @@ -13,7 +13,7 @@ describe("chain / bls / multithread queue", () => { const sets: ISignatureSet[] = []; const sameMessageSets: {publicKey: PublicKey; signature: Uint8Array}[] = []; const sameMessage = Buffer.alloc(32, 100); - const index2pubkey: PublicKey[] = []; + const pubkeyCache = createPubkeyCache(); beforeAll(() => { for (let i = 0; i < 3; i++) { @@ -31,7 +31,7 @@ describe("chain / bls / multithread queue", () => { publicKey: pk, signature: sk.sign(sameMessage).toBytes(), }); - index2pubkey.push(pk); + pubkeyCache.set(pubkeyCache.size, pk.toBytes()); } }); @@ -49,7 +49,7 @@ describe("chain / bls / multithread queue", () => { }); async function initializePool(): Promise { - const pool = new BlsMultiThreadWorkerPool({}, {logger, metrics: null, index2pubkey}); + const pool = new BlsMultiThreadWorkerPool({}, {logger, metrics: null, pubkeyCache}); // await terminating all workers afterEachCallbacks.push(() => pool.close()); // Wait until initialized diff --git a/packages/beacon-node/test/fixtures/capella.ts b/packages/beacon-node/test/fixtures/capella.ts index 0fed040a9f39..484d9f9df4d0 100644 --- a/packages/beacon-node/test/fixtures/capella.ts +++ b/packages/beacon-node/test/fixtures/capella.ts @@ -1,17 +1,19 @@ -import {CachedBeaconStateAltair, Index2PubkeyCache} from "@lodestar/state-transition"; +import {CachedBeaconStateAltair, PubkeyCache} from "@lodestar/state-transition"; import {capella} from "@lodestar/types"; export function generateBlsToExecutionChanges( - index2pubkey: Index2PubkeyCache, + pubkeyCache: PubkeyCache, state: CachedBeaconStateAltair, count: number ): capella.SignedBLSToExecutionChange[] { const result: capella.SignedBLSToExecutionChange[] = []; for (const validatorIndex of state.epochCtx.proposers) { + const pubkey = pubkeyCache.getOrThrow(validatorIndex); + result.push({ message: { - fromBlsPubkey: index2pubkey[validatorIndex].toBytes(), + fromBlsPubkey: pubkey.toBytes(), toExecutionAddress: Buffer.alloc(20), validatorIndex, }, diff --git a/packages/beacon-node/test/mocks/mockedBeaconChain.ts b/packages/beacon-node/test/mocks/mockedBeaconChain.ts index 2907e6da4081..9bbe71c9fea1 100644 --- a/packages/beacon-node/test/mocks/mockedBeaconChain.ts +++ b/packages/beacon-node/test/mocks/mockedBeaconChain.ts @@ -1,8 +1,8 @@ import {Mock, Mocked, vi} from "vitest"; -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {BeaconConfig, ChainForkConfig} from "@lodestar/config"; import {config as defaultConfig} from "@lodestar/config/default"; import {EpochDifference, ForkChoice, ProtoBlock} from "@lodestar/fork-choice"; +import {createPubkeyCache} from "@lodestar/state-transition"; import {Logger} from "@lodestar/utils"; import {BeaconProposerCache} from "../../src/chain/beaconProposerCache.js"; import {BeaconChain} from "../../src/chain/chain.js"; @@ -141,8 +141,7 @@ vi.mock("../../src/chain/chain.js", async (importActual) => { // @ts-expect-error seenBlockInputCache: new SeenBlockInput(), shufflingCache: new ShufflingCache(), - pubkey2index: new PubkeyIndexMap(), - index2pubkey: [], + pubkeyCache: createPubkeyCache(), produceCommonBlockBody: vi.fn(), getProposerHead: vi.fn(), produceBlock: vi.fn(), diff --git a/packages/beacon-node/test/perf/api/impl/validator/attester.test.ts b/packages/beacon-node/test/perf/api/impl/validator/attester.test.ts index 2fbd607b309d..0366692dd32a 100644 --- a/packages/beacon-node/test/perf/api/impl/validator/attester.test.ts +++ b/packages/beacon-node/test/perf/api/impl/validator/attester.test.ts @@ -32,7 +32,7 @@ describe("api / impl / validator", () => { noThreshold: true, fn: () => { for (let i = 0; i < reqCount; i++) { - const pubkey = state.epochCtx.index2pubkey[i]; + const pubkey = state.epochCtx.pubkeyCache.getOrThrow(i); pubkey.toBytes(); } }, diff --git a/packages/beacon-node/test/perf/chain/opPools/opPool.test.ts b/packages/beacon-node/test/perf/chain/opPools/opPool.test.ts index b3f0bf44a3d5..ff96e430199d 100644 --- a/packages/beacon-node/test/perf/chain/opPools/opPool.test.ts +++ b/packages/beacon-node/test/perf/chain/opPools/opPool.test.ts @@ -8,7 +8,7 @@ import { MAX_PROPOSER_SLASHINGS, MAX_VOLUNTARY_EXITS, } from "@lodestar/params"; -import {CachedBeaconStateAltair, Index2PubkeyCache} from "@lodestar/state-transition"; +import {CachedBeaconStateAltair, PubkeyCache} from "@lodestar/state-transition"; import {ssz} from "@lodestar/types"; import {generatePerfTestCachedStateAltair} from "../../../../../state-transition/test/perf/util.js"; import {BlockType} from "../../../../src/chain/interface.js"; @@ -38,8 +38,8 @@ describe("opPool", () => { fillAttesterSlashing(pool, originalState, MAX_ATTESTER_SLASHINGS); fillProposerSlashing(pool, originalState, MAX_PROPOSER_SLASHINGS); fillVoluntaryExits(pool, originalState, MAX_VOLUNTARY_EXITS); - // TODO: feed index2pubkey separately instead of getting from originalState - fillBlsToExecutionChanges(originalState.epochCtx.index2pubkey, pool, originalState, MAX_BLS_TO_EXECUTION_CHANGES); + // TODO: feed pubkeyCache separately instead of getting from originalState + fillBlsToExecutionChanges(originalState.epochCtx.pubkeyCache, pool, originalState, MAX_BLS_TO_EXECUTION_CHANGES); return pool; }, @@ -57,8 +57,8 @@ describe("opPool", () => { fillAttesterSlashing(pool, originalState, maxItemsInPool); fillProposerSlashing(pool, originalState, maxItemsInPool); fillVoluntaryExits(pool, originalState, maxItemsInPool); - // TODO: feed index2pubkey separately instead of getting from originalState - fillBlsToExecutionChanges(originalState.epochCtx.index2pubkey, pool, originalState, maxItemsInPool); + // TODO: feed pubkeyCache separately instead of getting from originalState + fillBlsToExecutionChanges(originalState.epochCtx.pubkeyCache, pool, originalState, maxItemsInPool); return pool; }, @@ -105,12 +105,12 @@ function fillVoluntaryExits(pool: OpPool, state: CachedBeaconStateAltair, count: // This does not set the `withdrawalCredentials` for the validator // So it will be in the pool but not returned from `getSlashingsAndExits` function fillBlsToExecutionChanges( - index2pubkey: Index2PubkeyCache, + pubkeyCache: PubkeyCache, pool: OpPool, state: CachedBeaconStateAltair, count: number ): OpPool { - for (const blsToExecution of generateBlsToExecutionChanges(index2pubkey, state, count)) { + for (const blsToExecution of generateBlsToExecutionChanges(pubkeyCache, state, count)) { pool.insertBlsToExecutionChange(blsToExecution); } diff --git a/packages/beacon-node/test/perf/chain/produceBlock/produceBlockBody.test.ts b/packages/beacon-node/test/perf/chain/produceBlock/produceBlockBody.test.ts index 304190626a9c..77425276e216 100644 --- a/packages/beacon-node/test/perf/chain/produceBlock/produceBlockBody.test.ts +++ b/packages/beacon-node/test/perf/chain/produceBlock/produceBlockBody.test.ts @@ -38,8 +38,7 @@ describe("produceBlockBody", () => { { privateKey: await generateKeyPair("secp256k1"), config: state.config, - pubkey2index: state.epochCtx.pubkey2index, - index2pubkey: state.epochCtx.index2pubkey, + pubkeyCache: state.epochCtx.pubkeyCache, db, dataDir: ".", dbName: ".", @@ -68,7 +67,7 @@ describe("produceBlockBody", () => { beforeEach: async () => { const head = chain.forkChoice.getHead(); const proposerIndex = state.epochCtx.getBeaconProposer(state.slot); - const proposerPubKey = state.epochCtx.index2pubkey[proposerIndex].toBytes(); + const proposerPubKey = state.epochCtx.pubkeyCache.getOrThrow(proposerIndex).toBytes(); return {chain, state, head, proposerIndex, proposerPubKey}; }, diff --git a/packages/beacon-node/test/perf/chain/verifyImportBlocks.test.ts b/packages/beacon-node/test/perf/chain/verifyImportBlocks.test.ts index 3a75d58d9fd6..a4a062300a30 100644 --- a/packages/beacon-node/test/perf/chain/verifyImportBlocks.test.ts +++ b/packages/beacon-node/test/perf/chain/verifyImportBlocks.test.ts @@ -91,8 +91,7 @@ describe.skip("verify+import blocks - range sync perf test", () => { { privateKey: await generateKeyPair("secp256k1"), config: state.config, - pubkey2index: state.epochCtx.pubkey2index, - index2pubkey: state.epochCtx.index2pubkey, + pubkeyCache: state.epochCtx.pubkeyCache, db, dataDir: ".", dbName: ".", diff --git a/packages/beacon-node/test/sim/electra-interop.test.ts b/packages/beacon-node/test/sim/electra-interop.test.ts index 02272874415e..7497926cd1fd 100644 --- a/packages/beacon-node/test/sim/electra-interop.test.ts +++ b/packages/beacon-node/test/sim/electra-interop.test.ts @@ -364,7 +364,7 @@ describe("executionEngine / ExecutionEngineHttp", () => { if (headState.validators.length !== 33 || headState.balances.length !== 33) { throw Error("New validator is not reflected in the beacon state at slot 5"); } - if (epochCtx.index2pubkey.length !== 33 || epochCtx.pubkey2index.size !== 33) { + if (epochCtx.pubkeyCache.size !== 33) { throw Error("Pubkey cache is not updated"); } @@ -391,7 +391,7 @@ describe("executionEngine / ExecutionEngineHttp", () => { if (headState.validators.length !== 33 || headState.balances.length !== 33) { throw Error("New validator is not reflected in the beacon state."); } - if (epochCtx.index2pubkey.length !== 33 || epochCtx.pubkey2index.size !== 33) { + if (epochCtx.pubkeyCache.size !== 33) { throw Error("New validator is not in pubkey cache"); } diff --git a/packages/beacon-node/test/spec/presets/fork_choice.test.ts b/packages/beacon-node/test/spec/presets/fork_choice.test.ts index 621cd9c2bd50..fd7d59f55bf9 100644 --- a/packages/beacon-node/test/spec/presets/fork_choice.test.ts +++ b/packages/beacon-node/test/spec/presets/fork_choice.test.ts @@ -1,7 +1,6 @@ import path from "node:path"; import {generateKeyPair} from "@libp2p/crypto/keys"; import {expect} from "vitest"; -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {toHexString} from "@chainsafe/ssz"; import {createBeaconConfig} from "@lodestar/config"; import {CheckpointWithHex, ForkChoice} from "@lodestar/fork-choice"; @@ -17,8 +16,8 @@ import { import {InputType} from "@lodestar/spec-test-util"; import { BeaconStateAllForks, - Index2PubkeyCache, createCachedBeaconState, + createPubkeyCache, isExecutionStateType, signedBlockToSignedHeader, syncPubkeys, @@ -97,15 +96,13 @@ const forkChoiceTest = }); const beaconConfig = createBeaconConfig(config, anchorState.genesisValidatorsRoot); - const pubkey2index = new PubkeyIndexMap(); - const index2pubkey: Index2PubkeyCache = []; - syncPubkeys(anchorState.validators.getAllReadonlyValues(), pubkey2index, index2pubkey); + const pubkeyCache = createPubkeyCache(); + syncPubkeys(pubkeyCache, anchorState.validators.getAllReadonlyValues()); const cachedState = createCachedBeaconState( anchorState, { config: beaconConfig, - pubkey2index, - index2pubkey, + pubkeyCache, }, {skipSyncPubkeys: true} ); @@ -133,8 +130,7 @@ const forkChoiceTest = { privateKey: await generateKeyPair("secp256k1"), config: beaconConfig, - pubkey2index, - index2pubkey, + pubkeyCache, db: getMockedBeaconDb(), dataDir: ".", dbName: ",", diff --git a/packages/beacon-node/test/unit/api/impl/beacon/state/utils.test.ts b/packages/beacon-node/test/unit/api/impl/beacon/state/utils.test.ts index 2a00c2eea290..59d6647da9c1 100644 --- a/packages/beacon-node/test/unit/api/impl/beacon/state/utils.test.ts +++ b/packages/beacon-node/test/unit/api/impl/beacon/state/utils.test.ts @@ -6,50 +6,50 @@ import {generateCachedAltairState} from "../../../../../utils/state.js"; describe("beacon state api utils", () => { describe("getStateValidatorIndex", () => { const state = generateCachedAltairState(); - const pubkey2index = state.epochCtx.pubkey2index; + const pubkeyCache = state.epochCtx.pubkeyCache; it("should return valid: false on invalid input", () => { // "invalid validator id number" - expect(getStateValidatorIndex("foo", state, pubkey2index).valid).toBe(false); + expect(getStateValidatorIndex("foo", state, pubkeyCache).valid).toBe(false); // "invalid hex" - expect(getStateValidatorIndex("0xfoo", state, pubkey2index).valid).toBe(false); + expect(getStateValidatorIndex("0xfoo", state, pubkeyCache).valid).toBe(false); }); it("should return valid: false on validator indices / pubkeys not in the state", () => { // "validator id not in state" - expect(getStateValidatorIndex(String(state.validators.length), state, pubkey2index).valid).toBe(false); + expect(getStateValidatorIndex(String(state.validators.length), state, pubkeyCache).valid).toBe(false); // "validator pubkey not in state" expect( getStateValidatorIndex( "0xa99af0913a2834ef4959637e8d7c4e17f0b63adc587d36ab43510452db3102d0771a4554ea4118a33913827d5ee80b76", state, - pubkey2index + pubkeyCache ).valid ).toBe(false); }); it("should return valid: true on validator indices / pubkeys in the state", () => { const index = state.validators.length - 1; - const resp1 = getStateValidatorIndex(String(index), state, pubkey2index); + const resp1 = getStateValidatorIndex(String(index), state, pubkeyCache); if (resp1.valid) { expect(resp1.validatorIndex).toBe(index); } else { expect.fail("validator index should be found - validator index as string input"); } - const resp2 = getStateValidatorIndex(index, state, pubkey2index); + const resp2 = getStateValidatorIndex(index, state, pubkeyCache); if (resp2.valid) { expect(resp2.validatorIndex).toBe(index); } else { expect.fail("validator index should be found - validator index as number input"); } const pubkey = state.validators.get(index).pubkey; - const resp3 = getStateValidatorIndex(pubkey, state, pubkey2index); + const resp3 = getStateValidatorIndex(pubkey, state, pubkeyCache); if (resp3.valid) { expect(resp3.validatorIndex).toBe(index); } else { expect.fail("validator index should be found - Uint8Array input"); } - const resp4 = getStateValidatorIndex(toHexString(pubkey), state, pubkey2index); + const resp4 = getStateValidatorIndex(toHexString(pubkey), state, pubkeyCache); if (resp4.valid) { expect(resp4.validatorIndex).toBe(index); } else { diff --git a/packages/beacon-node/test/unit/chain/bls/bls.test.ts b/packages/beacon-node/test/unit/chain/bls/bls.test.ts index 5a325c492228..c91831f74b11 100644 --- a/packages/beacon-node/test/unit/chain/bls/bls.test.ts +++ b/packages/beacon-node/test/unit/chain/bls/bls.test.ts @@ -1,6 +1,6 @@ import {beforeEach, describe, expect, it} from "vitest"; import {PublicKey, SecretKey, Signature} from "@chainsafe/blst"; -import {ISignatureSet, SignatureSetType} from "@lodestar/state-transition"; +import {ISignatureSet, SignatureSetType, createPubkeyCache} from "@lodestar/state-transition"; import {BlsMultiThreadWorkerPool} from "../../../../src/chain/bls/multithread/index.js"; import {BlsSingleThreadVerifier} from "../../../../src/chain/bls/singleThread.js"; import {testLogger} from "../../../utils/logger.js"; @@ -9,11 +9,14 @@ describe("BlsVerifier ", () => { // take time for creating thread pool const numKeys = 3; const secretKeys = Array.from({length: numKeys}, (_, i) => SecretKey.fromKeygen(Buffer.alloc(32, i))); - // Create a mock index2pubkey that maps indices to public keys - const index2pubkey = secretKeys.map((sk) => sk.toPublicKey()); + // Create a mock pubkeyCache that maps indices to public keys + const pubkeyCache = createPubkeyCache(); + for (const [i, sk] of secretKeys.entries()) { + pubkeyCache.set(i, sk.toPublicKey().toBytes()); + } const verifiers = [ - new BlsSingleThreadVerifier({metrics: null, index2pubkey}), - new BlsMultiThreadWorkerPool({}, {metrics: null, logger: testLogger(), index2pubkey}), + new BlsSingleThreadVerifier({metrics: null, pubkeyCache}), + new BlsMultiThreadWorkerPool({}, {metrics: null, logger: testLogger(), pubkeyCache}), ]; for (const verifier of verifiers) { diff --git a/packages/beacon-node/test/unit/chain/validation/attestation/validateGossipAttestationsSameAttData.test.ts b/packages/beacon-node/test/unit/chain/validation/attestation/validateGossipAttestationsSameAttData.test.ts index fa3e155af54d..7f8d858f9c51 100644 --- a/packages/beacon-node/test/unit/chain/validation/attestation/validateGossipAttestationsSameAttData.test.ts +++ b/packages/beacon-node/test/unit/chain/validation/attestation/validateGossipAttestationsSameAttData.test.ts @@ -1,7 +1,7 @@ import {afterEach, beforeEach, describe, expect, it, vi} from "vitest"; import {PublicKey, SecretKey} from "@chainsafe/blst"; import {ForkName} from "@lodestar/params"; -import {SignatureSetType} from "@lodestar/state-transition"; +import {SignatureSetType, createPubkeyCache} from "@lodestar/state-transition"; import {ssz} from "@lodestar/types"; import {BlsSingleThreadVerifier} from "../../../../../src/chain/bls/singleThread.js"; import {AttestationError, AttestationErrorCode, GossipAction} from "../../../../../src/chain/errors/index.js"; @@ -56,22 +56,22 @@ describe("validateGossipAttestationsSameAttData", () => { return keypair; } - // Build index2pubkey cache for test - const index2pubkey: PublicKey[] = []; + // Build pubkeyCache for test + const pubkeyCache = createPubkeyCache(); for (let i = 0; i < 10; i++) { - index2pubkey.push(getKeypair(i).publicKey); + pubkeyCache.set(i, getKeypair(i).publicKey.toBytes()); } // Add a special keypair for invalid signatures - index2pubkey[2023] = getKeypair(2023).publicKey; + pubkeyCache.set(2023, getKeypair(2023).publicKey.toBytes()); let chain: IBeaconChain; const signingRoot = Buffer.alloc(32, 1); beforeEach(() => { chain = { - bls: new BlsSingleThreadVerifier({metrics: null, index2pubkey}), + bls: new BlsSingleThreadVerifier({metrics: null, pubkeyCache}), seenAttesters: new SeenAttesters(), - index2pubkey, + pubkeyCache, opts: { minSameMessageSignatureSetsToBatch: 2, } as IBeaconChain["opts"], diff --git a/packages/beacon-node/test/utils/networkWithMockDb.ts b/packages/beacon-node/test/utils/networkWithMockDb.ts index edf388a1fdf2..fc927bd18145 100644 --- a/packages/beacon-node/test/utils/networkWithMockDb.ts +++ b/packages/beacon-node/test/utils/networkWithMockDb.ts @@ -1,7 +1,6 @@ import {generateKeyPair} from "@libp2p/crypto/keys"; -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {ChainForkConfig, createBeaconConfig} from "@lodestar/config"; -import {Index2PubkeyCache, createCachedBeaconState, syncPubkeys} from "@lodestar/state-transition"; +import {createCachedBeaconState, createPubkeyCache, syncPubkeys} from "@lodestar/state-transition"; import {ssz} from "@lodestar/types"; import {sleep} from "@lodestar/utils"; import {BeaconChain} from "../../src/chain/chain.js"; @@ -43,15 +42,13 @@ export async function getNetworkForTest( ); const beaconConfig = createBeaconConfig(config, state.genesisValidatorsRoot); - const pubkey2index = new PubkeyIndexMap(); - const index2pubkey: Index2PubkeyCache = []; - syncPubkeys(state.validators.getAllReadonlyValues(), pubkey2index, index2pubkey); + const pubkeyCache = createPubkeyCache(); + syncPubkeys(pubkeyCache, state.validators.getAllReadonlyValues()); const cachedState = createCachedBeaconState( state, { config: beaconConfig, - pubkey2index, - index2pubkey, + pubkeyCache, }, {skipSyncPubkeys: true} ); @@ -73,8 +70,7 @@ export async function getNetworkForTest( { privateKey, config: beaconConfig, - pubkey2index, - index2pubkey, + pubkeyCache, db, dataDir: ".", dbName: ".", diff --git a/packages/beacon-node/test/utils/node/beacon.ts b/packages/beacon-node/test/utils/node/beacon.ts index e97341004b43..3543ac7dd2ab 100644 --- a/packages/beacon-node/test/utils/node/beacon.ts +++ b/packages/beacon-node/test/utils/node/beacon.ts @@ -4,7 +4,6 @@ import deepmerge from "deepmerge"; import tmp from "tmp"; import {setHasher} from "@chainsafe/persistent-merkle-tree"; import {hasher} from "@chainsafe/persistent-merkle-tree/hasher/hashtree"; -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {ChainConfig, createBeaconConfig, createChainForkConfig} from "@lodestar/config"; import {config as minimalConfig} from "@lodestar/config/default"; import {LevelDbController} from "@lodestar/db/controller/level"; @@ -12,10 +11,10 @@ import {LoggerNode} from "@lodestar/logger/node"; import {ForkSeq, GENESIS_SLOT} from "@lodestar/params"; import { BeaconStateAllForks, - Index2PubkeyCache, computeAnchorCheckpoint, computeEpochAtSlot, createCachedBeaconState, + createPubkeyCache, syncPubkeys, } from "@lodestar/state-transition"; import {phase0, ssz} from "@lodestar/types"; @@ -133,15 +132,13 @@ export async function getDevBeaconNode( ); const beaconConfig = createBeaconConfig(config, anchorState.genesisValidatorsRoot); - const pubkey2index = new PubkeyIndexMap(); - const index2pubkey: Index2PubkeyCache = []; - syncPubkeys(anchorState.validators.getAllReadonlyValues(), pubkey2index, index2pubkey); + const pubkeyCache = createPubkeyCache(); + syncPubkeys(pubkeyCache, anchorState.validators.getAllReadonlyValues()); const cachedState = createCachedBeaconState( anchorState, { config: beaconConfig, - pubkey2index, - index2pubkey, + pubkeyCache, }, {skipSyncPubkeys: true} ); @@ -149,8 +146,7 @@ export async function getDevBeaconNode( return BeaconNode.init({ opts: options as IBeaconNodeOptions, config: beaconConfig, - pubkey2index, - index2pubkey, + pubkeyCache, db, logger, processShutdownCallback: () => {}, diff --git a/packages/beacon-node/test/utils/state.ts b/packages/beacon-node/test/utils/state.ts index 64ee9d9514f9..a6aa3aa7741e 100644 --- a/packages/beacon-node/test/utils/state.ts +++ b/packages/beacon-node/test/utils/state.ts @@ -1,5 +1,4 @@ import {SecretKey} from "@chainsafe/blst"; -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {ChainForkConfig, createBeaconConfig} from "@lodestar/config"; import {config as minimalConfig} from "@lodestar/config/default"; import {ExecutionStatus, ProtoBlock} from "@lodestar/fork-choice"; @@ -13,6 +12,7 @@ import { CachedBeaconStateElectra, DataAvailabilityStatus, createCachedBeaconState, + createPubkeyCache, } from "@lodestar/state-transition"; import {BeaconState, altair, bellatrix, electra, ssz} from "@lodestar/types"; import {ZERO_HASH_HEX} from "../../src/constants/constants.js"; @@ -114,8 +114,7 @@ export function generateCachedState(opts?: TestBeaconState): CachedBeaconStateAl return createCachedBeaconState(state, { config: createBeaconConfig(config, state.genesisValidatorsRoot), // This is a performance test, there's no need to have a global shared cache of keys - pubkey2index: new PubkeyIndexMap(), - index2pubkey: [], + pubkeyCache: createPubkeyCache(), }); } @@ -128,8 +127,7 @@ export function generateCachedAltairState(opts?: TestBeaconState, altairForkEpoc return createCachedBeaconState(state, { config: createBeaconConfig(config, state.genesisValidatorsRoot), // This is a performance test, there's no need to have a global shared cache of keys - pubkey2index: new PubkeyIndexMap(), - index2pubkey: [], + pubkeyCache: createPubkeyCache(), }); } @@ -142,8 +140,7 @@ export function generateCachedBellatrixState(opts?: TestBeaconState): CachedBeac return createCachedBeaconState(state as BeaconStateBellatrix, { config: createBeaconConfig(config, state.genesisValidatorsRoot), // This is a performance test, there's no need to have a global shared cache of keys - pubkey2index: new PubkeyIndexMap(), - index2pubkey: [], + pubkeyCache: createPubkeyCache(), }); } @@ -155,8 +152,7 @@ export function generateCachedElectraState(opts?: TestBeaconState, electraForkEp const state = generateState(opts, config); return createCachedBeaconState(state as BeaconStateElectra, { config: createBeaconConfig(config, state.genesisValidatorsRoot), - pubkey2index: new PubkeyIndexMap(), - index2pubkey: [], + pubkeyCache: createPubkeyCache(), }); } export const zeroProtoBlock: ProtoBlock = { diff --git a/packages/beacon-node/test/utils/validationData/attestation.ts b/packages/beacon-node/test/utils/validationData/attestation.ts index 3932d38a38cb..e626ef9a66e7 100644 --- a/packages/beacon-node/test/utils/validationData/attestation.ts +++ b/packages/beacon-node/test/utils/validationData/attestation.ts @@ -153,13 +153,13 @@ export function getAttestationValidData(opts: AttestationValidDataOpts): { seenAggregatedAttestations: new SeenAggregatedAttestations(null), seenAttestationDatas: new SeenAttestationDatas(null, 0, 0), bls: blsVerifyAllMainThread - ? new BlsSingleThreadVerifier({metrics: null, index2pubkey: state.epochCtx.index2pubkey}) + ? new BlsSingleThreadVerifier({metrics: null, pubkeyCache: state.epochCtx.pubkeyCache}) : new BlsMultiThreadWorkerPool( {}, - {logger: testLogger(), metrics: null, index2pubkey: state.epochCtx.index2pubkey} + {logger: testLogger(), metrics: null, pubkeyCache: state.epochCtx.pubkeyCache} ), waitForBlock: () => Promise.resolve(false), - index2pubkey: state.epochCtx.index2pubkey, + pubkeyCache: state.epochCtx.pubkeyCache, shufflingCache, opts: defaultChainOptions, } as Partial as IBeaconChain; diff --git a/packages/cli/src/cmds/beacon/handler.ts b/packages/cli/src/cmds/beacon/handler.ts index 93f9141a18da..6b617aa16a9c 100644 --- a/packages/cli/src/cmds/beacon/handler.ts +++ b/packages/cli/src/cmds/beacon/handler.ts @@ -2,13 +2,12 @@ import path from "node:path"; import {getHeapStatistics} from "node:v8"; import {SignableENR} from "@chainsafe/enr"; import {hasher} from "@chainsafe/persistent-merkle-tree"; -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {BeaconDb, BeaconNode} from "@lodestar/beacon-node"; import {ChainForkConfig, createBeaconConfig} from "@lodestar/config"; import {LevelDbController} from "@lodestar/db/controller/level"; import {LoggerNode, getNodeLogger} from "@lodestar/logger/node"; import {ACTIVE_PRESET, PresetName} from "@lodestar/params"; -import {Index2PubkeyCache, createCachedBeaconState, syncPubkeys} from "@lodestar/state-transition"; +import {createCachedBeaconState, createPubkeyCache, syncPubkeys} from "@lodestar/state-transition"; import {ErrorAborted, bytesToInt, formatBytes} from "@lodestar/utils"; import {ProcessShutdownCallback} from "@lodestar/validator"; import {BeaconNodeOptions, getBeaconConfigFromArgs} from "../../config/index.js"; @@ -80,15 +79,13 @@ export async function beaconHandler(args: BeaconArgs & GlobalArgs): Promise Number Map + * Couples both index→pubkey and pubkey→index lookups, keeping them in sync atomically. */ - pubkey2index: PubkeyIndexMap; - /** - * Unique globally shared pubkey registry. There should only exist one for the entire application. - * - * $VALIDATOR_COUNT x BLST deserialized pubkey (Jacobian coordinates) - */ - index2pubkey: Index2PubkeyCache; + pubkeyCache: PubkeyCache; /** * Indexes of the block proposers for the current epoch. * For pre-fulu, this is computed and cached from the current shuffling. @@ -249,8 +241,7 @@ export class EpochCache { constructor(data: { config: BeaconConfig; - pubkey2index: PubkeyIndexMap; - index2pubkey: Index2PubkeyCache; + pubkeyCache: PubkeyCache; proposers: number[]; proposersPrevEpoch: number[] | null; proposersNextEpoch: ProposersDeferred; @@ -280,8 +271,7 @@ export class EpochCache { syncPeriod: SyncPeriod; }) { this.config = data.config; - this.pubkey2index = data.pubkey2index; - this.index2pubkey = data.index2pubkey; + this.pubkeyCache = data.pubkeyCache; this.proposers = data.proposers; this.proposersPrevEpoch = data.proposersPrevEpoch; this.proposersNextEpoch = data.proposersNextEpoch; @@ -319,7 +309,7 @@ export class EpochCache { */ static createFromState( state: BeaconStateAllForks, - {config, pubkey2index, index2pubkey}: EpochCacheImmutableData, + {config, pubkeyCache}: EpochCacheImmutableData, opts?: EpochCacheOpts ): EpochCache { const currentEpoch = computeEpochAtSlot(state.slot); @@ -335,9 +325,9 @@ export class EpochCache { const validatorCount = validators.length; // syncPubkeys here to ensure EpochCacheImmutableData is popualted before computing the rest of caches - // - computeSyncCommitteeCache() needs a fully populated pubkey2index cache + // - computeSyncCommitteeCache() needs a fully populated pubkeyCache if (!opts?.skipSyncPubkeys) { - syncPubkeys(validators, pubkey2index, index2pubkey); + syncPubkeys(pubkeyCache, validators); } const effectiveBalanceIncrements = getEffectiveBalanceIncrementsWithLen(validatorCount); @@ -450,8 +440,8 @@ export class EpochCache { // Allow to skip populating sync committee for initializeBeaconStateFromEth1() if (afterAltairFork && !opts?.skipSyncCommitteeCache) { const altairState = state as BeaconStateAltair; - currentSyncCommitteeIndexed = computeSyncCommitteeCache(altairState.currentSyncCommittee, pubkey2index); - nextSyncCommitteeIndexed = computeSyncCommitteeCache(altairState.nextSyncCommittee, pubkey2index); + currentSyncCommitteeIndexed = computeSyncCommitteeCache(altairState.currentSyncCommittee, pubkeyCache); + nextSyncCommitteeIndexed = computeSyncCommitteeCache(altairState.nextSyncCommittee, pubkeyCache); } else { currentSyncCommitteeIndexed = new SyncCommitteeCacheEmpty(); nextSyncCommitteeIndexed = new SyncCommitteeCacheEmpty(); @@ -514,8 +504,7 @@ export class EpochCache { return new EpochCache({ config, - pubkey2index, - index2pubkey, + pubkeyCache, proposers, // On first epoch, set to null to prevent unnecessary work since this is only used for metrics proposersPrevEpoch: null, @@ -557,8 +546,7 @@ export class EpochCache { return new EpochCache({ config: this.config, // Common append-only structures shared with all states, no need to clone - pubkey2index: this.pubkey2index, - index2pubkey: this.index2pubkey, + pubkeyCache: this.pubkeyCache, // Immutable data proposers: this.proposers, proposersPrevEpoch: this.proposersPrevEpoch, @@ -882,16 +870,15 @@ export class EpochCache { * Return pubkey given the validator index. */ getPubkey(index: ValidatorIndex): PublicKey | undefined { - return this.index2pubkey[index]; + return this.pubkeyCache.get(index); } getValidatorIndex(pubkey: Uint8Array): ValidatorIndex | null { - return this.pubkey2index.get(pubkey); + return this.pubkeyCache.getIndex(pubkey); } addPubkey(index: ValidatorIndex, pubkey: Uint8Array): void { - this.pubkey2index.set(pubkey, index); - this.index2pubkey[index] = PublicKey.fromBytes(pubkey); // Optimize for aggregation + this.pubkeyCache.set(index, pubkey); } getShufflingAtSlot(slot: Slot): EpochShuffling { @@ -1099,7 +1086,6 @@ export function createEmptyEpochCacheImmutableData( return { config: createBeaconConfig(chainConfig, state.genesisValidatorsRoot), // This is a test state, there's no need to have a global shared cache of keys - pubkey2index: new PubkeyIndexMap(), - index2pubkey: [], + pubkeyCache: createPubkeyCache(), }; } diff --git a/packages/state-transition/src/cache/pubkeyCache.ts b/packages/state-transition/src/cache/pubkeyCache.ts index 62b7cf1e5c84..a392e5ae643c 100644 --- a/packages/state-transition/src/cache/pubkeyCache.ts +++ b/packages/state-transition/src/cache/pubkeyCache.ts @@ -1,33 +1,74 @@ import {PublicKey} from "@chainsafe/blst"; import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; -import {phase0} from "@lodestar/types"; +import {ValidatorIndex, phase0} from "@lodestar/types"; -export type Index2PubkeyCache = PublicKey[]; +/** + * Unified pubkey cache coupling index→pubkey and pubkey→index lookups. + * Both directions are kept in sync atomically via `set()`. + */ +export interface PubkeyCache { + /** Get deserialized PublicKey by validator index */ + get(index: ValidatorIndex): PublicKey | undefined; + /** Get deserialized PublicKey by validator index or throw if not found */ + getOrThrow(index: ValidatorIndex): PublicKey; + /** Get validator index by pubkey bytes */ + getIndex(pubkey: Uint8Array): ValidatorIndex | null; + /** Set both directions atomically. Takes raw pubkey bytes — deserialization is handled internally. */ + set(index: ValidatorIndex, pubkey: Uint8Array): void; + /** Number of entries */ + readonly size: number; +} + +class StandardPubkeyCache implements PubkeyCache { + private readonly pubkey2index: PubkeyIndexMap; + private readonly index2pubkey: (PublicKey | undefined)[]; + + constructor(pubkey2index?: PubkeyIndexMap, index2pubkey?: (PublicKey | undefined)[]) { + this.pubkey2index = pubkey2index ?? new PubkeyIndexMap(); + this.index2pubkey = index2pubkey ?? []; + } + + get size(): number { + return this.pubkey2index.size; + } + + get(index: ValidatorIndex): PublicKey | undefined { + return this.index2pubkey[index]; + } + + getOrThrow(index: ValidatorIndex): PublicKey { + const pubkey = this.get(index); + if (!pubkey) throw Error(`Missing pubkey for validator index ${index}`); + return pubkey; + } + + getIndex(pubkey: Uint8Array): ValidatorIndex | null { + return this.pubkey2index.get(pubkey); + } + + set(index: ValidatorIndex, pubkey: Uint8Array): void { + this.pubkey2index.set(pubkey, index); + // Pubkeys must be checked for group + inf. This must be done only once when the validator deposit is processed. + // Afterwards any public key in the state is considered validated. + // > Do not do any validation here + this.index2pubkey[index] = PublicKey.fromBytes(pubkey); // Optimize for aggregation + } +} + +export function createPubkeyCache(): PubkeyCache { + return new StandardPubkeyCache(); +} /** * Checks the pubkey indices against a state and adds missing pubkeys * - * Mutates `pubkey2index` and `index2pubkey` + * Mutates `pubkeyCache` * - * If pubkey caches are empty: SLOW CODE - 🐢 + * If pubkey cache is empty: SLOW CODE - 🐢 */ -export function syncPubkeys( - validators: phase0.Validator[], - pubkey2index: PubkeyIndexMap, - index2pubkey: Index2PubkeyCache -): void { - if (pubkey2index.size !== index2pubkey.length) { - throw new Error(`Pubkey indices have fallen out of sync: ${pubkey2index.size} != ${index2pubkey.length}`); - } - +export function syncPubkeys(pubkeyCache: PubkeyCache, validators: phase0.Validator[]): void { const newCount = validators.length; - index2pubkey.length = newCount; - for (let i = pubkey2index.size; i < newCount; i++) { - const pubkey = validators[i].pubkey; - pubkey2index.set(pubkey, i); - // Pubkeys must be checked for group + inf. This must be done only once when the validator deposit is processed. - // Afterwards any public key is the state consider validated. - // > Do not do any validation here - index2pubkey[i] = PublicKey.fromBytes(pubkey); // Optimize for aggregation + for (let i = pubkeyCache.size; i < newCount; i++) { + pubkeyCache.set(i, validators[i].pubkey); } } diff --git a/packages/state-transition/src/cache/stateCache.ts b/packages/state-transition/src/cache/stateCache.ts index a3c0d2325201..6054001a6027 100644 --- a/packages/state-transition/src/cache/stateCache.ts +++ b/packages/state-transition/src/cache/stateCache.ts @@ -1,4 +1,3 @@ -import {PublicKey} from "@chainsafe/blst"; import {BeaconConfig} from "@lodestar/config"; import {loadState} from "../util/loadState/loadState.js"; import {EpochCache, EpochCacheImmutableData, EpochCacheOpts} from "./epochCache.js"; @@ -180,22 +179,19 @@ export function loadCachedBeaconState( stateBytes, seedValidatorsBytes ); - const {pubkey2index, index2pubkey} = cachedSeedState.epochCtx; + const {pubkeyCache} = cachedSeedState.epochCtx; // Get the validators sub tree once for all the loop const validators = migratedState.validators; for (const validatorIndex of modifiedValidators) { const validator = validators.getReadonly(validatorIndex); - const pubkey = validator.pubkey; - pubkey2index.set(pubkey, validatorIndex); - index2pubkey[validatorIndex] = PublicKey.fromBytes(pubkey); + pubkeyCache.set(validatorIndex, validator.pubkey); } return createCachedBeaconState( migratedState, { config: cachedSeedState.config, - pubkey2index, - index2pubkey, + pubkeyCache, }, {...(opts ?? {}), ...{skipSyncPubkeys: true}} ) as T; diff --git a/packages/state-transition/src/cache/syncCommitteeCache.ts b/packages/state-transition/src/cache/syncCommitteeCache.ts index ee729d9d692b..f6de109bd76b 100644 --- a/packages/state-transition/src/cache/syncCommitteeCache.ts +++ b/packages/state-transition/src/cache/syncCommitteeCache.ts @@ -1,4 +1,3 @@ -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {CompositeViewDU} from "@chainsafe/ssz"; import {ValidatorIndex, ssz} from "@lodestar/types"; import {toPubkeyHex} from "@lodestar/utils"; @@ -39,9 +38,9 @@ export function getSyncCommitteeCache(validatorIndices: Uint32Array): SyncCommit export function computeSyncCommitteeCache( syncCommittee: CompositeViewDU, - pubkey2index: PubkeyIndexMap + pubkeyCache: {getIndex(pubkey: Uint8Array): number | null} ): SyncCommitteeCache { - const validatorIndices = computeSyncCommitteeValidatorIndices(syncCommittee, pubkey2index); + const validatorIndices = computeSyncCommitteeValidatorIndices(syncCommittee, pubkeyCache); const validatorIndexMap = computeValidatorSyncCommitteeIndexMap(validatorIndices); return { validatorIndices, @@ -79,12 +78,12 @@ export function computeValidatorSyncCommitteeIndexMap( */ function computeSyncCommitteeValidatorIndices( syncCommittee: CompositeViewDU, - pubkey2index: PubkeyIndexMap + pubkeyCache: {getIndex(pubkey: Uint8Array): number | null} ): Uint32Array { const pubkeys = syncCommittee.pubkeys.getAllReadonly(); const validatorIndices = new Uint32Array(pubkeys.length); for (const [i, pubkey] of pubkeys.entries()) { - const validatorIndex = pubkey2index.get(pubkey); + const validatorIndex = pubkeyCache.getIndex(pubkey); if (validatorIndex === null) { throw Error(`SyncCommittee pubkey is unknown ${toPubkeyHex(pubkey)}`); } diff --git a/packages/state-transition/src/index.ts b/packages/state-transition/src/index.ts index 23d2d6fd6e61..d2b23533eb59 100644 --- a/packages/state-transition/src/index.ts +++ b/packages/state-transition/src/index.ts @@ -25,7 +25,7 @@ export { createEmptyEpochCacheImmutableData, } from "./cache/epochCache.js"; export {type EpochTransitionCache, beforeProcessEpoch} from "./cache/epochTransitionCache.js"; -export {type Index2PubkeyCache, syncPubkeys} from "./cache/pubkeyCache.js"; +export {type PubkeyCache, createPubkeyCache, syncPubkeys} from "./cache/pubkeyCache.js"; // Main state caches export { type BeaconStateCache, diff --git a/packages/state-transition/src/rewards/attestationsRewards.ts b/packages/state-transition/src/rewards/attestationsRewards.ts index b3fea34633af..d7d27f58c11a 100644 --- a/packages/state-transition/src/rewards/attestationsRewards.ts +++ b/packages/state-transition/src/rewards/attestationsRewards.ts @@ -1,4 +1,3 @@ -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {BeaconConfig} from "@lodestar/config"; import { EFFECTIVE_BALANCE_INCREMENT, @@ -16,6 +15,7 @@ import { import {ValidatorIndex, rewards} from "@lodestar/types"; import {fromHex} from "@lodestar/utils"; import {EpochTransitionCache, beforeProcessEpoch} from "../cache/epochTransitionCache.js"; +import {PubkeyCache} from "../cache/pubkeyCache.js"; import {CachedBeaconStateAllForks, CachedBeaconStateAltair} from "../types.js"; import { FLAG_ELIGIBLE_ATTESTER, @@ -34,7 +34,7 @@ const defaultAttestationsPenalty = {target: 0, source: 0}; export async function computeAttestationsRewards( config: BeaconConfig, - pubkey2index: PubkeyIndexMap, + pubkeyCache: PubkeyCache, state: CachedBeaconStateAllForks, validatorIds?: (ValidatorIndex | string)[] ): Promise { @@ -53,7 +53,7 @@ export async function computeAttestationsRewards( ); const totalRewards = computeTotalAttestationsRewardsAltair( config, - pubkey2index, + pubkeyCache, stateAltair, transitionCache, idealRewards, @@ -142,7 +142,7 @@ function computeIdealAttestationsRewardsAndPenaltiesAltair( // Same calculation as `getRewardsAndPenaltiesAltair` but returns the breakdown of rewards instead of aggregated function computeTotalAttestationsRewardsAltair( config: BeaconConfig, - pubkey2index: PubkeyIndexMap, + pubkeyCache: PubkeyCache, state: CachedBeaconStateAltair, transitionCache: EpochTransitionCache, idealRewards: rewards.IdealAttestationsReward[], @@ -153,7 +153,7 @@ function computeTotalAttestationsRewardsAltair( const {flags} = transitionCache; const {epochCtx} = state; const validatorIndices = validatorIds - .map((id) => (typeof id === "number" ? id : pubkey2index.get(fromHex(id)))) + .map((id) => (typeof id === "number" ? id : pubkeyCache.getIndex(fromHex(id)))) .filter((index) => index !== undefined); // Validator indices to include in the result const inactivityPenaltyDenominator = config.INACTIVITY_SCORE_BIAS * INACTIVITY_PENALTY_QUOTIENT_ALTAIR; diff --git a/packages/state-transition/src/rewards/syncCommitteeRewards.ts b/packages/state-transition/src/rewards/syncCommitteeRewards.ts index 27e2c24e3dd9..e93d3f61c726 100644 --- a/packages/state-transition/src/rewards/syncCommitteeRewards.ts +++ b/packages/state-transition/src/rewards/syncCommitteeRewards.ts @@ -1,12 +1,12 @@ import {BeaconConfig} from "@lodestar/config"; import {ForkName, SYNC_COMMITTEE_SIZE} from "@lodestar/params"; import {BeaconBlock, ValidatorIndex, altair, rewards} from "@lodestar/types"; -import {Index2PubkeyCache} from "../cache/pubkeyCache.js"; +import {PubkeyCache} from "../cache/pubkeyCache.js"; import {CachedBeaconStateAllForks, CachedBeaconStateAltair} from "../cache/stateCache.js"; export async function computeSyncCommitteeRewards( config: BeaconConfig, - index2pubkey: Index2PubkeyCache, + pubkeyCache: PubkeyCache, block: BeaconBlock, preState: CachedBeaconStateAllForks, validatorIds: (ValidatorIndex | string)[] = [] @@ -47,9 +47,10 @@ export async function computeSyncCommitteeRewards( if (validatorIds.length) { const filtersSet = new Set(validatorIds); - return rewards.filter( - (reward) => filtersSet.has(reward.validatorIndex) || filtersSet.has(index2pubkey[reward.validatorIndex].toHex()) - ); + return rewards.filter((reward) => { + const pubkeyHex = pubkeyCache.get(reward.validatorIndex)?.toHex(); + return filtersSet.has(reward.validatorIndex) || (pubkeyHex !== undefined && filtersSet.has(pubkeyHex)); + }); } return rewards; diff --git a/packages/state-transition/src/signatureSets/proposer.ts b/packages/state-transition/src/signatureSets/proposer.ts index 8505ec22bd3c..f1538d312510 100644 --- a/packages/state-transition/src/signatureSets/proposer.ts +++ b/packages/state-transition/src/signatureSets/proposer.ts @@ -1,17 +1,17 @@ import {BeaconConfig} from "@lodestar/config"; import {DOMAIN_BEACON_PROPOSER} from "@lodestar/params"; import {SignedBeaconBlock, SignedBlindedBeaconBlock, Slot, isBlindedBeaconBlock, phase0, ssz} from "@lodestar/types"; -import {Index2PubkeyCache} from "../cache/pubkeyCache.js"; +import {PubkeyCache} from "../cache/pubkeyCache.js"; import {computeSigningRoot} from "../util/index.js"; import {ISignatureSet, SignatureSetType, verifySignatureSet} from "../util/signatureSets.js"; export function verifyProposerSignature( config: BeaconConfig, - index2pubkey: Index2PubkeyCache, + pubkeyCache: PubkeyCache, signedBlock: SignedBeaconBlock | SignedBlindedBeaconBlock ): boolean { const signatureSet = getBlockProposerSignatureSet(config, signedBlock); - return verifySignatureSet(signatureSet, index2pubkey); + return verifySignatureSet(signatureSet, pubkeyCache); } export function getBlockProposerSignatureSet( diff --git a/packages/state-transition/src/signatureSets/randao.ts b/packages/state-transition/src/signatureSets/randao.ts index d6a132c215b2..0a8372745f23 100644 --- a/packages/state-transition/src/signatureSets/randao.ts +++ b/packages/state-transition/src/signatureSets/randao.ts @@ -1,7 +1,7 @@ import {BeaconConfig} from "@lodestar/config"; import {DOMAIN_RANDAO} from "@lodestar/params"; import {BeaconBlock, ssz} from "@lodestar/types"; -import {Index2PubkeyCache} from "../cache/pubkeyCache.js"; +import {PubkeyCache} from "../cache/pubkeyCache.js"; import { ISignatureSet, SignatureSetType, @@ -10,12 +10,8 @@ import { verifySignatureSet, } from "../util/index.js"; -export function verifyRandaoSignature( - config: BeaconConfig, - index2pubkey: Index2PubkeyCache, - block: BeaconBlock -): boolean { - return verifySignatureSet(getRandaoRevealSignatureSet(config, block), index2pubkey); +export function verifyRandaoSignature(config: BeaconConfig, pubkeyCache: PubkeyCache, block: BeaconBlock): boolean { + return verifySignatureSet(getRandaoRevealSignatureSet(config, block), pubkeyCache); } /** diff --git a/packages/state-transition/src/signatureSets/voluntaryExits.ts b/packages/state-transition/src/signatureSets/voluntaryExits.ts index 2ed07c6caaee..0cedcd399b94 100644 --- a/packages/state-transition/src/signatureSets/voluntaryExits.ts +++ b/packages/state-transition/src/signatureSets/voluntaryExits.ts @@ -1,6 +1,6 @@ import {BeaconConfig} from "@lodestar/config"; import {SignedBeaconBlock, Slot, phase0, ssz} from "@lodestar/types"; -import {Index2PubkeyCache} from "../cache/pubkeyCache.js"; +import {PubkeyCache} from "../cache/pubkeyCache.js"; import { ISignatureSet, SignatureSetType, @@ -11,11 +11,11 @@ import { export function verifyVoluntaryExitSignature( config: BeaconConfig, - index2pubkey: Index2PubkeyCache, + pubkeyCache: PubkeyCache, stateSlot: Slot, signedVoluntaryExit: phase0.SignedVoluntaryExit ): boolean { - return verifySignatureSet(getVoluntaryExitSignatureSet(config, stateSlot, signedVoluntaryExit), index2pubkey); + return verifySignatureSet(getVoluntaryExitSignatureSet(config, stateSlot, signedVoluntaryExit), pubkeyCache); } /** diff --git a/packages/state-transition/src/stateTransition.ts b/packages/state-transition/src/stateTransition.ts index 9db9297db52e..d9993a2220ae 100644 --- a/packages/state-transition/src/stateTransition.ts +++ b/packages/state-transition/src/stateTransition.ts @@ -112,7 +112,7 @@ export function stateTransition( postState = processSlotsWithTransientCache(postState, blockSlot, options, {metrics, validatorMonitor}); // Verify proposer signature only - if (verifyProposer && !verifyProposerSignature(postState.config, postState.epochCtx.index2pubkey, signedBlock)) { + if (verifyProposer && !verifyProposerSignature(postState.config, postState.epochCtx.pubkeyCache, signedBlock)) { throw new Error("Invalid block signature"); } diff --git a/packages/state-transition/src/stateView/beaconStateView.ts b/packages/state-transition/src/stateView/beaconStateView.ts index b1c7151da227..85c3848111d2 100644 --- a/packages/state-transition/src/stateView/beaconStateView.ts +++ b/packages/state-transition/src/stateView/beaconStateView.ts @@ -1,5 +1,4 @@ import {CompactMultiProof, ProofType, Tree, createProof} from "@chainsafe/persistent-merkle-tree"; -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {ByteViews} from "@chainsafe/ssz"; import {BeaconConfig} from "@lodestar/config"; import {ForkSeq, SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params"; @@ -32,6 +31,7 @@ import {VoluntaryExitValidity, getVoluntaryExitValidity} from "../block/processV import {getExpectedWithdrawals} from "../block/processWithdrawals.js"; import {EffectiveBalanceIncrements} from "../cache/effectiveBalanceIncrements.js"; import {EpochTransitionCacheOpts} from "../cache/epochTransitionCache.js"; +import {PubkeyCache, createPubkeyCache} from "../cache/pubkeyCache.js"; import {RewardCache} from "../cache/rewardCache.js"; import { CachedBeaconStateAllForks, @@ -543,7 +543,7 @@ export class BeaconStateView implements IBeaconStateView { async computeAttestationsRewards(validatorIds?: (ValidatorIndex | string)[]): Promise { return computeAttestationsRewards( this.cachedState.config, - this.cachedState.epochCtx.pubkey2index, + this.cachedState.epochCtx.pubkeyCache, this.cachedState, validatorIds ); @@ -555,7 +555,7 @@ export class BeaconStateView implements IBeaconStateView { ): Promise { return computeSyncCommitteeRewards( this.cachedState.config, - this.cachedState.epochCtx.index2pubkey, + this.cachedState.epochCtx.pubkeyCache, block, this.cachedState, validatorIds @@ -641,8 +641,7 @@ export class BeaconStateView implements IBeaconStateView { { config: this.config, // as of Feb 2026, it's not necessary to sync pubkey cache as it's shared across states in Lodestar - pubkey2index: this.cachedState.epochCtx.pubkey2index, - index2pubkey: this.cachedState.epochCtx.index2pubkey, + pubkeyCache: this.cachedState.epochCtx.pubkeyCache, }, { skipSyncPubkeys: true, @@ -715,14 +714,13 @@ export function createBeaconStateViewForHistoricalRegen( ): IBeaconStateView { const state = getStateTypeFromBytes(config, stateBytes).deserializeToViewDU(stateBytes); - const pubkey2index = new PubkeyIndexMap(); - syncPubkeyCache(state, pubkey2index); + const pubkeyCache = createPubkeyCache(); + syncPubkeyCache(state, pubkeyCache); const cachedState = createCachedBeaconState( state, { config, - pubkey2index, - index2pubkey: [], + pubkeyCache, }, { skipSyncPubkeys: true, @@ -735,12 +733,12 @@ export function createBeaconStateViewForHistoricalRegen( /** * Populate a PubkeyIndexMap with any new entries based on a BeaconState */ -function syncPubkeyCache(state: BeaconStateAllForks, pubkey2index: PubkeyIndexMap): void { +function syncPubkeyCache(state: BeaconStateAllForks, pubkeyCache: PubkeyCache): void { // Get the validators sub tree once for all the loop const newCount = state.validators.length; - for (let i = pubkey2index.size; i < newCount; i++) { + for (let i = pubkeyCache.size; i < newCount; i++) { const pubkey = state.validators.getReadonly(i).pubkey; - pubkey2index.set(pubkey, i); + pubkeyCache.set(i, pubkey); } } diff --git a/packages/state-transition/src/util/signatureSets.ts b/packages/state-transition/src/util/signatureSets.ts index eb5a7e41977a..76700d1ea0a5 100644 --- a/packages/state-transition/src/util/signatureSets.ts +++ b/packages/state-transition/src/util/signatureSets.ts @@ -1,6 +1,6 @@ import {PublicKey, Signature, aggregatePublicKeys, fastAggregateVerify, verify} from "@chainsafe/blst"; import {Root} from "@lodestar/types"; -import {Index2PubkeyCache} from "../cache/pubkeyCache.js"; +import {PubkeyCache} from "../cache/pubkeyCache.js"; export enum SignatureSetType { single = "single", @@ -49,18 +49,21 @@ export type ISignatureSet = SingleSignatureSet | IndexedSignatureSet | Aggregate /** * Get the pubkey for a signature set, performing aggregation if necessary. - * Requires index2pubkey cache for indexed and aggregate sets. + * Requires pubkeyCache for indexed and aggregate sets. */ -export function getSignatureSetPubkey(signatureSet: ISignatureSet, index2pubkey: Index2PubkeyCache): PublicKey { +export function getSignatureSetPubkey(signatureSet: ISignatureSet, pubkeyCache: PubkeyCache): PublicKey { switch (signatureSet.type) { case SignatureSetType.single: return signatureSet.pubkey; - case SignatureSetType.indexed: - return index2pubkey[signatureSet.index]; + case SignatureSetType.indexed: { + return pubkeyCache.getOrThrow(signatureSet.index); + } case SignatureSetType.aggregate: { - const pubkeys = signatureSet.indices.map((i) => index2pubkey[i]); + const pubkeys = signatureSet.indices.map((i) => { + return pubkeyCache.getOrThrow(i); + }); return aggregatePublicKeys(pubkeys); } @@ -69,11 +72,11 @@ export function getSignatureSetPubkey(signatureSet: ISignatureSet, index2pubkey: } } -export function verifySignatureSet(signatureSet: SingleSignatureSet, index2pubkey?: Index2PubkeyCache): boolean; -export function verifySignatureSet(signatureSet: IndexedSignatureSet, index2pubkey: Index2PubkeyCache): boolean; -export function verifySignatureSet(signatureSet: AggregatedSignatureSet, index2pubkey: Index2PubkeyCache): boolean; -export function verifySignatureSet(signatureSet: ISignatureSet, index2pubkey: Index2PubkeyCache): boolean; -export function verifySignatureSet(signatureSet: ISignatureSet, index2pubkey?: Index2PubkeyCache): boolean { +export function verifySignatureSet(signatureSet: SingleSignatureSet, pubkeyCache?: PubkeyCache): boolean; +export function verifySignatureSet(signatureSet: IndexedSignatureSet, pubkeyCache: PubkeyCache): boolean; +export function verifySignatureSet(signatureSet: AggregatedSignatureSet, pubkeyCache: PubkeyCache): boolean; +export function verifySignatureSet(signatureSet: ISignatureSet, pubkeyCache: PubkeyCache): boolean; +export function verifySignatureSet(signatureSet: ISignatureSet, pubkeyCache?: PubkeyCache): boolean { // All signatures are not trusted and must be group checked (p2.subgroup_check) const signature = Signature.fromBytes(signatureSet.signature, true); @@ -82,17 +85,20 @@ export function verifySignatureSet(signatureSet: ISignatureSet, index2pubkey?: I return verify(signatureSet.signingRoot, signatureSet.pubkey, signature); case SignatureSetType.indexed: { - if (!index2pubkey) { - throw Error("index2pubkey required for indexed signature set"); + if (!pubkeyCache) { + throw Error("pubkeyCache required for indexed signature set"); } - return verify(signatureSet.signingRoot, index2pubkey[signatureSet.index], signature); + const pubkey = pubkeyCache.getOrThrow(signatureSet.index); + return verify(signatureSet.signingRoot, pubkey, signature); } case SignatureSetType.aggregate: { - if (!index2pubkey) { - throw Error("index2pubkey required for aggregate signature set"); + if (!pubkeyCache) { + throw Error("pubkeyCache required for aggregate signature set"); } - const pubkeys = signatureSet.indices.map((i) => index2pubkey[i]); + const pubkeys = signatureSet.indices.map((i) => { + return pubkeyCache.getOrThrow(i); + }); return fastAggregateVerify(signatureSet.signingRoot, pubkeys, signature); } diff --git a/packages/state-transition/test/perf/util.ts b/packages/state-transition/test/perf/util.ts index 4790be1b1346..fc644379c2f3 100644 --- a/packages/state-transition/test/perf/util.ts +++ b/packages/state-transition/test/perf/util.ts @@ -1,5 +1,4 @@ import {PublicKey, SecretKey} from "@chainsafe/blst"; -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {BitArray, fromHexString} from "@chainsafe/ssz"; import {createBeaconConfig, createChainForkConfig} from "@lodestar/config"; import {config} from "@lodestar/config/default"; @@ -19,6 +18,7 @@ import { computeCommitteeCount, computeEpochAtSlot, createCachedBeaconState, + createPubkeyCache, interopSecretKey, newFilledArray, processSlots, @@ -84,32 +84,21 @@ export function getSecretKeyFromIndexCached(validatorIndex: number): SecretKey { return sk; } -function getPubkeyCaches({pubkeysMod, pubkeysModObj}: ReturnType) { +function getPubkeyCaches({pubkeysMod}: ReturnType) { // Manually sync pubkeys to prevent doing BLS opts 110_000 times - const pubkey2index = new PubkeyIndexMap(); - const index2pubkey = [] as PublicKey[]; + const pubkeyCache = createPubkeyCache(); for (let i = 0; i < numValidators; i++) { const pubkey = pubkeysMod[i % keypairsMod]; - const pubkeyObj = pubkeysModObj[i % keypairsMod]; - pubkey2index.set(pubkey, i); - index2pubkey.push(pubkeyObj); + pubkeyCache.set(i, pubkey); } - // Since most pubkeys are equal the size of pubkey2index is not numValidators. - // Fill with junk up to numValidators - for (let i = pubkey2index.size; i < numValidators; i++) { - const buf = Buffer.alloc(48, 0); - buf.writeInt32LE(i); - pubkey2index.set(buf, i); - } - - return {pubkey2index, index2pubkey}; + return {pubkeyCache}; } export function generatePerfTestCachedStatePhase0(opts?: {goBackOneSlot: boolean}): CachedBeaconStatePhase0 { // Generate only some publicKeys const {pubkeys, pubkeysMod, pubkeysModObj} = getPubkeys(); - const {pubkey2index, index2pubkey} = getPubkeyCaches({pubkeys, pubkeysMod, pubkeysModObj}); + const {pubkeyCache} = getPubkeyCaches({pubkeys, pubkeysMod, pubkeysModObj}); if (!phase0State) { const state = buildPerformanceStatePhase0(); @@ -126,8 +115,7 @@ export function generatePerfTestCachedStatePhase0(opts?: {goBackOneSlot: boolean state.slot -= 1; phase0CachedState23637 = createCachedBeaconState(state, { config: createBeaconConfig(config, state.genesisValidatorsRoot), - pubkey2index, - index2pubkey, + pubkeyCache, }); const currentEpoch = computeEpochAtSlot(state.slot - 1); @@ -219,7 +207,7 @@ export function generatePerfTestCachedStateAltair(opts?: { vc?: number; }): CachedBeaconStateAltair { const {pubkeys, pubkeysMod, pubkeysModObj} = getPubkeys(opts?.vc); - const {pubkey2index, index2pubkey} = getPubkeyCaches({pubkeys, pubkeysMod, pubkeysModObj}); + const {pubkeyCache} = getPubkeyCaches({pubkeys, pubkeysMod, pubkeysModObj}); const altairConfig = createChainForkConfig({ALTAIR_FORK_EPOCH: 0}); @@ -230,8 +218,7 @@ export function generatePerfTestCachedStateAltair(opts?: { state.slot -= 1; altairCachedState23637 = createCachedBeaconState(state, { config: createBeaconConfig(altairConfig, state.genesisValidatorsRoot), - pubkey2index, - index2pubkey, + pubkeyCache, }); } if (!altairCachedState23638) { @@ -389,16 +376,13 @@ export function generateTestCachedBeaconStateOnlyValidators({ slot: Slot; }): CachedBeaconStateAllForks { // Generate only some publicKeys - const {pubkeys, pubkeysMod, pubkeysModObj} = getPubkeys(vc); + const {pubkeys, pubkeysMod} = getPubkeys(vc); // Manually sync pubkeys to prevent doing BLS opts 110_000 times - const pubkey2index = new PubkeyIndexMap(); - const index2pubkey = [] as PublicKey[]; + const pubkeyCache = createPubkeyCache(); for (let i = 0; i < vc; i++) { const pubkey = pubkeysMod[i % keypairsMod]; - const pubkeyObj = pubkeysModObj[i % keypairsMod]; - pubkey2index.set(pubkey, i); - index2pubkey.push(pubkeyObj); + pubkeyCache.set(i, pubkey); } const state = ssz.phase0.BeaconState.defaultViewDU(); @@ -438,8 +422,7 @@ export function generateTestCachedBeaconStateOnlyValidators({ state, { config: createBeaconConfig(config, state.genesisValidatorsRoot), - pubkey2index, - index2pubkey, + pubkeyCache, }, {skipSyncPubkeys: true} ); diff --git a/packages/state-transition/test/perf/util/loadState/loadState.test.ts b/packages/state-transition/test/perf/util/loadState/loadState.test.ts index 526b02bd30e5..29b15e6d975d 100644 --- a/packages/state-transition/test/perf/util/loadState/loadState.test.ts +++ b/packages/state-transition/test/perf/util/loadState/loadState.test.ts @@ -1,7 +1,5 @@ import {bench, describe} from "@chainsafe/benchmark"; -import {PublicKey} from "@chainsafe/blst"; -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; -import {Index2PubkeyCache} from "../../../../src/cache/pubkeyCache.js"; +import {PubkeyCache, createPubkeyCache} from "../../../../src/cache/pubkeyCache.js"; import {createCachedBeaconState} from "../../../../src/cache/stateCache.js"; import {loadState} from "../../../../src/util/loadState/loadState.js"; import {generatePerfTestCachedStateAltair} from "../../util.js"; @@ -49,21 +47,18 @@ describe("loadState", () => { migratedState.hashTreeRoot(); // Get the validators sub tree once for all the loop const validators = migratedState.validators; - const pubkey2index = new PubkeyIndexMap(); - const index2pubkey: Index2PubkeyCache = []; + const pubkeyCache: PubkeyCache = createPubkeyCache(); for (const validatorIndex of modifiedValidators) { const validator = validators.getReadonly(validatorIndex); const pubkey = validator.pubkey; - pubkey2index.set(pubkey, validatorIndex); - index2pubkey[validatorIndex] = PublicKey.fromBytes(pubkey); + pubkeyCache.set(validatorIndex, pubkey); } const shufflingGetter = () => seedState.epochCtx.currentShuffling; createCachedBeaconState( migratedState, { config: seedState.config, - pubkey2index, - index2pubkey, + pubkeyCache, }, {skipSyncPubkeys: true, skipSyncCommitteeCache: true, shufflingGetter} ); diff --git a/packages/state-transition/test/unit/block/isValidIndexedAttestation.test.ts b/packages/state-transition/test/unit/block/isValidIndexedAttestation.test.ts index 9afa10327b31..3943c825cc56 100644 --- a/packages/state-transition/test/unit/block/isValidIndexedAttestation.test.ts +++ b/packages/state-transition/test/unit/block/isValidIndexedAttestation.test.ts @@ -48,7 +48,7 @@ describe("validate indexed attestation", () => { expect( isValidIndexedAttestation( state.config, - state.epochCtx.index2pubkey, + state.epochCtx.pubkeyCache, state.slot, state.validators.length, indexedAttestation, diff --git a/packages/state-transition/test/unit/cachedBeaconState.test.ts b/packages/state-transition/test/unit/cachedBeaconState.test.ts index 5da5f698a5a2..d3c9c59b4243 100644 --- a/packages/state-transition/test/unit/cachedBeaconState.test.ts +++ b/packages/state-transition/test/unit/cachedBeaconState.test.ts @@ -1,10 +1,10 @@ import {describe, expect, it, vi} from "vitest"; -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {fromHexString} from "@chainsafe/ssz"; import {createBeaconConfig} from "@lodestar/config"; import {config as defaultConfig} from "@lodestar/config/default"; import {Epoch, RootHex, ssz} from "@lodestar/types"; import {toHexString} from "@lodestar/utils"; +import {createPubkeyCache} from "../../src/cache/pubkeyCache.js"; import {createCachedBeaconState, loadCachedBeaconState} from "../../src/cache/stateCache.js"; import {EpochShuffling, calculateShufflingDecisionRoot} from "../../src/util/epochShuffling.js"; import {modifyStateSameValidator, newStateWithValidators} from "../utils/capella.js"; @@ -92,8 +92,7 @@ describe("CachedBeaconState", () => { stateView, { config, - pubkey2index: new PubkeyIndexMap(), - index2pubkey: [], + pubkeyCache: createPubkeyCache(), }, {skipSyncCommitteeCache: true} ); @@ -195,8 +194,7 @@ describe("CachedBeaconState", () => { state, { config, - pubkey2index: new PubkeyIndexMap(), - index2pubkey: [], + pubkeyCache: createPubkeyCache(), }, {skipSyncCommitteeCache: true, shufflingGetter} ); @@ -207,8 +205,8 @@ describe("CachedBeaconState", () => { // confirm loadCachedBeaconState() result for (let i = 0; i < newCachedState.validators.length; i++) { - expect(newCachedState.epochCtx.pubkey2index.get(newCachedState.validators.get(i).pubkey)).toBe(i); - expect(newCachedState.epochCtx.index2pubkey[i].toBytes()).toEqual(pubkeys[i]); + expect(newCachedState.epochCtx.getValidatorIndex(newCachedState.validators.get(i).pubkey)).toBe(i); + expect(newCachedState.epochCtx.pubkeyCache.get(i)?.toBytes()).toEqual(pubkeys[i]); } }); } diff --git a/packages/state-transition/test/unit/upgradeState.test.ts b/packages/state-transition/test/unit/upgradeState.test.ts index d2f39bdb1a13..573d819e4ab6 100644 --- a/packages/state-transition/test/unit/upgradeState.test.ts +++ b/packages/state-transition/test/unit/upgradeState.test.ts @@ -1,9 +1,9 @@ import {describe, expect, it} from "vitest"; -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {ChainForkConfig, createBeaconConfig, createChainForkConfig} from "@lodestar/config"; import {config as chainConfig} from "@lodestar/config/default"; import {ForkName} from "@lodestar/params"; import {ssz} from "@lodestar/types"; +import {createPubkeyCache} from "../../src/cache/pubkeyCache.js"; import {createCachedBeaconState} from "../../src/cache/stateCache.js"; import {upgradeStateToDeneb} from "../../src/slot/upgradeStateToDeneb.js"; import {upgradeStateToElectra} from "../../src/slot/upgradeStateToElectra.js"; @@ -16,8 +16,7 @@ describe("upgradeState", () => { capellaState, { config: createBeaconConfig(config, capellaState.genesisValidatorsRoot), - pubkey2index: new PubkeyIndexMap(), - index2pubkey: [], + pubkeyCache: createPubkeyCache(), }, {skipSyncCommitteeCache: true} ); @@ -31,8 +30,7 @@ describe("upgradeState", () => { denebState, { config: createBeaconConfig(config, denebState.genesisValidatorsRoot), - pubkey2index: new PubkeyIndexMap(), - index2pubkey: [], + pubkeyCache: createPubkeyCache(), }, {skipSyncCommitteeCache: true} ); diff --git a/packages/state-transition/test/unit/util/cachedBeaconState.test.ts b/packages/state-transition/test/unit/util/cachedBeaconState.test.ts index c85a8c7a2ffd..a9668f09d6df 100644 --- a/packages/state-transition/test/unit/util/cachedBeaconState.test.ts +++ b/packages/state-transition/test/unit/util/cachedBeaconState.test.ts @@ -1,9 +1,8 @@ import {describe, it} from "vitest"; -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {createBeaconConfig} from "@lodestar/config"; import {config} from "@lodestar/config/default"; import {ssz} from "@lodestar/types"; -import {createCachedBeaconState} from "../../../src/index.js"; +import {createCachedBeaconState, createPubkeyCache} from "../../../src/index.js"; describe("CachedBeaconState", () => { it("Create empty CachedBeaconState", () => { @@ -11,8 +10,7 @@ describe("CachedBeaconState", () => { createCachedBeaconState(emptyState, { config: createBeaconConfig(config, emptyState.genesisValidatorsRoot), - pubkey2index: new PubkeyIndexMap(), - index2pubkey: [], + pubkeyCache: createPubkeyCache(), }); }); }); diff --git a/packages/state-transition/test/utils/state.ts b/packages/state-transition/test/utils/state.ts index 520c23669eb7..d8161bf69e37 100644 --- a/packages/state-transition/test/utils/state.ts +++ b/packages/state-transition/test/utils/state.ts @@ -1,4 +1,3 @@ -import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {ChainForkConfig, createBeaconConfig} from "@lodestar/config"; import {config, config as minimalConfig} from "@lodestar/config/default"; import { @@ -17,6 +16,7 @@ import { BeaconStatePhase0, CachedBeaconStateAllForks, createCachedBeaconState, + createPubkeyCache, } from "../../src/index.js"; import {newZeroedArray} from "../../src/util/index.js"; @@ -89,8 +89,7 @@ export function generateCachedState( return createCachedBeaconState(state, { config: createBeaconConfig(config, state.genesisValidatorsRoot), // This is a test state, there's no need to have a global shared cache of keys - pubkey2index: new PubkeyIndexMap(), - index2pubkey: [], + pubkeyCache: createPubkeyCache(), }); } @@ -104,8 +103,7 @@ export function createCachedBeaconStateTest( { config: createBeaconConfig(configCustom, state.genesisValidatorsRoot), // This is a test state, there's no need to have a global shared cache of keys - pubkey2index: new PubkeyIndexMap(), - index2pubkey: [], + pubkeyCache: createPubkeyCache(), }, opts ); From 95cf2edc4c2d1ba4d8407ffc255ddbdab5a9466b Mon Sep 17 00:00:00 2001 From: Cayman Date: Tue, 24 Feb 2026 07:31:50 -0500 Subject: [PATCH 05/63] chore: properly await async monitoring requests during shutdown (#8896) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - ~~**Await `backfillSync.close()`**: Store the internal sync loop promise and await it in `close()`, ensuring in-flight DB writes complete before shutdown continues. Moved after `network.close()` so in-flight req/resp calls are terminated first.~~ **Edit nflaig**: I removed that change, the code is unused - **Await `monitoring.close()`**: After aborting the fetch controller, await the pending HTTP request so it resolves cleanly 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.6 Co-authored-by: Nico Flaig --- packages/beacon-node/src/monitoring/service.ts | 5 +++-- packages/beacon-node/src/node/nodejs.ts | 5 ++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/beacon-node/src/monitoring/service.ts b/packages/beacon-node/src/monitoring/service.ts index 4a2a155b681b..75f5e28c1b66 100644 --- a/packages/beacon-node/src/monitoring/service.ts +++ b/packages/beacon-node/src/monitoring/service.ts @@ -89,9 +89,9 @@ export class MonitoringService { } /** - * Stop sending client stats + * Stop sending client stats and wait for any pending request to complete */ - close(): void { + async close(): Promise { if (this.status === Status.Closed) return; this.status = Status.Closed; @@ -103,6 +103,7 @@ export class MonitoringService { } if (this.pendingRequest) { this.fetchAbortController?.abort(FetchAbortReason.Close); + await this.pendingRequest; } } diff --git a/packages/beacon-node/src/node/nodejs.ts b/packages/beacon-node/src/node/nodejs.ts index 0a3c321d16c7..56e33c3179de 100644 --- a/packages/beacon-node/src/node/nodejs.ts +++ b/packages/beacon-node/src/node/nodejs.ts @@ -360,9 +360,12 @@ export class BeaconNode { if (this.restApi) await this.restApi.close(); await this.network.close(); if (this.metricsServer) await this.metricsServer.close(); - if (this.monitoring) this.monitoring.close(); + if (this.monitoring) await this.monitoring.close(); await this.chain.persistToDisk(); await this.chain.close(); + // Abort signal last: close() calls above clear intervals/timeouts so no new + // operations get scheduled. If we aborted first, a still-pending interval could + // fire and schedule a new operation after abort, leaving it stuck and delaying shutdown. if (this.controller) this.controller.abort(); await sleep(DELAY_BEFORE_CLOSING_DB_MS); await this.db.close(); From 3b98c59c6427949df467561ffdd0cc959fc077ac Mon Sep 17 00:00:00 2001 From: Lodekeeper <258435968+lodekeeper@users.noreply.github.com> Date: Tue, 24 Feb 2026 13:48:11 +0000 Subject: [PATCH 06/63] fix: bootstrap already-open libp2p connections at PeerManager startup (#8948) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary PeerManager.connectedPeers map was empty at startup even when libp2p already had established connections (e.g., from bootnodes). This caused STATUS not being sent to peers, violating the consensus spec requirement that the dialing client MUST send a Status request upon connection. This is the same issue addressed in PR #8770 (`bal-devnet-2`), ported to `unstable`. ## Changes 1. Extract core peer-tracking logic from `onLibp2pPeerConnect` into a reusable `trackLibp2pConnection()` helper 2. Add `bootstrapAlreadyOpenConnections()` that seeds `connectedPeers` from existing libp2p connections at init 3. Defer initial status/ping sweep via `setTimeout` to avoid listener-order race ## Test plan - Added E2E test verifying peers connected before PeerManager init are tracked - Type check: `pnpm --filter @lodestar/beacon-node check-types` ✅ - Validated in cross-client EPBS devnet (LH+LS): resolved immediate peer disconnect at startup Closes https://github.com/ChainSafe/lodestar/issues/8779 > [!NOTE] > This PR was authored with AI assistance (Claude/Codex). --------- Co-authored-by: lodekeeper --- .../src/network/peers/peerManager.ts | 92 +++++++++++++------ .../e2e/network/peers/peerManager.test.ts | 25 ++++- 2 files changed, 89 insertions(+), 28 deletions(-) diff --git a/packages/beacon-node/src/network/peers/peerManager.ts b/packages/beacon-node/src/network/peers/peerManager.ts index 8cdd26b7dab6..3d4f2c8aa922 100644 --- a/packages/beacon-node/src/network/peers/peerManager.ts +++ b/packages/beacon-node/src/network/peers/peerManager.ts @@ -9,6 +9,7 @@ import {prettyPrintIndices, toHex, withTimeout} from "@lodestar/utils"; import {GOODBYE_KNOWN_CODES, GoodByeReasonCode, Libp2pEvent} from "../../constants/index.js"; import {IClock} from "../../util/clock.js"; import {computeColumnsForCustodyGroup, getCustodyGroups} from "../../util/dataColumns.js"; +import {callInNextEventLoop} from "../../util/eventLoop.js"; import {NetworkCoreMetrics} from "../core/metrics.js"; import {LodestarDiscv5Opts} from "../discv5/types.js"; import {INetworkEventBus, NetworkEvent, NetworkEventData} from "../events.js"; @@ -196,6 +197,13 @@ export class PeerManager { this.lastStatus = this.statusCache.get(); + // A connection may already be open before listeners are attached. + // Seed those peers so they are tracked in connectedPeers immediately. + this.bootstrapAlreadyOpenConnections(); + // Defer status/ping to the next event loop tick so the heartbeat interval and + // event listeners are fully registered before we begin handshakes. + callInNextEventLoop(() => this.pingAndStatusTimeouts()); + // On start-up will connected to existing peers in libp2p.peerStore, same as autoDial behaviour this.heartbeat(); this.intervals = [ @@ -692,36 +700,39 @@ export class PeerManager { } } - /** - * The libp2p Upgrader has successfully upgraded a peer connection on a particular multiaddress - * This event is routed through the connectionManager - * - * Registers a peer as connected. The `direction` parameter determines if the peer is being - * dialed or connecting to us. - */ - private onLibp2pPeerConnect = async (evt: CustomEvent): Promise => { - const {direction, status, remotePeer} = evt.detail; + private bootstrapAlreadyOpenConnections(): void { + let bootstrapped = 0; + + for (const {value: connections} of getConnectionsMap(this.libp2p).values()) { + for (const connection of connections) { + // trackLibp2pConnection handles deduplication via overwriteExisting: false + if (this.trackLibp2pConnection(connection, {overwriteExisting: false, triggerHandshakeNow: false})) { + bootstrapped++; + } + } + } + + if (bootstrapped > 0) { + this.logger.verbose("Bootstrapped already-open libp2p peers", {bootstrapped}); + } + } + + private trackLibp2pConnection( + connection: Connection, + opts: {overwriteExisting: boolean; triggerHandshakeNow: boolean} + ): boolean { + const {direction, status, remotePeer} = connection; const remotePeerStr = remotePeer.toString(); const remotePeerPrettyStr = prettyPrintPeerId(remotePeer); - this.logger.verbose("peer connected", {peer: remotePeerPrettyStr, direction, status}); - // NOTE: The peerConnect event is not emitted here here, but after asserting peer relevance - this.metrics?.peerConnectedEvent.inc({direction, status}); - if (evt.detail.status !== "open") { + if (status !== "open") { this.logger.debug("Peer disconnected before identify protocol initiated", { peerId: remotePeerPrettyStr, - status: evt.detail.status, + status, }); - return; + return false; } - // On connection: - // - Outbound connections: send a STATUS and PING request - // - Inbound connections: expect to be STATUS'd, schedule STATUS and PING for latter - // NOTE: libp2p may emit two "peer:connect" events: One for inbound, one for outbound - // If that happens, it's okay. Only the "outbound" connection triggers immediate action - const now = Date.now(); - // Ethereum uses secp256k1 for node IDs, reject peers with other key types if (remotePeer.type !== "secp256k1") { this.logger.debug("Peer does not have secp256k1 key, disconnecting", { @@ -729,9 +740,19 @@ export class PeerManager { type: remotePeer.type, }); void this.goodbyeAndDisconnect(remotePeer, GoodByeReasonCode.IRRELEVANT_NETWORK); - return; + return false; } + if (!opts.overwriteExisting && this.connectedPeers.has(remotePeerStr)) { + return false; + } + + // On connection: + // - Outbound connections: send a STATUS and PING request + // - Inbound connections: expect to be STATUS'd, schedule STATUS and PING for later + // NOTE: libp2p may emit two "peer:connect" events: One for inbound, one for outbound + // If that happens, it's okay. Only the "outbound" connection triggers immediate action + const now = Date.now(); const nodeId = computeNodeId(remotePeer); const peerData: PeerData = { lastReceivedMsgUnixTsMs: direction === "outbound" ? 0 : now, @@ -750,14 +771,13 @@ export class PeerManager { }; this.connectedPeers.set(remotePeerStr, peerData); - if (direction === "outbound") { - // this.pingAndStatusTimeouts(); + if (direction === "outbound" && opts.triggerHandshakeNow) { void this.requestPing(remotePeer); void this.requestStatus(remotePeer, this.statusCache.get()); } this.libp2p.services.identify - .identify(evt.detail) + .identify(connection) .then((result) => { const agentVersion = result.agentVersion; if (agentVersion) { @@ -766,7 +786,7 @@ export class PeerManager { } }) .catch((err) => { - if (evt.detail.status !== "open") { + if (connection.status !== "open") { this.logger.debug("Peer disconnected during identify protocol", { peerId: remotePeerPrettyStr, error: (err as Error).message, @@ -775,6 +795,24 @@ export class PeerManager { this.logger.debug("Error setting agentVersion for the peer", {peerId: remotePeerPrettyStr}, err); } }); + + return true; + } + + /** + * The libp2p Upgrader has successfully upgraded a peer connection on a particular multiaddress + * This event is routed through the connectionManager + * + * Registers a peer as connected. The `direction` parameter determines if the peer is being + * dialed or connecting to us. + */ + private onLibp2pPeerConnect = (evt: CustomEvent): void => { + const {direction, status, remotePeer} = evt.detail; + this.logger.verbose("peer connected", {peer: prettyPrintPeerId(remotePeer), direction, status}); + // NOTE: The peerConnect event is not emitted here here, but after asserting peer relevance + this.metrics?.peerConnectedEvent.inc({direction, status}); + + this.trackLibp2pConnection(evt.detail, {overwriteExisting: true, triggerHandshakeNow: true}); }; /** diff --git a/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts b/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts index 4763957e6f85..ae1f0e86946a 100644 --- a/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts +++ b/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts @@ -34,7 +34,7 @@ describe("network / peers / PeerManager", () => { } }); - async function mockModules() { + async function mockModules(opts?: {preOpenConnections?: Connection[]}) { // Setup fake chain const block = ssz.phase0.SignedBeaconBlock.defaultValue(); const state = generateState({ @@ -66,6 +66,18 @@ describe("network / peers / PeerManager", () => { }); const reqResp = new ReqRespFake(); + reqResp.sendPing.mockResolvedValue(BigInt(0)); + reqResp.sendStatus.mockResolvedValue(status); + + if (opts?.preOpenConnections) { + for (const connection of opts.preOpenConnections) { + getConnectionsMap(libp2p).set(connection.remotePeer.toString(), { + key: connection.remotePeer, + value: [connection], + }); + } + } + const peerRpcScores = new PeerRpcScoreStore(); const networkEventBus = new NetworkEventBus(); const mockSubnetsService: IAttnetsService = { @@ -189,6 +201,17 @@ describe("network / peers / PeerManager", () => { await peerConnectedPromise; }); + it("Bootstraps already-open outbound connections at startup", async () => { + const {reqResp, peerManager} = await mockModules({preOpenConnections: [libp2pConnectionOutboud]}); + + // Constructor bootstrap is async (requestPing/requestStatus), allow microtasks to flush. + await sleep(0); + + expect(peerManager["connectedPeers"].has(peerId1.toString())).toBe(true); + expect(reqResp.sendPing).toHaveBeenCalledOnce(); + expect(reqResp.sendStatus).toHaveBeenCalledOnce(); + }); + it("On peerConnect handshake flow", async () => { const {statusCache, libp2p, reqResp, peerManager, networkEventBus} = await mockModules(); From ef310100c0b7cee091d6225db43c9238c3364c0c Mon Sep 17 00:00:00 2001 From: Cayman Date: Tue, 24 Feb 2026 09:35:50 -0500 Subject: [PATCH 07/63] chore: track forkchoice update results via metrics (#8895) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Add `engineNotifyForkchoiceUpdateResult` metric to track the outcome of `notifyForkchoiceUpdate` calls to the execution engine - Replace fire-and-forget `.catch()` with `.then(onSuccess, onError)` to increment the metric on success (`VALID`) or failure (`error`) - Enables operators to alert on FCU failures that could indicate EL/CL head mismatches ## Test plan - [ ] Verify TypeScript compilation passes - [ ] Confirm new metric `lodestar_execution_engine_notify_forkchoice_update_result_total` is exposed in Prometheus output - [ ] Validate metric increments on successful and failed FCU calls 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.6 Co-authored-by: Nico Flaig Co-authored-by: Lodekeeper <258435968+lodekeeper@users.noreply.github.com> Co-authored-by: lodekeeper --- dashboards/lodestar_execution_engine.json | 15 +++++++++++++-- packages/beacon-node/src/execution/engine/http.ts | 3 +++ .../beacon-node/src/metrics/metrics/lodestar.ts | 5 +++++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/dashboards/lodestar_execution_engine.json b/dashboards/lodestar_execution_engine.json index 3ff00be01c2e..15fb4be6f74d 100644 --- a/dashboards/lodestar_execution_engine.json +++ b/dashboards/lodestar_execution_engine.json @@ -1267,12 +1267,23 @@ }, "editorMode": "builder", "expr": "rate(lodestar_execution_engine_notify_new_payload_result_total[1h])", - "legendFormat": "{{result}}", + "legendFormat": "newPayload_{{result}}", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "rate(lodestar_execution_engine_notify_forkchoice_update_result_total[1h])", + "legendFormat": "fcu_{{result}}", + "range": true, + "refId": "B" } ], - "title": "notifyNewPayload result", + "title": "notifyNewPayload / notifyForkchoiceUpdate result", "type": "timeseries" }, { diff --git a/packages/beacon-node/src/execution/engine/http.ts b/packages/beacon-node/src/execution/engine/http.ts index 0885268606c4..69094ce2fa8b 100644 --- a/packages/beacon-node/src/execution/engine/http.ts +++ b/packages/beacon-node/src/execution/engine/http.ts @@ -128,6 +128,7 @@ const getClientVersionOpts: ReqOpts = {routeId: "getClientVersion"}; */ export class ExecutionEngineHttp implements IExecutionEngine { private logger: Logger; + private metrics: Metrics | null; // The default state is ONLINE, it will be updated to SYNCING once we receive the first payload // This assumption is better than the OFFLINE state, since we can't be sure if the EL is offline and being offline may trigger some notifications @@ -167,6 +168,7 @@ export class ExecutionEngineHttp implements IExecutionEngine { metrics?.engineHttpProcessorQueue ); this.logger = logger; + this.metrics = metrics ?? null; this.rpc.emitter.on(JsonRpcHttpClientEvent.ERROR, ({error}) => { this.updateEngineState(getExecutionEngineState({payloadError: error, oldState: this.state})); @@ -369,6 +371,7 @@ export class ExecutionEngineHttp implements IExecutionEngine { } = await request; this.updateEngineState(getExecutionEngineState({payloadStatus: status, oldState: this.state})); + this.metrics?.engineNotifyForkchoiceUpdateResult.inc({result: status}); switch (status) { case ExecutionPayloadStatus.VALID: diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index 54bb57984d8e..f6abe18eb96a 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -928,6 +928,11 @@ export function createLodestarMetrics( help: "The total result of calling notifyNewPayload execution engine api", labelNames: ["result"], }), + engineNotifyForkchoiceUpdateResult: register.gauge<{result: ExecutionPayloadStatus}>({ + name: "lodestar_execution_engine_notify_forkchoice_update_result_total", + help: "The total result of calling notifyForkchoiceUpdate execution engine api", + labelNames: ["result"], + }), backfillSync: { backfilledTillSlot: register.gauge({ name: "lodestar_backfill_till_slot", From 96f78af84b6b0925afb62b96df14da077c5f7ce0 Mon Sep 17 00:00:00 2001 From: Cayman Date: Tue, 24 Feb 2026 17:36:03 -0500 Subject: [PATCH 08/63] feat: libp2p v3 (#8890) ## Summary Migrate from libp2p v2 to libp2p v3 (`libp2p` 2.9.0 -> 3.1.3). ### Key changes **libp2p v3 dependency bumps** (some important dependencies listed here) - `libp2p` 2.9.0 -> 3.1.3 - `@libp2p/interface` ^2.7.0 -> ^3.1.0 - `@chainsafe/discv5` ^11 -> ^12, `@chainsafe/enr` ^5 -> ^6 - `@multiformats/multiaddr` ^12 -> ^13 - Migrated from `@chainsafe/libp2p-gossipsub` to `@libp2p/gossipsub` ^15 **ReqResp: Replace `it-pipe`/async-iterator pattern with `byteStream`** - Replaced `it-pipe`, `it-all`, and the `BufferedSource`/`abortableSource`/`onChunk` utilities with direct `byteStream(stream)` usage from `@libp2p/utils` - Request/response encoders now take `Stream` directly instead of being `Sink`/`Source` transforms - Removed custom `it-pipe` type declarations (`types/it-pipe/index.d.ts`) - Snappy decompression rewritten to read from `ByteStream` instead of `BufferedSource` **Gossipsub: Composition instead of inheritance** - `Eth2Gossipsub` no longer extends `GossipSub` -- it wraps an instance via composition (`this.gossipsub`) - Uses the new `gossipsub()` factory function API from `@libp2p/gossipsub` - Internal gossipsub fields (`mesh`, `peers`, `score`, `direct`, `topics`) and public methods accessed through a `GossipSubInternal` type cast **Peer management: Identify moved to post-STATUS and connection handling** - Identify is now triggered after successful STATUS exchange instead of on connection open, avoiding wasted streams on peers that close identify early or turn out to be irrelevant - Concurrent identify attempts for the same peer are deduplicated via an `identifyInProgress` map keyed by connection ID - A passive `onPeerIdentify` event listener captures identify results from libp2p (e.g. identify-push) as a fallback for peers not explicitly identified - `onLibp2pPeerConnect` preserves existing peer data (agentVersion, metadata, status, timestamps) across multiple `connection:open` events for the same peer - `onLibp2pPeerDisconnect` ignores disconnect events while other connections for the same peer remain open **libp2p v3 API changes** - Stream handler signature changed from `({connection, stream})` to `(stream, connection)` - Multiaddr API migrated to component-based: `getPeerId()`, `protoNames()`, and `tuples()` replaced with `getComponents()` - Discv5 event handler type updated (`ENRData` -> `Multiaddr`) --------- Co-authored-by: Nico Flaig Co-authored-by: Lodekeeper <258435968+lodekeeper@users.noreply.github.com> Co-authored-by: lodekeeper --- package.json | 2 +- packages/beacon-node/package.json | 49 +- .../beacon-node/src/api/impl/node/utils.ts | 6 +- packages/beacon-node/src/constants/network.ts | 1 + .../src/db/repositories/blockArchive.ts | 3 +- .../src/network/core/networkCore.ts | 6 +- .../network/core/networkCoreWorkerHandler.ts | 6 +- .../beacon-node/src/network/core/types.ts | 4 +- packages/beacon-node/src/network/events.ts | 3 +- .../src/network/gossip/encoding.ts | 6 +- .../src/network/gossip/gossipsub.ts | 111 +- .../src/network/gossip/interface.ts | 6 +- .../src/network/gossip/scoringParameters.ts | 8 +- packages/beacon-node/src/network/interface.ts | 6 +- .../beacon-node/src/network/libp2p/index.ts | 11 +- packages/beacon-node/src/network/network.ts | 6 +- packages/beacon-node/src/network/options.ts | 3 + .../src/network/peers/datastore.ts | 23 +- .../src/network/peers/peerManager.ts | 143 ++- .../network/peers/utils/prioritizePeers.ts | 6 +- .../network/processor/gossipValidatorFn.ts | 2 +- .../src/network/processor/types.ts | 2 +- .../beacon-node/src/network/reqresp/score.ts | 1 - packages/beacon-node/src/network/util.ts | 4 +- packages/beacon-node/src/util/workerEvents.ts | 2 +- .../onWorker/dataSerialization.test.ts | 2 +- .../e2e/network/peers/peerManager.test.ts | 239 +++- .../test/e2e/network/reqresp.test.ts | 57 +- .../test/e2e/network/reqrespEncode.test.ts | 71 +- .../test/perf/network/noise/sendData.test.ts | 28 +- .../test/unit/db/api/repository.test.ts | 3 +- .../network/gossip/scoringParameters.test.ts | 2 +- .../test/unit/network/reqresp/utils.ts | 79 +- .../beacon-node/test/unit/util/error.test.ts | 2 +- .../test/unit/util/itTrigger.test.ts | 7 +- packages/beacon-node/test/utils/node/p2p.ts | 4 +- packages/beacon-node/tsconfig.json | 1 - packages/cli/package.json | 12 +- .../cli/src/cmds/beacon/initPeerIdAndEnr.ts | 18 +- packages/cli/src/cmds/bootnode/handler.ts | 4 +- packages/db/package.json | 3 +- .../db/test/unit/controller/level.test.ts | 5 +- packages/era/src/util.ts | 14 +- packages/reqresp/README.md | 2 +- packages/reqresp/package.json | 16 +- packages/reqresp/src/ReqResp.ts | 8 +- .../reqresp/src/encoders/requestDecode.ts | 48 +- .../reqresp/src/encoders/requestEncode.ts | 2 +- .../reqresp/src/encoders/responseDecode.ts | 140 ++- .../reqresp/src/encoders/responseEncode.ts | 46 +- .../reqresp/src/encodingStrategies/index.ts | 14 +- .../encodingStrategies/sszSnappy/decode.ts | 164 ++- .../encodingStrategies/sszSnappy/encode.ts | 4 +- .../encodingStrategies/sszSnappy/errors.ts | 4 - .../src/encodingStrategies/sszSnappy/index.ts | 1 - packages/reqresp/src/metrics.ts | 18 +- packages/reqresp/src/request/errors.ts | 7 +- packages/reqresp/src/request/index.ts | 160 ++- packages/reqresp/src/response/index.ts | 116 +- packages/reqresp/src/types.ts | 4 +- packages/reqresp/src/utils/abortableSource.ts | 80 -- packages/reqresp/src/utils/bufferedSource.ts | 46 - .../reqresp/src/utils/collectMaxResponse.ts | 3 +- packages/reqresp/src/utils/errorMessage.ts | 27 +- packages/reqresp/src/utils/index.ts | 4 +- packages/reqresp/src/utils/onChunk.ts | 12 - packages/reqresp/src/utils/protocolId.ts | 4 +- packages/reqresp/src/utils/snappy.ts | 2 - packages/reqresp/src/utils/snappyCompress.ts | 2 +- packages/reqresp/src/utils/snappyIndex.ts | 2 +- .../reqresp/src/utils/snappyUncompress.ts | 148 ++- packages/reqresp/src/utils/stream.ts | 34 + .../test/fixtures/encodingStrategies.ts | 11 +- packages/reqresp/test/unit/ReqResp.test.ts | 33 +- .../test/unit/encoders/reqestEncode.test.ts | 6 +- .../test/unit/encoders/requestDecode.test.ts | 8 +- .../test/unit/encoders/responseDecode.test.ts | 17 +- .../test/unit/encoders/responseEncode.test.ts | 7 +- .../sszSnappy/decode.test.ts | 19 +- .../sszSnappy/encode.test.ts | 10 +- .../sszSnappy/snappyFrames/uncompress.test.ts | 99 +- .../reqresp/test/unit/request/index.test.ts | 206 +++- .../reqresp/test/unit/response/index.test.ts | 16 +- .../test/unit/utils/errorMessage.test.ts | 2 +- packages/reqresp/test/utils/index.ts | 44 +- packages/reqresp/test/utils/mockStream.ts | 48 + packages/reqresp/test/utils/response.ts | 9 +- packages/reqresp/tsconfig.json | 1 - pnpm-lock.yaml | 1053 ++++++++--------- tsconfig.build.json | 4 +- tsconfig.e2e.json | 4 +- types/it-pipe/index.d.ts | 120 -- 92 files changed, 2023 insertions(+), 1783 deletions(-) delete mode 100644 packages/reqresp/src/utils/abortableSource.ts delete mode 100644 packages/reqresp/src/utils/bufferedSource.ts delete mode 100644 packages/reqresp/src/utils/onChunk.ts delete mode 100644 packages/reqresp/src/utils/snappy.ts create mode 100644 packages/reqresp/src/utils/stream.ts create mode 100644 packages/reqresp/test/utils/mockStream.ts delete mode 100644 types/it-pipe/index.d.ts diff --git a/package.json b/package.json index ef1622ec947c..1c5eeee965b1 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,7 @@ "https-browserify": "^1.0.0", "inquirer": "^9.1.5", "jsdom": "^23.0.1", - "libp2p": "2.9.0", + "libp2p": "3.1.3", "node-gyp": "^9.4.0", "npm-run-all": "^4.1.5", "path-browserify": "^1.0.1", diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index da0a4cc3fb5b..65834918361b 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -110,10 +110,9 @@ "dependencies": { "@chainsafe/as-sha256": "^1.2.0", "@chainsafe/blst": "^2.2.0", - "@chainsafe/discv5": "^11.0.4", - "@chainsafe/enr": "^5.0.1", - "@chainsafe/libp2p-gossipsub": "^14.1.2", - "@chainsafe/libp2p-noise": "^16.1.5", + "@chainsafe/discv5": "^12.0.0", + "@chainsafe/enr": "^6.0.0", + "@chainsafe/libp2p-noise": "^17.0.0", "@chainsafe/persistent-merkle-tree": "^1.2.1", "@chainsafe/prometheus-gc-stats": "^1.0.0", "@chainsafe/pubkey-index-map": "^3.0.0", @@ -125,15 +124,16 @@ "@fastify/cors": "^10.0.1", "@fastify/swagger": "^9.0.0", "@fastify/swagger-ui": "^5.0.1", - "@libp2p/bootstrap": "^11.0.32", - "@libp2p/crypto": "^5.0.15", - "@libp2p/identify": "^3.0.27", - "@libp2p/interface": "^2.7.0", - "@libp2p/mdns": "^11.0.32", - "@libp2p/mplex": "^11.0.32", - "@libp2p/peer-id": "^5.1.0", - "@libp2p/prometheus-metrics": "^4.3.15", - "@libp2p/tcp": "^10.1.8", + "@libp2p/bootstrap": "^12.0.11", + "@libp2p/crypto": "^5.1.13", + "@libp2p/gossipsub": "^15.0.12", + "@libp2p/identify": "^4.0.10", + "@libp2p/interface": "^3.1.0", + "@libp2p/mdns": "^12.0.11", + "@libp2p/mplex": "^12.0.11", + "@libp2p/peer-id": "^6.0.4", + "@libp2p/prometheus-metrics": "^5.0.10", + "@libp2p/tcp": "^11.0.10", "@lodestar/api": "workspace:^", "@lodestar/config": "workspace:^", "@lodestar/db": "workspace:^", @@ -146,18 +146,16 @@ "@lodestar/types": "workspace:^", "@lodestar/utils": "workspace:^", "@lodestar/validator": "workspace:^", - "@multiformats/multiaddr": "^12.1.3", - "datastore-core": "^10.0.2", - "datastore-fs": "^10.0.6", - "datastore-level": "^11.0.3", + "@multiformats/multiaddr": "^13.0.1", + "datastore-core": "^11.0.2", + "datastore-fs": "^11.0.2", + "datastore-level": "^12.0.2", "deepmerge": "^4.3.1", "fastify": "^5.7.4", - "interface-datastore": "^8.3.0", - "it-all": "^3.0.4", - "it-pipe": "^3.0.1", + "interface-datastore": "^9.0.2", "jwt-simple": "0.5.6", - "libp2p": "2.9.0", - "multiformats": "^11.0.1", + "libp2p": "3.1.3", + "multiformats": "^13.4.2", "prom-client": "^15.1.0", "qs": "^6.11.1", "strict-event-emitter-types": "^2.0.0", @@ -167,14 +165,13 @@ }, "devDependencies": { "@chainsafe/swap-or-not-shuffle": "^1.2.1", - "@libp2p/interface-internal": "^2.3.18", - "@libp2p/logger": "^5.1.21", + "@libp2p/interface-internal": "^3.0.10", + "@libp2p/logger": "^6.2.2", + "@libp2p/utils": "^7.0.10", "@lodestar/spec-test-util": "workspace:^", "@types/js-yaml": "^4.0.5", "@types/qs": "^6.9.7", "@types/tmp": "^0.2.3", - "it-drain": "^3.0.3", - "it-pair": "^2.0.6", "js-yaml": "^4.1.0", "rewiremock": "^3.14.5", "rimraf": "^4.4.1", diff --git a/packages/beacon-node/src/api/impl/node/utils.ts b/packages/beacon-node/src/api/impl/node/utils.ts index 36c651879730..1a852e4cad1c 100644 --- a/packages/beacon-node/src/api/impl/node/utils.ts +++ b/packages/beacon-node/src/api/impl/node/utils.ts @@ -1,4 +1,4 @@ -import {Connection, StreamStatus} from "@libp2p/interface"; +import type {Connection, ConnectionStatus} from "@libp2p/interface"; import {routes} from "@lodestar/api"; /** @@ -24,7 +24,7 @@ export function formatNodePeer(peerIdStr: string, connections: Connection[]): ro * - Otherwise, the first closed connection */ export function getRelevantConnection(connections: Connection[]): Connection | null { - const byStatus = new Map(); + const byStatus = new Map(); for (const conn of connections) { if (conn.status === "open") return conn; if (!byStatus.has(conn.status)) byStatus.set(conn.status, conn); @@ -37,7 +37,7 @@ export function getRelevantConnection(connections: Connection[]): Connection | n * Map libp2p connection status to the API's peer state notation * @param status */ -function getPeerState(status: StreamStatus): routes.node.PeerState { +function getPeerState(status: ConnectionStatus): routes.node.PeerState { switch (status) { case "open": return "connected"; diff --git a/packages/beacon-node/src/constants/network.ts b/packages/beacon-node/src/constants/network.ts index c786bf25a3bc..e26bbe69ee6b 100644 --- a/packages/beacon-node/src/constants/network.ts +++ b/packages/beacon-node/src/constants/network.ts @@ -30,4 +30,5 @@ export const GOODBYE_KNOWN_CODES: Record = { export enum Libp2pEvent { connectionOpen = "connection:open", connectionClose = "connection:close", + peerIdentify = "peer:identify", } diff --git a/packages/beacon-node/src/db/repositories/blockArchive.ts b/packages/beacon-node/src/db/repositories/blockArchive.ts index 1f087cea7e93..a2f9adfef591 100644 --- a/packages/beacon-node/src/db/repositories/blockArchive.ts +++ b/packages/beacon-node/src/db/repositories/blockArchive.ts @@ -1,4 +1,3 @@ -import all from "it-all"; import {ChainForkConfig} from "@lodestar/config"; import {Db, FilterOptions, KeyValue, Repository} from "@lodestar/db"; import {Root, SignedBeaconBlock, Slot, ssz} from "@lodestar/types"; @@ -121,7 +120,7 @@ export class BlockArchiveRepository extends Repository } async values(opts?: BlockFilterOptions): Promise { - return all(this.valuesStream(opts)); + return await Array.fromAsync(this.valuesStream(opts)); } // INDEX diff --git a/packages/beacon-node/src/network/core/networkCore.ts b/packages/beacon-node/src/network/core/networkCore.ts index d5ec1f6e9abe..6e78e1acc3dc 100644 --- a/packages/beacon-node/src/network/core/networkCore.ts +++ b/packages/beacon-node/src/network/core/networkCore.ts @@ -1,8 +1,8 @@ -import {Connection, PrivateKey} from "@libp2p/interface"; +import type {PeerScoreStatsDump} from "@libp2p/gossipsub/score"; +import type {PublishOpts} from "@libp2p/gossipsub/types"; +import type {Connection, PrivateKey} from "@libp2p/interface"; import {peerIdFromPrivateKey} from "@libp2p/peer-id"; import {multiaddr} from "@multiformats/multiaddr"; -import {PeerScoreStatsDump} from "@chainsafe/libp2p-gossipsub/score"; -import {PublishOpts} from "@chainsafe/libp2p-gossipsub/types"; import {routes} from "@lodestar/api"; import {BeaconConfig, ForkBoundary} from "@lodestar/config"; import type {LoggerNode} from "@lodestar/logger/node"; diff --git a/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts b/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts index 194a7a9ea73a..d137f565a5c5 100644 --- a/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts +++ b/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts @@ -1,9 +1,9 @@ import path from "node:path"; import workerThreads from "node:worker_threads"; import {privateKeyToProtobuf} from "@libp2p/crypto/keys"; -import {PrivateKey} from "@libp2p/interface"; -import {PeerScoreStatsDump} from "@chainsafe/libp2p-gossipsub/score"; -import {PublishOpts} from "@chainsafe/libp2p-gossipsub/types"; +import type {PeerScoreStatsDump} from "@libp2p/gossipsub/score"; +import type {PublishOpts} from "@libp2p/gossipsub/types"; +import type {PrivateKey} from "@libp2p/interface"; import {ModuleThread, Thread, Worker, spawn} from "@chainsafe/threads"; import {routes} from "@lodestar/api"; import {BeaconConfig, chainConfigToJson} from "@lodestar/config"; diff --git a/packages/beacon-node/src/network/core/types.ts b/packages/beacon-node/src/network/core/types.ts index 1763b2e6bd61..57992c6d4a7f 100644 --- a/packages/beacon-node/src/network/core/types.ts +++ b/packages/beacon-node/src/network/core/types.ts @@ -1,5 +1,5 @@ -import {PeerScoreStatsDump} from "@chainsafe/libp2p-gossipsub/score"; -import {PublishOpts} from "@chainsafe/libp2p-gossipsub/types"; +import type {PeerScoreStatsDump} from "@libp2p/gossipsub/score"; +import type {PublishOpts} from "@libp2p/gossipsub/types"; import {routes} from "@lodestar/api"; import {SpecJson} from "@lodestar/config"; import {LoggerNodeOpts} from "@lodestar/logger/node"; diff --git a/packages/beacon-node/src/network/events.ts b/packages/beacon-node/src/network/events.ts index 20d34b9966ea..545625a43972 100644 --- a/packages/beacon-node/src/network/events.ts +++ b/packages/beacon-node/src/network/events.ts @@ -1,5 +1,6 @@ import {EventEmitter} from "node:events"; -import {PeerId, TopicValidatorResult} from "@libp2p/interface"; +import type {TopicValidatorResult} from "@libp2p/gossipsub"; +import type {PeerId} from "@libp2p/interface"; import {CustodyIndex, Status} from "@lodestar/types"; import {PeerIdStr} from "../util/peerId.js"; import {StrictEventEmitterSingleArg} from "../util/strictEvents.js"; diff --git a/packages/beacon-node/src/network/gossip/encoding.ts b/packages/beacon-node/src/network/gossip/encoding.ts index 59b5ff665fe7..9b495804c44e 100644 --- a/packages/beacon-node/src/network/gossip/encoding.ts +++ b/packages/beacon-node/src/network/gossip/encoding.ts @@ -1,9 +1,9 @@ -import {Message} from "@libp2p/interface"; +import type {Message} from "@libp2p/gossipsub"; +import type {RPC} from "@libp2p/gossipsub/message"; +import type {DataTransform} from "@libp2p/gossipsub/types"; // snappyjs is better for compression for smaller payloads import xxhashFactory from "xxhash-wasm"; import {digest} from "@chainsafe/as-sha256"; -import {RPC} from "@chainsafe/libp2p-gossipsub/message"; -import {DataTransform} from "@chainsafe/libp2p-gossipsub/types"; import snappyWasm from "@chainsafe/snappy-wasm"; import {ForkName} from "@lodestar/params"; import {intToBytes} from "@lodestar/utils"; diff --git a/packages/beacon-node/src/network/gossip/gossipsub.ts b/packages/beacon-node/src/network/gossip/gossipsub.ts index 2018591f781e..eb543ba13cc0 100644 --- a/packages/beacon-node/src/network/gossip/gossipsub.ts +++ b/packages/beacon-node/src/network/gossip/gossipsub.ts @@ -1,10 +1,18 @@ +import { + type GossipSub, + type GossipSubEvents, + type PublishResult, + StrictNoSign, + type TopicValidatorResult, + gossipsub, +} from "@libp2p/gossipsub"; +import type {MetricsRegister, TopicLabel, TopicStrToLabel} from "@libp2p/gossipsub/metrics"; +import type {PeerScoreParams, PeerScoreStatsDump} from "@libp2p/gossipsub/score"; +import type {AddrInfo, PublishOpts, TopicStr} from "@libp2p/gossipsub/types"; +import type {PeerId} from "@libp2p/interface"; import {peerIdFromString} from "@libp2p/peer-id"; import {multiaddr} from "@multiformats/multiaddr"; import {ENR} from "@chainsafe/enr"; -import {GossipSub, GossipsubEvents} from "@chainsafe/libp2p-gossipsub"; -import {MetricsRegister, TopicLabel, TopicStrToLabel} from "@chainsafe/libp2p-gossipsub/metrics"; -import {PeerScoreParams} from "@chainsafe/libp2p-gossipsub/score"; -import {AddrInfo, SignaturePolicy, TopicStr} from "@chainsafe/libp2p-gossipsub/types"; import {routes} from "@lodestar/api"; import {BeaconConfig, ForkBoundary} from "@lodestar/config"; import {ATTESTATION_SUBNET_COUNT, SLOTS_PER_EPOCH, SYNC_COMMITTEE_SUBNET_COUNT} from "@lodestar/params"; @@ -69,6 +77,24 @@ export type Eth2GossipsubOpts = { export type ForkBoundaryLabel = string; +// Many of the internal properties we need are not available on the public interface, +// so we create an extended type here to avoid excessive type assertions throughout the codebase. +// Mind that any updates to the gossipsub package may require updates to this type. +type GossipSubInternal = GossipSub & { + mesh: Map>; + peers: Map; + score: {score: (peerIdStr: string) => number}; + direct: Set; + topics: Map>; + start: () => Promise; + stop: () => Promise; + publish: (topic: TopicStr, data: Uint8Array, opts?: PublishOpts) => Promise; + getMeshPeers: (topic: TopicStr) => string[]; + dumpPeerScoreStats: () => PeerScoreStatsDump; + getScore: (peerIdStr: string) => number; + reportMessageValidationResult: (msgId: string, propagationSource: string, acceptance: TopicValidatorResult) => void; +}; + /** * Wrapper around js-libp2p-gossipsub with the following extensions: * - Eth2 message id @@ -82,13 +108,14 @@ export type ForkBoundaryLabel = string; * * See https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/p2p-interface.md#the-gossip-domain-gossipsub */ -export class Eth2Gossipsub extends GossipSub { +export class Eth2Gossipsub { readonly scoreParams: Partial; private readonly config: BeaconConfig; private readonly logger: Logger; private readonly peersData: PeersData; private readonly events: NetworkEventBus; private readonly libp2p: Libp2p; + private readonly gossipsub: GossipSubInternal; // Internal caches private readonly gossipTopicCache: GossipTopicCache; @@ -103,9 +130,6 @@ export class Eth2Gossipsub extends GossipSub { let metrics: Eth2GossipsubMetrics | null = null; if (metricsRegister) { metrics = createEth2GossipsubMetrics(metricsRegister); - metrics.gossipMesh.peersByType.addCollect(() => - this.onScrapeLodestarMetrics(metrics as Eth2GossipsubMetrics, networkConfig) - ); } // Parse direct peers from multiaddr strings to AddrInfo objects @@ -113,8 +137,8 @@ export class Eth2Gossipsub extends GossipSub { // Gossipsub parameters defined here: // https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/p2p-interface.md#the-gossip-domain-gossipsub - super(modules.libp2p.services.components, { - globalSignaturePolicy: SignaturePolicy.StrictNoSign, + const gossipsubInstance = gossipsub({ + globalSignaturePolicy: StrictNoSign, allowPublishToZeroTopicPeers: allowPublishToZeroPeers, D: gossipsubD ?? GOSSIP_D, Dlo: gossipsubDLow ?? GOSSIP_D_LOW, @@ -155,7 +179,12 @@ export class Eth2Gossipsub extends GossipSub { // This should be large enough to not send IDONTWANT for "small" messages // See https://github.com/ChainSafe/lodestar/pull/7077#issuecomment-2383679472 idontwantMinDataSize: 16829, - }); + })(modules.libp2p.services.components) as GossipSubInternal; + + if (metrics) { + metrics.gossipMesh.peersByType.addCollect(() => this.onScrapeLodestarMetrics(metrics, networkConfig)); + } + this.gossipsub = gossipsubInstance; this.scoreParams = scoreParams; this.config = config; this.logger = logger; @@ -164,7 +193,7 @@ export class Eth2Gossipsub extends GossipSub { this.libp2p = modules.libp2p; this.gossipTopicCache = gossipTopicCache; - this.addEventListener("gossipsub:message", this.onGossipsubMessage.bind(this)); + this.gossipsub.addEventListener("gossipsub:message", this.onGossipsubMessage.bind(this)); this.events.on(NetworkEvent.gossipMessageValidationResult, this.onValidationResult.bind(this)); // Having access to this data is CRUCIAL for debugging. While this is a massive log, it must not be deleted. @@ -174,6 +203,38 @@ export class Eth2Gossipsub extends GossipSub { } } + async start(): Promise { + await this.gossipsub.start(); + } + + async stop(): Promise { + await this.gossipsub.stop(); + } + + get mesh(): Map> { + return this.gossipsub.mesh; + } + + getTopics(): TopicStr[] { + return this.gossipsub.getTopics(); + } + + getMeshPeers(topic: TopicStr): string[] { + return this.gossipsub.getMeshPeers(topic); + } + + publish(topic: TopicStr, data: Uint8Array, opts?: PublishOpts): Promise { + return this.gossipsub.publish(topic, data, opts); + } + + dumpPeerScoreStats(): PeerScoreStatsDump { + return this.gossipsub.dumpPeerScoreStats(); + } + + getScore(peerIdStr: string): number { + return this.gossipsub.getScore(peerIdStr); + } + /** * Subscribe to a `GossipTopic` */ @@ -183,7 +244,7 @@ export class Eth2Gossipsub extends GossipSub { this.gossipTopicCache.setTopic(topicStr, topic); this.logger.verbose("Subscribe to gossipsub topic", {topic: topicStr}); - this.subscribe(topicStr); + this.gossipsub.subscribe(topicStr); } /** @@ -192,15 +253,14 @@ export class Eth2Gossipsub extends GossipSub { unsubscribeTopic(topic: GossipTopic): void { const topicStr = stringifyGossipTopic(this.config, topic); this.logger.verbose("Unsubscribe to gossipsub topic", {topic: topicStr}); - this.unsubscribe(topicStr); + this.gossipsub.unsubscribe(topicStr); } private onScrapeLodestarMetrics(metrics: Eth2GossipsubMetrics, networkConfig: NetworkConfig): void { - const mesh = this.mesh; - // biome-ignore lint/complexity/useLiteralKeys: `topics` is a private attribute - const topics = this["topics"] as Map>; - const peers = this.peers; - const score = this.score; + const mesh = this.gossipsub.mesh; + const topics = this.gossipsub.topics; + const peers = this.gossipsub.peers; + const score = this.gossipsub.score; const meshPeersByClient = new Map(); const meshPeerIdStrs = new Set(); @@ -305,7 +365,7 @@ export class Eth2Gossipsub extends GossipSub { metrics.gossipPeer.score.set(gossipScores); } - private onGossipsubMessage(event: GossipsubEvents["gossipsub:message"]): void { + private onGossipsubMessage(event: GossipSubEvents["gossipsub:message"]): void { const {propagationSource, msgId, msg} = event.detail; // Also validates that the topicStr is known @@ -341,7 +401,7 @@ export class Eth2Gossipsub extends GossipSub { // Without this we'll have huge event loop lag // See https://github.com/ChainSafe/lodestar/issues/5604 callInNextEventLoop(() => { - this.reportMessageValidationResult(data.msgId, data.propagationSource, data.acceptance); + this.gossipsub.reportMessageValidationResult(data.msgId, data.propagationSource, data.acceptance); }); } @@ -379,7 +439,7 @@ export class Eth2Gossipsub extends GossipSub { } // Add to direct peers set only after addresses are stored - this.direct.add(peerIdStr); + this.gossipsub.direct.add(peerIdStr); this.logger.info("Added direct peer via API", {peerId: peerIdStr}); return peerIdStr; @@ -389,7 +449,7 @@ export class Eth2Gossipsub extends GossipSub { * Remove a peer from direct peers. */ removeDirectPeer(peerIdStr: string): boolean { - const removed = this.direct.delete(peerIdStr); + const removed = this.gossipsub.direct.delete(peerIdStr); if (removed) { this.logger.info("Removed direct peer via API", {peerId: peerIdStr}); } @@ -400,7 +460,7 @@ export class Eth2Gossipsub extends GossipSub { * Get list of current direct peer IDs. */ getDirectPeers(): string[] { - return Array.from(this.direct); + return Array.from(this.gossipsub.direct); } } @@ -498,7 +558,8 @@ export function parseDirectPeers(directPeerStrs: routes.lodestar.DirectPeer[], l try { const ma = multiaddr(peerStr); - const peerIdStr = ma.getPeerId(); + const peerIdComponent = ma.getComponents().findLast((component) => component.name === "p2p"); + const peerIdStr = peerIdComponent?.value; if (!peerIdStr) { logger.warn("Direct peer multiaddr must contain /p2p/ component with peer ID", {multiaddr: peerStr}); continue; diff --git a/packages/beacon-node/src/network/gossip/interface.ts b/packages/beacon-node/src/network/gossip/interface.ts index f7544b91f67b..76584a0ebc51 100644 --- a/packages/beacon-node/src/network/gossip/interface.ts +++ b/packages/beacon-node/src/network/gossip/interface.ts @@ -1,6 +1,6 @@ -import {Message, TopicValidatorResult} from "@libp2p/interface"; -import {Libp2p} from "libp2p"; -import {PeerIdStr} from "@chainsafe/libp2p-gossipsub/types"; +import type {Message, TopicValidatorResult} from "@libp2p/gossipsub"; +import type {PeerIdStr} from "@libp2p/gossipsub/types"; +import type {Libp2p} from "libp2p"; import {BeaconConfig, ForkBoundary} from "@lodestar/config"; import { AttesterSlashing, diff --git a/packages/beacon-node/src/network/gossip/scoringParameters.ts b/packages/beacon-node/src/network/gossip/scoringParameters.ts index b9f2f24adf03..669d188ae8c9 100644 --- a/packages/beacon-node/src/network/gossip/scoringParameters.ts +++ b/packages/beacon-node/src/network/gossip/scoringParameters.ts @@ -1,9 +1,9 @@ import { - PeerScoreParams, - PeerScoreThresholds, - TopicScoreParams, + type PeerScoreParams, + type PeerScoreThresholds, + type TopicScoreParams, defaultTopicScoreParams, -} from "@chainsafe/libp2p-gossipsub/score"; +} from "@libp2p/gossipsub/score"; import {BeaconConfig} from "@lodestar/config"; import {ATTESTATION_SUBNET_COUNT, PTC_SIZE, SLOTS_PER_EPOCH, TARGET_AGGREGATORS_PER_COMMITTEE} from "@lodestar/params"; import {computeCommitteeCount} from "@lodestar/state-transition"; diff --git a/packages/beacon-node/src/network/interface.ts b/packages/beacon-node/src/network/interface.ts index 998948644e5f..225202317620 100644 --- a/packages/beacon-node/src/network/interface.ts +++ b/packages/beacon-node/src/network/interface.ts @@ -1,5 +1,5 @@ -import {Identify} from "@libp2p/identify"; -import { +import type {Identify} from "@libp2p/identify"; +import type { ComponentLogger, ConnectionGater, ConnectionProtector, @@ -16,7 +16,7 @@ import { } from "@libp2p/interface"; import type {AddressManager, ConnectionManager, Registrar, TransportManager} from "@libp2p/interface-internal"; import type {Datastore} from "interface-datastore"; -import {Libp2p as ILibp2p} from "libp2p"; +import type {Libp2p as ILibp2p} from "libp2p"; import { AttesterSlashing, DataColumnSidecar, diff --git a/packages/beacon-node/src/network/libp2p/index.ts b/packages/beacon-node/src/network/libp2p/index.ts index ec47acd06d90..9aa53d6fd10b 100644 --- a/packages/beacon-node/src/network/libp2p/index.ts +++ b/packages/beacon-node/src/network/libp2p/index.ts @@ -1,6 +1,6 @@ import {bootstrap} from "@libp2p/bootstrap"; import {identify} from "@libp2p/identify"; -import {PrivateKey} from "@libp2p/interface"; +import type {PrivateKey} from "@libp2p/interface"; import {mdns} from "@libp2p/mdns"; import {mplex} from "@libp2p/mplex"; import {prometheusMetrics} from "@libp2p/prometheus-metrics"; @@ -39,6 +39,7 @@ export async function createNodeJsLibp2p( nodeJsLibp2pOpts: NodeJsLibp2pOpts = {} ): Promise { const localMultiaddrs = networkOpts.localMultiaddrs || defaultNetworkOptions.localMultiaddrs; + const disconnectThreshold = networkOpts.disconnectThreshold ?? defaultNetworkOptions.disconnectThreshold; const {peerStoreDir, disablePeerDiscovery} = nodeJsLibp2pOpts; let datastore: undefined | Eth2PeerDataStore = undefined; @@ -74,6 +75,11 @@ export async function createNodeJsLibp2p( return createLibp2p({ privateKey, + nodeInfo: { + name: "lodestar", + version: networkOpts.version ?? "unknown", + userAgent: networkOpts.private ? "" : networkOpts.version ? `lodestar/${networkOpts.version}` : "lodestar", + }, addresses: { listen: localMultiaddrs, announce: [], @@ -93,7 +99,7 @@ export async function createNodeJsLibp2p( }, }), ], - streamMuxers: [mplex({maxInboundStreams: 256, disconnectThreshold: networkOpts.disconnectThreshold})], + streamMuxers: [mplex({disconnectThreshold})], peerDiscovery, metrics: nodeJsLibp2pOpts.metrics ? prometheusMetrics({ @@ -124,7 +130,6 @@ export async function createNodeJsLibp2p( datastore, services: { identify: identify({ - agentVersion: networkOpts.private ? "" : networkOpts.version ? `lodestar/${networkOpts.version}` : "lodestar", runOnConnectionOpen: false, }), // individual components are specified because the components object is a Proxy diff --git a/packages/beacon-node/src/network/network.ts b/packages/beacon-node/src/network/network.ts index 4aeccb04f8bf..340ba2f82a73 100644 --- a/packages/beacon-node/src/network/network.ts +++ b/packages/beacon-node/src/network/network.ts @@ -1,7 +1,7 @@ -import {PeerId, PrivateKey} from "@libp2p/interface"; +import type {PeerScoreStatsDump} from "@libp2p/gossipsub/score"; +import type {PublishOpts} from "@libp2p/gossipsub/types"; +import type {PeerId, PrivateKey} from "@libp2p/interface"; import {peerIdFromPrivateKey} from "@libp2p/peer-id"; -import {PeerScoreStatsDump} from "@chainsafe/libp2p-gossipsub/score"; -import {PublishOpts} from "@chainsafe/libp2p-gossipsub/types"; import {routes} from "@lodestar/api"; import {BeaconConfig} from "@lodestar/config"; import {LoggerNode} from "@lodestar/logger/node"; diff --git a/packages/beacon-node/src/network/options.ts b/packages/beacon-node/src/network/options.ts index e3989ebd9502..f6e7e4374081 100644 --- a/packages/beacon-node/src/network/options.ts +++ b/packages/beacon-node/src/network/options.ts @@ -72,4 +72,7 @@ export const defaultNetworkOptions: NetworkOptions = { // - for fusaka-devnets, we have 25-30 peers per subnet // - for public testnets or mainnet, average number of peers per group is SAMPLES_PER_SLOT * targetPeers / NUMBER_OF_CUSTODY_GROUPS = 6.25 so this should not be an issue targetGroupPeers: 6, + // Keep this high enough for normal req/resp bursts on stable connections. + // libp2p-mplex default (5) is too low and can cause frequent connection resets. + disconnectThreshold: 50, }; diff --git a/packages/beacon-node/src/network/peers/datastore.ts b/packages/beacon-node/src/network/peers/datastore.ts index f57e66bf5caa..d21d145cec1f 100644 --- a/packages/beacon-node/src/network/peers/datastore.ts +++ b/packages/beacon-node/src/network/peers/datastore.ts @@ -8,6 +8,9 @@ type MemoryItem = { data: Uint8Array; }; +// biome-ignore lint/suspicious/noExplicitAny: used below (copied from upstream) +type AwaitGenerator = Generator | AsyncGenerator; + /** * Before libp2p 0.35, peerstore stays in memory and periodically write to db after n dirty items * This has a memory issue because all peer data stays in memory and loaded at startup time @@ -93,7 +96,7 @@ export class Eth2PeerDataStore extends BaseDatastore { * This throws error if not found * see https://github.com/ipfs/js-datastore-level/blob/38f44058dd6be858e757a1c90b8edb31590ec0bc/src/index.js#L102 */ - async get(key: Key): Promise { + async get(key: Key, options?: AbortOptions): Promise { const keyStr = key.toString(); const memoryItem = this._memoryDatastore.get(keyStr); if (memoryItem) { @@ -102,16 +105,16 @@ export class Eth2PeerDataStore extends BaseDatastore { } // this throws error if not found - const dbValue = await this._dbDatastore.get(key); + const dbValue = await this._dbDatastore.get(key, options); // don't call this._memoryDatastore.set directly // we want to get through prune() logic with fromDb as true await this._put(key, dbValue, true); return dbValue; } - async has(key: Key): Promise { + async has(key: Key, options?: AbortOptions): Promise { try { - await this.get(key); + await this.get(key, options); } catch (err) { // this is the same to how js-datastore-level handles notFound error // https://github.com/ipfs/js-datastore-level/blob/38f44058dd6be858e757a1c90b8edb31590ec0bc/src/index.js#L121 @@ -121,26 +124,26 @@ export class Eth2PeerDataStore extends BaseDatastore { return true; } - async delete(key: Key): Promise { + async delete(key: Key, options?: AbortOptions): Promise { this._memoryDatastore.delete(key.toString()); - await this._dbDatastore.delete(key); + await this._dbDatastore.delete(key, options); } - async *_all(q: Query): AsyncIterable { + async *_all(q: Query, options?: AbortOptions): AwaitGenerator { for (const [key, value] of this._memoryDatastore.entries()) { yield { key: new Key(key), value: value.data, }; } - yield* this._dbDatastore.query(q); + yield* this._dbDatastore.query(q, options); } - async *_allKeys(q: KeyQuery): AsyncIterable { + async *_allKeys(q: KeyQuery, options?: AbortOptions): AwaitGenerator { for (const key of this._memoryDatastore.keys()) { yield new Key(key); } - yield* this._dbDatastore.queryKeys(q); + yield* this._dbDatastore.queryKeys(q, options); } private async _addDirtyItem(keyStr: string): Promise { diff --git a/packages/beacon-node/src/network/peers/peerManager.ts b/packages/beacon-node/src/network/peers/peerManager.ts index 3d4f2c8aa922..86f19c979a6d 100644 --- a/packages/beacon-node/src/network/peers/peerManager.ts +++ b/packages/beacon-node/src/network/peers/peerManager.ts @@ -1,4 +1,4 @@ -import {Connection, PeerId, PrivateKey} from "@libp2p/interface"; +import {Connection, type IdentifyResult, PeerId, PrivateKey} from "@libp2p/interface"; import {BitArray} from "@chainsafe/ssz"; import {BeaconConfig} from "@lodestar/config"; import {LoggerNode} from "@lodestar/logger/node"; @@ -162,6 +162,8 @@ export class PeerManager { // A single map of connected peers with all necessary data to handle PINGs, STATUS, and metrics private connectedPeers: Map; + /** Track one in-flight identify call per peer/connection id */ + private readonly identifyInProgress = new Map(); private opts: PeerManagerOpts; private intervals: NodeJS.Timeout[] = []; @@ -193,6 +195,7 @@ export class PeerManager { this.libp2p.services.components.events.addEventListener(Libp2pEvent.connectionOpen, this.onLibp2pPeerConnect); this.libp2p.services.components.events.addEventListener(Libp2pEvent.connectionClose, this.onLibp2pPeerDisconnect); + this.libp2p.services.components.events.addEventListener(Libp2pEvent.peerIdentify, this.onPeerIdentify); this.networkEventBus.on(NetworkEvent.reqRespRequest, this.onRequest); this.lastStatus = this.statusCache.get(); @@ -236,6 +239,7 @@ export class PeerManager { Libp2pEvent.connectionClose, this.onLibp2pPeerDisconnect ); + this.libp2p.services.components.events.removeEventListener(Libp2pEvent.peerIdentify, this.onPeerIdentify); this.networkEventBus.off(NetworkEvent.reqRespRequest, this.onRequest); for (const interval of this.intervals) clearInterval(interval); } @@ -480,6 +484,32 @@ export class PeerManager { clientAgent, custodyColumns, }); + + // Identify peer after status proves the connection is usable. + // This is the only place we trigger identify — avoids wasted streams on + // peers that close identify right after connection open or turn out to be + // irrelevant. + if (peerData?.agentVersion === null) { + const peerIdStr = peer.toString(); + const connection = getConnection(this.libp2p, peerIdStr); + if (!connection || connection.status !== "open") { + this.logger.debug("Peer has no open connection for identify", {peerId: prettyPrintPeerId(peer)}); + return; + } + + const identifyKey = connection.id; + if (this.identifyInProgress.get(peerIdStr) === identifyKey) { + return; + } + + this.identifyInProgress.set(peerIdStr, identifyKey); + void this.identifyPeer(peerIdStr, prettyPrintPeerId(peer), connection, identifyKey).finally(() => { + // Clear only if this identify attempt is still the active one for this peer + if (this.identifyInProgress.get(peerIdStr) === identifyKey) { + this.identifyInProgress.delete(peerIdStr); + } + }); + } } } @@ -753,21 +783,26 @@ export class PeerManager { // NOTE: libp2p may emit two "peer:connect" events: One for inbound, one for outbound // If that happens, it's okay. Only the "outbound" connection triggers immediate action const now = Date.now(); + const existingPeerData = this.connectedPeers.get(remotePeerStr); const nodeId = computeNodeId(remotePeer); const peerData: PeerData = { - lastReceivedMsgUnixTsMs: direction === "outbound" ? 0 : now, + // Keep existing timestamps if this peer already had another open connection. + // libp2p may emit multiple connection:open events per peer. + lastReceivedMsgUnixTsMs: existingPeerData?.lastReceivedMsgUnixTsMs ?? (direction === "outbound" ? 0 : now), // If inbound, request after STATUS_INBOUND_GRACE_PERIOD - lastStatusUnixTsMs: direction === "outbound" ? 0 : now - STATUS_INTERVAL_MS + STATUS_INBOUND_GRACE_PERIOD, - connectedUnixTsMs: now, - relevantStatus: RelevantPeerStatus.Unknown, + lastStatusUnixTsMs: + existingPeerData?.lastStatusUnixTsMs ?? + (direction === "outbound" ? 0 : now - STATUS_INTERVAL_MS + STATUS_INBOUND_GRACE_PERIOD), + connectedUnixTsMs: existingPeerData?.connectedUnixTsMs ?? now, + relevantStatus: existingPeerData?.relevantStatus ?? RelevantPeerStatus.Unknown, direction, nodeId, peerId: remotePeer, - status: null, - metadata: null, - agentVersion: null, - agentClient: null, - encodingPreference: null, + status: existingPeerData?.status ?? null, + metadata: existingPeerData?.metadata ?? null, + agentVersion: existingPeerData?.agentVersion ?? null, + agentClient: existingPeerData?.agentClient ?? null, + encodingPreference: existingPeerData?.encodingPreference ?? null, }; this.connectedPeers.set(remotePeerStr, peerData); @@ -776,26 +811,6 @@ export class PeerManager { void this.requestStatus(remotePeer, this.statusCache.get()); } - this.libp2p.services.identify - .identify(connection) - .then((result) => { - const agentVersion = result.agentVersion; - if (agentVersion) { - peerData.agentVersion = agentVersion; - peerData.agentClient = getKnownClientFromAgentVersion(agentVersion); - } - }) - .catch((err) => { - if (connection.status !== "open") { - this.logger.debug("Peer disconnected during identify protocol", { - peerId: remotePeerPrettyStr, - error: (err as Error).message, - }); - } else { - this.logger.debug("Error setting agentVersion for the peer", {peerId: remotePeerPrettyStr}, err); - } - }); - return true; } @@ -822,6 +837,19 @@ export class PeerManager { const {direction, status, remotePeer} = evt.detail; const peerIdStr = remotePeer.toString(); + const openConnections = + getConnectionsMap(this.libp2p) + .get(peerIdStr) + ?.value.filter((connection) => connection.status === "open") ?? []; + if (openConnections.length > 0) { + this.logger.debug("Ignoring peer disconnect event while another connection is still open", { + peerId: prettyPrintPeerIdStr(peerIdStr), + direction, + status, + }); + return; + } + let logMessage = "onLibp2pPeerDisconnect"; const logContext: Record = { peerId: prettyPrintPeerIdStr(peerIdStr), @@ -839,6 +867,7 @@ export class PeerManager { // remove the ping and status timer for the peer this.connectedPeers.delete(peerIdStr); + this.identifyInProgress.delete(peerIdStr); this.logger.verbose(logMessage, logContext); this.networkEventBus.emit(NetworkEvent.peerDisconnected, {peer: peerIdStr}); @@ -856,6 +885,60 @@ export class PeerManager { } } + /** + * Consume successful identify results from libp2p events. + * This captures agentVersion from identify-push or successful inbound/outbound identify, + * even if our explicit identify request failed earlier. + */ + private onPeerIdentify = (evt: CustomEvent): void => { + const {peerId, agentVersion} = evt.detail; + if (!agentVersion) return; + + const peerIdStr = peerId.toString(); + const peerData = this.connectedPeers.get(peerIdStr); + if (!peerData) return; + + peerData.agentVersion = agentVersion; + peerData.agentClient = getKnownClientFromAgentVersion(agentVersion); + this.identifyInProgress.delete(peerIdStr); + }; + + private async identifyPeer( + peerIdStr: string, + peerIdPretty: string, + connection: Connection, + identifyKey: string + ): Promise { + if (this.identifyInProgress.get(peerIdStr) !== identifyKey) { + return; + } + + if (connection.status !== "open") { + this.logger.debug("Peer has no open connection for identify", {peerId: peerIdPretty}); + return; + } + + try { + const result = await this.libp2p.services.identify.identify(connection); + + // A newer identify attempt may have superseded this one (e.g. reconnect). + if (this.identifyInProgress.get(peerIdStr) !== identifyKey) { + return; + } + + const agentVersion = result.agentVersion; + if (agentVersion) { + const connectedPeerData = this.connectedPeers.get(peerIdStr); + if (connectedPeerData) { + connectedPeerData.agentVersion = agentVersion; + connectedPeerData.agentClient = getKnownClientFromAgentVersion(agentVersion); + } + } + } catch (e) { + this.logger.debug("Error setting agentVersion for the peer", {peerId: peerIdPretty}, e as Error); + } + } + private async goodbyeAndDisconnect(peer: PeerId, goodbye: GoodByeReasonCode): Promise { const reason = GOODBYE_KNOWN_CODES[goodbye.toString()] || ""; const peerIdStr = peer.toString(); diff --git a/packages/beacon-node/src/network/peers/utils/prioritizePeers.ts b/packages/beacon-node/src/network/peers/utils/prioritizePeers.ts index 057398d19bc2..571af140214b 100644 --- a/packages/beacon-node/src/network/peers/utils/prioritizePeers.ts +++ b/packages/beacon-node/src/network/peers/utils/prioritizePeers.ts @@ -1,4 +1,4 @@ -import {Direction, PeerId} from "@libp2p/interface"; +import type {MessageStreamDirection, PeerId} from "@libp2p/interface"; import {BitArray} from "@chainsafe/ssz"; import {ChainConfig} from "@lodestar/config"; import {ATTESTATION_SUBNET_COUNT, SYNC_COMMITTEE_SUBNET_COUNT} from "@lodestar/params"; @@ -95,7 +95,7 @@ function computeStatusScore(ours: Status, theirs: Status | null, opts: Prioritiz type PeerInfo = { id: PeerId; - direction: Direction | null; + direction: MessageStreamDirection | null; statusScore: StatusScore; attnets: phase0.AttestationSubnets; syncnets: altair.SyncSubnets; @@ -137,7 +137,7 @@ export enum ExcessPeerDisconnectReason { export function prioritizePeers( connectedPeersInfo: { id: PeerId; - direction: Direction | null; + direction: MessageStreamDirection | null; status: Status | null; attnets: phase0.AttestationSubnets | null; syncnets: altair.SyncSubnets | null; diff --git a/packages/beacon-node/src/network/processor/gossipValidatorFn.ts b/packages/beacon-node/src/network/processor/gossipValidatorFn.ts index cf6a7be07be2..2ee57fc693ed 100644 --- a/packages/beacon-node/src/network/processor/gossipValidatorFn.ts +++ b/packages/beacon-node/src/network/processor/gossipValidatorFn.ts @@ -1,4 +1,4 @@ -import {TopicValidatorResult} from "@libp2p/interface"; +import {TopicValidatorResult} from "@libp2p/gossipsub"; import {ChainForkConfig} from "@lodestar/config"; import {Logger} from "@lodestar/utils"; import {AttestationError, GossipAction, GossipActionError} from "../../chain/errors/index.js"; diff --git a/packages/beacon-node/src/network/processor/types.ts b/packages/beacon-node/src/network/processor/types.ts index b6c01a253f2e..49ec7b0243e5 100644 --- a/packages/beacon-node/src/network/processor/types.ts +++ b/packages/beacon-node/src/network/processor/types.ts @@ -1,4 +1,4 @@ -import {Message} from "@libp2p/interface"; +import type {Message} from "@libp2p/gossipsub"; import {ForkName} from "@lodestar/params"; import {Slot, SlotOptionalRoot} from "@lodestar/types"; import {PeerIdStr} from "../../util/peerId.js"; diff --git a/packages/beacon-node/src/network/reqresp/score.ts b/packages/beacon-node/src/network/reqresp/score.ts index 308249ad8629..e9f7cee3607d 100644 --- a/packages/beacon-node/src/network/reqresp/score.ts +++ b/packages/beacon-node/src/network/reqresp/score.ts @@ -38,7 +38,6 @@ export function onOutgoingReqRespError(e: RequestError, method: ReqRespMethod): : PeerAction.LowToleranceError; // TODO: Detect SSZDecodeError and return PeerAction.Fatal - case RequestErrorCode.TTFB_TIMEOUT: case RequestErrorCode.RESP_TIMEOUT: switch (method) { case ReqRespMethod.Ping: diff --git a/packages/beacon-node/src/network/util.ts b/packages/beacon-node/src/network/util.ts index 38a10a2ac052..0a905f2c60c8 100644 --- a/packages/beacon-node/src/network/util.ts +++ b/packages/beacon-node/src/network/util.ts @@ -23,7 +23,7 @@ export function getConnection(libp2p: Libp2p, peerIdStr: string): Connection | u return getConnectionsMap(libp2p).get(peerIdStr)?.value[0] ?? undefined; } -// https://github.com/ChainSafe/js-libp2p-gossipsub/blob/3475242ed254f7647798ab7f36b21909f6cb61da/src/index.ts#L2009 +// https://github.com/libp2p/js-libp2p/blob/f87cba928991736d9646b3e054c367f55cab315c/packages/gossipsub/src/gossipsub.ts#L2076 export function isPublishToZeroPeersError(e: Error): boolean { - return e.message.includes("PublishError.InsufficientPeers"); + return e.message.includes("PublishError.NoPeersSubscribedToTopic"); } diff --git a/packages/beacon-node/src/util/workerEvents.ts b/packages/beacon-node/src/util/workerEvents.ts index 24941bd31699..d67017089763 100644 --- a/packages/beacon-node/src/util/workerEvents.ts +++ b/packages/beacon-node/src/util/workerEvents.ts @@ -1,5 +1,5 @@ import {MessagePort, Worker} from "node:worker_threads"; -import {Message} from "@libp2p/interface"; +import type {Message} from "@libp2p/gossipsub"; import {Thread} from "@chainsafe/threads"; import {Logger} from "@lodestar/logger"; import {sleep} from "@lodestar/utils"; diff --git a/packages/beacon-node/test/e2e/network/onWorker/dataSerialization.test.ts b/packages/beacon-node/test/e2e/network/onWorker/dataSerialization.test.ts index cbef30901e57..3d96df834caf 100644 --- a/packages/beacon-node/test/e2e/network/onWorker/dataSerialization.test.ts +++ b/packages/beacon-node/test/e2e/network/onWorker/dataSerialization.test.ts @@ -1,4 +1,4 @@ -import {TopicValidatorResult} from "@libp2p/interface"; +import {TopicValidatorResult} from "@libp2p/gossipsub"; import {afterAll, beforeAll, describe, expect, it} from "vitest"; import {BitArray} from "@chainsafe/ssz"; import {routes} from "@lodestar/api"; diff --git a/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts b/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts index ae1f0e86946a..2cde954688f6 100644 --- a/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts +++ b/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts @@ -8,6 +8,7 @@ import {phase0, ssz} from "@lodestar/types"; import {sleep} from "@lodestar/utils"; import {Eth2Gossipsub, NetworkEvent, NetworkEventBus, getConnectionsMap} from "../../../../src/network/index.js"; import {NetworkConfig} from "../../../../src/network/networkConfig.js"; +import {ClientKind} from "../../../../src/network/peers/client.js"; import {IReqRespBeaconNodePeerManager, PeerManager, PeerRpcScoreStore} from "../../../../src/network/peers/index.js"; import {PeersData} from "../../../../src/network/peers/peersData.js"; import {ReqRespMethod} from "../../../../src/network/reqresp/ReqRespBeaconNode.js"; @@ -179,7 +180,9 @@ describe("network / peers / PeerManager", () => { direction: "outbound", status: "open", remotePeer: peerId1, - } as Connection; + close: async () => {}, + abort: () => {}, + } as unknown as Connection; it("Should emit peer connected event on relevant peer status", async () => { const {statusCache, libp2p, networkEventBus} = await mockModules(); @@ -259,4 +262,238 @@ describe("network / peers / PeerManager", () => { expect(peerManager["connectedPeers"].get(peerId1.toString())?.metadata).toEqual(remoteMetadata); }); + + it("Should identify peer after successful status", async () => { + const {libp2p, peerManager, statusCache, networkEventBus} = await mockModules(); + + vi.spyOn(libp2p.services.identify, "identify").mockImplementation( + () => Promise.resolve({agentVersion: "Lighthouse/v6.0.1"}) as ReturnType + ); + + const inboundConnection = { + id: "connection-1", + direction: "inbound", + status: "open", + remotePeer: peerId1, + close: async () => {}, + abort: () => {}, + } as unknown as Connection; + + // Connection open does NOT trigger identify + getConnectionsMap(libp2p).set(peerId1.toString(), {key: peerId1, value: [inboundConnection]}); + await peerManager["onLibp2pPeerConnect"](new CustomEvent("evt", {detail: inboundConnection})); + await sleep(0); + expect(libp2p.services.identify.identify).not.toHaveBeenCalled(); + + // Status proves the connection is usable — triggers identify + const remoteStatus = statusCache.get(); + networkEventBus.emit(NetworkEvent.reqRespRequest, { + request: {method: ReqRespMethod.Status, body: remoteStatus}, + peer: peerId1, + peerClient: "Unknown", + }); + await sleep(0); + + expect(libp2p.services.identify.identify).toHaveBeenCalledTimes(1); + const peerData = peerManager["connectedPeers"].get(peerId1.toString()); + expect(peerData?.agentVersion).toBe("Lighthouse/v6.0.1"); + expect(peerData?.agentClient).toBe(ClientKind.Lighthouse); + }); + + it("Should not re-identify after second status if agentVersion is already known", async () => { + const {libp2p, peerManager, statusCache, networkEventBus} = await mockModules(); + + vi.spyOn(libp2p.services.identify, "identify").mockImplementation( + () => Promise.resolve({agentVersion: "Nimbus/v25.0.0"}) as ReturnType + ); + + const inboundConnection = { + id: "connection-1", + direction: "inbound", + status: "open", + remotePeer: peerId1, + close: async () => {}, + abort: () => {}, + } as unknown as Connection; + + getConnectionsMap(libp2p).set(peerId1.toString(), {key: peerId1, value: [inboundConnection]}); + await peerManager["onLibp2pPeerConnect"](new CustomEvent("evt", {detail: inboundConnection})); + + // First status triggers identify + const remoteStatus = statusCache.get(); + networkEventBus.emit(NetworkEvent.reqRespRequest, { + request: {method: ReqRespMethod.Status, body: remoteStatus}, + peer: peerId1, + peerClient: "Unknown", + }); + await sleep(0); + expect(libp2p.services.identify.identify).toHaveBeenCalledTimes(1); + + // Second status should NOT trigger identify again since agentVersion is already known + networkEventBus.emit(NetworkEvent.reqRespRequest, { + request: {method: ReqRespMethod.Status, body: remoteStatus}, + peer: peerId1, + peerClient: "Unknown", + }); + await sleep(0); + expect(libp2p.services.identify.identify).toHaveBeenCalledTimes(1); + + const peerData = peerManager["connectedPeers"].get(peerId1.toString()); + expect(peerData?.agentVersion).toBe("Nimbus/v25.0.0"); + expect(peerData?.agentClient).toBe(ClientKind.Nimbus); + }); + + it("Should deduplicate in-flight identify requests for the same connection", async () => { + const {libp2p, peerManager, statusCache, networkEventBus} = await mockModules(); + + let resolveIdentify!: (value: {agentVersion: string}) => void; + const identifyPromise = new Promise<{agentVersion: string}>((resolve) => { + resolveIdentify = resolve; + }); + + vi.spyOn(libp2p.services.identify, "identify").mockImplementation( + () => identifyPromise as ReturnType + ); + + const connection = { + id: "connection-1", + direction: "inbound", + status: "open", + remotePeer: peerId1, + close: async () => {}, + abort: () => {}, + } as unknown as Connection; + + getConnectionsMap(libp2p).set(peerId1.toString(), {key: peerId1, value: [connection]}); + await peerManager["onLibp2pPeerConnect"](new CustomEvent("evt", {detail: connection})); + + const remoteStatus = statusCache.get(); + networkEventBus.emit(NetworkEvent.reqRespRequest, { + request: {method: ReqRespMethod.Status, body: remoteStatus}, + peer: peerId1, + peerClient: "Unknown", + }); + networkEventBus.emit(NetworkEvent.reqRespRequest, { + request: {method: ReqRespMethod.Status, body: remoteStatus}, + peer: peerId1, + peerClient: "Unknown", + }); + + await sleep(0); + expect(libp2p.services.identify.identify).toHaveBeenCalledTimes(1); + + resolveIdentify({agentVersion: "Prysm/v6.0.0"}); + await sleep(0); + + const peerData = peerManager["connectedPeers"].get(peerId1.toString()); + expect(peerData?.agentVersion).toBe("Prysm/v6.0.0"); + expect(peerData?.agentClient).toBe(ClientKind.Prysm); + }); + + it("Should allow a new identify attempt after reconnect and ignore stale previous result", async () => { + const {libp2p, peerManager, statusCache, networkEventBus} = await mockModules(); + + let resolveFirstIdentify!: (value: {agentVersion: string}) => void; + const firstIdentifyPromise = new Promise<{agentVersion: string}>((resolve) => { + resolveFirstIdentify = resolve; + }); + + vi.spyOn(libp2p.services.identify, "identify") + .mockImplementationOnce(() => firstIdentifyPromise as ReturnType) + .mockImplementationOnce( + () => Promise.resolve({agentVersion: "Teku/v24.9.0"}) as ReturnType + ); + + const connection1 = { + id: "connection-1", + direction: "inbound", + status: "open", + remotePeer: peerId1, + close: async () => {}, + abort: () => {}, + } as unknown as Connection; + + getConnectionsMap(libp2p).set(peerId1.toString(), {key: peerId1, value: [connection1]}); + await peerManager["onLibp2pPeerConnect"](new CustomEvent("evt", {detail: connection1})); + + const remoteStatus = statusCache.get(); + networkEventBus.emit(NetworkEvent.reqRespRequest, { + request: {method: ReqRespMethod.Status, body: remoteStatus}, + peer: peerId1, + peerClient: "Unknown", + }); + await sleep(0); + + const closedConnection1 = {...connection1, status: "closed"} as Connection; + getConnectionsMap(libp2p).set(peerId1.toString(), {key: peerId1, value: [closedConnection1]}); + await peerManager["onLibp2pPeerDisconnect"](new CustomEvent("evt", {detail: closedConnection1})); + + const connection2 = { + id: "connection-2", + direction: "inbound", + status: "open", + remotePeer: peerId1, + close: async () => {}, + abort: () => {}, + } as unknown as Connection; + getConnectionsMap(libp2p).set(peerId1.toString(), {key: peerId1, value: [connection2]}); + await peerManager["onLibp2pPeerConnect"](new CustomEvent("evt", {detail: connection2})); + + networkEventBus.emit(NetworkEvent.reqRespRequest, { + request: {method: ReqRespMethod.Status, body: remoteStatus}, + peer: peerId1, + peerClient: "Unknown", + }); + await sleep(0); + + expect(libp2p.services.identify.identify).toHaveBeenCalledTimes(2); + + // Resolve old identify last; it must not overwrite new connection's identify result. + resolveFirstIdentify({agentVersion: "Lighthouse/v6.0.1"}); + await sleep(0); + + const peerData = peerManager["connectedPeers"].get(peerId1.toString()); + expect(peerData?.agentVersion).toBe("Teku/v24.9.0"); + expect(peerData?.agentClient).toBe(ClientKind.Teku); + }); + + it("Should update agentVersion via peer:identify event even if explicit identify fails", async () => { + const {libp2p, peerManager, statusCache, networkEventBus} = await mockModules(); + + vi.spyOn(libp2p.services.identify, "identify").mockRejectedValue(new Error("Unexpected EOF")); + + const connection = { + id: "connection-1", + direction: "inbound", + status: "open", + remotePeer: peerId1, + close: async () => {}, + abort: () => {}, + } as unknown as Connection; + + getConnectionsMap(libp2p).set(peerId1.toString(), {key: peerId1, value: [connection]}); + await peerManager["onLibp2pPeerConnect"](new CustomEvent("evt", {detail: connection})); + + const remoteStatus = statusCache.get(); + networkEventBus.emit(NetworkEvent.reqRespRequest, { + request: {method: ReqRespMethod.Status, body: remoteStatus}, + peer: peerId1, + peerClient: "Unknown", + }); + await sleep(0); + + libp2p.services.components.events.dispatchEvent( + new CustomEvent("peer:identify", { + detail: { + peerId: peerId1, + agentVersion: "Lighthouse/v6.0.1", + }, + }) + ); + await sleep(0); + + const peerData = peerManager["connectedPeers"].get(peerId1.toString()); + expect(peerData?.agentVersion).toBe("Lighthouse/v6.0.1"); + expect(peerData?.agentClient).toBe(ClientKind.Lighthouse); + }); }); diff --git a/packages/beacon-node/test/e2e/network/reqresp.test.ts b/packages/beacon-node/test/e2e/network/reqresp.test.ts index 154d5f282296..0d68bc1d3760 100644 --- a/packages/beacon-node/test/e2e/network/reqresp.test.ts +++ b/packages/beacon-node/test/e2e/network/reqresp.test.ts @@ -9,7 +9,6 @@ import {sleep} from "@lodestar/utils"; import {Network, ReqRespBeaconNodeOpts} from "../../../src/network/index.js"; import {GetReqRespHandlerFn, ReqRespMethod} from "../../../src/network/reqresp/types.js"; import {PeerIdStr} from "../../../src/util/peerId.js"; -import {arrToSource} from "../../unit/network/reqresp/utils.js"; import {expectRejectedWithLodestarError} from "../../utils/errors.js"; import {connect, getPeerIdOf, onPeerConnect} from "../../utils/network.js"; import {getNetworkForTest} from "../../utils/networkWithMockDb.js"; @@ -186,7 +185,7 @@ function runTests({useWorker}: {useWorker: boolean}): void { (method) => async function* onRequest() { if (method === ReqRespMethod.LightClientUpdatesByRange) { - yield* arrToSource(lightClientUpdates); + yield* lightClientUpdates; } } ); @@ -242,29 +241,28 @@ function runTests({useWorker}: {useWorker: boolean}): void { ); }); - it("should trigger TTFB_TIMEOUT error if first response is delayed", async () => { - const ttfbTimeoutMs = 250; + it("should trigger a RESP_TIMEOUT error if first response is delayed", async () => { + const respTimeoutMs = 250; const [netA, _, _0, peerIdB] = await createAndConnectPeers( (method) => async function* onRequest() { if (method === ReqRespMethod.BeaconBlocksByRange) { // Wait for too long before sending first response chunk - await sleep(ttfbTimeoutMs * 10, controller.signal); + await sleep(respTimeoutMs * 10, controller.signal); yield wrapBlockAsEncodedPayload(config, config.getForkTypes(0).SignedBeaconBlock.defaultValue()); } }, - {ttfbTimeoutMs} + {respTimeoutMs} ); await expectRejectedWithLodestarError( netA.sendBeaconBlocksByRange(peerIdB, {startSlot: 0, step: 1, count: 1}), - new RequestError({code: RequestErrorCode.TTFB_TIMEOUT}) + new RequestError({code: RequestErrorCode.RESP_TIMEOUT}) ); }); - it("should trigger a RESP_TIMEOUT error if first byte is on time but later delayed", async () => { - const ttfbTimeoutMs = 250; + it("should trigger a RESP_TIMEOUT error if later response is delayed", async () => { const respTimeoutMs = 300; const [netA, _, _0, peerIdB] = await createAndConnectPeers( @@ -277,46 +275,7 @@ function runTests({useWorker}: {useWorker: boolean}): void { yield getEmptyEncodedPayloadSignedBeaconBlock(config); } }, - {ttfbTimeoutMs, respTimeoutMs} - ); - - await expectRejectedWithLodestarError( - netA.sendBeaconBlocksByRange(peerIdB, {startSlot: 0, step: 1, count: 2}), - new RequestError({code: RequestErrorCode.RESP_TIMEOUT}) - ); - }); - - it("should trigger TTFB_TIMEOUT error if respTimeoutMs and ttfbTimeoutMs is the same", async () => { - const ttfbTimeoutMs = 250; - const respTimeoutMs = 250; - - const [netA, _, _0, peerIdB] = await createAndConnectPeers( - (method) => - // biome-ignore lint/correctness/useYield: No need for yield in test context - async function* onRequest() { - if (method === ReqRespMethod.BeaconBlocksByRange) { - await sleep(100000000, controller.signal); - } - }, - {respTimeoutMs, ttfbTimeoutMs} - ); - - await expectRejectedWithLodestarError( - netA.sendBeaconBlocksByRange(peerIdB, {startSlot: 0, step: 1, count: 2}), - new RequestError({code: RequestErrorCode.TTFB_TIMEOUT}) - ); - }); - - it("should trigger a RESP_TIMEOUT error if first byte is on time but sleep infinite", async () => { - const [netA, _, _0, peerIdB] = await createAndConnectPeers( - (method) => - async function* onRequest() { - if (method === ReqRespMethod.BeaconBlocksByRange) { - yield getEmptyEncodedPayloadSignedBeaconBlock(config); - await sleep(100000000, controller.signal); - } - }, - {respTimeoutMs: 250, ttfbTimeoutMs: 250} + {respTimeoutMs} ); await expectRejectedWithLodestarError( diff --git a/packages/beacon-node/test/e2e/network/reqrespEncode.test.ts b/packages/beacon-node/test/e2e/network/reqrespEncode.test.ts index 64b128eee389..5d82185666b7 100644 --- a/packages/beacon-node/test/e2e/network/reqrespEncode.test.ts +++ b/packages/beacon-node/test/e2e/network/reqrespEncode.test.ts @@ -1,11 +1,12 @@ import {generateKeyPair} from "@libp2p/crypto/keys"; -import {PrivateKey} from "@libp2p/interface"; +import type {PrivateKey} from "@libp2p/interface"; import {mplex} from "@libp2p/mplex"; import {peerIdFromPrivateKey} from "@libp2p/peer-id"; import {tcp} from "@libp2p/tcp"; -import {Multiaddr, multiaddr} from "@multiformats/multiaddr"; -import all from "it-all"; -import {Libp2p, createLibp2p} from "libp2p"; +import {byteStream} from "@libp2p/utils"; +import type {Multiaddr} from "@multiformats/multiaddr"; +import type {Libp2p} from "libp2p"; +import {createLibp2p} from "libp2p"; import {afterEach, describe, expect, it} from "vitest"; import {noise} from "@chainsafe/libp2p-noise"; import {createBeaconConfig} from "@lodestar/config"; @@ -30,8 +31,6 @@ import {CustodyConfig} from "../../../src/util/dataColumns.js"; import {testLogger} from "../../utils/logger.js"; describe("reqresp encoder", () => { - let port = 60000; - const afterEachCallbacks: (() => Promise | void)[] = []; afterEach(async () => { while (afterEachCallbacks.length > 0) { @@ -41,18 +40,20 @@ describe("reqresp encoder", () => { }); async function getLibp2p(privateKey?: PrivateKey) { - const listen = `/ip4/127.0.0.1/tcp/${port++}`; const libp2p = await createLibp2p({ privateKey, transports: [tcp()], - streamMuxers: [mplex()], + // Increase disconnectThreshold to prevent mplex from closing the connection + // when it receives messages for already-closed streams + streamMuxers: [mplex({disconnectThreshold: Infinity})], connectionEncrypters: [noise()], addresses: { - listen: [listen], + listen: ["/ip4/127.0.0.1/tcp/0"], }, }); afterEachCallbacks.push(() => libp2p.stop()); - return {libp2p, multiaddr: multiaddr(`${listen}/p2p/${libp2p.peerId.toString()}`)}; + const listenMultiaddr = libp2p.getMultiaddrs()[0]; + return {libp2p, multiaddr: listenMultiaddr}; } async function getReqResp(getHandler?: GetReqRespHandlerFn) { @@ -107,17 +108,33 @@ describe("reqresp encoder", () => { expectedChunks: string[]; }) { const stream = await dialer.dialProtocol(toMultiaddr, protocol); + // Use byteStream to read response - it attaches event listeners immediately, + // avoiding race conditions with the async iterator where remoteCloseWrite + // events can be lost if the server responds in the same macrotask + const bytes = byteStream(stream); + if (requestChunks) { - await stream.sink(requestChunks.map(fromHex)); + for (const chunk of requestChunks) { + await bytes.write(fromHex(chunk)); + } + } + + const chunks: Uint8Array[] = []; + while (true) { + const chunk = await bytes.read({signal: AbortSignal.timeout(2000)}); + if (chunk === null) break; + chunks.push(chunk.subarray()); } - const chunks = await all(stream.source); + // Abort for fast cleanup instead of graceful close which can be slow + stream.abort(new Error("test done")); + const join = (c: string[]): string => c.join("").replace(/0x/g, ""); - const chunksHex = chunks.map((chunk) => toHex(chunk.slice(0, chunk.byteLength))); + const chunksHex = chunks.map((chunk) => toHex(chunk)); expect(join(chunksHex)).toEqual(join(expectedChunks)); } - it("assert correct handler switch between metadata v2 and v1", async () => { + it("assert correct handler for metadata v3", async () => { const {multiaddr: serverMultiaddr, reqresp} = await getReqResp(); reqresp.registerProtocolsAtBoundary({fork: ForkName.phase0, epoch: GENESIS_EPOCH}); await sleep(0); // Sleep to resolve register handler promises @@ -126,20 +143,34 @@ describe("reqresp encoder", () => { reqresp["metadataController"].attnets.set(8, true); reqresp["metadataController"].syncnets.set(1, true); - const privateKey = await generateKeyPair("secp256k1"); - const {libp2p: dialer} = await getLibp2p(privateKey); + const {libp2p: dialer} = await getLibp2p(); await dialProtocol({ dialer, toMultiaddr: serverMultiaddr, - protocol: "/eth2/beacon_chain/req/metadata/1/ssz_snappy", - expectedChunks: ["0x00", "0x10", "0xff060000734e615070590114000077b18d3800000000000000000101000000000000"], + protocol: "/eth2/beacon_chain/req/metadata/3/ssz_snappy", + expectedChunks: [ + "0x00", + "0x19", + "0xff060000734e61507059001b000082e4dd0e1900000d01400101000000000000020400000000000000", + ], }); + }); + + it("assert correct handler for metadata v1", async () => { + const {multiaddr: serverMultiaddr, reqresp} = await getReqResp(); + reqresp.registerProtocolsAtBoundary({fork: ForkName.phase0, epoch: GENESIS_EPOCH}); + await sleep(0); // Sleep to resolve register handler promises + + reqresp["metadataController"].attnets.set(0, true); + reqresp["metadataController"].attnets.set(8, true); + reqresp["metadataController"].syncnets.set(1, true); + const {libp2p: dialer} = await getLibp2p(); await dialProtocol({ dialer, toMultiaddr: serverMultiaddr, - protocol: "/eth2/beacon_chain/req/metadata/2/ssz_snappy", - expectedChunks: ["0x00", "0x11", "0xff060000734e615070590013000080f865931100000d0120010100000000000002"], + protocol: "/eth2/beacon_chain/req/metadata/1/ssz_snappy", + expectedChunks: ["0x00", "0x10", "0xff060000734e615070590114000077b18d3800000000000000000101000000000000"], }); }); diff --git a/packages/beacon-node/test/perf/network/noise/sendData.test.ts b/packages/beacon-node/test/perf/network/noise/sendData.test.ts index 804d2416d620..62c6f8d0949d 100644 --- a/packages/beacon-node/test/perf/network/noise/sendData.test.ts +++ b/packages/beacon-node/test/perf/network/noise/sendData.test.ts @@ -1,11 +1,8 @@ import {generateKeyPair} from "@libp2p/crypto/keys"; -import {Upgrader} from "@libp2p/interface"; +import type {Upgrader} from "@libp2p/interface"; import {defaultLogger} from "@libp2p/logger"; import {peerIdFromPrivateKey} from "@libp2p/peer-id"; -import drain from "it-drain"; -import {duplexPair} from "it-pair/duplex"; -import {pipe} from "it-pipe"; -import {Uint8ArrayList} from "uint8arraylist"; +import {streamPair} from "@libp2p/utils"; import {bench, describe} from "@chainsafe/benchmark"; import {noise} from "@chainsafe/libp2p-noise"; @@ -34,24 +31,29 @@ describe("network / noise / sendData", () => { const noiseA = noise()({logger: defaultLogger(), privateKey: privateKeyA, peerId: peerA, upgrader}); const noiseB = noise()({logger: defaultLogger(), privateKey: privateKeyB, peerId: peerB, upgrader}); - const [inboundConnection, outboundConnection] = duplexPair(); + const [outboundConnection, inboundConnection] = await streamPair(); const [outbound, inbound] = await Promise.all([ noiseA.secureOutbound(outboundConnection, {remotePeer: peerB}), noiseB.secureInbound(inboundConnection, {remotePeer: peerA}), ]); - return {connA: outbound.conn, connB: inbound.conn, data: new Uint8Array(messageLength)}; + return {connA: outbound.connection, connB: inbound.connection, data: new Uint8Array(messageLength)}; }, fn: async ({connA, connB, data}) => { await Promise.all([ - // - pipe(connB.source, connB.sink), - pipe(function* () { + (async () => { for (let i = 0; i < numberOfMessages; i++) { - yield data; + if (!connA.send(data)) { + await connA.onDrain(); + } } - }, connA.sink), - pipe(connB.source, drain), + await connA.close(); + })(), + (async () => { + for await (const _chunk of connB) { + // Drain inbound messages + } + })(), ]); }, }); diff --git a/packages/beacon-node/test/unit/db/api/repository.test.ts b/packages/beacon-node/test/unit/db/api/repository.test.ts index 3bed3675456d..36da34839947 100644 --- a/packages/beacon-node/test/unit/db/api/repository.test.ts +++ b/packages/beacon-node/test/unit/db/api/repository.test.ts @@ -1,4 +1,3 @@ -import all from "it-all"; import {MockedObject, afterEach, beforeEach, describe, expect, it, vi} from "vitest"; import {ContainerType} from "@chainsafe/ssz"; import {config} from "@lodestar/config/default"; @@ -145,7 +144,7 @@ describe("database repository", () => { } controller.valuesStream.mockReturnValue(sample()); - const result = await all(repository.valuesStream()); + const result = await Array.fromAsync(repository.valuesStream()); expect(result.length).toBe(2); }); }); diff --git a/packages/beacon-node/test/unit/network/gossip/scoringParameters.test.ts b/packages/beacon-node/test/unit/network/gossip/scoringParameters.test.ts index 168e4f346d67..aa05f51a96c4 100644 --- a/packages/beacon-node/test/unit/network/gossip/scoringParameters.test.ts +++ b/packages/beacon-node/test/unit/network/gossip/scoringParameters.test.ts @@ -1,5 +1,5 @@ +import type {TopicScoreParams} from "@libp2p/gossipsub/score"; import {describe, expect, it} from "vitest"; -import {TopicScoreParams} from "@chainsafe/libp2p-gossipsub/score"; import {createBeaconConfig} from "@lodestar/config"; import {mainnetChainConfig} from "@lodestar/config/configs"; import {ATTESTATION_SUBNET_COUNT, ForkName, GENESIS_EPOCH, SLOTS_PER_EPOCH} from "@lodestar/params"; diff --git a/packages/beacon-node/test/unit/network/reqresp/utils.ts b/packages/beacon-node/test/unit/network/reqresp/utils.ts index 614789f8916c..9b483bd9284d 100644 --- a/packages/beacon-node/test/unit/network/reqresp/utils.ts +++ b/packages/beacon-node/test/unit/network/reqresp/utils.ts @@ -1,5 +1,4 @@ -import {Direction, ReadStatus, Stream, StreamStatus, WriteStatus} from "@libp2p/interface"; -import {logger} from "@libp2p/logger"; +import type {Stream} from "@libp2p/interface"; import {Uint8ArrayList} from "uint8arraylist"; import {expect} from "vitest"; import {toHexString} from "@chainsafe/ssz"; @@ -13,16 +12,6 @@ export function generateRoots(count: number, offset = 0): Root[] { return roots; } -/** - * Helper for it-pipe when first argument is an array. - * it-pipe does not convert the chunks array to a generator and BufferedSource breaks - */ -export async function* arrToSource(arr: T[]): AsyncGenerator { - for (const item of arr) { - yield item; - } -} - /** * Wrapper for type-safety to ensure and array of Buffers is equal with a diff in hex */ @@ -30,34 +19,44 @@ export function expectEqualByteChunks(chunks: Uint8Array[], expectedChunks: Uint expect(chunks.map(toHexString)).toEqual(expectedChunks.map(toHexString)); } +type SourceChunk = Uint8Array | Uint8ArrayList; + +function toUint8ArrayList(chunk: SourceChunk): Uint8ArrayList { + return chunk instanceof Uint8ArrayList ? chunk : new Uint8ArrayList(chunk); +} + +function toUint8Array(chunk: SourceChunk): Uint8Array { + return chunk instanceof Uint8ArrayList ? chunk.subarray() : chunk; +} + /** - * Useful to simulate a LibP2P stream source emitting prepared bytes - * and capture the response with a sink accessible via `this.resultChunks` + * Minimal stream test double for reqresp unit tests. + * It captures sent chunks and yields a provided source for reads. */ -export class MockLibP2pStream implements Stream { - id = "mock"; - log = logger("mock"); - direction: Direction = "inbound"; - timeline = { - open: Date.now(), - }; - status: StreamStatus = "open"; - readStatus: ReadStatus = "ready"; - writeStatus: WriteStatus = "ready"; - metadata = {}; - source: Stream["source"]; - resultChunks: Uint8Array[] = []; - - constructor(requestChunks: Uint8ArrayList[]) { - this.source = arrToSource(requestChunks); - } - sink: Stream["sink"] = async (source) => { - for await (const chunk of source) { - this.resultChunks.push(chunk.subarray()); - } - }; - close: Stream["close"] = async () => {}; - closeRead = async (): Promise => {}; - closeWrite = async (): Promise => {}; - abort: Stream["abort"] = () => this.close(); +export function createMockStream({ + protocol = "", + source = (async function* (): AsyncIterable {})(), +}: { + protocol?: string; + source?: AsyncIterable; +} = {}): {stream: Stream; sentChunks: Uint8Array[]} { + const sentChunks: Uint8Array[] = []; + + const stream = { + protocol, + send(chunk: SourceChunk): boolean { + sentChunks.push(toUint8Array(chunk)); + return true; + }, + async onDrain(): Promise {}, + async close(): Promise {}, + abort(): void {}, + async *[Symbol.asyncIterator](): AsyncGenerator { + for await (const chunk of source) { + yield toUint8ArrayList(chunk); + } + }, + } as unknown as Stream; + + return {stream, sentChunks}; } diff --git a/packages/beacon-node/test/unit/util/error.test.ts b/packages/beacon-node/test/unit/util/error.test.ts index 02e99df0cac3..37b4edec010b 100644 --- a/packages/beacon-node/test/unit/util/error.test.ts +++ b/packages/beacon-node/test/unit/util/error.test.ts @@ -9,7 +9,7 @@ function structuredClone(value: T): T { describe("ThreadBoundaryError", () => { it("should clone RequestError through thread boundary", () => { - const requestError = new RequestError({code: RequestErrorCode.TTFB_TIMEOUT}); + const requestError = new RequestError({code: RequestErrorCode.RESP_TIMEOUT}); const threadBoundaryError = toThreadBoundaryError(requestError); const clonedError = structuredClone(threadBoundaryError); expect(clonedError.error).toBeNull(); diff --git a/packages/beacon-node/test/unit/util/itTrigger.test.ts b/packages/beacon-node/test/unit/util/itTrigger.test.ts index ff76cf13eafa..baf5f0f69181 100644 --- a/packages/beacon-node/test/unit/util/itTrigger.test.ts +++ b/packages/beacon-node/test/unit/util/itTrigger.test.ts @@ -1,4 +1,3 @@ -import all from "it-all"; import {describe, expect, it} from "vitest"; import {ItTrigger} from "../../../src/util/itTrigger.js"; @@ -10,7 +9,7 @@ describe("util / itTrigger", () => { itTrigger.trigger(); itTrigger.end(); - const res = await all(itTrigger); + const res = await Array.fromAsync(itTrigger); expect(res).toHaveLength(0); }); @@ -27,7 +26,7 @@ describe("util / itTrigger", () => { }, 5); }, 5); - const res = await all(itTrigger); + const res = await Array.fromAsync(itTrigger); expect(res).toHaveLength(2); }); @@ -43,7 +42,7 @@ describe("util / itTrigger", () => { }, 5); }, 5); - await expect(all(itTrigger)).rejects.toThrow(testError); + await expect(Array.fromAsync(itTrigger)).rejects.toThrow(testError); }); it("ItTrigger as a single thread processor", async () => { diff --git a/packages/beacon-node/test/utils/node/p2p.ts b/packages/beacon-node/test/utils/node/p2p.ts index 1a6e1d987b29..c4399f65f8c0 100644 --- a/packages/beacon-node/test/utils/node/p2p.ts +++ b/packages/beacon-node/test/utils/node/p2p.ts @@ -1,10 +1,10 @@ -import {Direction, PeerId} from "@libp2p/interface"; +import type {MessageStreamDirection, PeerId} from "@libp2p/interface"; import {routes} from "@lodestar/api"; export function lodestarNodePeer( peer: PeerId, state: routes.node.PeerState, - direction: Direction | null + direction: MessageStreamDirection | null ): routes.lodestar.LodestarNodePeer { return { peerId: peer.toString(), diff --git a/packages/beacon-node/tsconfig.json b/packages/beacon-node/tsconfig.json index d1c8b717b54f..a0f4f2a31e93 100644 --- a/packages/beacon-node/tsconfig.json +++ b/packages/beacon-node/tsconfig.json @@ -1,6 +1,5 @@ { "extends": "../../tsconfig.json", - "exclude": ["../../node_modules/it-pipe"], "include": ["src", "test"], "compilerOptions": { "outDir": "lib" diff --git a/packages/cli/package.json b/packages/cli/package.json index de708f1b7f97..ef7608b55bc8 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -60,15 +60,15 @@ "@chainsafe/bls-keygen": "^0.4.0", "@chainsafe/bls-keystore": "^3.1.0", "@chainsafe/blst": "^2.2.0", - "@chainsafe/discv5": "^11.0.4", - "@chainsafe/enr": "^5.0.1", + "@chainsafe/discv5": "^12.0.0", + "@chainsafe/enr": "^6.0.0", "@chainsafe/persistent-merkle-tree": "^1.2.1", "@chainsafe/pubkey-index-map": "^3.0.0", "@chainsafe/ssz": "^1.2.2", "@chainsafe/threads": "^1.11.3", - "@libp2p/crypto": "^5.0.15", - "@libp2p/interface": "^2.7.0", - "@libp2p/peer-id": "^5.1.0", + "@libp2p/crypto": "^5.1.13", + "@libp2p/interface": "^3.1.0", + "@libp2p/peer-id": "^6.0.4", "@lodestar/api": "workspace:^", "@lodestar/beacon-node": "workspace:^", "@lodestar/config": "workspace:^", @@ -80,7 +80,7 @@ "@lodestar/types": "workspace:^", "@lodestar/utils": "workspace:^", "@lodestar/validator": "workspace:^", - "@multiformats/multiaddr": "^12.1.3", + "@multiformats/multiaddr": "^13.0.1", "deepmerge": "^4.3.1", "ethers": "^6.7.0", "find-up": "^6.3.0", diff --git a/packages/cli/src/cmds/beacon/initPeerIdAndEnr.ts b/packages/cli/src/cmds/beacon/initPeerIdAndEnr.ts index 4d54643cffe9..fa833a194179 100644 --- a/packages/cli/src/cmds/beacon/initPeerIdAndEnr.ts +++ b/packages/cli/src/cmds/beacon/initPeerIdAndEnr.ts @@ -18,27 +18,19 @@ import {BeaconArgs} from "./options.js"; export function isLocalMultiAddr(multiaddr: Multiaddr | undefined): boolean { if (!multiaddr) return false; - const protoNames = multiaddr.protoNames(); - if (protoNames.length !== 2 && protoNames[1] !== "udp") { + const components = multiaddr.getComponents(); + if (components.length !== 2 && components[1].name !== "udp") { throw new Error("Invalid udp multiaddr"); } const interfaces = os.networkInterfaces(); - const tuples = multiaddr.tuples(); - const family = tuples[0][0]; - const isIPv4: boolean = family === 4; - const ip = tuples[0][1]; + const family = components[0].name === "ip4" ? 4 : 6; + const ipStr = components[0].value; - if (!ip) { + if (!ipStr) { return false; } - const ipStr = isIPv4 - ? Array.from(ip).join(".") - : Array.from(Uint16Array.from(ip)) - .map((n) => n.toString(16)) - .join(":"); - for (const networkInterfaces of Object.values(interfaces)) { for (const networkInterface of networkInterfaces || []) { // since node version 18, the netowrkinterface family returns 4 | 6 instead of ipv4 | ipv6, diff --git a/packages/cli/src/cmds/bootnode/handler.ts b/packages/cli/src/cmds/bootnode/handler.ts index 639e022bea2c..47ba4daeb707 100644 --- a/packages/cli/src/cmds/bootnode/handler.ts +++ b/packages/cli/src/cmds/bootnode/handler.ts @@ -2,7 +2,7 @@ import path from "node:path"; import {PrivateKey} from "@libp2p/interface"; import {Multiaddr, multiaddr} from "@multiformats/multiaddr"; import {Discv5, Discv5EventEmitter} from "@chainsafe/discv5"; -import {ENR, ENRData, SignableENR} from "@chainsafe/enr"; +import {ENR, SignableENR} from "@chainsafe/enr"; import { HttpMetricsServer, IBeaconNodeOptions, @@ -87,7 +87,7 @@ export async function bootnodeHandler(args: BootnodeArgs & GlobalArgs): Promise< void discv5.findRandomNode(); } - discv5.on("multiaddrUpdated", (addr: ENRData) => { + discv5.on("multiaddrUpdated", (addr: Multiaddr) => { logger.info("Advertised socket address updated", {addr: addr.toString()}); }); diff --git a/packages/db/package.json b/packages/db/package.json index dba1d1c5af7d..fabcf0365e35 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -46,8 +46,7 @@ "@chainsafe/ssz": "^1.2.2", "@lodestar/config": "workspace:^", "@lodestar/utils": "workspace:^", - "classic-level": "^1.4.1", - "it-all": "^3.0.4" + "classic-level": "^1.4.1" }, "devDependencies": { "@lodestar/logger": "workspace:^" diff --git a/packages/db/test/unit/controller/level.test.ts b/packages/db/test/unit/controller/level.test.ts index 9bd0b68f6dbd..f265e562c149 100644 --- a/packages/db/test/unit/controller/level.test.ts +++ b/packages/db/test/unit/controller/level.test.ts @@ -1,6 +1,5 @@ import {execSync} from "node:child_process"; import os from "node:os"; -import all from "it-all"; import {afterEach, beforeEach, describe, expect, it} from "vitest"; import {getEnvLogger} from "@lodestar/logger/env"; import {LevelDbController} from "../../../src/index.js"; @@ -168,7 +167,7 @@ describe("LevelDB controller", () => { gte: k1, lte: k2, }); - const result = await all(resultStream); + const result = await Array.fromAsync(resultStream); expect(result.length).toBe(2); }); @@ -180,7 +179,7 @@ describe("LevelDB controller", () => { const result = await db.entries({limit: 3}); expect(result.length).toBe(3); const resultStream = db.entriesStream({limit: 3}); - expect((await all(resultStream)).length).toBe(3); + expect((await Array.fromAsync(resultStream)).length).toBe(3); }); it("test reverse", async () => { diff --git a/packages/era/src/util.ts b/packages/era/src/util.ts index d8afcc4e5540..02df02258162 100644 --- a/packages/era/src/util.ts +++ b/packages/era/src/util.ts @@ -1,5 +1,4 @@ -import {Uint8ArrayList} from "uint8arraylist"; -import {SnappyFramesUncompress, encodeSnappy} from "@lodestar/reqresp/utils"; +import {decodeSnappyFrames, encodeSnappy} from "@lodestar/reqresp/utils"; /** Read 48-bit signed integer (little-endian) at offset. */ export function readInt48(bytes: Uint8Array, offset: number): number { @@ -38,16 +37,7 @@ export function writeUint32(target: Uint8Array, offset: number, v: number): void /** Decompress snappy-framed data */ export function snappyUncompress(compressedData: Uint8Array): Uint8Array { - const decompressor = new SnappyFramesUncompress(); - - const input = new Uint8ArrayList(compressedData); - const result = decompressor.uncompress(input); - - if (result === null) { - throw new Error("Snappy decompression failed - no data returned"); - } - - return result.subarray(); + return decodeSnappyFrames(compressedData).subarray(); } /** Compress data using snappy framing */ diff --git a/packages/reqresp/README.md b/packages/reqresp/README.md index b28d9641c43f..87aafef978b4 100644 --- a/packages/reqresp/README.md +++ b/packages/reqresp/README.md @@ -7,7 +7,7 @@ > This package is part of [ChainSafe's Lodestar](https://lodestar.chainsafe.io) project -Typescript implementation of the [Ethereum Consensus Req/Resp protocol](https://github.com/ethereum/consensus-specs/blob/v1.4.0/specs/phase0/p2p-interface.md#reqresp) +Typescript implementation of the [Ethereum Consensus Req/Resp protocol](https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/p2p-interface.md#reqresp) ## Usage diff --git a/packages/reqresp/package.json b/packages/reqresp/package.json index 174de78130e3..f72fabd5a7f3 100644 --- a/packages/reqresp/package.json +++ b/packages/reqresp/package.json @@ -45,12 +45,11 @@ }, "dependencies": { "@chainsafe/fast-crc32c": "^4.2.0", - "@libp2p/interface": "^2.7.0", + "@libp2p/interface": "^3.1.0", + "@libp2p/utils": "^7.0.10", "@lodestar/config": "workspace:^", "@lodestar/params": "workspace:^", "@lodestar/utils": "workspace:^", - "it-all": "^3.0.4", - "it-pipe": "^3.0.1", "snappy": "^7.2.2", "snappyjs": "^0.7.0", "uint8-varint": "^2.0.2", @@ -58,16 +57,15 @@ }, "devDependencies": { "@chainsafe/ssz": "^1.2.2", - "@libp2p/crypto": "^5.1.7", - "@libp2p/logger": "^5.1.21", - "@libp2p/peer-id": "^5.1.8", + "@libp2p/crypto": "^5.1.13", + "@libp2p/logger": "^6.2.2", + "@libp2p/peer-id": "^6.0.4", "@lodestar/logger": "workspace:^", "@lodestar/types": "workspace:^", - "it-stream-types": "^2.0.2", - "libp2p": "2.9.0" + "libp2p": "3.1.3" }, "peerDependencies": { - "libp2p": "~2.9.0" + "libp2p": "^3.1.3" }, "keywords": [ "ethereum", diff --git a/packages/reqresp/src/ReqResp.ts b/packages/reqresp/src/ReqResp.ts index 6e1115041fca..756a735cb0f6 100644 --- a/packages/reqresp/src/ReqResp.ts +++ b/packages/reqresp/src/ReqResp.ts @@ -37,8 +37,8 @@ export interface ReqRespOpts extends SendRequestOpts, ReqRespRateLimiterOpts { /** * Implementation of Ethereum Consensus p2p Req/Resp domain. * For the spec that this code is based on, see: - * https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/p2p-interface.md#the-reqresp-domain - * https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/p2p-interface.md#the-reqresp-domain + * https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/p2p-interface.md#the-reqresp-domain + * https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/altair/light-client/p2p-interface.md#the-reqresp-domain */ export class ReqResp { // protected to be usable by extending class @@ -221,7 +221,7 @@ export class ReqResp { } private getRequestHandler(protocol: MixedProtocol, protocolID: string) { - return async ({connection, stream}: {connection: Connection; stream: Stream}) => { + return async (stream: Stream, connection: Connection) => { if (this.dialOnlyProtocols.get(protocolID)) { throw new Error(`Received request on dial only protocol '${protocolID}'`); } @@ -281,7 +281,7 @@ export class ReqResp { * ``` * /ProtocolPrefix/MessageName/SchemaVersion/Encoding * ``` - * https://github.com/ethereum/consensus-specs/blob/v1.2.0/specs/phase0/p2p-interface.md#protocol-identification + * https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/p2p-interface.md#protocol-identification */ protected formatProtocolID(protocol: Pick): string { return formatProtocolID(this.protocolPrefix, protocol.method, protocol.version, protocol.encoding); diff --git a/packages/reqresp/src/encoders/requestDecode.ts b/packages/reqresp/src/encoders/requestDecode.ts index d50f1ee37b5b..1fb102a2a4f5 100644 --- a/packages/reqresp/src/encoders/requestDecode.ts +++ b/packages/reqresp/src/encoders/requestDecode.ts @@ -1,8 +1,8 @@ -import type {Sink} from "it-stream-types"; -import {Uint8ArrayList} from "uint8arraylist"; +import type {Stream} from "@libp2p/interface"; +import {byteStream} from "@libp2p/utils"; import {readEncodedPayload} from "../encodingStrategies/index.js"; import {MixedProtocol} from "../types.js"; -import {BufferedSource} from "../utils/index.js"; +import {drainByteStream} from "../utils/stream.ts"; const EMPTY_DATA = new Uint8Array(); @@ -12,18 +12,34 @@ const EMPTY_DATA = new Uint8Array(); * request ::= | * ``` */ -export function requestDecode( - protocol: MixedProtocol -): Sink, Promise> { - return async function requestDecodeSink(source) { - const type = protocol.requestSizes; - if (type === null) { - // method has no body - return EMPTY_DATA; - } +export async function requestDecode( + protocol: MixedProtocol, + stream: Stream, + signal?: AbortSignal +): Promise { + const type = protocol.requestSizes; + if (type === null) { + // method has no body + return EMPTY_DATA; + } - // Request has a single payload, so return immediately - const bufferedSource = new BufferedSource(source as AsyncGenerator); - return readEncodedPayload(bufferedSource, protocol.encoding, type); - }; + // Request has a single payload, so return immediately + const bytes = byteStream(stream); + let requestReadDone = false; + try { + const requestBody = await readEncodedPayload(bytes, protocol.encoding, type, signal); + requestReadDone = true; + return requestBody; + } finally { + try { + if (!requestReadDone) { + // Do not push partial bytes back into the stream on decode failure/abort. + // This stream is consumed by req/resp only once. + drainByteStream(bytes); + } + bytes.unwrap(); + } catch { + // Ignore unwrap errors - stream may already be closed + } + } } diff --git a/packages/reqresp/src/encoders/requestEncode.ts b/packages/reqresp/src/encoders/requestEncode.ts index 752924c10118..76f9ed262ef9 100644 --- a/packages/reqresp/src/encoders/requestEncode.ts +++ b/packages/reqresp/src/encoders/requestEncode.ts @@ -9,7 +9,7 @@ import {MixedProtocol} from "../types.js"; * Requests may contain no payload (e.g. /eth2/beacon_chain/req/metadata/1/) * if so, it would yield no byte chunks */ -export async function* requestEncode(protocol: MixedProtocol, requestBody: Uint8Array): AsyncGenerator { +export function* requestEncode(protocol: MixedProtocol, requestBody: Uint8Array): Generator { const type = protocol.requestSizes; if (type && requestBody !== null) { diff --git a/packages/reqresp/src/encoders/responseDecode.ts b/packages/reqresp/src/encoders/responseDecode.ts index affe77a15134..ed8a2a2ee91e 100644 --- a/packages/reqresp/src/encoders/responseDecode.ts +++ b/packages/reqresp/src/encoders/responseDecode.ts @@ -1,4 +1,6 @@ -import {Uint8ArrayList} from "uint8arraylist"; +import type {Stream} from "@libp2p/interface"; +import type {ByteStream} from "@libp2p/utils"; +import {byteStream} from "@libp2p/utils"; import {ForkName} from "@lodestar/params"; import {readEncodedPayload} from "../encodingStrategies/index.js"; import {RespStatus} from "../interface.js"; @@ -10,7 +12,7 @@ import { MixedProtocol, ResponseIncoming, } from "../types.js"; -import {BufferedSource, decodeErrorMessage} from "../utils/index.js"; +import {decodeErrorMessage, drainByteStream} from "../utils/index.js"; /** * Internal helper type to signal stream ended early @@ -27,22 +29,17 @@ enum StreamStatus { * result ::= "0" | "1" | "2" | ["128" ... "255"] * ``` */ -export function responseDecode( +export async function* responseDecode( protocol: MixedProtocol, - cbs: { - onFirstHeader: () => void; - onFirstResponseChunk: () => void; - } -): (source: AsyncIterable) => AsyncIterable { - return async function* responseDecodeSink(source) { - const bufferedSource = new BufferedSource(source as AsyncGenerator); - - let readFirstHeader = false; - let readFirstResponseChunk = false; + stream: Stream, + opts: {signal?: AbortSignal; getError?: () => Error} = {} +): AsyncIterable { + const bytes = byteStream(stream); + let responseReadDone = false; - // Consumers of `responseDecode()` may limit the number of and break out of the while loop - while (!bufferedSource.isDone) { - const status = await readResultHeader(bufferedSource); + try { + while (true) { + const status = await readResultHeader(bytes, opts.signal); // Stream is only allowed to end at the start of a block // The happens when source ends before readResultHeader() can fetch 1 byte @@ -50,34 +47,41 @@ export function responseDecode( break; } - if (!readFirstHeader) { - cbs.onFirstHeader(); - readFirstHeader = true; - } - // For multiple chunks, only the last chunk is allowed to have a non-zero error // code (i.e. The chunk stream is terminated once an error occurs if (status !== RespStatus.SUCCESS) { - const errorMessage = await readErrorMessage(bufferedSource); + const errorMessage = await readErrorMessage(bytes, opts.signal); throw new ResponseError(status, errorMessage); } - const forkName = await readContextBytes(protocol.contextBytes, bufferedSource); + const forkName = await readContextBytes(protocol.contextBytes, bytes, opts.signal); const typeSizes = protocol.responseSizes(forkName); - const chunkData = await readEncodedPayload(bufferedSource, protocol.encoding, typeSizes); + const chunkData = await readEncodedPayload(bytes, protocol.encoding, typeSizes, opts.signal); yield { data: chunkData, fork: forkName, protocolVersion: protocol.version, }; - - if (!readFirstResponseChunk) { - cbs.onFirstResponseChunk(); - readFirstResponseChunk = true; + } + responseReadDone = true; + } catch (e) { + if (opts.signal?.aborted && opts.getError) { + throw opts.getError(); + } + throw e; + } finally { + try { + if (!responseReadDone) { + // Do not push partial bytes back into the stream on decode failure/abort. + // This stream is consumed by req/resp only once. + drainByteStream(bytes); } + bytes.unwrap(); + } catch { + // Ignore unwrap errors - stream may already be closed } - }; + } } /** @@ -87,18 +91,17 @@ export function responseDecode( * ``` * `` starts with a single-byte response code which determines the contents of the response_chunk */ -export async function readResultHeader(bufferedSource: BufferedSource): Promise { - for await (const buffer of bufferedSource) { - const status = buffer.get(0); - buffer.consume(1); - - // If first chunk had zero bytes status === null, get next - if (status !== null) { - return status; - } - } - - return StreamStatus.Ended; +export async function readResultHeader( + bytes: ByteStream, + signal?: AbortSignal +): Promise { + const chunk = await bytes.read({bytes: 1, signal}).catch((e) => { + if ((e as Error).name === "UnexpectedEOFError") return null; + throw e; + }); + if (chunk === null) return StreamStatus.Ended; + + return chunk.get(0); } /** @@ -108,28 +111,29 @@ export async function readResultHeader(bufferedSource: BufferedSource): Promise< * result ::= "1" | "2" | ["128" ... "255"] * ``` */ -export async function readErrorMessage(bufferedSource: BufferedSource): Promise { - // Read at least 256 or wait for the stream to end - let length: number | undefined; - for await (const buffer of bufferedSource) { - // Wait for next chunk with bytes or for the stream to end - // Note: The entire is expected to be in the same chunk - if (buffer.length >= 256) { - length = 256; +export async function readErrorMessage(bytes: ByteStream, signal?: AbortSignal): Promise { + const chunks: Uint8Array[] = []; + let total = 0; + + while (total < 256) { + const chunk = await bytes.read({signal}).catch((e) => { + if ((e as Error).name === "UnexpectedEOFError") return null; + throw e; + }); + if (chunk === null) { + // If EOF is reached while satisfying a larger read, libp2p v3 may still have + // buffered bytes available. Drain them so error_message matches pre-v3 behavior. + const remaining = drainByteStream(bytes); + if (remaining) { + chunks.push(remaining); + } break; } - length = buffer.length; + chunks.push(chunk.subarray()); + total += chunk.byteLength; } - // biome-ignore lint/complexity/useLiteralKeys: It is a private attribute - const bytes = bufferedSource["buffer"].slice(0, length); - - try { - return decodeErrorMessage(bytes); - } catch (_e) { - // Error message is optional and may not be included in the response stream - return Buffer.prototype.toString.call(bytes, "hex"); - } + return decodeErrorMessage(Buffer.concat(chunks).subarray(0, 256)); } /** @@ -139,14 +143,15 @@ export async function readErrorMessage(bufferedSource: BufferedSource): Promise< */ export async function readContextBytes( contextBytes: ContextBytesFactory, - bufferedSource: BufferedSource + bytes: ByteStream, + signal?: AbortSignal ): Promise { switch (contextBytes.type) { case ContextBytesType.Empty: return ForkName.phase0; case ContextBytesType.ForkDigest: { - const forkDigest = await readContextBytesForkDigest(bufferedSource); + const forkDigest = await readContextBytesForkDigest(bytes, signal); return contextBytes.config.forkDigest2ForkBoundary(forkDigest).fork; } } @@ -155,15 +160,6 @@ export async function readContextBytes( /** * Consumes a stream source to read ``, where it's a fixed-width 4 byte */ -export async function readContextBytesForkDigest(bufferedSource: BufferedSource): Promise { - for await (const buffer of bufferedSource) { - if (buffer.length >= CONTEXT_BYTES_FORK_DIGEST_LENGTH) { - const bytes = buffer.slice(0, CONTEXT_BYTES_FORK_DIGEST_LENGTH); - buffer.consume(CONTEXT_BYTES_FORK_DIGEST_LENGTH); - return bytes; - } - } - - // TODO: Use typed error - throw Error("Source ended while reading context bytes"); +export async function readContextBytesForkDigest(bytes: ByteStream, signal?: AbortSignal): Promise { + return (await bytes.read({bytes: CONTEXT_BYTES_FORK_DIGEST_LENGTH, signal})).subarray(); } diff --git a/packages/reqresp/src/encoders/responseEncode.ts b/packages/reqresp/src/encoders/responseEncode.ts index 90406164830b..c1c6f40ed6cf 100644 --- a/packages/reqresp/src/encoders/responseEncode.ts +++ b/packages/reqresp/src/encoders/responseEncode.ts @@ -14,30 +14,23 @@ const SUCCESS_BUFFER = Buffer.from([RespStatus.SUCCESS]); * ``` * Note: `response` has zero or more chunks (denoted by `<>*`) */ -export function responseEncodeSuccess( +export async function* responseEncodeSuccess( protocol: Protocol, - cbs: {onChunk: (chunkIndex: number) => void} -): (source: AsyncIterable) => AsyncIterable { - return async function* responseEncodeSuccessTransform(source) { - let chunkIndex = 0; + source: AsyncIterable +): AsyncIterable { + for await (const chunk of source) { + // + yield SUCCESS_BUFFER; - for await (const chunk of source) { - // Postfix increment, return 0 as first chunk - cbs.onChunk(chunkIndex++); - - // - yield SUCCESS_BUFFER; - - // - from altair - const contextBytes = getContextBytes(protocol.contextBytes, chunk); - if (contextBytes) { - yield contextBytes as Buffer; - } - - // | - yield* writeEncodedPayload(chunk.data, protocol.encoding); + // - from altair + const contextBytes = getContextBytes(protocol.contextBytes, chunk); + if (contextBytes) { + yield contextBytes; } - }; + + // | + yield* writeEncodedPayload(chunk.data, protocol.encoding); + } } /** @@ -50,20 +43,15 @@ export function responseEncodeSuccess( * Only the last `` is allowed to have a non-zero error code, so this * fn yields exactly one `` and afterwards the stream must be terminated */ -export async function* responseEncodeError( +export function* responseEncodeError( protocol: Pick, status: RpcResponseStatusError, errorMessage: string -): AsyncGenerator { - if (!errorMessage) { - yield Buffer.from([status]); - return; - } - +): Generator { // Combine and into a single chunk for atomic delivery. // Yielding them separately causes a race condition where the stream closes after the // status byte but before the error message arrives on the reader side. - const errorMessageBuffer = await encodeErrorMessageToBuffer(errorMessage, protocol.encoding); + const errorMessageBuffer = encodeErrorMessageToBuffer(errorMessage, protocol.encoding); yield Buffer.concat([Buffer.from([status]), errorMessageBuffer]); } diff --git a/packages/reqresp/src/encodingStrategies/index.ts b/packages/reqresp/src/encodingStrategies/index.ts index 209359a7c22d..c78ea0a45ba9 100644 --- a/packages/reqresp/src/encodingStrategies/index.ts +++ b/packages/reqresp/src/encodingStrategies/index.ts @@ -1,10 +1,11 @@ +import type {Stream} from "@libp2p/interface"; +import type {ByteStream} from "@libp2p/utils"; import {Encoding, TypeSizes} from "../types.js"; -import {BufferedSource} from "../utils/index.js"; import {readSszSnappyPayload} from "./sszSnappy/decode.js"; import {writeSszSnappyPayload} from "./sszSnappy/encode.js"; // For more info about Ethereum Consensus request/response encoding strategies, see: -// https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/p2p-interface.md#encoding-strategies +// https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/p2p-interface.md#encoding-strategies // Supported encoding strategies: // - ssz_snappy @@ -15,13 +16,14 @@ import {writeSszSnappyPayload} from "./sszSnappy/encode.js"; * ``` */ export async function readEncodedPayload( - bufferedSource: BufferedSource, + stream: ByteStream, encoding: Encoding, - type: TypeSizes + type: TypeSizes, + signal?: AbortSignal ): Promise { switch (encoding) { case Encoding.SSZ_SNAPPY: - return readSszSnappyPayload(bufferedSource, type); + return readSszSnappyPayload(stream, type, signal); default: throw Error("Unsupported encoding"); @@ -34,7 +36,7 @@ export async function readEncodedPayload( * | * ``` */ -export async function* writeEncodedPayload(chunkData: Uint8Array, encoding: Encoding): AsyncGenerator { +export function* writeEncodedPayload(chunkData: Uint8Array, encoding: Encoding): Generator { switch (encoding) { case Encoding.SSZ_SNAPPY: yield* writeSszSnappyPayload(chunkData); diff --git a/packages/reqresp/src/encodingStrategies/sszSnappy/decode.ts b/packages/reqresp/src/encodingStrategies/sszSnappy/decode.ts index 254a9268da9e..1e63509ccae3 100644 --- a/packages/reqresp/src/encodingStrategies/sszSnappy/decode.ts +++ b/packages/reqresp/src/encodingStrategies/sszSnappy/decode.ts @@ -1,8 +1,9 @@ +import type {Stream} from "@libp2p/interface"; +import type {ByteStream} from "@libp2p/utils"; import {decode as varintDecode, encodingLength as varintEncodingLength} from "uint8-varint"; import {Uint8ArrayList} from "uint8arraylist"; import {TypeSizes} from "../../types.js"; -import {BufferedSource} from "../../utils/index.js"; -import {SnappyFramesUncompress} from "../../utils/snappyIndex.js"; +import {ChunkType, decodeSnappyFrameData, parseSnappyFrameHeader} from "../../utils/snappyIndex.js"; import {SszSnappyError, SszSnappyErrorCode} from "./errors.js"; import {maxEncodedLen} from "./utils.js"; @@ -15,76 +16,115 @@ export const MAX_VARINT_BYTES = 10; * | * ``` */ -export async function readSszSnappyPayload(bufferedSource: BufferedSource, type: TypeSizes): Promise { - const sszDataLength = await readSszSnappyHeader(bufferedSource, type); - - return readSszSnappyBody(bufferedSource, sszDataLength); +export async function readSszSnappyPayload( + stream: ByteStream, + type: TypeSizes, + signal?: AbortSignal +): Promise { + const sszDataLength = await readSszSnappyHeader(stream, type, signal); + + return readSszSnappyBody(stream, sszDataLength, signal); } /** * Reads `` for ssz-snappy. * encoding-header ::= the length of the raw SSZ bytes, encoded as an unsigned protobuf varint */ -export async function readSszSnappyHeader(bufferedSource: BufferedSource, type: TypeSizes): Promise { - for await (const buffer of bufferedSource) { - // Get next bytes if empty - if (buffer.length === 0) { - continue; - } +export async function readSszSnappyHeader( + stream: ByteStream, + type: TypeSizes, + signal?: AbortSignal +): Promise { + const varintBytes: number[] = []; - let sszDataLength: number; - try { - sszDataLength = varintDecode(buffer.subarray()); - } catch (_e) { - throw new SszSnappyError({code: SszSnappyErrorCode.INVALID_VARINT_BYTES_COUNT, bytes: Infinity}); - } + while (true) { + const byte = await readExactOrSourceAborted(stream, 1, signal); - // MUST validate: the unsigned protobuf varint used for the length-prefix MUST not be longer than 10 bytes - // encodingLength function only returns 1-8 inclusive - const varintBytes = varintEncodingLength(sszDataLength); - buffer.consume(varintBytes); + const value = byte.get(0); - // MUST validate: the length-prefix is within the expected size bounds derived from the payload SSZ type. - const minSize = type.minSize; - const maxSize = type.maxSize; - if (sszDataLength < minSize) { - throw new SszSnappyError({code: SszSnappyErrorCode.UNDER_SSZ_MIN_SIZE, minSize, sszDataLength}); - } - if (sszDataLength > maxSize) { - throw new SszSnappyError({code: SszSnappyErrorCode.OVER_SSZ_MAX_SIZE, maxSize, sszDataLength}); + varintBytes.push(value); + if (varintBytes.length > MAX_VARINT_BYTES) { + throw new SszSnappyError({code: SszSnappyErrorCode.INVALID_VARINT_BYTES_COUNT, bytes: varintBytes.length}); } - return sszDataLength; + // MSB not set => varint terminated + if ((value & 0x80) === 0) break; + } + + let sszDataLength: number; + try { + sszDataLength = varintDecode(Uint8Array.from(varintBytes)); + } catch { + throw new SszSnappyError({code: SszSnappyErrorCode.INVALID_VARINT_BYTES_COUNT, bytes: Infinity}); } - throw new SszSnappyError({code: SszSnappyErrorCode.SOURCE_ABORTED}); + // MUST validate: the unsigned protobuf varint used for the length-prefix MUST not be longer than 10 bytes + // encodingLength function only returns 1-8 inclusive + const varintByteLength = varintEncodingLength(sszDataLength); + if (varintByteLength > MAX_VARINT_BYTES) { + throw new SszSnappyError({code: SszSnappyErrorCode.INVALID_VARINT_BYTES_COUNT, bytes: varintByteLength}); + } + + // MUST validate: the length-prefix is within the expected size bounds derived from the payload SSZ type. + const minSize = type.minSize; + const maxSize = type.maxSize; + if (sszDataLength < minSize) { + throw new SszSnappyError({code: SszSnappyErrorCode.UNDER_SSZ_MIN_SIZE, minSize, sszDataLength}); + } + if (sszDataLength > maxSize) { + throw new SszSnappyError({code: SszSnappyErrorCode.OVER_SSZ_MAX_SIZE, maxSize, sszDataLength}); + } + + return sszDataLength; } /** * Reads `` for ssz-snappy and decompress. * The returned bytes can be SSZ deseralized */ -export async function readSszSnappyBody(bufferedSource: BufferedSource, sszDataLength: number): Promise { - const decompressor = new SnappyFramesUncompress(); +export async function readSszSnappyBody( + stream: ByteStream, + sszDataLength: number, + signal?: AbortSignal +): Promise { const uncompressedData = new Uint8ArrayList(); - let readBytes = 0; + let encodedBytesRead = 0; + const maxBytes = maxEncodedLen(sszDataLength); + let foundIdentifier = false; + + while (uncompressedData.length < sszDataLength) { + const header = await readExactOrSourceAborted(stream, 4, signal); - for await (const buffer of bufferedSource) { // SHOULD NOT read more than max_encoded_len(n) bytes after reading the SSZ length-prefix n from the header - readBytes += buffer.length; - if (readBytes > maxEncodedLen(sszDataLength)) { - throw new SszSnappyError({code: SszSnappyErrorCode.TOO_MUCH_BYTES_READ, readBytes, sszDataLength}); + encodedBytesRead = addEncodedBytesReadOrThrow(encodedBytesRead, header.length, maxBytes, sszDataLength); + + let headerParsed: {type: ChunkType; frameSize: number}; + try { + headerParsed = parseSnappyFrameHeader(header.subarray()); + if (!foundIdentifier && headerParsed.type !== ChunkType.IDENTIFIER) { + throw new Error("malformed input: must begin with an identifier"); + } + } catch (e) { + throw new SszSnappyError({code: SszSnappyErrorCode.DECOMPRESSOR_ERROR, decompressorError: e as Error}); } - // No bytes left to consume, get next - if (buffer.length === 0) { - continue; + if (headerParsed.frameSize > maxBytes - encodedBytesRead) { + throw new SszSnappyError({ + code: SszSnappyErrorCode.TOO_MUCH_BYTES_READ, + readBytes: encodedBytesRead + headerParsed.frameSize, + sszDataLength, + }); } + const frame = await readExactOrSourceAborted(stream, headerParsed.frameSize, signal); + + encodedBytesRead = addEncodedBytesReadOrThrow(encodedBytesRead, frame.length, maxBytes, sszDataLength); - // stream contents can be passed through a buffered Snappy reader to decompress frame by frame try { - const uncompressed = decompressor.uncompress(buffer); - buffer.consume(buffer.length); + if (headerParsed.type === ChunkType.IDENTIFIER) { + foundIdentifier = true; + } + + const uncompressed = decodeSnappyFrameData(headerParsed.type, frame.subarray()); if (uncompressed !== null) { uncompressedData.append(uncompressed); } @@ -96,16 +136,34 @@ export async function readSszSnappyBody(bufferedSource: BufferedSource, sszDataL if (uncompressedData.length > sszDataLength) { throw new SszSnappyError({code: SszSnappyErrorCode.TOO_MANY_BYTES, sszDataLength}); } + } - // Keep reading chunks until `n` SSZ bytes - if (uncompressedData.length < sszDataLength) { - continue; - } + // buffer.length === n + return uncompressedData.subarray(0, sszDataLength); +} - // buffer.length === n - return uncompressedData.subarray(0, sszDataLength); +function addEncodedBytesReadOrThrow( + encodedBytesRead: number, + bytesToAdd: number, + maxBytes: number, + sszDataLength: number +): number { + const nextReadBytes = encodedBytesRead + bytesToAdd; + if (nextReadBytes > maxBytes) { + throw new SszSnappyError({code: SszSnappyErrorCode.TOO_MUCH_BYTES_READ, readBytes: nextReadBytes, sszDataLength}); } + return nextReadBytes; +} - // SHOULD consider invalid: An early EOF before fully reading the declared length-prefix worth of SSZ bytes - throw new SszSnappyError({code: SszSnappyErrorCode.SOURCE_ABORTED}); +async function readExactOrSourceAborted( + stream: ByteStream, + bytes: number, + signal?: AbortSignal +): Promise { + return stream.read({bytes, signal}).catch((e) => { + if ((e as Error).name === "UnexpectedEOFError") { + throw new SszSnappyError({code: SszSnappyErrorCode.SOURCE_ABORTED}); + } + throw e; + }); } diff --git a/packages/reqresp/src/encodingStrategies/sszSnappy/encode.ts b/packages/reqresp/src/encodingStrategies/sszSnappy/encode.ts index edea19b3c198..6f34462d5fa5 100644 --- a/packages/reqresp/src/encodingStrategies/sszSnappy/encode.ts +++ b/packages/reqresp/src/encodingStrategies/sszSnappy/encode.ts @@ -8,12 +8,12 @@ import {encodeSnappy} from "../../utils/snappyIndex.js"; * | * ``` */ -export const writeSszSnappyPayload = encodeSszSnappy as (bytes: Uint8Array) => AsyncGenerator; +export const writeSszSnappyPayload = encodeSszSnappy as (bytes: Uint8Array) => Generator; /** * Buffered Snappy writer */ -export async function* encodeSszSnappy(bytes: Buffer): AsyncGenerator { +export function* encodeSszSnappy(bytes: Buffer): Generator { // MUST encode the length of the raw SSZ bytes, encoded as an unsigned protobuf varint const varint = varintEncode(bytes.length); yield Buffer.from(varint.buffer, varint.byteOffset, varint.byteLength); diff --git a/packages/reqresp/src/encodingStrategies/sszSnappy/errors.ts b/packages/reqresp/src/encodingStrategies/sszSnappy/errors.ts index f9d6c4a8e9b5..bfd3b9417b87 100644 --- a/packages/reqresp/src/encodingStrategies/sszSnappy/errors.ts +++ b/packages/reqresp/src/encodingStrategies/sszSnappy/errors.ts @@ -9,8 +9,6 @@ export enum SszSnappyErrorCode { OVER_SSZ_MAX_SIZE = "SSZ_SNAPPY_ERROR_OVER_SSZ_MAX_SIZE", TOO_MUCH_BYTES_READ = "SSZ_SNAPPY_ERROR_TOO_MUCH_BYTES_READ", DECOMPRESSOR_ERROR = "SSZ_SNAPPY_ERROR_DECOMPRESSOR_ERROR", - DESERIALIZE_ERROR = "SSZ_SNAPPY_ERROR_DESERIALIZE_ERROR", - SERIALIZE_ERROR = "SSZ_SNAPPY_ERROR_SERIALIZE_ERROR", /** Received more bytes than specified sszDataLength */ TOO_MANY_BYTES = "SSZ_SNAPPY_ERROR_TOO_MANY_BYTES", /** Source aborted before reading sszDataLength bytes */ @@ -23,8 +21,6 @@ type SszSnappyErrorType = | {code: SszSnappyErrorCode.OVER_SSZ_MAX_SIZE; maxSize: number; sszDataLength: number} | {code: SszSnappyErrorCode.TOO_MUCH_BYTES_READ; readBytes: number; sszDataLength: number} | {code: SszSnappyErrorCode.DECOMPRESSOR_ERROR; decompressorError: Error} - | {code: SszSnappyErrorCode.DESERIALIZE_ERROR; deserializeError: Error} - | {code: SszSnappyErrorCode.SERIALIZE_ERROR; serializeError: Error} | {code: SszSnappyErrorCode.TOO_MANY_BYTES; sszDataLength: number} | {code: SszSnappyErrorCode.SOURCE_ABORTED}; diff --git a/packages/reqresp/src/encodingStrategies/sszSnappy/index.ts b/packages/reqresp/src/encodingStrategies/sszSnappy/index.ts index 8aaf7fc5e6f8..e8a76688571c 100644 --- a/packages/reqresp/src/encodingStrategies/sszSnappy/index.ts +++ b/packages/reqresp/src/encodingStrategies/sszSnappy/index.ts @@ -1,4 +1,3 @@ -export {SnappyFramesUncompress, encodeSnappy} from "../../utils/snappyIndex.js"; export * from "./decode.js"; export * from "./encode.js"; export * from "./errors.js"; diff --git a/packages/reqresp/src/metrics.ts b/packages/reqresp/src/metrics.ts index 7d8386742979..564e29300dd6 100644 --- a/packages/reqresp/src/metrics.ts +++ b/packages/reqresp/src/metrics.ts @@ -4,7 +4,7 @@ import {RequestErrorCode} from "./request/errors.js"; export type Metrics = ReturnType; /** - * A collection of metrics used throughout the Gossipsub behaviour. + * A collection of metrics used throughout the Req/Resp domain. */ export function getMetrics(register: MetricsRegisterExtra) { // Using function style instead of class to prevent having to re-declare all MetricsPrometheus types. @@ -29,7 +29,6 @@ export function getMetrics(register: MetricsRegisterExtra) { name: "beacon_reqresp_outgoing_request_roundtrip_time_seconds", help: "Histogram of outgoing requests round-trip time", labelNames: ["method"], - // Spec sets RESP_TIMEOUT = 10 sec buckets: [0.1, 0.2, 0.5, 1, 5, 10, 15, 60], }), outgoingErrors: register.gauge<{method: string}>({ @@ -61,7 +60,6 @@ export function getMetrics(register: MetricsRegisterExtra) { name: "beacon_reqresp_incoming_request_handler_time_seconds", help: "Histogram of incoming requests internal handling time", labelNames: ["method"], - // Spec sets RESP_TIMEOUT = 10 sec buckets: [0.1, 0.2, 0.5, 1, 5, 10], }), incomingErrors: register.gauge<{method: string}>({ @@ -69,20 +67,6 @@ export function getMetrics(register: MetricsRegisterExtra) { help: "Counts total failed responses handled per method", labelNames: ["method"], }), - outgoingResponseTTFB: register.histogram<{method: string}>({ - name: "beacon_reqresp_outgoing_response_ttfb_seconds", - help: "Time to first byte (TTFB) for outgoing responses", - labelNames: ["method"], - // Spec sets TTFB_TIMEOUT = 5 sec - buckets: [0.1, 1, 5], - }), - incomingResponseTTFB: register.histogram<{method: string}>({ - name: "beacon_reqresp_incoming_response_ttfb_seconds", - help: "Time to first byte (TTFB) for incoming responses", - labelNames: ["method"], - // Spec sets TTFB_TIMEOUT = 5 sec - buckets: [0.1, 1, 5], - }), dialErrors: register.gauge({ name: "beacon_reqresp_dial_errors_total", help: "Count total dial errors", diff --git a/packages/reqresp/src/request/errors.ts b/packages/reqresp/src/request/errors.ts index 2f5c7722f463..5d84b051f004 100644 --- a/packages/reqresp/src/request/errors.ts +++ b/packages/reqresp/src/request/errors.ts @@ -21,13 +21,9 @@ export enum RequestErrorCode { REQUEST_TIMEOUT = "REQUEST_ERROR_REQUEST_TIMEOUT", /** Error when sending request to responder */ REQUEST_ERROR = "REQUEST_ERROR_REQUEST_ERROR", - /** Reponder did not deliver a full reponse before max maxTotalResponseTimeout() */ - RESPONSE_TIMEOUT = "REQUEST_ERROR_RESPONSE_TIMEOUT", /** A single-response method returned 0 chunks */ EMPTY_RESPONSE = "REQUEST_ERROR_EMPTY_RESPONSE", - /** Time to first byte timeout */ - TTFB_TIMEOUT = "REQUEST_ERROR_TTFB_TIMEOUT", - /** Timeout between `` exceed */ + /** Response transfer timeout exceeded */ RESP_TIMEOUT = "REQUEST_ERROR_RESP_TIMEOUT", /** Request rate limited */ REQUEST_RATE_LIMITED = "REQUEST_ERROR_RATE_LIMITED", @@ -50,7 +46,6 @@ type RequestErrorType = | {code: RequestErrorCode.REQUEST_TIMEOUT} | {code: RequestErrorCode.REQUEST_ERROR; error: Error} | {code: RequestErrorCode.EMPTY_RESPONSE} - | {code: RequestErrorCode.TTFB_TIMEOUT} | {code: RequestErrorCode.RESP_TIMEOUT} | {code: RequestErrorCode.REQUEST_RATE_LIMITED} | {code: RequestErrorCode.REQUEST_SELF_RATE_LIMITED} diff --git a/packages/reqresp/src/request/index.ts b/packages/reqresp/src/request/index.ts index 0e37d36ed0dc..3745c12424ab 100644 --- a/packages/reqresp/src/request/index.ts +++ b/packages/reqresp/src/request/index.ts @@ -1,31 +1,46 @@ +import type {Stream} from "@libp2p/interface"; import {PeerId} from "@libp2p/interface"; -import {pipe} from "it-pipe"; import type {Libp2p} from "libp2p"; -import {Uint8ArrayList} from "uint8arraylist"; import {ErrorAborted, Logger, TimeoutError, withTimeout} from "@lodestar/utils"; import {requestEncode} from "../encoders/requestEncode.js"; import {responseDecode} from "../encoders/responseDecode.js"; import {Metrics} from "../metrics.js"; import {ResponseError} from "../response/index.js"; import {MixedProtocol, ResponseIncoming} from "../types.js"; -import {abortableSource, prettyPrintPeerId} from "../utils/index.js"; +import {prettyPrintPeerId, sendChunks} from "../utils/index.js"; import {RequestError, RequestErrorCode, responseStatusErrorToRequestError} from "./errors.js"; export {RequestError, RequestErrorCode}; -// Default spec values from https://github.com/ethereum/consensus-specs/blob/v1.2.0/specs/phase0/p2p-interface.md#configuration +// https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/p2p-interface.md#the-reqresp-domain export const DEFAULT_DIAL_TIMEOUT = 5 * 1000; // 5 sec export const DEFAULT_REQUEST_TIMEOUT = 5 * 1000; // 5 sec -export const DEFAULT_TTFB_TIMEOUT = 5 * 1000; // 5 sec export const DEFAULT_RESP_TIMEOUT = 10 * 1000; // 10 sec +function getStreamNotFullyConsumedError(): Error { + return new Error("ReqResp stream was not fully consumed"); +} + +function scheduleStreamAbortIfNotClosed(stream: Stream, timeoutMs: number): void { + const onClose = (): void => { + clearTimeout(timeout); + }; + + const timeout = setTimeout(() => { + stream.removeEventListener("close", onClose); + if (stream.status === "open" && stream.remoteWriteStatus === "writable") { + stream.abort(getStreamNotFullyConsumedError()); + } + }, timeoutMs); + + stream.addEventListener("close", onClose, {once: true}); +} + export interface SendRequestOpts { /** The maximum time for complete response transfer. */ respTimeoutMs?: number; /** Non-spec timeout from sending request until write stream closed by responder */ requestTimeoutMs?: number; - /** The maximum time to wait for first byte of request response (time-to-first-byte). */ - ttfbTimeoutMs?: number; /** Non-spec timeout from dialing protocol until stream opened */ dialTimeoutMs?: number; } @@ -64,7 +79,6 @@ export async function* sendRequest( const DIAL_TIMEOUT = opts?.dialTimeoutMs ?? DEFAULT_DIAL_TIMEOUT; const REQUEST_TIMEOUT = opts?.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT; - const TTFB_TIMEOUT = opts?.ttfbTimeoutMs ?? DEFAULT_TTFB_TIMEOUT; const RESP_TIMEOUT = opts?.respTimeoutMs ?? DEFAULT_RESP_TIMEOUT; const peerIdStrShort = prettyPrintPeerId(peerId); @@ -83,17 +97,6 @@ export async function* sendRequest( // the picked protocol in `connection.protocol` const protocolsMap = new Map(protocols.map((protocol, i) => [protocolIDs[i], protocol])); - // As of October 2020 we can't rely on libp2p.dialProtocol timeout to work so - // this function wraps the dialProtocol promise with an extra timeout - // - // > The issue might be: you add the peer's addresses to the AddressBook, - // which will result in autoDial to kick in and dial your peer. In parallel, - // you do a manual dial and it will wait for the previous one without using - // the abort signal: - // - // https://github.com/ChainSafe/lodestar/issues/1597#issuecomment-703394386 - - // DIAL_TIMEOUT: Non-spec timeout from dialing protocol until stream opened const stream = await withTimeout( async (timeoutAndParentSignal) => { const protocolIds = Array.from(protocolsMap.keys()); @@ -112,9 +115,6 @@ export async function* sendRequest( metrics?.outgoingOpenedStreams?.inc({method}); - // TODO: Does the TTFB timer start on opening stream or after receiving request - const timerTTFB = metrics?.outgoingResponseTTFB.startTimer({method}); - // Parse protocol selected by the responder const protocolId = stream.protocol ?? "unknown"; const protocol = protocolsMap.get(protocolId); @@ -126,21 +126,24 @@ export async function* sendRequest( logger.debug("Req sending request", logCtx); // Spec: The requester MUST close the write side of the stream once it finishes writing the request message - // Impl: stream.sink is closed automatically by js-libp2p-mplex when piped source is exhausted // REQUEST_TIMEOUT: Non-spec timeout from sending request until write stream closed by responder - // Note: libp2p.stop() will close all connections, so not necessary to abort this pipe on parent stop - await withTimeout(() => pipe(requestEncode(protocol, requestBody), stream.sink), REQUEST_TIMEOUT, signal).catch( - (e) => { - // Must close the stream read side (stream.source) manually AND the write side - stream.abort(e); - - if (e instanceof TimeoutError) { - throw new RequestError({code: RequestErrorCode.REQUEST_TIMEOUT}); - } - throw new RequestError({code: RequestErrorCode.REQUEST_ERROR, error: e as Error}); + // Note: libp2p.stop() will close all connections, so not necessary to abort this send on parent stop + await withTimeout( + async (timeoutAndParentSignal) => { + await sendChunks(stream, requestEncode(protocol, requestBody), timeoutAndParentSignal); + await stream.close({signal: timeoutAndParentSignal}); + }, + REQUEST_TIMEOUT, + signal + ).catch((e) => { + stream.abort(e as Error); + + if (e instanceof TimeoutError) { + throw new RequestError({code: RequestErrorCode.REQUEST_TIMEOUT}); } - ); + throw new RequestError({code: RequestErrorCode.REQUEST_ERROR, error: e as Error}); + }); logger.debug("Req request sent", logCtx); @@ -150,67 +153,52 @@ export async function* sendRequest( return; } - // - TTFB_TIMEOUT: The requester MUST wait a maximum of TTFB_TIMEOUT for the first response byte to arrive - // - RESP_TIMEOUT: Requester allows a further RESP_TIMEOUT for each subsequent response_chunk - // - Max total timeout: This timeout is not required by the spec. It may not be necessary, but it's kept as - // safe-guard to close. streams in case of bugs on other timeout mechanisms. - const ttfbTimeoutController = new AbortController(); - const respTimeoutController = new AbortController(); + // RESP_TIMEOUT: Maximum time for complete response transfer + const respSignal = signal + ? AbortSignal.any([signal, AbortSignal.timeout(RESP_TIMEOUT)]) + : AbortSignal.timeout(RESP_TIMEOUT); - let timeoutRESP: NodeJS.Timeout | null = null; - - const timeoutTTFB = setTimeout(() => { - // If we abort on first byte delay, don't need to abort for response delay - if (timeoutRESP) clearTimeout(timeoutRESP); - ttfbTimeoutController.abort(); - }, TTFB_TIMEOUT); - - const restartRespTimeout = (): void => { - if (timeoutRESP) clearTimeout(timeoutRESP); - timeoutRESP = setTimeout(() => respTimeoutController.abort(), RESP_TIMEOUT); - }; + let responseError: Error | null = null; + let responseFullyConsumed = false; try { - // Note: libp2p.stop() will close all connections, so not necessary to abort this pipe on parent stop - yield* pipe( - abortableSource(stream.source as AsyncIterable, [ - { - signal: ttfbTimeoutController.signal, - getError: () => new RequestError({code: RequestErrorCode.TTFB_TIMEOUT}), - }, - { - signal: respTimeoutController.signal, - getError: () => new RequestError({code: RequestErrorCode.RESP_TIMEOUT}), - }, - ]), - - // Transforms `Buffer` chunks to yield `ResponseBody` chunks - responseDecode(protocol, { - onFirstHeader() { - // On first byte, cancel the single use TTFB_TIMEOUT, and start RESP_TIMEOUT - clearTimeout(timeoutTTFB); - timerTTFB?.(); - restartRespTimeout(); - }, - onFirstResponseChunk() { - // On , cancel this chunk's RESP_TIMEOUT and start next's - restartRespTimeout(); - }, - }) - ); + yield* responseDecode(protocol, stream, { + signal: respSignal, + getError: () => + signal?.aborted ? new ErrorAborted("sendRequest") : new RequestError({code: RequestErrorCode.RESP_TIMEOUT}), + }); + responseFullyConsumed = true; // NOTE: Only log once per request to verbose, intermediate steps to debug // NOTE: Do not log the response, logs get extremely cluttered // NOTE: add double space after "Req " to align log with the "Resp " log logger.verbose("Req done", logCtx); + } catch (e) { + responseError = e as Error; + throw e; } finally { - clearTimeout(timeoutTTFB); - if (timeoutRESP !== null) clearTimeout(timeoutRESP); - - // Necessary to call `stream.close()` since collectResponses() may break out of the source before exhausting it - // `stream.close()` libp2p-mplex will .end() the source (it-pushable instance) - // If collectResponses() exhausts the source, it-pushable.end() can be safely called multiple times - await stream.close(); + // On decode/timeout failures abort immediately so mplex can reclaim stream state. + // On normal early consumer exit, close gracefully to avoid stream-id desync with peers. + if (responseError !== null || signal?.aborted) { + stream.abort(responseError ?? new ErrorAborted("sendRequest")); + } else { + await stream.close().catch((e) => { + stream.abort(e as Error); + }); + + if (!responseFullyConsumed) { + // Stop buffering unread inbound data after caller exits early. + // mplex does not support propagating closeRead to the remote, so still + // abort later if the remote never closes write. + await stream.closeRead().catch(() => { + // Ignore closeRead errors - close/abort path below will reclaim stream. + }); + + if (stream.remoteWriteStatus === "writable") { + scheduleStreamAbortIfNotClosed(stream, RESP_TIMEOUT); + } + } + } metrics?.outgoingClosedStreams?.inc({method}); logger.verbose("Req stream closed", logCtx); } diff --git a/packages/reqresp/src/response/index.ts b/packages/reqresp/src/response/index.ts index 4226570c2d33..7f032e03c5f3 100644 --- a/packages/reqresp/src/response/index.ts +++ b/packages/reqresp/src/response/index.ts @@ -1,5 +1,4 @@ import {PeerId, Stream} from "@libp2p/interface"; -import {pipe} from "it-pipe"; import {Logger, TimeoutError, withTimeout} from "@lodestar/utils"; import {requestDecode} from "../encoders/requestDecode.js"; import {responseEncodeError, responseEncodeSuccess} from "../encoders/responseEncode.js"; @@ -8,12 +7,12 @@ import {Metrics} from "../metrics.js"; import {ReqRespRateLimiter} from "../rate_limiter/ReqRespRateLimiter.js"; import {RequestError, RequestErrorCode} from "../request/errors.js"; import {Protocol, ReqRespRequest} from "../types.js"; -import {prettyPrintPeerId} from "../utils/index.js"; +import {prettyPrintPeerId, sendChunks} from "../utils/index.js"; import {ResponseError} from "./errors.js"; export {ResponseError}; -// Default spec values from https://github.com/ethereum/consensus-specs/blob/v1.2.0/specs/phase0/p2p-interface.md#configuration +// https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/p2p-interface.md#the-reqresp-domain export const DEFAULT_REQUEST_TIMEOUT = 5 * 1000; // 5 sec export interface HandleRequestOpts { @@ -28,7 +27,7 @@ export interface HandleRequestOpts { requestId?: number; /** Peer client type for logging and metrics: 'prysm' | 'lighthouse' */ peerClient?: string; - /** Non-spec timeout from sending request until write stream closed by responder */ + /** Timeout for reading the incoming request payload */ requestTimeoutMs?: number; } @@ -67,71 +66,66 @@ export async function handleRequest({ metrics?.incomingOpenedStreams.inc({method: protocol.method}); let responseError: Error | null = null; - await pipe( + let streamSendError: Error | null = null; + + try { // Yields success chunks and error chunks in the same generator - // This syntax allows to recycle stream.sink to send success and error chunks without returning + // This syntax allows to recycle stream to send success and error chunks without returning // in case request whose body is a List fails at chunk_i > 0, without breaking out of the for..await..of - (async function* requestHandlerSource() { - try { - // TODO: Does the TTFB timer start on opening stream or after receiving request - const timerTTFB = metrics?.outgoingResponseTTFB.startTimer({method: protocol.method}); - - const requestBody = await withTimeout( - () => pipe(stream.source, requestDecode(protocol)), - REQUEST_TIMEOUT, - signal - ).catch((e: unknown) => { - if (e instanceof TimeoutError) { - throw e; // Let outter catch (_e) {} re-type the error as SERVER_ERROR + await sendChunks( + stream, + (async function* requestHandlerSource() { + try { + const requestBody = await withTimeout( + (timeoutAndParentSignal) => requestDecode(protocol, stream, timeoutAndParentSignal), + REQUEST_TIMEOUT, + signal + ).catch((e: unknown) => { + if (e instanceof TimeoutError) { + throw e; // Let outer catch re-type the error as SERVER_ERROR + } + throw new ResponseError(RespStatus.INVALID_REQUEST, (e as Error).message); + }); + + logger.debug("Req received", logCtx); + + // Max count by request for byRange and byRoot + const requestCount = protocol?.inboundRateLimits?.getRequestCount?.(requestBody) ?? 1; + + if (!rateLimiter.allows(peerId, protocolID, requestCount)) { + throw new RequestError({code: RequestErrorCode.REQUEST_RATE_LIMITED}); } - throw new ResponseError(RespStatus.INVALID_REQUEST, (e as Error).message); - }); - logger.debug("Req received", logCtx); + const requestChunk: ReqRespRequest = { + data: requestBody, + version: protocol.version, + }; - // Max count by request for byRange and byRoot - const requestCount = protocol?.inboundRateLimits?.getRequestCount?.(requestBody) ?? 1; + yield* responseEncodeSuccess(protocol, protocol.handler(requestChunk, peerId, peerClient)); + } catch (e) { + const status = e instanceof ResponseError ? e.status : RespStatus.SERVER_ERROR; + yield* responseEncodeError(protocol, status, (e as Error).message); - if (!rateLimiter.allows(peerId, protocolID, requestCount)) { - throw new RequestError({code: RequestErrorCode.REQUEST_RATE_LIMITED}); + responseError = e as Error; } + })(), + signal + ); + } catch (e) { + streamSendError = e as Error; + throw e; + } finally { + if (streamSendError) { + stream.abort(streamSendError); + } else { + await stream.close().catch((e) => { + stream.abort(e as Error); + }); + } + metrics?.incomingClosedStreams.inc({method: protocol.method}); + } - const requestChunk: ReqRespRequest = { - data: requestBody, - version: protocol.version, - }; - - yield* pipe( - // TODO: Debug the reason for type conversion here - protocol.handler(requestChunk, peerId, peerClient), - // NOTE: Do not log the resp chunk contents, logs get extremely cluttered - // Note: Not logging on each chunk since after 1 year it hasn't add any value when debugging - // onChunk(() => logger.debug("Resp sending chunk", logCtx)), - responseEncodeSuccess(protocol, { - onChunk(chunkIndex) { - if (chunkIndex === 0) timerTTFB?.(); - }, - }) - ); - } catch (e) { - const status = e instanceof ResponseError ? e.status : RespStatus.SERVER_ERROR; - yield* responseEncodeError(protocol, status, (e as Error).message); - - // Should not throw an error here or libp2p-mplex throws with 'AbortError: stream reset' - // throw e; - responseError = e as Error; - } - })(), - stream.sink - ); - - // If streak.sink throws, libp2p-mplex will close stream.source - // If `requestDecode()` throws the stream.source must be closed manually - // To ensure the stream.source it-pushable instance is always closed, stream.close() is called always - await stream.close(); - metrics?.incomingClosedStreams.inc({method: protocol.method}); - - // TODO: It may happen that stream.sink returns before returning stream.source first, + // TODO: It may happen that the response write completes before the request is fully read, // so you never see "Resp received request" in the logs and the response ends without // sending any chunk, triggering EMPTY_RESPONSE error on the requesting side // It has only happened when doing a request too fast upon immediate connection on inbound peer diff --git a/packages/reqresp/src/types.ts b/packages/reqresp/src/types.ts index 4d97e7874a01..09daeff248cc 100644 --- a/packages/reqresp/src/types.ts +++ b/packages/reqresp/src/types.ts @@ -4,11 +4,9 @@ import {ForkName} from "@lodestar/params"; import {LodestarError} from "@lodestar/utils"; import {RateLimiterQuota} from "./rate_limiter/rateLimiterGRCA.js"; -export const protocolPrefix = "/eth2/beacon_chain/req"; - /** * Available request/response encoding strategies: - * https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/p2p-interface.md#encoding-strategies + * https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/p2p-interface.md#encoding-strategies */ export enum Encoding { SSZ_SNAPPY = "ssz_snappy", diff --git a/packages/reqresp/src/utils/abortableSource.ts b/packages/reqresp/src/utils/abortableSource.ts deleted file mode 100644 index 127339bbd597..000000000000 --- a/packages/reqresp/src/utils/abortableSource.ts +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Wraps an AsyncIterable and rejects early if any signal aborts. - * Throws the error returned by `getError()` of each signal options. - * - * Simplified fork of `"abortable-iterator"`. - * Read function's source for reasoning of the fork. - */ -export function abortableSource( - sourceArg: AsyncIterable, - signals: { - signal: AbortSignal; - getError: () => Error; - }[] -): AsyncIterable { - const source = sourceArg as AsyncGenerator; - - async function* abortable(): AsyncIterable { - // Handler that will hold a reference to the `abort()` promise, - // necessary for the signal abort listeners to reject the iterable promise - let nextAbortHandler: ((error: Error) => void) | null = null; - - // For each signal register an abortHandler(), and prepare clean-up with `onDoneCbs` - const onDoneCbs: (() => void)[] = []; - for (const {signal, getError} of signals) { - const abortHandler = (): void => { - if (nextAbortHandler) nextAbortHandler(getError()); - }; - signal.addEventListener("abort", abortHandler); - onDoneCbs.push(() => { - signal.removeEventListener("abort", abortHandler); - }); - } - - try { - while (true) { - // Abort early if any signal is aborted - for (const {signal, getError} of signals) { - if (signal.aborted) { - throw getError(); - } - } - - // Race the iterator and the abort signals - const result = await Promise.race([ - new Promise((_, reject) => { - nextAbortHandler = (error) => reject(error); - }), - source.next(), - ]); - - // source.next() resolved first - nextAbortHandler = null; - - if (result.done) { - return; - } - - yield result.value; - } - } catch (err) { - // End the iterator if it is a generator - if (typeof source.return === "function") { - // This source.return() function may never resolve depending on the source AsyncGenerator implementation. - // This is the main reason to fork "abortable-iterator", which caused our node to get stuck during Sync. - // We choose to call .return() but not await it. In general, source.return should never throw. If it does, - // it a problem of the source implementor, and thus logged as an unhandled rejection. If that happens, - // the source implementor should fix the upstream code. - void source.return(null); - } - - throw err; - } finally { - for (const cb of onDoneCbs) { - cb(); - } - } - } - - return abortable(); -} diff --git a/packages/reqresp/src/utils/bufferedSource.ts b/packages/reqresp/src/utils/bufferedSource.ts deleted file mode 100644 index e5daca1d2f67..000000000000 --- a/packages/reqresp/src/utils/bufferedSource.ts +++ /dev/null @@ -1,46 +0,0 @@ -import {Uint8ArrayList} from "uint8arraylist"; - -/** - * Wraps a buffer chunk stream source with another async iterable - * so it can be reused in multiple for..of statements. - * - * Uses a BufferList internally to make sure all chunks are consumed - * when switching consumers - */ -export class BufferedSource { - isDone = false; - private buffer: Uint8ArrayList; - private source: AsyncGenerator; - - constructor(source: AsyncGenerator) { - this.buffer = new Uint8ArrayList(); - this.source = source; - } - - [Symbol.asyncIterator](): AsyncIterator { - const that = this; - - let firstNext = true; - - return { - async next() { - // Prevent fetching a new chunk if there are pending bytes - // not processed by a previous consumer of this BufferedSource - if (firstNext && that.buffer.length > 0) { - firstNext = false; - return {done: false, value: that.buffer}; - } - - const {done, value: chunk} = await that.source.next(); - if (done === true) { - that.isDone = true; - return {done: true, value: undefined}; - } - - // Concat new chunk and return a reference to its BufferList instance - that.buffer.append(chunk); - return {done: false, value: that.buffer}; - }, - }; - } -} diff --git a/packages/reqresp/src/utils/collectMaxResponse.ts b/packages/reqresp/src/utils/collectMaxResponse.ts index 42ce2c1c8611..9fafa2b00c46 100644 --- a/packages/reqresp/src/utils/collectMaxResponse.ts +++ b/packages/reqresp/src/utils/collectMaxResponse.ts @@ -6,12 +6,11 @@ * Collects a bounded list of responses up to `maxResponses` */ export async function collectMaxResponse(source: AsyncIterable, maxResponses: number): Promise { - // else: zero or more responses const responses: T[] = []; for await (const response of source) { responses.push(response); - if (maxResponses !== undefined && responses.length >= maxResponses) { + if (responses.length >= maxResponses) { break; } } diff --git a/packages/reqresp/src/utils/errorMessage.ts b/packages/reqresp/src/utils/errorMessage.ts index bcfbe2e9742a..be529e010569 100644 --- a/packages/reqresp/src/utils/errorMessage.ts +++ b/packages/reqresp/src/utils/errorMessage.ts @@ -1,8 +1,7 @@ import {decode as varintDecode, encodingLength as varintEncodingLength} from "uint8-varint"; -import {Uint8ArrayList} from "uint8arraylist"; import {writeSszSnappyPayload} from "../encodingStrategies/sszSnappy/encode.js"; import {Encoding} from "../types.js"; -import {SnappyFramesUncompress} from "./snappyIndex.js"; +import {decodeSnappyFrames} from "./snappyIndex.js"; // ErrorMessage schema: // @@ -13,18 +12,21 @@ import {SnappyFramesUncompress} from "./snappyIndex.js"; // By convention, the error_message is a sequence of bytes that MAY be interpreted as a // UTF-8 string (for debugging purposes). Clients MUST treat as valid any byte sequences // -// Spec v1.1.10 https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/p2p-interface.md#responding-side +// https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/p2p-interface.md#responding-side /** * Encodes a UTF-8 string to 256 bytes max */ -export async function* encodeErrorMessage(errorMessage: string, encoding: Encoding): AsyncGenerator { +export function* encodeErrorMessage(errorMessage: string, encoding: Encoding): Generator { const encoder = new TextEncoder(); const bytes = encoder.encode(errorMessage).slice(0, 256); switch (encoding) { case Encoding.SSZ_SNAPPY: yield* writeSszSnappyPayload(bytes); + break; + default: + throw Error("Unsupported encoding"); } } @@ -32,9 +34,9 @@ export async function* encodeErrorMessage(errorMessage: string, encoding: Encodi * Encodes a UTF-8 error message string into a single buffer (max 256 bytes before encoding). * Unlike `encodeErrorMessage`, this collects all encoded chunks into one buffer. */ -export async function encodeErrorMessageToBuffer(errorMessage: string, encoding: Encoding): Promise { +export function encodeErrorMessageToBuffer(errorMessage: string, encoding: Encoding): Buffer { const chunks: Buffer[] = []; - for await (const chunk of encodeErrorMessage(errorMessage, encoding)) { + for (const chunk of encodeErrorMessage(errorMessage, encoding)) { chunks.push(chunk); } return Buffer.concat(chunks); @@ -43,21 +45,20 @@ export async function encodeErrorMessageToBuffer(errorMessage: string, encoding: /** * Decodes error message from network bytes and removes non printable, non ascii characters. */ -export async function decodeErrorMessage(encodedErrorMessage: Uint8Array): Promise { - const encoder = new TextDecoder(); +export function decodeErrorMessage(encodedErrorMessage: Uint8Array): string { + const decoder = new TextDecoder(); let sszDataLength: number; try { sszDataLength = varintDecode(encodedErrorMessage); - const decompressor = new SnappyFramesUncompress(); const varintBytes = varintEncodingLength(sszDataLength); - const errorMessage = decompressor.uncompress(new Uint8ArrayList(encodedErrorMessage.subarray(varintBytes))); - if (errorMessage == null || errorMessage.length !== sszDataLength) { + const errorMessage = decodeSnappyFrames(encodedErrorMessage.subarray(varintBytes)); + if (errorMessage.length !== sszDataLength) { throw new Error("Malformed input: data length mismatch"); } // remove non ascii characters from string - return encoder.decode(errorMessage.subarray(0)).replace(/[^\x20-\x7F]/g, ""); + return decoder.decode(errorMessage.subarray(0)).replace(/[^\x20-\x7F]/g, ""); } catch (_e) { // remove non ascii characters from string - return encoder.decode(encodedErrorMessage.slice(0, 256)).replace(/[^\x20-\x7F]/g, ""); + return decoder.decode(encodedErrorMessage.slice(0, 256)).replace(/[^\x20-\x7F]/g, ""); } } diff --git a/packages/reqresp/src/utils/index.ts b/packages/reqresp/src/utils/index.ts index a0831d4f8647..2655eff6ed90 100644 --- a/packages/reqresp/src/utils/index.ts +++ b/packages/reqresp/src/utils/index.ts @@ -1,9 +1,7 @@ -export * from "./abortableSource.js"; -export * from "./bufferedSource.js"; export * from "./collectExactOne.js"; export * from "./collectMaxResponse.js"; export * from "./errorMessage.js"; -export * from "./onChunk.js"; export * from "./peerId.js"; export * from "./protocolId.js"; export * from "./snappyIndex.js"; +export * from "./stream.js"; diff --git a/packages/reqresp/src/utils/onChunk.ts b/packages/reqresp/src/utils/onChunk.ts deleted file mode 100644 index 4ca4a2ceed94..000000000000 --- a/packages/reqresp/src/utils/onChunk.ts +++ /dev/null @@ -1,12 +0,0 @@ -/** - * Calls `callback` with each `chunk` received from the `source` AsyncIterable - * Useful for logging, or cancelling timeouts - */ -export function onChunk(callback: (chunk: T) => void): (source: AsyncIterable) => AsyncIterable { - return async function* onChunkTransform(source) { - for await (const chunk of source) { - callback(chunk); - yield chunk; - } - }; -} diff --git a/packages/reqresp/src/utils/protocolId.ts b/packages/reqresp/src/utils/protocolId.ts index a86de251f050..9d85eea2e154 100644 --- a/packages/reqresp/src/utils/protocolId.ts +++ b/packages/reqresp/src/utils/protocolId.ts @@ -1,14 +1,14 @@ import {Encoding, ProtocolAttributes} from "../types.js"; /** - * https://github.com/ethereum/consensus-specs/blob/v1.2.0/specs/phase0/p2p-interface.md#protocol-identification + * https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/p2p-interface.md#protocol-identification */ export function formatProtocolID(protocolPrefix: string, method: string, version: number, encoding: Encoding): string { return `${protocolPrefix}/${method}/${version}/${encoding}`; } /** - * https://github.com/ethereum/consensus-specs/blob/v1.2.0/specs/phase0/p2p-interface.md#protocol-identification + * https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/p2p-interface.md#protocol-identification */ export function parseProtocolID(protocolId: string): ProtocolAttributes { const result = protocolId.split("/"); diff --git a/packages/reqresp/src/utils/snappy.ts b/packages/reqresp/src/utils/snappy.ts deleted file mode 100644 index 21bb5a7dfaea..000000000000 --- a/packages/reqresp/src/utils/snappy.ts +++ /dev/null @@ -1,2 +0,0 @@ -export {compressSync as compress} from "snappy"; -export {uncompress} from "snappyjs"; diff --git a/packages/reqresp/src/utils/snappyCompress.ts b/packages/reqresp/src/utils/snappyCompress.ts index e3ac57c62bde..07f7e772e678 100644 --- a/packages/reqresp/src/utils/snappyCompress.ts +++ b/packages/reqresp/src/utils/snappyCompress.ts @@ -3,7 +3,7 @@ import {compressSync} from "snappy"; import {ChunkType, IDENTIFIER_FRAME, UNCOMPRESSED_CHUNK_SIZE, crc} from "./snappyCommon.js"; // The logic in this file is largely copied (in simplified form) from https://github.com/ChainSafe/node-snappy-stream/ -export async function* encodeSnappy(bytes: Buffer): AsyncGenerator { +export function* encodeSnappy(bytes: Buffer): Generator { yield IDENTIFIER_FRAME; for (let i = 0; i < bytes.length; i += UNCOMPRESSED_CHUNK_SIZE) { diff --git a/packages/reqresp/src/utils/snappyIndex.ts b/packages/reqresp/src/utils/snappyIndex.ts index fa79b7ca2a05..c07c566d7886 100644 --- a/packages/reqresp/src/utils/snappyIndex.ts +++ b/packages/reqresp/src/utils/snappyIndex.ts @@ -1,3 +1,3 @@ export * from "./snappyCommon.js"; export {encodeSnappy} from "./snappyCompress.js"; -export {SnappyFramesUncompress} from "./snappyUncompress.js"; +export * from "./snappyUncompress.js"; diff --git a/packages/reqresp/src/utils/snappyUncompress.ts b/packages/reqresp/src/utils/snappyUncompress.ts index af9d5b890e0c..61e8db053fcf 100644 --- a/packages/reqresp/src/utils/snappyUncompress.ts +++ b/packages/reqresp/src/utils/snappyUncompress.ts @@ -2,96 +2,94 @@ import {uncompress} from "snappyjs"; import {Uint8ArrayList} from "uint8arraylist"; import {ChunkType, IDENTIFIER, UNCOMPRESSED_CHUNK_SIZE, crc} from "./snappyCommon.js"; -export class SnappyFramesUncompress { - private buffer = new Uint8ArrayList(); - - private state: UncompressState = { - foundIdentifier: false, - }; - - /** - * Accepts chunk of data containing some part of snappy frames stream - * @param chunk - * @return Buffer if there is one or more whole frames, null if it's partial - */ - uncompress(chunk: Uint8ArrayList): Uint8ArrayList | null { - this.buffer.append(chunk); - const result = new Uint8ArrayList(); - while (this.buffer.length > 0) { - if (this.buffer.length < 4) break; +export function parseSnappyFrameHeader(header: Uint8Array): {type: ChunkType; frameSize: number} { + if (header.length !== 4) { + throw new Error("malformed input: incomplete frame header"); + } - const type = getChunkType(this.buffer.get(0)); + const type = getChunkType(header[0]); + const frameSize = header[1] + (header[2] << 8) + (header[3] << 16); + return {type, frameSize}; +} - if (!this.state.foundIdentifier && type !== ChunkType.IDENTIFIER) { - throw "malformed input: must begin with an identifier"; +export function decodeSnappyFrameData(type: ChunkType, frame: Uint8Array): Uint8Array | null { + switch (type) { + case ChunkType.IDENTIFIER: { + if (!Buffer.prototype.equals.call(frame, IDENTIFIER)) { + throw new Error("malformed input: bad identifier"); + } + return null; + } + case ChunkType.PADDING: + case ChunkType.SKIPPABLE: + return null; + case ChunkType.COMPRESSED: { + if (frame.length < 4) { + throw new Error("malformed input: too short"); } - const frameSize = getFrameSize(this.buffer, 1); + const checksum = frame.subarray(0, 4); + const data = frame.subarray(4); + const uncompressed = uncompress(data, UNCOMPRESSED_CHUNK_SIZE); + if (crc(uncompressed).compare(checksum) !== 0) { + throw new Error("malformed input: bad checksum"); + } + return uncompressed; + } + case ChunkType.UNCOMPRESSED: { + if (frame.length < 4) { + throw new Error("malformed input: too short"); + } - if (this.buffer.length - 4 < frameSize) { - break; + const checksum = frame.subarray(0, 4); + const uncompressed = frame.subarray(4); + if (uncompressed.length > UNCOMPRESSED_CHUNK_SIZE) { + throw new Error("malformed input: too large"); + } + if (crc(uncompressed).compare(checksum) !== 0) { + throw new Error("malformed input: bad checksum"); } + return uncompressed; + } + } +} - const frame = this.buffer.subarray(4, 4 + frameSize); - this.buffer.consume(4 + frameSize); +export function decodeSnappyFrames(data: Uint8Array): Uint8ArrayList { + const out = new Uint8ArrayList(); + let foundIdentifier = false; + let offset = 0; - switch (type) { - case ChunkType.IDENTIFIER: { - if (!Buffer.prototype.equals.call(frame, IDENTIFIER)) { - throw "malformed input: bad identifier"; - } - this.state.foundIdentifier = true; - continue; - } - case ChunkType.PADDING: - case ChunkType.SKIPPABLE: - continue; - case ChunkType.COMPRESSED: { - const checksum = frame.subarray(0, 4); - const data = frame.subarray(4); + while (offset < data.length) { + const remaining = data.length - offset; + if (remaining < 4) { + throw new Error("malformed input: incomplete frame header"); + } - const uncompressed = uncompress(data, UNCOMPRESSED_CHUNK_SIZE); - if (crc(uncompressed).compare(checksum) !== 0) { - throw "malformed input: bad checksum"; - } - result.append(uncompressed); - break; - } - case ChunkType.UNCOMPRESSED: { - const checksum = frame.subarray(0, 4); - const uncompressed = frame.subarray(4); + const {type, frameSize} = parseSnappyFrameHeader(data.subarray(offset, offset + 4)); + if (!foundIdentifier && type !== ChunkType.IDENTIFIER) { + throw new Error("malformed input: must begin with an identifier"); + } - if (uncompressed.length > UNCOMPRESSED_CHUNK_SIZE) { - throw "malformed input: too large"; - } - if (crc(uncompressed).compare(checksum) !== 0) { - throw "malformed input: bad checksum"; - } - result.append(uncompressed); - break; - } - } + offset += 4; + + if (data.length - offset < frameSize) { + throw new Error("malformed input: incomplete frame"); } - if (result.length === 0) { - return null; + + const frame = data.subarray(offset, offset + frameSize); + offset += frameSize; + + if (type === ChunkType.IDENTIFIER) { + foundIdentifier = true; } - return result; - } - reset(): void { - this.buffer = new Uint8ArrayList(); - this.state = { - foundIdentifier: false, - }; + const uncompressed = decodeSnappyFrameData(type, frame); + if (uncompressed !== null) { + out.append(uncompressed); + } } -} - -type UncompressState = { - foundIdentifier: boolean; -}; -function getFrameSize(buffer: Uint8ArrayList, offset: number): number { - return buffer.get(offset) + (buffer.get(offset + 1) << 8) + (buffer.get(offset + 2) << 16); + return out; } function getChunkType(value: number): ChunkType { diff --git a/packages/reqresp/src/utils/stream.ts b/packages/reqresp/src/utils/stream.ts new file mode 100644 index 000000000000..a335059692b6 --- /dev/null +++ b/packages/reqresp/src/utils/stream.ts @@ -0,0 +1,34 @@ +import type {Stream} from "@libp2p/interface"; +import {ByteStream} from "@libp2p/utils"; +import {Uint8ArrayList} from "uint8arraylist"; +import {ErrorAborted} from "@lodestar/utils"; + +export async function sendChunks( + stream: Stream, + source: Iterable | AsyncIterable, + signal?: AbortSignal +): Promise { + for await (const chunk of source) { + if (signal?.aborted) { + throw new ErrorAborted("sendChunks"); + } + + if (!stream.send(chunk)) { + await stream.onDrain({signal}); + } + } +} + +export function drainByteStream(bytes: ByteStream): Uint8Array | undefined { + const readBuffer = ( + bytes as unknown as { + readBuffer?: {byteLength: number; subarray: () => Uint8Array; consume: (bytes: number) => void}; + } + ).readBuffer; + if (readBuffer && readBuffer.byteLength > 0) { + const drained = readBuffer.subarray(); + readBuffer.consume(readBuffer.byteLength); + return drained; + } + return undefined; +} diff --git a/packages/reqresp/test/fixtures/encodingStrategies.ts b/packages/reqresp/test/fixtures/encodingStrategies.ts index bbcb01a6e346..dce2c6c24a6e 100644 --- a/packages/reqresp/test/fixtures/encodingStrategies.ts +++ b/packages/reqresp/test/fixtures/encodingStrategies.ts @@ -71,7 +71,16 @@ export const encodingStrategiesDecodingErrorCases: { id: "if it read more than maxEncodedLen", type: ssz.phase0.Ping, error: SszSnappyErrorCode.TOO_MUCH_BYTES_READ, - chunks: [Buffer.from(varintEncode(ssz.phase0.Ping.minSize)), Buffer.alloc(100)], + chunks: [ + Buffer.from(varintEncode(ssz.phase0.Ping.minSize)), + Buffer.concat([ + // snappy stream identifier frame + Buffer.from([0xff, 0x06, 0x00, 0x00]), + Buffer.from("sNaPpY"), + // many empty skippable frames to force maxEncodedLen overflow + ...new Array(32).fill(Buffer.from([0x80, 0x00, 0x00, 0x00])), + ]), + ], }, { id: "if failed ssz snappy input malformed", diff --git a/packages/reqresp/test/unit/ReqResp.test.ts b/packages/reqresp/test/unit/ReqResp.test.ts index 2cb821b534e1..9e1f9760dd89 100644 --- a/packages/reqresp/test/unit/ReqResp.test.ts +++ b/packages/reqresp/test/unit/ReqResp.test.ts @@ -1,4 +1,4 @@ -import {Libp2p} from "libp2p"; +import type {Libp2p} from "libp2p"; import {afterEach, beforeEach, describe, expect, it, vi} from "vitest"; import {getEmptyLogger} from "@lodestar/logger/empty"; import {Logger} from "@lodestar/utils"; @@ -8,7 +8,7 @@ import {RateLimiterQuota} from "../../src/rate_limiter/rateLimiterGRCA.js"; import {Protocol} from "../../src/types.js"; import {getEmptyHandler, sszSnappyPing} from "../fixtures/messages.js"; import {numberToStringProtocol, numberToStringProtocolDialOnly, pingProtocol} from "../fixtures/protocols.js"; -import {MockLibP2pStream} from "../utils/index.js"; +import {createMockStream} from "../utils/mockStream.js"; import {responseEncode} from "../utils/response.js"; describe("ResResp", () => { @@ -19,19 +19,22 @@ describe("ResResp", () => { beforeEach(() => { libp2p = { - dialProtocol: vi.fn().mockResolvedValue( - new MockLibP2pStream( - responseEncode( - [ - { - status: RespStatus.SUCCESS, - payload: sszSnappyPing.binaryPayload, - }, - ], - ping - ), - ping.method - ) + dialProtocol: vi.fn().mockImplementation( + async () => + ( + await createMockStream({ + protocol: ping.method, + source: responseEncode( + [ + { + status: RespStatus.SUCCESS, + payload: sszSnappyPing.binaryPayload, + }, + ], + ping + ), + }) + ).stream ), handle: vi.fn(), } as unknown as Libp2p; diff --git a/packages/reqresp/test/unit/encoders/reqestEncode.test.ts b/packages/reqresp/test/unit/encoders/reqestEncode.test.ts index 8771e26913cd..0bb7d8edb398 100644 --- a/packages/reqresp/test/unit/encoders/reqestEncode.test.ts +++ b/packages/reqresp/test/unit/encoders/reqestEncode.test.ts @@ -1,5 +1,3 @@ -import all from "it-all"; -import {pipe} from "it-pipe"; import {describe, it} from "vitest"; import {requestEncode} from "../../../src/encoders/requestEncode.js"; import {requestEncodersCases} from "../../fixtures/encoders.js"; @@ -8,9 +6,9 @@ import {expectEqualByteChunks} from "../../utils/index.js"; describe("encoders / requestEncode", () => { describe("valid cases", () => { it.each(requestEncodersCases)("$id", async ({protocol, requestBody, chunks}) => { - const encodedChunks = await pipe(requestEncode(protocol, requestBody), all); + const encodedChunks = await Array.fromAsync(requestEncode(protocol, requestBody)); expectEqualByteChunks( - encodedChunks as Uint8Array[], + encodedChunks, chunks.map((c) => c.subarray()) ); }); diff --git a/packages/reqresp/test/unit/encoders/requestDecode.test.ts b/packages/reqresp/test/unit/encoders/requestDecode.test.ts index cd8c7bc303e6..623f3d498260 100644 --- a/packages/reqresp/test/unit/encoders/requestDecode.test.ts +++ b/packages/reqresp/test/unit/encoders/requestDecode.test.ts @@ -1,21 +1,23 @@ -import {pipe} from "it-pipe"; import {describe, expect, it} from "vitest"; import {requestDecode} from "../../../src/encoders/requestDecode.js"; import {requestEncodersCases, requestEncodersErrorCases} from "../../fixtures/encoders.js"; import {expectRejectedWithLodestarError} from "../../utils/errors.js"; import {arrToSource} from "../../utils/index.js"; +import {createMockStream} from "../../utils/mockStream.js"; describe("encoders / requestDecode", () => { describe("valid cases", () => { it.each(requestEncodersCases)("$id", async ({protocol, requestBody, chunks}) => { - const decodedBody = await pipe(arrToSource(chunks), requestDecode(protocol)); + const {stream} = await createMockStream({source: arrToSource(chunks)}); + const decodedBody = await requestDecode(protocol, stream); expect(decodedBody).toEqual(requestBody); }); }); describe("error cases", () => { it.each(requestEncodersErrorCases.filter((r) => r.errorDecode))("$id", async ({protocol, errorDecode, chunks}) => { - await expectRejectedWithLodestarError(pipe(arrToSource(chunks), requestDecode(protocol)), errorDecode); + const {stream} = await createMockStream({source: arrToSource(chunks)}); + await expectRejectedWithLodestarError(requestDecode(protocol, stream), errorDecode); }); }); }); diff --git a/packages/reqresp/test/unit/encoders/responseDecode.test.ts b/packages/reqresp/test/unit/encoders/responseDecode.test.ts index e8ed906c942f..0a8b13616d46 100644 --- a/packages/reqresp/test/unit/encoders/responseDecode.test.ts +++ b/packages/reqresp/test/unit/encoders/responseDecode.test.ts @@ -1,5 +1,3 @@ -import all from "it-all"; -import {pipe} from "it-pipe"; import {describe, expect, it} from "vitest"; import {LodestarError} from "@lodestar/utils"; import {responseDecode} from "../../../src/encoders/responseDecode.js"; @@ -7,15 +5,13 @@ import {ResponseIncoming} from "../../../src/types.js"; import {responseEncodersErrorTestCases, responseEncodersTestCases} from "../../fixtures/encoders.js"; import {expectRejectedWithLodestarError} from "../../utils/errors.js"; import {arrToSource, onlySuccessResp} from "../../utils/index.js"; +import {createMockStream} from "../../utils/mockStream.js"; describe("encoders / responseDecode", () => { describe("valid cases", () => { it.each(responseEncodersTestCases)("$id", async ({protocol, responseChunks, chunks}) => { - const responses = (await pipe( - arrToSource(chunks), - responseDecode(protocol, {onFirstHeader: () => {}, onFirstResponseChunk: () => {}}), - all - )) as ResponseIncoming[]; + const {stream} = await createMockStream({source: arrToSource(chunks)}); + const responses = (await Array.fromAsync(responseDecode(protocol, stream))) as ResponseIncoming[]; const expectedResponses = responseChunks.filter(onlySuccessResp).map((r) => r.payload); expect(responses.map((r) => ({...r, data: Buffer.from(r.data)}))).toEqual( @@ -28,12 +24,9 @@ describe("encoders / responseDecode", () => { it.each(responseEncodersErrorTestCases.filter((r) => r.decodeError !== undefined))( "$id", async ({protocol, chunks, decodeError}) => { + const {stream} = await createMockStream({source: arrToSource(chunks as Uint8Array[])}); await expectRejectedWithLodestarError( - pipe( - arrToSource(chunks as Uint8Array[]), - responseDecode(protocol, {onFirstHeader: () => {}, onFirstResponseChunk: () => {}}), - all - ), + Array.fromAsync(responseDecode(protocol, stream)), decodeError as LodestarError ); } diff --git a/packages/reqresp/test/unit/encoders/responseEncode.test.ts b/packages/reqresp/test/unit/encoders/responseEncode.test.ts index 9580d0b2fcd8..f43d3bd21949 100644 --- a/packages/reqresp/test/unit/encoders/responseEncode.test.ts +++ b/packages/reqresp/test/unit/encoders/responseEncode.test.ts @@ -1,5 +1,3 @@ -import all from "it-all"; -import {pipe} from "it-pipe"; import {describe, it} from "vitest"; import {Protocol} from "../../../src/types.js"; import {responseEncodersTestCases} from "../../fixtures/encoders.js"; @@ -11,10 +9,9 @@ describe("encoders / responseEncode", () => { it.each(responseEncodersTestCases.filter((f) => !f.skipEncoding))( "$id", async ({protocol, responseChunks, chunks}) => { - const encodedChunks = await pipe(responseEncode(responseChunks, protocol as Protocol), all); - + const encodedChunks = await Array.fromAsync(responseEncode(responseChunks, protocol as Protocol)); expectEqualByteChunks( - encodedChunks as Uint8Array[], + encodedChunks, chunks.map((c) => c.subarray()) ); } diff --git a/packages/reqresp/test/unit/encodingStrategies/sszSnappy/decode.test.ts b/packages/reqresp/test/unit/encodingStrategies/sszSnappy/decode.test.ts index f315cc26316b..6cf4d69f78c7 100644 --- a/packages/reqresp/test/unit/encodingStrategies/sszSnappy/decode.test.ts +++ b/packages/reqresp/test/unit/encodingStrategies/sszSnappy/decode.test.ts @@ -1,19 +1,21 @@ +import {byteStream} from "@libp2p/utils"; import {encode as varintEncode} from "uint8-varint"; import {Uint8ArrayList} from "uint8arraylist"; import {describe, expect, it} from "vitest"; import {readSszSnappyPayload} from "../../../../src/encodingStrategies/sszSnappy/index.js"; -import {BufferedSource} from "../../../../src/utils/index.js"; import { encodingStrategiesDecodingErrorCases, encodingStrategiesMainnetTestCases, encodingStrategiesTestCases, } from "../../../fixtures/index.js"; import {arrToSource} from "../../../utils/index.js"; +import {createMockStream} from "../../../utils/mockStream.js"; describe("encodingStrategies / sszSnappy / decode", () => { it.each(encodingStrategiesTestCases)("$id", async ({type, binaryPayload, chunks}) => { - const bufferedSource = new BufferedSource(arrToSource(chunks)); - const bodyResult = await readSszSnappyPayload(bufferedSource, type); + const {stream} = await createMockStream({source: arrToSource(chunks)}); + const bytes = byteStream(stream); + const bodyResult = await readSszSnappyPayload(bytes, type).finally(() => bytes.unwrap()); expect(bodyResult).toEqual(binaryPayload.data); }); @@ -23,8 +25,9 @@ describe("encodingStrategies / sszSnappy / decode", () => { const streamedBytes = new Uint8ArrayList(Buffer.concat([Buffer.from(varintEncode(bodySize)), streamedBody])); it(id, async () => { - const bufferedSource = new BufferedSource(arrToSource([streamedBytes])); - const bodyResult = await readSszSnappyPayload(bufferedSource, serializer); + const {stream} = await createMockStream({source: arrToSource([streamedBytes])}); + const bytes = byteStream(stream); + const bodyResult = await readSszSnappyPayload(bytes, serializer).finally(() => bytes.unwrap()); expect(bodyResult).toEqual(new Uint8Array(payload.data)); }); @@ -34,8 +37,10 @@ describe("encodingStrategies / sszSnappy / decode", () => { describe("error cases", () => { for (const {id, type, error, chunks} of encodingStrategiesDecodingErrorCases) { it(id, async () => { - const bufferedSource = new BufferedSource(arrToSource([new Uint8ArrayList(...chunks)])); - await expect(readSszSnappyPayload(bufferedSource, type)).rejects.toThrow(error); + const {stream} = await createMockStream({source: arrToSource([new Uint8ArrayList(...chunks)])}); + const bytes = byteStream(stream); + await expect(readSszSnappyPayload(bytes, type)).rejects.toThrow(error); + bytes.unwrap(); }); } }); diff --git a/packages/reqresp/test/unit/encodingStrategies/sszSnappy/encode.test.ts b/packages/reqresp/test/unit/encodingStrategies/sszSnappy/encode.test.ts index 68f5c4e5299d..410cfc581d14 100644 --- a/packages/reqresp/test/unit/encodingStrategies/sszSnappy/encode.test.ts +++ b/packages/reqresp/test/unit/encodingStrategies/sszSnappy/encode.test.ts @@ -1,5 +1,3 @@ -import all from "it-all"; -import {pipe} from "it-pipe"; import {encode as varintEncode} from "uint8-varint"; import {describe, expect, it} from "vitest"; import {writeSszSnappyPayload} from "../../../../src/encodingStrategies/sszSnappy/encode.js"; @@ -8,9 +6,9 @@ import {expectEqualByteChunks} from "../../../utils/index.js"; describe("encodingStrategies / sszSnappy / encode", () => { it.each(encodingStrategiesTestCases)("$id", async ({binaryPayload, chunks}) => { - const encodedChunks = await pipe(writeSszSnappyPayload(Buffer.from(binaryPayload.data)), all); + const encodedChunks = await Array.fromAsync(writeSszSnappyPayload(Buffer.from(binaryPayload.data))); expectEqualByteChunks( - encodedChunks as Uint8Array[], + encodedChunks, chunks.map((c) => c.subarray()) ); }); @@ -19,8 +17,8 @@ describe("encodingStrategies / sszSnappy / encode", () => { it.each(encodingStrategiesMainnetTestCases)("$id", async ({payload, streamedBody}) => { const bodySize = payload.data.length; - const encodedChunks = await pipe(writeSszSnappyPayload(Buffer.from(payload.data)), all); - const encodedStream = Buffer.concat(encodedChunks as Uint8Array[]); + const encodedChunks = await Array.fromAsync(writeSszSnappyPayload(Buffer.from(payload.data))); + const encodedStream = Buffer.concat(encodedChunks); const expectedStreamed = Buffer.concat([Buffer.from(varintEncode(bodySize)), streamedBody]); expect(encodedStream).toEqual(expectedStreamed); }); diff --git a/packages/reqresp/test/unit/encodingStrategies/sszSnappy/snappyFrames/uncompress.test.ts b/packages/reqresp/test/unit/encodingStrategies/sszSnappy/snappyFrames/uncompress.test.ts index 56715a48cb40..8cf7fc307c10 100644 --- a/packages/reqresp/test/unit/encodingStrategies/sszSnappy/snappyFrames/uncompress.test.ts +++ b/packages/reqresp/test/unit/encodingStrategies/sszSnappy/snappyFrames/uncompress.test.ts @@ -1,65 +1,48 @@ -import {pipe} from "it-pipe"; import {Uint8ArrayList} from "uint8arraylist"; import {describe, expect, it} from "vitest"; import { ChunkType, IDENTIFIER_FRAME, - SnappyFramesUncompress, crc, + decodeSnappyFrameData, + decodeSnappyFrames, encodeSnappy, + parseSnappyFrameHeader, } from "../../../../../src/utils/snappyIndex.js"; describe("encodingStrategies / sszSnappy / snappy frames / uncompress", () => { - it("should work with short input", () => - new Promise((done) => { - const testData = "Small test data"; - const compressIterable = encodeSnappy(Buffer.from(testData)); - - const decompress = new SnappyFramesUncompress(); - - void pipe(compressIterable, async (source) => { - for await (const data of source) { - const result = decompress.uncompress(new Uint8ArrayList(data)); - if (result) { - expect(result.subarray().toString()).toBe(testData); - done(); - } - } - }); - })); - - it("should work with huge input", () => - new Promise((done) => { - const testData = Buffer.alloc(100000, 4).toString(); - const compressIterable = encodeSnappy(Buffer.from(testData)); - let result = Buffer.alloc(0); - const decompress = new SnappyFramesUncompress(); - - void pipe(compressIterable, async (source) => { - for await (const data of source) { - // testData will come compressed as two or more chunks - result = Buffer.concat([ - result, - decompress.uncompress(new Uint8ArrayList(data))?.subarray() ?? Buffer.alloc(0), - ]); - if (result.length === testData.length) { - expect(result.toString()).toBe(testData); - done(); - } - } - }); - })); + it("should work with short input", async () => { + const testData = "Small test data"; + const compressIterable = encodeSnappy(Buffer.from(testData)); + const encoded: Uint8Array[] = []; - it("should detect malformed input", () => { - const decompress = new SnappyFramesUncompress(); + for await (const data of compressIterable) { + encoded.push(data); + } - expect(() => decompress.uncompress(new Uint8ArrayList(Buffer.alloc(32, 5)))).toThrow(); + const result = decodeSnappyFrames(Buffer.concat(encoded)); + expect(Buffer.from(result.subarray()).toString()).toBe(testData); }); - it("should return null if not enough data", () => { - const decompress = new SnappyFramesUncompress(); + it("should work with huge input", async () => { + const testData = Buffer.alloc(100000, 4).toString(); + const compressIterable = encodeSnappy(Buffer.from(testData)); + const encoded: Uint8Array[] = []; - expect(decompress.uncompress(new Uint8ArrayList(Buffer.alloc(3, 1)))).toBe(null); + for await (const data of compressIterable) { + encoded.push(data); + } + + const result = decodeSnappyFrames(Buffer.concat(encoded)); + expect(Buffer.from(result.subarray()).toString()).toBe(testData); + }); + + it("should detect malformed input", () => { + expect(() => decodeSnappyFrames(Buffer.alloc(32, 5))).toThrow(); + }); + + it("should return null if not enough data", () => { + expect(() => parseSnappyFrameHeader(Buffer.alloc(3, 1))).toThrow(/incomplete frame header/); }); it("should detect invalid checksum", () => { @@ -71,8 +54,7 @@ describe("encodingStrategies / sszSnappy / snappy frames / uncompress", () => { // 0xffffffff is clearly an invalid checksum chunks.append(Uint8Array.from(Array.from({length: 0x80}, () => 0xff))); - const decompress = new SnappyFramesUncompress(); - expect(() => decompress.uncompress(chunks)).toThrow(/checksum/); + expect(() => decodeSnappyFrames(chunks.subarray())).toThrow(/checksum/); }); it("should detect skippable frames", () => { @@ -82,8 +64,7 @@ describe("encodingStrategies / sszSnappy / snappy frames / uncompress", () => { chunks.append(Uint8Array.from([ChunkType.SKIPPABLE, 0x80, 0x00, 0x00])); chunks.append(Uint8Array.from(Array.from({length: 0x80}, () => 0xff))); - const decompress = new SnappyFramesUncompress(); - expect(decompress.uncompress(chunks)).toBeNull(); + expect(decodeSnappyFrames(chunks.subarray()).length).toBe(0); }); it("should detect large data", () => { @@ -97,7 +78,19 @@ describe("encodingStrategies / sszSnappy / snappy frames / uncompress", () => { chunks.append(checksum); chunks.append(data); - const decompress = new SnappyFramesUncompress(); - expect(() => decompress.uncompress(chunks)).toThrow(/large/); + expect(() => decodeSnappyFrames(chunks.subarray())).toThrow(/large/); + }); + + it("should parse header and decode uncompressed frame", () => { + const payload = Uint8Array.from([1, 2, 3, 4]); + const checksum = crc(payload); + const frame = Buffer.concat([checksum, payload]); + + const header = Uint8Array.from([ChunkType.UNCOMPRESSED, frame.length, 0x00, 0x00]); + const parsed = parseSnappyFrameHeader(header); + const decoded = decodeSnappyFrameData(parsed.type, frame); + + expect(parsed.frameSize).toBe(frame.length); + expect(Buffer.from(decoded ?? [])).toEqual(Buffer.from(payload)); }); }); diff --git a/packages/reqresp/test/unit/request/index.test.ts b/packages/reqresp/test/unit/request/index.test.ts index 578feec130b6..82457f7f24bd 100644 --- a/packages/reqresp/test/unit/request/index.test.ts +++ b/packages/reqresp/test/unit/request/index.test.ts @@ -1,7 +1,5 @@ import {PeerId} from "@libp2p/interface"; -import all from "it-all"; -import {pipe} from "it-pipe"; -import {Libp2p} from "libp2p"; +import type {Libp2p} from "libp2p"; import {afterEach, beforeEach, describe, expect, it, vi} from "vitest"; import {getEmptyLogger} from "@lodestar/logger/empty"; import {LodestarError, sleep} from "@lodestar/utils"; @@ -11,7 +9,7 @@ import {MixedProtocol, Protocol, ResponseIncoming} from "../../../src/types.js"; import {getEmptyHandler, sszSnappyPing} from "../../fixtures/messages.js"; import {pingProtocol} from "../../fixtures/protocols.js"; import {expectRejectedWithLodestarError} from "../../utils/errors.js"; -import {MockLibP2pStream} from "../../utils/index.js"; +import {createMockStream} from "../../utils/mockStream.js"; import {getValidPeerId} from "../../utils/peer.js"; import {responseEncode} from "../../utils/response.js"; @@ -27,7 +25,6 @@ describe("request / sendRequest", () => { id: string; protocols: MixedProtocol[]; requestBody: ResponseIncoming; - maxResponses?: number; expectedReturn: unknown[]; }[] = [ { @@ -36,13 +33,6 @@ describe("request / sendRequest", () => { requestBody: sszSnappyPing.binaryPayload, expectedReturn: [{...sszSnappyPing.binaryPayload, data: Buffer.from(sszSnappyPing.binaryPayload.data)}], }, - // limit to max responses is no longer the responsibility of this package - // { - // id: "Return up to maxResponses for a multi-chunk method", - // protocols: [customProtocol({})], - // requestBody: sszSnappySignedBeaconBlockPhase0.binaryPayload, - // expectedReturn: [sszSnappySignedBeaconBlockPhase0.binaryPayload], - // }, ]; beforeEach(() => { @@ -57,18 +47,25 @@ describe("request / sendRequest", () => { for (const {id, protocols, expectedReturn, requestBody} of testCases) { it(id, async () => { + const encodedResponse = await Array.fromAsync( + responseEncode([{status: RespStatus.SUCCESS, payload: requestBody}], protocols[0] as Protocol) + ); + libp2p = { - dialProtocol: vi - .fn() - .mockResolvedValue( - new MockLibP2pStream( - responseEncode([{status: RespStatus.SUCCESS, payload: requestBody}], protocols[0] as Protocol), - protocols[0].method - ) - ), + dialProtocol: vi.fn().mockImplementation( + async () => + ( + await createMockStream({ + protocol: protocols[0].method, + source: (async function* (): AsyncIterable { + yield Buffer.concat(encodedResponse); + })(), + }) + ).stream + ), } as unknown as Libp2p; - const responses = await pipe( + const responses = await Array.fromAsync( sendRequest( {logger, libp2p, metrics: null}, peerId, @@ -76,13 +73,152 @@ describe("request / sendRequest", () => { protocols.map((p) => p.method), EMPTY_REQUEST, controller.signal - ), - all + ) ); - expect(responses).toEqual(expectedReturn); + expect(responses.map((r) => ({...r, data: Buffer.from(r.data)}))).toEqual(expectedReturn); }); } + it("closes stream gracefully when caller stops consuming responses early", async () => { + const encodedResponse = await Array.fromAsync( + responseEncode( + [ + {status: RespStatus.SUCCESS, payload: sszSnappyPing.binaryPayload}, + {status: RespStatus.SUCCESS, payload: sszSnappyPing.binaryPayload}, + ], + emptyProtocol as Protocol + ) + ); + + let abortCalled = false; + let getStreamState = (): {status: string; readStatus: string; writeStatus: string} => ({ + status: "not-created", + readStatus: "not-created", + writeStatus: "not-created", + }); + libp2p = { + dialProtocol: vi.fn().mockImplementation(async () => { + const streamResult = await createMockStream({ + protocol: emptyProtocol.method, + source: (async function* (): AsyncIterable { + yield Buffer.concat(encodedResponse); + await sleep(100000, controller.signal); + })(), + }); + const reqStream = streamResult.stream as unknown as { + status: string; + readStatus: string; + writeStatus: string; + abort: (error: Error) => void; + }; + const abort = reqStream.abort.bind(reqStream); + reqStream.abort = (error: Error): void => { + abortCalled = true; + abort(error); + }; + getStreamState = () => ({ + status: reqStream.status, + readStatus: reqStream.readStatus, + writeStatus: reqStream.writeStatus, + }); + return streamResult.stream; + }), + } as unknown as Libp2p; + + let responseCount = 0; + for await (const _ of sendRequest( + {logger, libp2p, metrics: null}, + peerId, + [emptyProtocol], + [emptyProtocol.method], + EMPTY_REQUEST, + controller.signal + )) { + responseCount++; + break; + } + + expect(responseCount).toBe(1); + expect(abortCalled).toBe(false); + const streamState = getStreamState(); + expect(streamState.status).not.toBe("aborted"); + expect(streamState.readStatus).toBe("closed"); + expect(streamState.writeStatus).toBe("closed"); + }); + + it("aborts stream if remote never closes after early consumer exit", async () => { + const encodedResponse = await Array.fromAsync( + responseEncode([{status: RespStatus.SUCCESS, payload: sszSnappyPing.binaryPayload}], emptyProtocol as Protocol) + ); + + let getStreamStatus = (): string => "not-created"; + libp2p = { + dialProtocol: vi.fn().mockImplementation(async () => { + const streamResult = await createMockStream({ + protocol: emptyProtocol.method, + source: (async function* (): AsyncIterable { + yield Buffer.concat(encodedResponse); + await sleep(100000, controller.signal); + })(), + }); + const reqStream = streamResult.stream as unknown as {status: string}; + getStreamStatus = () => reqStream.status; + return streamResult.stream; + }), + } as unknown as Libp2p; + + for await (const _ of sendRequest( + {logger, libp2p, metrics: null}, + peerId, + [emptyProtocol], + [emptyProtocol.method], + EMPTY_REQUEST, + controller.signal, + {respTimeoutMs: 20} + )) { + break; + } + + expect(getStreamStatus()).not.toBe("aborted"); + await sleep(50, controller.signal); + expect(getStreamStatus()).toBe("aborted"); + }); + + it("aborts stream on RESP_TIMEOUT", async () => { + const testMethod = "req/test"; + let getStreamStatus = (): string => "not-created"; + libp2p = { + dialProtocol: vi.fn().mockImplementation(async () => { + const streamResult = await createMockStream({ + protocol: testMethod, + source: (async function* (): AsyncIterable { + await sleep(100000, controller.signal); + yield new Uint8Array(); + })(), + }); + const reqStream = streamResult.stream as unknown as {status: string}; + getStreamStatus = () => reqStream.status; + return streamResult.stream; + }), + } as unknown as Libp2p; + + await expectRejectedWithLodestarError( + Array.fromAsync( + sendRequest( + {logger, libp2p, metrics: null}, + peerId, + [emptyProtocol], + [testMethod], + EMPTY_REQUEST, + controller.signal, + {respTimeoutMs: 1} + ) + ), + new RequestError({code: RequestErrorCode.RESP_TIMEOUT}) + ); + expect(getStreamStatus()).toBe("aborted"); + }); + describe("timeout cases", () => { const peerId = getValidPeerId(); const testMethod = "req/test"; @@ -94,13 +230,13 @@ describe("request / sendRequest", () => { error?: LodestarError; }[] = [ { - id: "trigger a TTFB_TIMEOUT", - opts: {ttfbTimeoutMs: 0}, + id: "trigger a RESP_TIMEOUT when first response is delayed", + opts: {respTimeoutMs: 0}, source: async function* () { await sleep(30); // Pause for too long before first byte yield sszSnappyPing.chunks[0]; }, - error: new RequestError({code: RequestErrorCode.TTFB_TIMEOUT}), + error: new RequestError({code: RequestErrorCode.RESP_TIMEOUT}), }, { id: "trigger a RESP_TIMEOUT", @@ -113,18 +249,17 @@ describe("request / sendRequest", () => { error: new RequestError({code: RequestErrorCode.RESP_TIMEOUT}), }, { - // Upstream "abortable-iterator" never throws with an infinite sleep. id: "Infinite sleep on first byte", - opts: {ttfbTimeoutMs: 1, respTimeoutMs: 1}, + opts: {respTimeoutMs: 1}, source: async function* () { await sleep(100000, controller.signal); yield sszSnappyPing.chunks[0]; }, - error: new RequestError({code: RequestErrorCode.TTFB_TIMEOUT}), + error: new RequestError({code: RequestErrorCode.RESP_TIMEOUT}), }, { id: "Infinite sleep on second chunk", - opts: {ttfbTimeoutMs: 1, respTimeoutMs: 1}, + opts: {respTimeoutMs: 1}, source: async function* () { yield sszSnappyPing.chunks[0]; await sleep(100000, controller.signal); @@ -136,11 +271,13 @@ describe("request / sendRequest", () => { for (const {id, source, opts, error} of timeoutTestCases) { it(id, async () => { libp2p = { - dialProtocol: vi.fn().mockResolvedValue(new MockLibP2pStream(source(), testMethod)), + dialProtocol: vi + .fn() + .mockImplementation(async () => (await createMockStream({protocol: testMethod, source: source()})).stream), } as unknown as Libp2p; await expectRejectedWithLodestarError( - pipe( + Array.fromAsync( sendRequest( {logger, libp2p, metrics: null}, peerId, @@ -149,8 +286,7 @@ describe("request / sendRequest", () => { EMPTY_REQUEST, controller.signal, opts - ), - all + ) ), error as LodestarError ); diff --git a/packages/reqresp/test/unit/response/index.test.ts b/packages/reqresp/test/unit/response/index.test.ts index f527961e078f..f808ac0d05a7 100644 --- a/packages/reqresp/test/unit/response/index.test.ts +++ b/packages/reqresp/test/unit/response/index.test.ts @@ -9,7 +9,8 @@ import {handleRequest} from "../../../src/response/index.js"; import {sszSnappyPing} from "../../fixtures/messages.js"; import {pingProtocol} from "../../fixtures/protocols.js"; import {expectRejectedWithLodestarError} from "../../utils/errors.js"; -import {MockLibP2pStream, expectEqualByteChunks} from "../../utils/index.js"; +import {expectEqualByteChunks} from "../../utils/index.js"; +import {createMockStream} from "../../utils/mockStream.js"; import {getValidPeerId} from "../../utils/peer.js"; const testCases: { @@ -59,7 +60,13 @@ describe("response / handleRequest", () => { afterEach(() => controller.abort()); it.each(testCases)("$id", async ({requestChunks, protocol, expectedResponseChunks, expectedError}) => { - const stream = new MockLibP2pStream(requestChunks as any); + const {stream, sentChunks} = await createMockStream({ + source: (async function* (): AsyncIterable { + for (const chunk of requestChunks) { + yield chunk; + } + })(), + }); const rateLimiter = new ReqRespRateLimiter({rateLimitMultiplier: 0}); const resultPromise = handleRequest({ @@ -80,6 +87,9 @@ describe("response / handleRequest", () => { await expect(resultPromise).resolves.toBeUndefined(); } - expectEqualByteChunks(stream.resultChunks, expectedResponseChunks, "Wrong response chunks"); + // Allow stream message events to be delivered before asserting on captured chunks + await new Promise((resolve) => setTimeout(resolve, 0)); + + expectEqualByteChunks(sentChunks, expectedResponseChunks, "Wrong response chunks"); }); }); diff --git a/packages/reqresp/test/unit/utils/errorMessage.test.ts b/packages/reqresp/test/unit/utils/errorMessage.test.ts index 763eaee5526e..4f833d612a0d 100644 --- a/packages/reqresp/test/unit/utils/errorMessage.test.ts +++ b/packages/reqresp/test/unit/utils/errorMessage.test.ts @@ -41,7 +41,7 @@ describe("encode and decode error message", () => { } const encodedMessage = accu.subarray(0); - const decodedErrorMessage = await decodeErrorMessage(encodedMessage); + const decodedErrorMessage = decodeErrorMessage(encodedMessage); expect(decodedErrorMessage).toBe(errorMessage); }); } diff --git a/packages/reqresp/test/utils/index.ts b/packages/reqresp/test/utils/index.ts index 67c5e5875c91..f8a05212c309 100644 --- a/packages/reqresp/test/utils/index.ts +++ b/packages/reqresp/test/utils/index.ts @@ -1,6 +1,3 @@ -import {Direction, ReadStatus, Stream, StreamStatus, WriteStatus} from "@libp2p/interface"; -import {logger} from "@libp2p/logger"; -import {Uint8ArrayList} from "uint8arraylist"; import {expect} from "vitest"; import {toHexString} from "@chainsafe/ssz"; import {fromHex} from "@lodestar/utils"; @@ -8,8 +5,7 @@ import {RespStatus, ResponseIncoming} from "../../src/index.js"; import {ResponseChunk} from "../fixtures/index.js"; /** - * Helper for it-pipe when first argument is an array. - * it-pipe does not convert the chunks array to a generator and BufferedSource breaks + * Converts an array to an async source. */ export async function* arrToSource(arr: T[]): AsyncGenerator { for (const item of arr) { @@ -41,44 +37,6 @@ export function expectInEqualByteChunks(chunks: Uint8Array[], expectedChunks: Ui } } -/** - * Useful to simulate a LibP2P stream source emitting prepared bytes - * and capture the response with a sink accessible via `this.resultChunks` - */ -export class MockLibP2pStream implements Stream { - protocol: string; - id = "mock"; - log = logger("mock"); - direction: Direction = "inbound"; - status: StreamStatus = "open"; - readStatus: ReadStatus = "ready"; - writeStatus: WriteStatus = "ready"; - timeline = { - open: Date.now(), - }; - metadata = {}; - source: Stream["source"]; - resultChunks: Uint8Array[] = []; - - constructor(requestChunks: Uint8ArrayList[] | AsyncIterable | AsyncGenerator, protocol?: string) { - this.source = Array.isArray(requestChunks) - ? arrToSource(requestChunks) - : (requestChunks as AsyncGenerator); - this.protocol = protocol ?? "mock"; - } - - sink: Stream["sink"] = async (source) => { - for await (const chunk of source) { - this.resultChunks.push(chunk.subarray()); - } - }; - - close: Stream["close"] = async () => {}; - closeRead = async (): Promise => {}; - closeWrite = async (): Promise => {}; - abort: Stream["abort"] = () => this.close(); -} - export function fromHexBuf(hex: string): Buffer { return Buffer.from(fromHex(hex)); } diff --git a/packages/reqresp/test/utils/mockStream.ts b/packages/reqresp/test/utils/mockStream.ts new file mode 100644 index 000000000000..f618460b4a65 --- /dev/null +++ b/packages/reqresp/test/utils/mockStream.ts @@ -0,0 +1,48 @@ +import type {Stream} from "@libp2p/interface"; +import {StreamMessageEvent} from "@libp2p/interface"; +import {streamPair} from "@libp2p/utils"; +import {Uint8ArrayList} from "uint8arraylist"; + +type SourceChunk = Uint8Array | Uint8ArrayList; + +/** + * Minimal stream test double for reqresp tests backed by upstream streamPair. + * Pumps source data into one side and returns the other for the code-under-test. + * Captures data sent back through the stream in sentChunks. + */ +export async function createMockStream({ + protocol = "", + source = (async function* (): AsyncIterable {})(), +}: { + protocol?: string; + source?: AsyncIterable; +} = {}): Promise<{stream: Stream; sentChunks: Uint8Array[]}> { + const [writer, reader] = await streamPair({protocol, delay: 0}); + const sentChunks: Uint8Array[] = []; + + // streamPair only sets protocol on the outbound stream; mirror it to the inbound + reader.protocol = protocol; + + // Capture data sent back through the reader stream (received on writer side) + writer.addEventListener("message", (evt: StreamMessageEvent) => { + const data = evt.data; + sentChunks.push(data instanceof Uint8ArrayList ? data.subarray() : data); + }); + + // Pump source data into the writer side so the reader can consume it. + // Use close() to signal EOF — AbstractStream.close() sends closeWrite to the + // remote, which is what byteStream uses to detect end-of-stream. + void (async () => { + try { + for await (const chunk of source) { + const data = chunk instanceof Uint8ArrayList ? chunk : new Uint8ArrayList(chunk); + writer.send(data); + } + } catch { + // Sources may abort on purpose to simulate timeouts + } + await writer.close(); + })(); + + return {stream: reader, sentChunks}; +} diff --git a/packages/reqresp/test/utils/response.ts b/packages/reqresp/test/utils/response.ts index 0452fb6006e4..ce2d636ee470 100644 --- a/packages/reqresp/test/utils/response.ts +++ b/packages/reqresp/test/utils/response.ts @@ -1,4 +1,3 @@ -import {pipe} from "it-pipe"; import {responseEncodeError, responseEncodeSuccess} from "../../src/encoders/responseEncode.js"; import {RespStatus} from "../../src/interface.js"; import {Protocol} from "../../src/types.js"; @@ -6,15 +5,15 @@ import {ResponseChunk} from "../fixtures/encoders.js"; import {beaconConfig} from "../fixtures/messages.js"; import {arrToSource} from "../utils/index.js"; -export async function* responseEncode(responseChunks: ResponseChunk[], protocol: Protocol): AsyncIterable { +export async function* responseEncode(responseChunks: ResponseChunk[], protocol: Protocol): AsyncIterable { for (const chunk of responseChunks) { if (chunk.status === RespStatus.SUCCESS) { const payload = chunk.payload; - yield* pipe( + yield* responseEncodeSuccess( + protocol, arrToSource([ {...payload, boundary: beaconConfig.getForkBoundaryAtEpoch(beaconConfig.forks[payload.fork].epoch)}, - ]), - responseEncodeSuccess(protocol, {onChunk: () => {}}) + ]) ); } else { yield* responseEncodeError(protocol, chunk.status, chunk.errorMessage); diff --git a/packages/reqresp/tsconfig.json b/packages/reqresp/tsconfig.json index 4ee2e1092000..d0d3bd757da9 100644 --- a/packages/reqresp/tsconfig.json +++ b/packages/reqresp/tsconfig.json @@ -1,7 +1,6 @@ { "extends": "../../tsconfig.json", "include": ["src", "test"], - "exclude": ["../../node_modules/it-pipe"], "compilerOptions": { "rootDir": "../..", "outDir": "lib", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d69daae11aaf..d2b20b93e3a2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -105,8 +105,8 @@ importers: specifier: ^23.0.1 version: 23.0.1 libp2p: - specifier: 2.9.0 - version: 2.9.0 + specifier: 3.1.3 + version: 3.1.3 node-gyp: specifier: ^9.4.0 version: 9.4.1 @@ -220,17 +220,14 @@ importers: specifier: ^2.2.0 version: 2.2.0 '@chainsafe/discv5': - specifier: ^11.0.4 - version: 11.0.4 + specifier: ^12.0.0 + version: 12.0.0 '@chainsafe/enr': - specifier: ^5.0.1 - version: 5.0.1 - '@chainsafe/libp2p-gossipsub': - specifier: ^14.1.2 - version: 14.1.2 + specifier: ^6.0.0 + version: 6.0.0 '@chainsafe/libp2p-noise': - specifier: ^16.1.5 - version: 16.1.5 + specifier: ^17.0.0 + version: 17.0.0 '@chainsafe/persistent-merkle-tree': specifier: ^1.2.1 version: 1.2.1 @@ -265,32 +262,35 @@ importers: specifier: ^5.0.1 version: 5.0.1 '@libp2p/bootstrap': - specifier: ^11.0.32 - version: 11.0.46 + specifier: ^12.0.11 + version: 12.0.11 '@libp2p/crypto': - specifier: ^5.0.15 - version: 5.1.7 + specifier: ^5.1.13 + version: 5.1.13 + '@libp2p/gossipsub': + specifier: ^15.0.12 + version: 15.0.12 '@libp2p/identify': - specifier: ^3.0.27 - version: 3.0.38 + specifier: ^4.0.10 + version: 4.0.10 '@libp2p/interface': - specifier: ^2.7.0 - version: 2.10.5 + specifier: ^3.1.0 + version: 3.1.0 '@libp2p/mdns': - specifier: ^11.0.32 - version: 11.0.46 + specifier: ^12.0.11 + version: 12.0.11 '@libp2p/mplex': - specifier: ^11.0.32 - version: 11.0.46 + specifier: ^12.0.11 + version: 12.0.11 '@libp2p/peer-id': - specifier: ^5.1.0 - version: 5.1.8 + specifier: ^6.0.4 + version: 6.0.4 '@libp2p/prometheus-metrics': - specifier: ^4.3.15 - version: 4.3.29 + specifier: ^5.0.10 + version: 5.0.10 '@libp2p/tcp': - specifier: ^10.1.8 - version: 10.1.18 + specifier: ^11.0.10 + version: 11.0.10 '@lodestar/api': specifier: workspace:^ version: link:../api @@ -328,17 +328,17 @@ importers: specifier: workspace:^ version: link:../validator '@multiformats/multiaddr': - specifier: ^12.1.3 - version: 12.5.1 + specifier: ^13.0.1 + version: 13.0.1 datastore-core: - specifier: ^10.0.2 - version: 10.0.2 + specifier: ^11.0.2 + version: 11.0.2 datastore-fs: - specifier: ^10.0.6 - version: 10.0.6 + specifier: ^11.0.2 + version: 11.0.2 datastore-level: - specifier: ^11.0.3 - version: 11.0.4 + specifier: ^12.0.2 + version: 12.0.2 deepmerge: specifier: ^4.3.1 version: 4.3.1 @@ -346,23 +346,17 @@ importers: specifier: ^5.7.4 version: 5.7.4 interface-datastore: - specifier: ^8.3.0 - version: 8.3.1 - it-all: - specifier: ^3.0.4 - version: 3.0.9 - it-pipe: - specifier: ^3.0.1 - version: 3.0.1 + specifier: ^9.0.2 + version: 9.0.2 jwt-simple: specifier: 0.5.6 version: 0.5.6 libp2p: - specifier: 2.9.0 - version: 2.9.0 + specifier: 3.1.3 + version: 3.1.3 multiformats: - specifier: ^11.0.1 - version: 11.0.2 + specifier: ^13.4.2 + version: 13.4.2 prom-client: specifier: ^15.1.0 version: 15.1.3 @@ -386,11 +380,14 @@ importers: specifier: ^1.2.1 version: 1.2.1 '@libp2p/interface-internal': - specifier: ^2.3.18 - version: 2.3.18 + specifier: ^3.0.10 + version: 3.0.10 '@libp2p/logger': - specifier: ^5.1.21 - version: 5.1.21 + specifier: ^6.2.2 + version: 6.2.2 + '@libp2p/utils': + specifier: ^7.0.10 + version: 7.0.10 '@lodestar/spec-test-util': specifier: workspace:^ version: link:../spec-test-util @@ -403,12 +400,6 @@ importers: '@types/tmp': specifier: ^0.2.3 version: 0.2.3 - it-drain: - specifier: ^3.0.3 - version: 3.0.10 - it-pair: - specifier: ^2.0.6 - version: 2.0.6 js-yaml: specifier: ^4.1.0 version: 4.1.1 @@ -440,11 +431,11 @@ importers: specifier: ^2.2.0 version: 2.2.0 '@chainsafe/discv5': - specifier: ^11.0.4 - version: 11.0.4 + specifier: ^12.0.0 + version: 12.0.0 '@chainsafe/enr': - specifier: ^5.0.1 - version: 5.0.1 + specifier: ^6.0.0 + version: 6.0.0 '@chainsafe/persistent-merkle-tree': specifier: ^1.2.1 version: 1.2.1 @@ -458,14 +449,14 @@ importers: specifier: ^1.11.3 version: 1.11.3 '@libp2p/crypto': - specifier: ^5.0.15 - version: 5.1.7 + specifier: ^5.1.13 + version: 5.1.13 '@libp2p/interface': - specifier: ^2.7.0 - version: 2.10.5 + specifier: ^3.1.0 + version: 3.1.0 '@libp2p/peer-id': - specifier: ^5.1.0 - version: 5.1.8 + specifier: ^6.0.4 + version: 6.0.4 '@lodestar/api': specifier: workspace:^ version: link:../api @@ -500,8 +491,8 @@ importers: specifier: workspace:^ version: link:../validator '@multiformats/multiaddr': - specifier: ^12.1.3 - version: 12.5.1 + specifier: ^13.0.1 + version: 13.0.1 deepmerge: specifier: ^4.3.1 version: 4.3.1 @@ -611,9 +602,6 @@ importers: classic-level: specifier: ^1.4.1 version: 1.4.1 - it-all: - specifier: ^3.0.4 - version: 3.0.9 devDependencies: '@lodestar/logger': specifier: workspace:^ @@ -899,8 +887,11 @@ importers: specifier: ^4.2.0 version: 4.2.0 '@libp2p/interface': - specifier: ^2.7.0 - version: 2.10.5 + specifier: ^3.1.0 + version: 3.1.0 + '@libp2p/utils': + specifier: ^7.0.10 + version: 7.0.10 '@lodestar/config': specifier: workspace:^ version: link:../config @@ -910,12 +901,6 @@ importers: '@lodestar/utils': specifier: workspace:^ version: link:../utils - it-all: - specifier: ^3.0.4 - version: 3.0.9 - it-pipe: - specifier: ^3.0.1 - version: 3.0.1 snappy: specifier: ^7.2.2 version: 7.2.2 @@ -933,26 +918,23 @@ importers: specifier: ^1.2.2 version: 1.2.2 '@libp2p/crypto': - specifier: ^5.1.7 - version: 5.1.7 + specifier: ^5.1.13 + version: 5.1.13 '@libp2p/logger': - specifier: ^5.1.21 - version: 5.1.21 + specifier: ^6.2.2 + version: 6.2.2 '@libp2p/peer-id': - specifier: ^5.1.8 - version: 5.1.8 + specifier: ^6.0.4 + version: 6.0.4 '@lodestar/logger': specifier: workspace:^ version: link:../logger '@lodestar/types': specifier: workspace:^ version: link:../types - it-stream-types: - specifier: ^2.0.2 - version: 2.0.2 libp2p: - specifier: 2.9.0 - version: 2.9.0 + specifier: 3.1.3 + version: 3.1.3 packages/spec-test-util: dependencies: @@ -1396,11 +1378,11 @@ packages: resolution: {integrity: sha512-VBaQoNE2a9d9+skAjQKv3Suk0yGKqp3mZM0YWYJNPj/Ae/f6lAyeVSgKqo2LrsNQBzD/LqrJLKUY8rJT3vDKLA==} engines: {node: '>= 16'} - '@chainsafe/discv5@11.0.4': - resolution: {integrity: sha512-8bUqQXlh3+VBO3f3kzvFL0CLOeShZLjUexlmjCyfXfOgwpP1e46F4G6a9tneEEXaU+AMLPMz8uRCwtP/YsESbw==} + '@chainsafe/discv5@12.0.0': + resolution: {integrity: sha512-ESP0UU1K9WySq35/9d6usB1B3jNP2QYtlbknriddQFwWMtp7bBY9aIK4X2/g6ccutD1ILt1RYxdBrccj555nXg==} - '@chainsafe/enr@5.0.1': - resolution: {integrity: sha512-RP5XxJOSYt0f775ne6NqlaG33Qia+IaMjWlP7PlWagz6SOpXlfbm+vaqHx3by6lKPb9aW4UQtO+fiRxz85XwaQ==} + '@chainsafe/enr@6.0.0': + resolution: {integrity: sha512-uRnTpjJlafTQI6RinhN1EgEGZL3VMOcHUcnpDCL9LclTglKUmDB7RdTi+Qqj8tJbYEehYXhCwZiWq2dgbnpKww==} '@chainsafe/fast-crc32c@4.2.0': resolution: {integrity: sha512-wbVJ3q1DBuYqs1VGwLe9o5VIDQtxGpSVzr7UY9Hq2SQ283fU6AouII/pwioS9MbiiIK42jDxtY0uLFmTuenb0g==} @@ -1452,12 +1434,8 @@ packages: '@chainsafe/is-ip@2.1.0': resolution: {integrity: sha512-KIjt+6IfysQ4GCv66xihEitBjvhU/bixbbbFxdJ1sqCp4uJ0wuZiYBPhksZoy4lfaF0k9cwNzY5upEW/VWdw3w==} - '@chainsafe/libp2p-gossipsub@14.1.2': - resolution: {integrity: sha512-HuyB1dO+GwqpbvKhq2TdrsvrtNt19iOWLEfjy2N3ItYY1gA2F67KtCOLmZ1NlEcCbjFDg+qRffQMjgkDwHSFOg==} - engines: {npm: '>=8.7.0'} - - '@chainsafe/libp2p-noise@16.1.5': - resolution: {integrity: sha512-yac0bknwfuYdXXOAFGVL4fYR0de0p1Uk7W0gIkuxlj8JSqhkHr+kEt3958PfnfmX880QJAIogsh/ZupFjg0uJA==} + '@chainsafe/libp2p-noise@17.0.0': + resolution: {integrity: sha512-vwrmY2Y+L1xYhIDiEpl61KHxwrLCZoXzTpwhyk34u+3+6zCAZPL3GxH3i2cs+u5IYNoyLptORdH17RKFXy7upA==} '@chainsafe/netmask@2.0.0': resolution: {integrity: sha512-I3Z+6SWUoaljh3TBzCnCxjlUyN8tA+NAk5L6m9IxvCf1BENQTePzPMis97CoN/iMW1St3WN+AWCCRp+TTBRiDg==} @@ -1848,16 +1826,16 @@ packages: resolution: {integrity: sha512-9PzshkvwO8YBkSD9+vyhJuzM6hxfZlljGnuUbXQlTSGEod7we8BRyzJW53W7nw/WRw5U6wf9Q2fpWypfZFkrbw==} engines: {node: '>=14'} + '@ethereumjs/rlp@10.1.1': + resolution: {integrity: sha512-jbnWTEwcpoY+gE0r+wxfDG9zgiu54DcTcwnc9sX3DsqKR4l5K7x2V8mQL3Et6hURa4DuT9g7z6ukwpBLFchszg==} + engines: {node: '>=20'} + hasBin: true + '@ethereumjs/rlp@4.0.1': resolution: {integrity: sha512-tqsQiBQDQdmPWE1xkkBq4rlSW5QZpLOUJ5RJh2/9fug+q9tnUhuZoVLk7s0scUIKTOzEtR72DFBXI4WiZcMpvw==} engines: {node: '>=14'} hasBin: true - '@ethereumjs/rlp@5.0.2': - resolution: {integrity: sha512-DziebCdg4JpGlEqEdGgXmjqcFoJi+JGulUXwEjsZGAscAQ7MyD/7LE/GVCP29vEQxKc7AAwjT3A2ywHp2xfoCA==} - engines: {node: '>=18'} - hasBin: true - '@ethereumjs/statemanager@1.0.5': resolution: {integrity: sha512-TVkx9Kgc2NtObCzUTTqrpUggNLnftdmxZybzKPd565Bh98FJJB30FrVkWdPwaIV8oB1d9ADtthttfx5Y/kY9gw==} @@ -2212,56 +2190,57 @@ packages: resolution: {integrity: sha512-hRFGB47RWHNKZkh9t+ka34kNlv5DQwrIDBvjI+mmzZqZK7UAtPRsJoE+MlQHVlQApDFwUPiSTdxY9UXzJrYFdQ==} engines: {node: ^20.17.0 || >=22.9.0} - '@libp2p/bootstrap@11.0.46': - resolution: {integrity: sha512-VxWHWjryu6YCv5JEylL5L9z2p38DvZPjEhH9jR382Ts19Dg0G+30DAcv7A3Ad1R2Bg8q2acxZGE3W0nwXrpJxQ==} + '@libp2p/bootstrap@12.0.11': + resolution: {integrity: sha512-ZIG8QKS+4w7ugK7a1ftdopjIA+NvOPKUq7JY1OsRxaiLdCdxgghPTiNIbinYsVv5iHULBnFZe4o5l+5L7+Hssw==} - '@libp2p/crypto@5.1.7': - resolution: {integrity: sha512-7DO0piidLEKfCuNfS420BlHG0e2tH7W/zugdsPSiC/1Apa/s1B1dBkaIEgfDkGjrRP4S/8Or86Rtq7zXeEu67g==} + '@libp2p/crypto@5.1.13': + resolution: {integrity: sha512-8NN9cQP3jDn+p9+QE9ByiEoZ2lemDFf/unTgiKmS3JF93ph240EUVdbCyyEgOMfykzb0okTM4gzvwfx9osJebQ==} - '@libp2p/identify@3.0.38': - resolution: {integrity: sha512-2eCZU5CSTvyt78jtMLQ88m6Rlrcrt7BVccGHsdjx8hguvFYWqBjq19RQ5Zm47bbHlNzjLwrAUi6vfKx2r7fKIg==} + '@libp2p/gossipsub@15.0.12': + resolution: {integrity: sha512-jhyzujQeH/o+WMS9L+m785ygF8W5++Lay6pgnfkgHj1qrtc/IJtMIgsASZaoTTQY+yC6Zy2EL4r5dJ131mBVNQ==} + engines: {npm: '>=8.7.0'} - '@libp2p/interface-internal@2.3.18': - resolution: {integrity: sha512-tnZ20IFASXLbDc2JxeUPZNIXDuN5Ge7be6BU458WLvmquf93NlSqZkWs6xFdi+0yXUrw7GGTgzIP5v+1LnDUmA==} + '@libp2p/identify@4.0.10': + resolution: {integrity: sha512-DROyV+bZIlz9czCCHJdeVtm1+hEOKUigJHyTzzA/cuwwyvtm8Dco8F+VRYcrwpafuVtjv7yN7CskN4oIys56jw==} - '@libp2p/interface@2.10.5': - resolution: {integrity: sha512-Z52n04Mph/myGdwyExbFi5S/HqrmZ9JOmfLc2v4r2Cik3GRdw98vrGH19PFvvwjLwAjaqsweCtlGaBzAz09YDw==} + '@libp2p/interface-internal@3.0.10': + resolution: {integrity: sha512-Gd/eQAoAlXqeCRJ6wOwcnTQ/SDe95bQow8osY8zq0nbfFBu26aChQHjAd+CjcCADJRh+Sd+7+dYG7BrhpxGt1A==} - '@libp2p/logger@5.1.21': - resolution: {integrity: sha512-V1TWlZM5BuKkiGQ7En4qOnseVP82JwDIpIfNjceUZz1ArL32A5HXJjLQnJchkZ3VW8PVciJzUos/vP6slhPY6Q==} + '@libp2p/interface@3.1.0': + resolution: {integrity: sha512-RE7/XyvC47fQBe1cHxhMvepYKa5bFCUyFrrpj8PuM0E7JtzxU7F+Du5j4VXbg2yLDcToe0+j8mB7jvwE2AThYw==} - '@libp2p/mdns@11.0.46': - resolution: {integrity: sha512-7E3kkdzXe6x+6lBc3ogniJJubaxz6SCAASi70Ne2NWZSMCOYaCjtGtnh5/Ddtel80ZJl92C/0nj9T4Gtgtr+Zg==} + '@libp2p/logger@6.2.2': + resolution: {integrity: sha512-XtanXDT+TuMuZoCK760HGV1AmJsZbwAw5AiRUxWDbsZPwAroYq64nb41AHRu9Gyc0TK9YD+p72+5+FIxbw0hzw==} - '@libp2p/mplex@11.0.46': - resolution: {integrity: sha512-01b077JuqqsDBUcSE68h+6kI1+/9HZM0eog6vM8EOuVttPwDhmyObpuUiBmBTTK8ggCWQrct2KWxR7+PHJDJIw==} + '@libp2p/mdns@12.0.11': + resolution: {integrity: sha512-OB6am5A21Yc5c7KBZONQhTao4BHRDc3MurZ1qHzqU4FQidi719cNRw4ac6TVk4dcdtOYx+1ef8pvvLX+57hXAQ==} - '@libp2p/multistream-select@6.0.28': - resolution: {integrity: sha512-ILu65FAX2Hak7x40DXb0gYptF6BmlGGW2kNgGeKIcNeseuvsAkBPO8k0CHwr8MU5mnHamTiweLJh5jD0iVZJ1A==} + '@libp2p/mplex@12.0.11': + resolution: {integrity: sha512-jD77lX3FkgHM4FdznF5G2aN8G6BoQrPZPTdox56KvFOUjUKs04KoiC58kuSHVuEo/25gBTpjTLnYJN07ixC3vg==} - '@libp2p/peer-collections@6.0.34': - resolution: {integrity: sha512-rw8gDGhou4sF6W6i9ntmRARFePX19Dw9MMVpZHr6Kx9q2kvBJq91IXUzsXP06roexEOu1CUlZwxtUAqOBy+Eww==} + '@libp2p/multistream-select@7.0.10': + resolution: {integrity: sha512-6RAFctqWzwQ/qPaN3CxoueSs1b7pBVMZ+0n6G0kcsqVBj0wc4eB+dcJyUNrTV1NGgMCAl6tVAGztZaE8XZc9lw==} - '@libp2p/peer-id@5.1.8': - resolution: {integrity: sha512-pGaM4BwjnXdGtAtd84L4/wuABpsnFYE+AQ+h3GxNFme0IsTaTVKWd1jBBE5YFeKHBHGUOhF3TlHsdjFfjQA7TA==} + '@libp2p/peer-collections@7.0.10': + resolution: {integrity: sha512-OvlSY5N3J6q8U+EbTrQGbW8zdyOa3y7nz9Y3IbuE55tIiMd7pwm1U3Lknfb6IPkOWkHNfQDfCGGfGVQcMRodvQ==} - '@libp2p/peer-record@8.0.34': - resolution: {integrity: sha512-GqvRBpvclscoKuF0JUfLyZTv+BwzICBBe50LFiAKio8LijZMBr43b+AcEaSEwFWDwlWmaKU73q8EQLrCb/e67Q==} + '@libp2p/peer-id@6.0.4': + resolution: {integrity: sha512-Z3xK0lwwKn4bPg3ozEpPr1HxsRi2CxZdghOL+MXoFah/8uhJJHxHFA8A/jxtKn4BB8xkk6F8R5vKNIS05yaCYw==} - '@libp2p/peer-store@11.2.6': - resolution: {integrity: sha512-3Lc982/7drqlXa51s9l1/DFHD48zzIjMMYajxFM2KbobyStH+lztYnFc3kNGB9sZijULaW1480PvbTMm9WaJ0g==} + '@libp2p/peer-record@9.0.5': + resolution: {integrity: sha512-disk23OO00yD52O4VmItbDkjJZ/YZJsKbMsqNgVhr+D3PcM+KRpu9VVbiCnN5Tzn9XvFEHhrMJY7BPE+rvT5MQ==} - '@libp2p/prometheus-metrics@4.3.29': - resolution: {integrity: sha512-TPRtOpDZNaeTko8uF7INtumE7gVNwuETPFlLP2C2BVbyp1SrpH8oJSxoioMzmAXRRwfY8esMxx3KUkdHd5BdaA==} + '@libp2p/peer-store@12.0.10': + resolution: {integrity: sha512-fe/6m0vXny9pvCyaSjg2GisdSVgxtHYZtp6op1WNm8dBvYqRXLuqSYi0QGEbLtSDSL4SeE8BKZyadyk/tYAqfg==} - '@libp2p/pubsub@10.1.17': - resolution: {integrity: sha512-hQMld/RiU7HbK81rGO///aNaM5yjBrEaULSo2m1pGB0Qy9SAhXdYJYgFP3ln+Pc18DwEU7wEjDIkE32CCYzpGA==} + '@libp2p/prometheus-metrics@5.0.10': + resolution: {integrity: sha512-93VjiHcu/D5DaTbwV2zikkR65WXBa3HEFNWWUjSJzhHCNYt8518r2nH5zmQVPyCNNK+lnXc/R7HkLbRAq/6mIw==} - '@libp2p/tcp@10.1.18': - resolution: {integrity: sha512-tB7rjETju5UMiYpyzAEOsX7qGVahJwaoBlsMAfpb35a8lH+KU0AaxBMMK8P4cFbKZZQSuDN86Is0mb5EG7npVA==} + '@libp2p/tcp@11.0.10': + resolution: {integrity: sha512-vp1XvbRUU6JyVZMDfrr8UX+xs1sybT2r3PFoN5m07r3GSrMMPOKpWN2HkhT2pCBZWJG6ADQOy5+K0tBRE782oA==} - '@libp2p/utils@6.7.1': - resolution: {integrity: sha512-x3WImvw4unmx1ZeAedj8AkRe4UImUlkw0ZItYAiKiekElMNUXwv+Yt48dI/LmB38JIof8sng29XvUeCVU3F6OA==} + '@libp2p/utils@7.0.10': + resolution: {integrity: sha512-+mzD+7yLMoZ8+34y/iS9d1CnwHjJJ/qEsao9FckHf9T9tnVXEyLLu9TpzBCcGRm4fUK/QCSHK2AcZH50kkAFkw==} '@lukeed/ms@2.0.2': resolution: {integrity: sha512-9I2Zn6+NJLfaGoz9jN3lpwDgAYvfGeNYdbAIjJOqzs4Tpc+VU3Jqq4IofSUBKajiDS8k9fZIg18/z13mpk1bsA==} @@ -2283,14 +2262,11 @@ packages: '@multiformats/dns@1.0.6': resolution: {integrity: sha512-nt/5UqjMPtyvkG9BQYdJ4GfLK3nMqGpFZOzf4hAmIa0sJh2LlS9YKXZ4FgwBDsaHvzZqR/rUFIywIc7pkHNNuw==} - '@multiformats/mafmt@12.1.6': - resolution: {integrity: sha512-tlJRfL21X+AKn9b5i5VnaTD6bNttpSpcqwKVmDmSHLwxoz97fAHaepqFOk/l1fIu94nImIXneNbhsJx/RQNIww==} - - '@multiformats/multiaddr-matcher@2.0.1': - resolution: {integrity: sha512-rhEYax74GlL0YPAZo61tK/hu0hwU7Hf9maXpoKNlpekmOPv+eckD+UlSeoTDCpGP8hQqMwLkX8e64MuDpy2Vlg==} + '@multiformats/multiaddr-matcher@3.0.1': + resolution: {integrity: sha512-jvjwzCPysVTQ53F4KqwmcqZw73BqHMk0UUZrMP9P4OtJ/YHrfs122ikTqhVA2upe0P/Qz9l8HVlhEifVYB2q9A==} - '@multiformats/multiaddr@12.5.1': - resolution: {integrity: sha512-+DDlr9LIRUS8KncI1TX/FfUn8F2dl6BIxJgshS/yFQCNB5IAF0OGzcwB39g5NLE22s4qqDePv0Qof6HdpJ/4aQ==} + '@multiformats/multiaddr@13.0.1': + resolution: {integrity: sha512-XToN915cnfr6Lr9EdGWakGJbPT0ghpg/850HvdC+zFX8XvpLZElwa8synCiwa8TuvKNnny6m8j8NVBNCxhIO3g==} '@napi-rs/snappy-android-arm-eabi@7.2.2': resolution: {integrity: sha512-H7DuVkPCK5BlAr1NfSU8bDEN7gYs+R78pSHhDng83QxRnCLmVIZk33ymmIwurmoA1HrdTxbkbuNl+lMvNqnytw==} @@ -2373,17 +2349,25 @@ packages: '@napi-rs/wasm-runtime@0.2.6': resolution: {integrity: sha512-z8YVS3XszxFTO73iwvFDNpQIzdMmSDTP/mB3E/ucR37V3Sx57hSExcXyMoNwaucWxnsWf4xfbZv0iZ30jr0M4Q==} - '@noble/ciphers@1.2.1': - resolution: {integrity: sha512-rONPWMC7PeExE077uLE4oqWrZ1IvAfz3oH9LibVAcVCopJiA9R62uavnbEzdkVmJYI6M6Zgkbeb07+tWjlq2XA==} + '@noble/ciphers@1.3.0': + resolution: {integrity: sha512-2I0gnIVPtfnMw9ee9h1dJG7tp81+8Ob3OJb3Mv37rx5L40/b0i7djjCVvGOVqc9AEIQyvyu1i6ypKdFw8R8gQw==} engines: {node: ^14.21.3 || >=16} + '@noble/ciphers@2.1.1': + resolution: {integrity: sha512-bysYuiVfhxNJuldNXlFEitTVdNnYUc+XNJZd7Qm2a5j1vZHgY+fazadNFWFaMK/2vye0JVlxV3gHmC0WDfAOQw==} + engines: {node: '>= 20.19.0'} + '@noble/curves@1.4.2': resolution: {integrity: sha512-TavHr8qycMChk8UwMld0ZDRvatedkzWfH8IiaeGCfymOP5i0hSCozz9vHOL0nkwk7HRMlFnAiKpS2jrUmSybcw==} - '@noble/curves@1.9.2': - resolution: {integrity: sha512-HxngEd2XUcg9xi20JkwlLCtYwfoFw4JGkuZpT+WlsPD4gB/cxkvTD8fSsoAnphGZhFdZYKeQIPCuFlWPm1uE0g==} + '@noble/curves@1.9.0': + resolution: {integrity: sha512-7YDlXiNMdO1YZeH6t/kvopHHbIZzlxrCV9WLqCY6QhcXOoXiNCMDqJIglZ9Yjx5+w7Dz30TITFrlTjnRg7sKEg==} engines: {node: ^14.21.3 || >=16} + '@noble/curves@2.0.1': + resolution: {integrity: sha512-vs1Az2OOTBiP4q0pwjW5aF0xp9n4MxVrmkFBxc6EKZc6ddYx5gaZiAsZoq0uRRXWbi3AT/sBqn05eRPtn1JCPw==} + engines: {node: '>= 20.19.0'} + '@noble/hashes@1.1.2': resolution: {integrity: sha512-KYRCASVTv6aeUi1tsF8/vpyR7zpfs3FUzy2Jqm+MU+LmUKhQ0y2FpfwqkCcxSg2ua4GALJd8k2R76WxwZGbQpA==} @@ -2395,11 +2379,15 @@ packages: resolution: {integrity: sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==} engines: {node: ^14.21.3 || >=16} + '@noble/hashes@2.0.1': + resolution: {integrity: sha512-XlOlEbQcE9fmuXxrVTXCTlG2nlRXa9Rj3rr5Ue/+tX+nmkgbX720YHh0VR3hBF9xDvwnb8D2shVGOwNx+ulArw==} + engines: {node: '>= 20.19.0'} + '@noble/secp256k1@1.7.1': resolution: {integrity: sha512-hOUk6AyBFmqVrv7k5WAw/LpszxVbj9gGN4JRkIX52fdFAj1UA61KXmZDvqVEm+pOyec3+fIeZB02LYa/pWOArw==} - '@noble/secp256k1@2.2.3': - resolution: {integrity: sha512-l7r5oEQym9Us7EAigzg30/PQAvynhMt2uoYtT3t26eGDVm9Yii5mZ5jWSWmZ/oSIR2Et0xfc6DXrG0bZ787V3w==} + '@noble/secp256k1@3.0.0': + resolution: {integrity: sha512-NJBaR352KyIvj3t6sgT/+7xrNyF9Xk9QlLSIqUGVUYlsnDTAUqY8LOmwpcgEx4AMJXRITQ5XEVHD+mMaPfr3mg==} '@node-rs/crc32-android-arm-eabi@1.10.6': resolution: {integrity: sha512-vZAMuJXm3TpWPOkkhxdrofWDv+Q+I2oO7ucLRbXyAPmXFNDhHtBxbO1rk9Qzz+M3eep8ieS4/+jCL1Q0zacNMQ==} @@ -2886,15 +2874,24 @@ packages: '@scure/base@1.1.8': resolution: {integrity: sha512-6CyAclxj3Nb0XT7GHK6K4zK6k2xJm6E4Ft0Ohjt4WgegiFUHEtFb2CGzmPmGBwoIhrLsqNLYfLr04Y1GePrzZg==} - '@scure/base@1.2.4': - resolution: {integrity: sha512-5Yy9czTO47mqz+/J8GM6GIId4umdCk1wc1q8rKERQulIoc8VP9pzDcghv10Tl2E7R96ZUx/PhND3ESYUQX8NuQ==} + '@scure/base@1.2.6': + resolution: {integrity: sha512-g/nm5FgUa//MCj1gV09zTJTaM6KBAHqLN907YVQqf7zC49+DcO4B1so4ZX07Ef10Twr6nuqYEH9GEggFXA4Fmg==} + + '@scure/base@2.0.0': + resolution: {integrity: sha512-3E1kpuZginKkek01ovG8krQ0Z44E3DHPjc5S2rjJw9lZn3KSQOs8S7wqikF/AH7iRanHypj85uGyxk0XAyC37w==} '@scure/bip32@1.4.0': resolution: {integrity: sha512-sVUpc0Vq3tXCkDGYVWGIZTRfnvu8LoTDaev7vbwh0omSvVORONr960MQWdKqJDCReIEmTj3PAr73O3aoxz7OPg==} + '@scure/bip32@1.7.0': + resolution: {integrity: sha512-E4FFX/N3f4B80AKWp5dP6ow+flD1LQZo/w8UnLGYZO674jS6YnYeepycOOksv+vLPSpgN35wgKgy+ybfTb2SMw==} + '@scure/bip39@1.3.0': resolution: {integrity: sha512-disdg7gHuTDZtY+ZdkmLpPCk7fxZSu3gBiEGuoC1XYxv9cGx3Z6cpTggCgW6odSOOIXCiDjuGejW+aJKCY/pIQ==} + '@scure/bip39@1.6.0': + resolution: {integrity: sha512-+lF0BbLiJNwVlev4eKelw1WWLaiKXw7sSl8T6FvBlWkdX+94aGJ4o8XjUdlyhTCjd8c+B3KT3JfS8P0bLRNU6A==} + '@sec-ant/readable-stream@0.4.1': resolution: {integrity: sha512-831qok9r2t8AlxLko40y2ebgSDhenenCatLVeW/uBtnHPyhHOvG0C7TvfgecV+wHzIm5KUICgzmVpWS+IMEAeg==} @@ -3146,8 +3143,8 @@ packages: '@types/retry@0.12.2': resolution: {integrity: sha512-XISRgDJ2Tc5q4TRqvgJtzsRkFYNJzZrhTdtMoGVBttwzzQJkPnS3WWTFc7kuDRoPtPakl+T+OfdEUjYJj7Jbow==} - '@types/sinon@17.0.4': - resolution: {integrity: sha512-RHnIrhfPO3+tJT0s7cFaXGZvsL4bbR3/k7z3P312qMS4JaS2Tk+KiwiLx1S0rQ56ERj00u1/BtdyVd0FY+Pdew==} + '@types/sinon@20.0.0': + resolution: {integrity: sha512-etYGUC6IEevDGSWvR9WrECRA01ucR2/Oi9XMBUAdV0g4bLkNf4HlZWGiGlDOq5lgwXRwcV+PSeKgFcW4QzzYOg==} '@types/sinonjs__fake-timers@6.0.2': resolution: {integrity: sha512-dIPoZ3g5gcx9zZEszaxLSVTvMReD3xxyyDnQUjA6IYDG9Ba2AV0otMPs+77sG9ojB4Qr2N2Vk5RnKeuA0X/0bg==} @@ -3287,6 +3284,10 @@ packages: resolution: {integrity: sha512-eUP/6pbXBkMbXFdx4IH2fVgvB7M0JvR7/lIL33zcs0IBcwjdzSSl31TOJsaCzmKSSDF9h8QYSOJux4Nd4YJqFg==} engines: {node: '>=12'} + abstract-level@3.1.1: + resolution: {integrity: sha512-CW2gKbJFTuX1feMvOrvsVMmijAOgI9kg2Ie9Dq3gOcMt/dVVoVmqNlLcEUCT13NxHFMEajcUcVBIplbyDroDiw==} + engines: {node: '>=18'} + abstract-logging@2.0.1: resolution: {integrity: sha512-2BjRTZxTPvheOvGbBslFSYOUkr+SjPtOnrLP33f+VIWLzezQpZcqVg7ja3L4dBXmzzgwT+a029jRx5PCi3JuiA==} @@ -3516,6 +3517,9 @@ packages: browser-level@1.0.1: resolution: {integrity: sha512-XECYKJ+Dbzw0lbydyQuJzwNXtOpbMSq737qxJN11sIRTErOMShvDpbzTlgju7orJKvx4epULolZAuJGLzCmWRQ==} + browser-level@3.0.0: + resolution: {integrity: sha512-kGXtLh29jMwqKaskz5xeDLtCtN1KBz/DbQSqmvH7QdJiyGRC7RAM8PPg6gvUiNMa+wVnaxS9eSmEtP/f5ajOVw==} + browser-resolve@2.0.0: resolution: {integrity: sha512-7sWsQlYL2rGLy2IWm8WL8DCTJvYLc/qlOnsakDac87SOoCd16WLsaAMdCiAqsTNHIe+SXfaqyxyo6THoWqs8WQ==} @@ -3615,6 +3619,10 @@ packages: resolution: {integrity: sha512-K7Qy8O9p76sL3/3m7/zLKbRkyOlSZAgzEaLhyj2mXS8PsCud2Eo4hAb8aLtZqHh0QGqLcb9dlJSu6lHRVENm1w==} engines: {node: '>=6'} + cborg@4.5.8: + resolution: {integrity: sha512-6/viltD51JklRhq4L7jC3zgy6gryuG5xfZ3kzpE+PravtyeQLeQmCYLREhQH7pWENg5pY4Yu/XCd6a7dKScVlw==} + hasBin: true + chai@6.2.0: resolution: {integrity: sha512-aUTnJc/JipRzJrNADXVvpVqi6CO0dn3nx4EVPxijri+fj3LUUDyZQOgVeW54Ob3Y1Xh9Iz8f+CgaCl8v0mn9bA==} engines: {node: '>=18'} @@ -3654,6 +3662,10 @@ packages: resolution: {integrity: sha512-qGx/KJl3bvtOHrGau2WklEZuXhS3zme+jf+fsu6Ej7W7IP/C49v7KNlWIsT1jZu0YnfzSIYDGcEWpCa1wKGWXQ==} engines: {node: '>=12'} + classic-level@3.0.0: + resolution: {integrity: sha512-yGy8j8LjPbN0Bh3+ygmyYvrmskVita92pD/zCoalfcC9XxZj6iDtZTAnz+ot7GG8p9KLTG+MZ84tSA4AhkgVZQ==} + engines: {node: '>=18'} + clean-stack@2.2.0: resolution: {integrity: sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==} engines: {node: '>=6'} @@ -3878,14 +3890,14 @@ packages: resolution: {integrity: sha512-ZYP5VBHshaDAiVZxjbRVcFJpc+4xGgT0bK3vzy1HLN8jTO975HEbuYzZJcHoQEY5K1a0z8YayJkyVETa08eNTg==} engines: {node: '>=18'} - datastore-core@10.0.2: - resolution: {integrity: sha512-B3WXxI54VxJkpXxnYibiF17si3bLXE1XOjrJB7wM5co9fx2KOEkiePDGiCCEtnapFHTnmAnYCPdA7WZTIpdn/A==} + datastore-core@11.0.2: + resolution: {integrity: sha512-0pN4hMcaCWcnUBo5OL/8j14Lt1l/p1v2VvzryRYeJAKRLqnFrzy2FhAQ7y0yTA63ki760ImQHfm2XlZrfIdFpQ==} - datastore-fs@10.0.6: - resolution: {integrity: sha512-809BIs3wxJMg8/BYDwm734c06oKfrP5svD5CahPPHOIOOk9fSJc3j0OyP1yrb+oUk1OU/V5m1kmeO+eU0tHNCQ==} + datastore-fs@11.0.2: + resolution: {integrity: sha512-5jBSqcc0PzrraaNL1pcoB4nrVc+sNyD7E4axDQoTlkm07wE778rGPHhZNodN8nJEWrvnccOlzqc84w7SLyInKg==} - datastore-level@11.0.4: - resolution: {integrity: sha512-vwiOglxBXMhhEa2r5sh3iQrkrnK3HDYT/7V9dAb+/IvKotqHFsyio+tii9XNuWes1bj3yOCIQ/w8kdV3LZ/Glg==} + datastore-level@12.0.2: + resolution: {integrity: sha512-BMXRvFhDfx7CxlUj7XBAtPt6V2D9CWnKwRGTvZ3KQ4BBTJoXL66UDTxi52rzYLKfd2QppwaO2S+ZhLWhF5PbQQ==} de-indent@1.0.2: resolution: {integrity: sha512-e/1zu3xH5MQryN2zdVaF0OrdNLUbvWxzMbi+iNA6Bky7l1RoP8a2fIbRocyHclXt/arDrrR6lL3TqFD9pMQTsg==} @@ -3949,9 +3961,9 @@ packages: resolution: {integrity: sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==} engines: {node: '>= 0.4'} - delay@6.0.0: - resolution: {integrity: sha512-2NJozoOHQ4NuZuVIr5CWd0iiLVIRSDepakaovIN+9eIDHEhdCAEvSy2cuf1DCrPPQLvHmbqTHODlhHg8UCy4zw==} - engines: {node: '>=16'} + delay@7.0.0: + resolution: {integrity: sha512-C3vaGs818qzZjCvVJ98GQUMVyWeg7dr5w2Nwwb2t5K8G98jOyyVO2ti2bKYk5yoYElqH3F2yA53ykuEnwD6MCg==} + engines: {node: '>=20'} delayed-stream@1.0.0: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} @@ -4140,6 +4152,10 @@ packages: ethereum-cryptography@2.2.1: resolution: {integrity: sha512-r/W8lkHSiTLxUxW8Rf3u4HGB0xQweG2RyETjywylKZSzLWoWAijRz8WCuOtJ6wah+avllXBqZuk29HCCvhEIRg==} + ethereum-cryptography@3.2.0: + resolution: {integrity: sha512-Urr5YVsalH+Jo0sYkTkv1MyI9bLYZwW8BENZCeE1QYaTHETEYx0Nv/SVsWkSqpYrzweg6d8KMY1wTjH/1m/BIg==} + engines: {node: ^14.21.3 || >=16, npm: '>=9'} + ethers@5.7.2: resolution: {integrity: sha512-wswUsmWo1aOK8rR7DIKiWSw9DbLWe6x98Jrn8wcTflTVvaXhAMaB5zGAXy0GYQEQp9iO1iSHWVyARQm11zUtyg==} @@ -4410,9 +4426,6 @@ packages: resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==} engines: {node: '>= 0.4'} - get-iterator@2.0.1: - resolution: {integrity: sha512-7HuY/hebu4gryTDT7O/XY/fvY9wRByEGdK6QOa4of8npTcv0+NS6frFKABcf6S9EBAsveTuKTsZQQBFMMNILIg==} - get-proto@1.0.1: resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} engines: {node: '>= 0.4'} @@ -4700,11 +4713,11 @@ packages: resolution: {integrity: sha512-3ygAIh8gcZavV9bj6MTdYddG2zPSYswP808fKS46NOwlF0zZljVpnLCHODDqItWJDbDpLb3aouAxGaJbkxoppA==} engines: {node: '>=14.18.0'} - interface-datastore@8.3.1: - resolution: {integrity: sha512-3r0ETmHIi6HmvM5sc09QQiCD3gUfwtEM/AAChOyAd/UAKT69uk8LXfTSUBufbUIO/dU65Vj8nb9O6QjwW8vDSQ==} + interface-datastore@9.0.2: + resolution: {integrity: sha512-jebn+GV/5LTDDoyicNIB4D9O0QszpPqT09Z/MpEWvf3RekjVKpXJCDguM5Au2fwIFxFDAQMZe5bSla0jMamCNg==} - interface-store@6.0.2: - resolution: {integrity: sha512-KSFCXtBlNoG0hzwNa0RmhHtrdhzexp+S+UY2s0rWTBJyfdEIgn6i6Zl9otVqrcFYbYrneBT7hbmHQ8gE0C3umA==} + interface-store@7.0.1: + resolution: {integrity: sha512-OPRRUO3Cs6Jr/t98BrJLQp1jUTPgrRH0PqFfuNoPAqd+J7ABN1tjFVjQdaOBiybYJTS/AyBSZnZVWLPvp3dW3w==} internal-slot@1.0.5: resolution: {integrity: sha512-Y+R5hJrzs52QCG2laLn4udYVnxsfny9CpOhNhUvk/SSSVyF6T27FzRbF0sroPidSu3X8oEAkOn2K804mjpt6UQ==} @@ -4924,42 +4937,25 @@ packages: it-batch@3.0.9: resolution: {integrity: sha512-z6p89Q8gm2urBtF3JcpnbJogacijWk3m1uc3xZYI3x0eJUoYLUbgF8IxJ2fnuVObV7yRv3SixfwGCufaZY1NCg==} - it-byte-stream@2.0.3: - resolution: {integrity: sha512-h7FFcn4DWiWsJw1dCJhuPdiY8cGi1z8g4aLAfFspTaJbwQxvEMlEBFG/f8lIVGwM8YK26ClM4/9lxLVhF33b8g==} - it-drain@3.0.10: resolution: {integrity: sha512-0w/bXzudlyKIyD1+rl0xUKTI7k4cshcS43LTlBiGFxI8K1eyLydNPxGcsVLsFVtKh1/ieS8AnVWt6KwmozxyEA==} it-filter@3.1.4: resolution: {integrity: sha512-80kWEKgiFEa4fEYD3mwf2uygo1dTQ5Y5midKtL89iXyjinruA/sNXl6iFkTcdNedydjvIsFhWLiqRPQP4fAwWQ==} - it-foreach@2.1.4: - resolution: {integrity: sha512-gFntBbNLpVK9uDmaHusugICD8/Pp+OCqbF5q1Z8K+B8WaG20YgMePWbMxI1I25+JmNWWr3hk0ecKyiI9pOLgeA==} - it-glob@3.0.4: resolution: {integrity: sha512-73PbGBTK/dHp5PX4l8pkQH1ozCONP0U+PB3qMqltxPonRJQNomINE3Hn9p02m2GOu95VoeVvSZdHI2N+qub0pw==} - it-length-prefixed-stream@2.0.3: - resolution: {integrity: sha512-Ns3jNFy2mcFnV59llCYitJnFHapg8wIcOsWkEaAwOkG9v4HBCk24nze/zGDQjiJdDTyFXTT5GOY3M/uaksot3w==} - it-length-prefixed@10.0.1: resolution: {integrity: sha512-BhyluvGps26u9a7eQIpOI1YN7mFgi8lFwmiPi07whewbBARKAG9LE09Odc8s1Wtbt2MB6rNUrl7j9vvfXTJwdQ==} engines: {node: '>=16.0.0', npm: '>=7.0.0'} - it-length-prefixed@9.1.1: - resolution: {integrity: sha512-O88nBweT6M9ozsmok68/auKH7ik/slNM4pYbM9lrfy2z5QnpokW5SlrepHZDKtN71llhG2sZvd6uY4SAl+lAQg==} - engines: {node: '>=16.0.0', npm: '>=7.0.0'} - it-map@3.1.4: resolution: {integrity: sha512-QB9PYQdE9fUfpVFYfSxBIyvKynUCgblb143c+ktTK6ZuKSKkp7iH58uYFzagqcJ5HcqIfn1xbfaralHWam+3fg==} it-merge@3.0.12: resolution: {integrity: sha512-nnnFSUxKlkZVZD7c0jYw6rDxCcAQYcMsFj27thf7KkDhpj0EA0g9KHPxbFzHuDoc6US2EPS/MtplkNj8sbCx4Q==} - it-pair@2.0.6: - resolution: {integrity: sha512-5M0t5RAcYEQYNG5BV7d7cqbdwbCAp5yLdzvkxsZmkuZsLbTdZzah6MQySYfaAQjNDCq6PUnDt0hqBZ4NwMfW6g==} - engines: {node: '>=16.0.0', npm: '>=7.0.0'} - it-parallel-batch@3.0.9: resolution: {integrity: sha512-TszXWqqLG8IG5DUEnC4cgH9aZI6CsGS7sdkXTiiacMIj913bFy7+ohU3IqsFURCcZkpnXtNLNzrYnXISsKBhbQ==} @@ -4974,9 +4970,6 @@ packages: resolution: {integrity: sha512-sIoNrQl1qSRg2seYSBH/3QxWhJFn9PKYvOf/bHdtCBF0bnghey44VyASsWzn5dAx0DCDDABq1hZIuzKmtBZmKA==} engines: {node: '>=16.0.0', npm: '>=7.0.0'} - it-protobuf-stream@2.0.3: - resolution: {integrity: sha512-Dus9qyylOSnC7l75/3qs6j3Fe9MCM2K5luXi9o175DYijFRne5FPucdOGIYdwaDBDQ4Oy34dNCuFobOpcusvEQ==} - it-pushable@3.2.3: resolution: {integrity: sha512-gzYnXYK8Y5t5b/BnJUr7glfQLO4U5vyb05gPx/TyTw+4Bv1zM9gFk4YsOrnulWefMewlphCjKkakFvj1y99Tcg==} @@ -5106,10 +5099,18 @@ packages: resolution: {integrity: sha512-PbXpve8rKeNcZ9C1mUicC9auIYFyGpkV9/i6g76tLgANwWhtG2v7I4xNBUlkn3lE2/dZF3Pi0ygYGtLc4RXXdA==} engines: {node: '>=12'} + level-supports@6.2.0: + resolution: {integrity: sha512-QNxVXP0IRnBmMsJIh+sb2kwNCYcKciQZJEt+L1hPCHrKNELllXhvrlClVHXBYZVT+a7aTSM6StgNXdAldoab3w==} + engines: {node: '>=16'} + level-transcoder@1.0.1: resolution: {integrity: sha512-t7bFwFtsQeD8cl8NIoQ2iwxA0CL/9IFw7/9gAjOonH0PWTTiRfY7Hq+Ejbsxh86tXobDQ6IOiddjNYIfOBs06w==} engines: {node: '>=12'} + level@10.0.0: + resolution: {integrity: sha512-aZJvdfRr/f0VBbSRF5C81FHON47ZsC2TkGxbBezXpGGXAUEL/s6+GP73nnhAYRSCIqUNsmJjfeOF4lzRDKbUig==} + engines: {node: '>=18'} + level@8.0.1: resolution: {integrity: sha512-oPBGkheysuw7DmzFQYyFe8NAia5jFLAgEnkgWnK3OXAuJr8qFT+xBQIwokAZPME2bhPFzS8hlYcL16m8UZrtwQ==} engines: {node: '>=12'} @@ -5122,8 +5123,8 @@ packages: resolution: {integrity: sha512-NVPTth/71cfbdYHqypcO9Lt5WFGTzFEcx81lWd7GDJIgZ95ERdYHGUfCtFejHCyqodKsQkNEx2JCkMpreDty/A==} engines: {node: ^20.17.0 || >=22.9.0} - libp2p@2.9.0: - resolution: {integrity: sha512-gzRnhLY+k9KjYifWQCYbdEfmWqCFdM0TZ5Q7qqdY13sAUKXixK0MF5+Z9LMrm5ELGDPWX7pRVLGK8BOSv5/v3Q==} + libp2p@3.1.3: + resolution: {integrity: sha512-Jgl6Km1PfFTKR7krDNDxuuxQ6ya3D6VHFOi/XYJA539F62PmbxOQLd+nqbqozwB9BgJVTxaXRVmGTKo7dyrdQw==} light-my-request@6.0.0: resolution: {integrity: sha512-kFkFXrmKCL0EEeOmJybMH5amWFd+AFvlvMlvFTRxCUwbhfapZqDmeLMPoWihntnYY6JpoQDE9k+vOzObF1fDqg==} @@ -5204,6 +5205,10 @@ packages: resolution: {integrity: sha512-B5Y16Jr9LB9dHVkh6ZevG+vAbOsNOYCX+sXvFWFu7B3Iz5mijW3zdbMyhsh8ANd2mSWBYdJgnqi+mL7/LrOPYg==} engines: {node: 20 || >=22} + lru-cache@11.2.5: + resolution: {integrity: sha512-vFrFJkWtJvJnD5hg+hJvVE8Lh/TcMzKnTgCWmtBipwI5yLX/iX+5UB2tfuyODF5E7k9xEzMdYgGqaSb1c0c5Yw==} + engines: {node: 20 || >=22} + lru-cache@5.1.1: resolution: {integrity: sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==} @@ -5247,6 +5252,10 @@ packages: resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} engines: {node: '>= 0.4'} + maybe-combine-errors@1.0.0: + resolution: {integrity: sha512-eefp6IduNPT6fVdwPp+1NgD0PML1NU5P6j1Mj5nz1nidX8/sWY7119WL8vTAHgqfsY74TzW0w1XPgdYEKkGZ5A==} + engines: {node: '>=10'} + mcl-wasm@0.7.9: resolution: {integrity: sha512-iJIUcQWA88IJB/5L15GnJVnSQJmf/YaxxV6zRavv83HILHaJQb6y0iFyDMdDO0gN8X37tdxmAOrH/P8B6RB8sQ==} engines: {node: '>=8.9.0'} @@ -5441,12 +5450,8 @@ packages: resolution: {integrity: sha512-2eznPJP8z2BFLX50tf0LuODrpINqP1RVIm/CObbTcBRITQgmC/TjcREF1NeTBzIcR5XO/ukWo+YHOjBbFwIupg==} hasBin: true - multiformats@11.0.2: - resolution: {integrity: sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==} - engines: {node: '>=16.0.0', npm: '>=7.0.0'} - - multiformats@13.3.7: - resolution: {integrity: sha512-meL9DERHj+fFVWoOX9fXqfcYcSpUfSYJPcFvDPKrxitICbwAoWR+Ut4j5NO9zAT917HUHLQmqzQbAsGNHlDcxQ==} + multiformats@13.4.2: + resolution: {integrity: sha512-eh6eHCrRi1+POZ3dA+Dq1C6jhP1GNtr9CRINMb67OKzqW9I5DUuZM/3jLPlzhgpGeiNUlEGEbkCYChXMCc/8DQ==} mute-stream@1.0.0: resolution: {integrity: sha512-avsJQhyd+680gKXyG/sQc0nXaC6rBkPOfyHYcFb9+hdkqQkR9bdnkJ0AMZhke0oesPqIO+mFFJ+IdBc7mst4IA==} @@ -5662,9 +5667,9 @@ packages: resolution: {integrity: sha512-Mr5KC5efvAK5VUptYEIopP1bakB85k2IWXaRC0rsh1uwn1L6M0LVml8OIQ4Gudg4oyZakf7FmeRLkMMtZW1i5A==} engines: {node: '>=12'} - p-event@6.0.1: - resolution: {integrity: sha512-Q6Bekk5wpzW5qIyUP4gdMEujObYstZl6DMMOSenwBvV0BlE5LkDwkjs5yHbZmdCEq2o4RJx4tE1vwxFVf2FG1w==} - engines: {node: '>=16.17'} + p-event@7.1.0: + resolution: {integrity: sha512-/lkPs5W1aC3cp6vqZefpdosOn65J571sWodyfOQiF0+tmDCpU+H8Atwpu0vQROCVUlZuToDN5eyTLsMLLc54mg==} + engines: {node: '>=20'} p-limit@2.3.0: resolution: {integrity: sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==} @@ -5718,9 +5723,9 @@ packages: resolution: {integrity: sha512-xsrIUgI0Kn6iyDYm9StOpOeK29XM1aboGji26+QEortiFST1hGZaUQOLhtEbqHErPpGW/aSz6allwK2qcptp0Q==} engines: {node: '>=12'} - p-retry@6.2.1: - resolution: {integrity: sha512-hEt02O4hUct5wtwg4H4KcWgDdm+l1bOaEy/hWzd8xtXB9BqxTWBBhb+2ImAtH4Cv4rPjV76xN3Zumqk3k3AhhQ==} - engines: {node: '>=16.17'} + p-retry@7.1.1: + resolution: {integrity: sha512-J5ApzjyRkkf601HpEeykoiCvzHQjWxPAHhyjFcEUP2SWq0+35NKh8TLhpLw+Dkq5TZBFvUM6UigdE9hIVYTl5w==} + engines: {node: '>=20'} p-timeout@6.1.2: resolution: {integrity: sha512-UbD77BuZ9Bc9aABo74gfXhNvzC9Tx7SxtHSh1fxvx3jTLLYvmVhiQZZrJzqqU0jKbN32kb5VOKiLEQI/3bIjgQ==} @@ -5895,21 +5900,11 @@ packages: engines: {node: '>=18'} hasBin: true - playwright-core@1.58.2: - resolution: {integrity: sha512-yZkEtftgwS8CsfYo7nm0KE8jsvm6i/PTgVtB8DL726wNf6H2IMsDuxCpJj59KDaxCtSnrWan2AeDqM7JBaultg==} - engines: {node: '>=18'} - hasBin: true - playwright@1.56.1: resolution: {integrity: sha512-aFi5B0WovBHTEvpM3DzXTUaeN6eN0qWnTkKx4NQaH4Wvcmc153PdaY2UBdSYKaGYw+UyWXSVyxDUg5DoPEttjw==} engines: {node: '>=18'} hasBin: true - playwright@1.58.2: - resolution: {integrity: sha512-vA30H8Nvkq/cPBnNw4Q8TWz1EJyqgpuinBcHET0YVJVFldr8JDNiU9LaWAE1KqSkRYazuaBhTpB5ZzShOezQ6A==} - engines: {node: '>=18'} - hasBin: true - pngjs@7.0.0: resolution: {integrity: sha512-LKWqWJRhstyYo9pGvgor/ivk2w94eSjE3RGVuzLGlr3NmD8bf7RcYGze1mNdEHRP6TRP6rMuDHk5t44hnTRyow==} engines: {node: '>=14.19.0'} @@ -6010,6 +6005,9 @@ packages: protons-runtime@5.5.0: resolution: {integrity: sha512-EsALjF9QsrEk6gbCx3lmfHxVN0ah7nG3cY7GySD4xf4g8cr7g543zB88Foh897Sr1RQJ9yDCUsoT1i1H/cVUFA==} + protons-runtime@5.6.0: + resolution: {integrity: sha512-/Kde+sB9DsMFrddJT/UZWe6XqvL7SL5dbag/DBCElFKhkwDj7XKt53S+mzLyaDP5OqS0wXjV5SA572uWDaT0Hg==} + proxy-from-env@1.1.0: resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} @@ -6070,6 +6068,10 @@ packages: race-signal@2.0.0: resolution: {integrity: sha512-P31bLhE4ByBX/70QDXMutxnqgwrF1WUXea1O8DXuviAgkdbQ1iQMQotNgzJIBC9yUSn08u/acZrMUhgw7w6GpA==} + random-int@3.1.0: + resolution: {integrity: sha512-h8CRz8cpvzj0hC/iH/1Gapgcl2TQ6xtnCpyOI5WvWfXf/yrDx2DOU+tD9rX23j36IF11xg1KqB9W11Z18JPMdw==} + engines: {node: '>=12'} + randombytes@2.1.0: resolution: {integrity: sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==} @@ -6151,10 +6153,6 @@ packages: resolution: {integrity: sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==} engines: {node: '>= 4'} - retry@0.13.1: - resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==} - engines: {node: '>= 4'} - reusify@1.0.4: resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==} engines: {iojs: '>=1.0.0', node: '>=0.10.0'} @@ -6480,9 +6478,6 @@ packages: stream-http@3.2.0: resolution: {integrity: sha512-Oq1bLqisTyK3TSCXpPbT4sdeYNdmyZJv1LxpEm2vu1ZhK89kSE5YXwZc3cWk0MagGaKriBh9mCFbVGtO+vY29A==} - stream-to-it@1.0.1: - resolution: {integrity: sha512-AqHYAYPHcmvMrcLNgncE/q0Aj/ajP6A4qGhxP6EVn7K3YTNs0bJpJyk57wc2Heb7MUL64jurvmnmui8D9kjZgA==} - strict-event-emitter-types@2.0.0: resolution: {integrity: sha512-Nk/brWYpD85WlOgzw5h173aci0Teyv8YdIAEtV+N88nDB0dLlazZyJMIsN6eo1/AR61l+p6CJTG1JIyFaoNEEA==} @@ -6559,6 +6554,10 @@ packages: engines: {node: '>=6.4.0'} deprecated: Please upgrade to supertest v7.1.3+, see release notes at https://github.com/forwardemail/supertest/releases/tag/v7.1.3 - maintenance is supported by Forward Email @ https://forwardemail.net + supports-color@10.2.2: + resolution: {integrity: sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==} + engines: {node: '>=18'} + supports-color@5.5.0: resolution: {integrity: sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==} engines: {node: '>=4'} @@ -6571,10 +6570,6 @@ packages: resolution: {integrity: sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==} engines: {node: '>=10'} - supports-color@9.4.0: - resolution: {integrity: sha512-VL+lNrEoIXww1coLPOmiEmK/0sGigko5COxI09KzHc2VJXJsQ37UaQ+8quuxjDeA7+KnLGTWRyOXSLLR2Wb4jw==} - engines: {node: '>=12'} - supports-preserve-symlinks-flag@1.0.0: resolution: {integrity: sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==} engines: {node: '>= 0.4'} @@ -6852,6 +6847,10 @@ packages: resolution: {integrity: sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==} engines: {node: '>= 10.0.0'} + unlimited-timeout@0.1.0: + resolution: {integrity: sha512-D4g+mxFeQGQHzCfnvij+R35ukJ0658Zzudw7j16p4tBBbNasKkKM4SocYxqhwT5xA7a9JYWDzKkEFyMlRi5sng==} + engines: {node: '>=20'} + upath@2.0.1: resolution: {integrity: sha512-1uEe95xksV1O0CYKXo8vQvN1JEbtJp7lb7C5U9HMsIp6IVwntkH/oNUzyVNQSd4S1sYk2FpSSW44FqMc8qee5w==} engines: {node: '>=4'} @@ -7035,8 +7034,8 @@ packages: wcwidth@1.0.1: resolution: {integrity: sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg==} - weald@1.0.4: - resolution: {integrity: sha512-+kYTuHonJBwmFhP1Z4YQK/dGi3jAnJGCYhyODFpHK73rbxnp9lnZQj7a2m+WVgn8fXr5bJaxUpF6l8qZpPeNWQ==} + weald@1.1.1: + resolution: {integrity: sha512-PaEQShzMCz8J/AD2N3dJMc1hTZWkJeLKS2NMeiVkV5KDHwgZe7qXLEzyodsT/SODxWDdXJJqocuwf3kHzcXhSQ==} web3-core@4.0.3: resolution: {integrity: sha512-KJaH1+ajm/gelvhImkXZx8HrBaGZDERqhOCRpikuwReVDTf4X3TlXqF+oKt153qf5HUXWR4CUL6NkNKNQWjhbA==} @@ -7678,31 +7677,30 @@ snapshots: '@chainsafe/blst-linux-x64-musl': 2.2.0 '@chainsafe/blst-win32-x64-msvc': 2.2.0 - '@chainsafe/discv5@11.0.4': - dependencies: - '@chainsafe/enr': 5.0.1 - '@ethereumjs/rlp': 5.0.2 - '@libp2p/crypto': 5.1.7 - '@libp2p/interface': 2.10.5 - '@multiformats/multiaddr': 12.5.1 - '@noble/hashes': 1.8.0 - '@noble/secp256k1': 2.2.3 - debug: 4.4.3 - lru-cache: 10.4.3 + '@chainsafe/discv5@12.0.0': + dependencies: + '@chainsafe/enr': 6.0.0 + '@ethereumjs/rlp': 10.1.1 + '@libp2p/crypto': 5.1.13 + '@libp2p/interface': 3.1.0 + '@libp2p/peer-id': 6.0.4 + '@multiformats/multiaddr': 13.0.1 + '@noble/hashes': 2.0.1 + '@noble/secp256k1': 3.0.0 + ethereum-cryptography: 3.2.0 + lru-cache: 11.2.5 strict-event-emitter-types: 2.0.0 - transitivePeerDependencies: - - supports-color + weald: 1.1.1 - '@chainsafe/enr@5.0.1': + '@chainsafe/enr@6.0.0': dependencies: - '@ethereumjs/rlp': 5.0.2 - '@libp2p/crypto': 5.1.7 - '@libp2p/interface': 2.10.5 - '@libp2p/peer-id': 5.1.8 - '@multiformats/multiaddr': 12.5.1 - '@scure/base': 1.2.4 - ethereum-cryptography: 2.2.1 - uint8-varint: 2.0.4 + '@ethereumjs/rlp': 10.1.1 + '@libp2p/crypto': 5.1.13 + '@libp2p/interface': 3.1.0 + '@libp2p/peer-id': 6.0.4 + '@multiformats/multiaddr': 13.0.1 + '@scure/base': 2.0.0 + ethereum-cryptography: 3.2.0 '@chainsafe/fast-crc32c@4.2.0': optionalDependencies: @@ -7737,39 +7735,18 @@ snapshots: '@chainsafe/is-ip@2.1.0': {} - '@chainsafe/libp2p-gossipsub@14.1.2': - dependencies: - '@libp2p/crypto': 5.1.7 - '@libp2p/interface': 2.10.5 - '@libp2p/interface-internal': 2.3.18 - '@libp2p/peer-id': 5.1.8 - '@libp2p/pubsub': 10.1.17 - '@multiformats/multiaddr': 12.5.1 - denque: 2.1.0 - it-length-prefixed: 9.1.1 - it-pipe: 3.0.1 - it-pushable: 3.2.3 - multiformats: 13.3.7 - protons-runtime: 5.5.0 - uint8arraylist: 2.4.8 - uint8arrays: 5.1.0 - - '@chainsafe/libp2p-noise@16.1.5': + '@chainsafe/libp2p-noise@17.0.0': dependencies: '@chainsafe/as-chacha20poly1305': 0.1.0 '@chainsafe/as-sha256': 1.2.0 - '@libp2p/crypto': 5.1.7 - '@libp2p/interface': 2.10.5 - '@libp2p/peer-id': 5.1.8 - '@noble/ciphers': 1.2.1 - '@noble/curves': 1.9.2 - '@noble/hashes': 1.8.0 - it-length-prefixed: 10.0.1 - it-length-prefixed-stream: 2.0.3 - it-pair: 2.0.6 - it-pipe: 3.0.1 - it-stream-types: 2.0.2 - protons-runtime: 5.5.0 + '@libp2p/crypto': 5.1.13 + '@libp2p/interface': 3.1.0 + '@libp2p/peer-id': 6.0.4 + '@libp2p/utils': 7.0.10 + '@noble/ciphers': 2.1.1 + '@noble/curves': 2.0.1 + '@noble/hashes': 2.0.1 + protons-runtime: 5.6.0 uint8arraylist: 2.4.8 uint8arrays: 5.1.0 wherearewe: 2.0.1 @@ -8103,9 +8080,9 @@ snapshots: - supports-color - utf-8-validate - '@ethereumjs/rlp@4.0.1': {} + '@ethereumjs/rlp@10.1.1': {} - '@ethereumjs/rlp@5.0.2': {} + '@ethereumjs/rlp@4.0.1': {} '@ethereumjs/statemanager@1.0.5': dependencies: @@ -8870,210 +8847,195 @@ snapshots: - conventional-commits-filter - supports-color - '@libp2p/bootstrap@11.0.46': + '@libp2p/bootstrap@12.0.11': dependencies: - '@libp2p/interface': 2.10.5 - '@libp2p/interface-internal': 2.3.18 - '@libp2p/peer-id': 5.1.8 - '@multiformats/mafmt': 12.1.6 - '@multiformats/multiaddr': 12.5.1 + '@libp2p/interface': 3.1.0 + '@libp2p/interface-internal': 3.0.10 + '@libp2p/peer-id': 6.0.4 + '@multiformats/multiaddr': 13.0.1 + '@multiformats/multiaddr-matcher': 3.0.1 main-event: 1.0.1 - '@libp2p/crypto@5.1.7': + '@libp2p/crypto@5.1.13': dependencies: - '@libp2p/interface': 2.10.5 - '@noble/curves': 1.9.2 - '@noble/hashes': 1.8.0 - multiformats: 13.3.7 + '@libp2p/interface': 3.1.0 + '@noble/curves': 2.0.1 + '@noble/hashes': 2.0.1 + multiformats: 13.4.2 + protons-runtime: 5.6.0 + uint8arraylist: 2.4.8 + uint8arrays: 5.1.0 + + '@libp2p/gossipsub@15.0.12': + dependencies: + '@libp2p/crypto': 5.1.13 + '@libp2p/interface': 3.1.0 + '@libp2p/interface-internal': 3.0.10 + '@libp2p/peer-id': 6.0.4 + '@libp2p/utils': 7.0.10 + '@multiformats/multiaddr': 13.0.1 + denque: 2.1.0 + it-length-prefixed: 10.0.1 + it-pipe: 3.0.1 + it-pushable: 3.2.3 + multiformats: 13.4.2 protons-runtime: 5.5.0 uint8arraylist: 2.4.8 uint8arrays: 5.1.0 - '@libp2p/identify@3.0.38': + '@libp2p/identify@4.0.10': dependencies: - '@libp2p/crypto': 5.1.7 - '@libp2p/interface': 2.10.5 - '@libp2p/interface-internal': 2.3.18 - '@libp2p/peer-id': 5.1.8 - '@libp2p/peer-record': 8.0.34 - '@libp2p/utils': 6.7.1 - '@multiformats/multiaddr': 12.5.1 - '@multiformats/multiaddr-matcher': 2.0.1 + '@libp2p/crypto': 5.1.13 + '@libp2p/interface': 3.1.0 + '@libp2p/interface-internal': 3.0.10 + '@libp2p/peer-id': 6.0.4 + '@libp2p/peer-record': 9.0.5 + '@libp2p/utils': 7.0.10 + '@multiformats/multiaddr': 13.0.1 + '@multiformats/multiaddr-matcher': 3.0.1 it-drain: 3.0.10 it-parallel: 3.0.13 - it-protobuf-stream: 2.0.3 main-event: 1.0.1 - protons-runtime: 5.5.0 + protons-runtime: 5.6.0 uint8arraylist: 2.4.8 uint8arrays: 5.1.0 - '@libp2p/interface-internal@2.3.18': + '@libp2p/interface-internal@3.0.10': dependencies: - '@libp2p/interface': 2.10.5 - '@libp2p/peer-collections': 6.0.34 - '@multiformats/multiaddr': 12.5.1 + '@libp2p/interface': 3.1.0 + '@libp2p/peer-collections': 7.0.10 + '@multiformats/multiaddr': 13.0.1 progress-events: 1.0.1 - '@libp2p/interface@2.10.5': + '@libp2p/interface@3.1.0': dependencies: '@multiformats/dns': 1.0.6 - '@multiformats/multiaddr': 12.5.1 - it-pushable: 3.2.3 - it-stream-types: 2.0.2 + '@multiformats/multiaddr': 13.0.1 main-event: 1.0.1 - multiformats: 13.3.7 + multiformats: 13.4.2 progress-events: 1.0.1 uint8arraylist: 2.4.8 - '@libp2p/logger@5.1.21': + '@libp2p/logger@6.2.2': dependencies: - '@libp2p/interface': 2.10.5 - '@multiformats/multiaddr': 12.5.1 - interface-datastore: 8.3.1 - multiformats: 13.3.7 - weald: 1.0.4 + '@libp2p/interface': 3.1.0 + '@multiformats/multiaddr': 13.0.1 + interface-datastore: 9.0.2 + multiformats: 13.4.2 + weald: 1.1.1 - '@libp2p/mdns@11.0.46': + '@libp2p/mdns@12.0.11': dependencies: - '@libp2p/interface': 2.10.5 - '@libp2p/interface-internal': 2.3.18 - '@libp2p/peer-id': 5.1.8 - '@libp2p/utils': 6.7.1 - '@multiformats/multiaddr': 12.5.1 + '@libp2p/interface': 3.1.0 + '@libp2p/interface-internal': 3.0.10 + '@libp2p/peer-id': 6.0.4 + '@libp2p/utils': 7.0.10 + '@multiformats/multiaddr': 13.0.1 '@types/multicast-dns': 7.2.4 dns-packet: 5.6.1 main-event: 1.0.1 multicast-dns: 7.2.5 - '@libp2p/mplex@11.0.46': + '@libp2p/mplex@12.0.11': dependencies: - '@libp2p/interface': 2.10.5 - '@libp2p/utils': 6.7.1 - it-pipe: 3.0.1 + '@libp2p/interface': 3.1.0 + '@libp2p/utils': 7.0.10 it-pushable: 3.2.3 - it-stream-types: 2.0.2 uint8-varint: 2.0.4 uint8arraylist: 2.4.8 uint8arrays: 5.1.0 - '@libp2p/multistream-select@6.0.28': + '@libp2p/multistream-select@7.0.10': dependencies: - '@libp2p/interface': 2.10.5 + '@libp2p/interface': 3.1.0 + '@libp2p/utils': 7.0.10 it-length-prefixed: 10.0.1 - it-length-prefixed-stream: 2.0.3 - it-stream-types: 2.0.2 - p-defer: 4.0.1 - race-signal: 1.1.3 - uint8-varint: 2.0.4 uint8arraylist: 2.4.8 uint8arrays: 5.1.0 - '@libp2p/peer-collections@6.0.34': + '@libp2p/peer-collections@7.0.10': dependencies: - '@libp2p/interface': 2.10.5 - '@libp2p/peer-id': 5.1.8 - '@libp2p/utils': 6.7.1 - multiformats: 13.3.7 + '@libp2p/interface': 3.1.0 + '@libp2p/peer-id': 6.0.4 + '@libp2p/utils': 7.0.10 + multiformats: 13.4.2 - '@libp2p/peer-id@5.1.8': + '@libp2p/peer-id@6.0.4': dependencies: - '@libp2p/crypto': 5.1.7 - '@libp2p/interface': 2.10.5 - multiformats: 13.3.7 + '@libp2p/crypto': 5.1.13 + '@libp2p/interface': 3.1.0 + multiformats: 13.4.2 uint8arrays: 5.1.0 - '@libp2p/peer-record@8.0.34': + '@libp2p/peer-record@9.0.5': dependencies: - '@libp2p/crypto': 5.1.7 - '@libp2p/interface': 2.10.5 - '@libp2p/peer-id': 5.1.8 - '@libp2p/utils': 6.7.1 - '@multiformats/multiaddr': 12.5.1 - multiformats: 13.3.7 - protons-runtime: 5.5.0 + '@libp2p/crypto': 5.1.13 + '@libp2p/interface': 3.1.0 + '@libp2p/peer-id': 6.0.4 + '@multiformats/multiaddr': 13.0.1 + multiformats: 13.4.2 + protons-runtime: 5.6.0 uint8-varint: 2.0.4 uint8arraylist: 2.4.8 uint8arrays: 5.1.0 - '@libp2p/peer-store@11.2.6': + '@libp2p/peer-store@12.0.10': dependencies: - '@libp2p/crypto': 5.1.7 - '@libp2p/interface': 2.10.5 - '@libp2p/peer-collections': 6.0.34 - '@libp2p/peer-id': 5.1.8 - '@libp2p/peer-record': 8.0.34 - '@multiformats/multiaddr': 12.5.1 - interface-datastore: 8.3.1 + '@libp2p/crypto': 5.1.13 + '@libp2p/interface': 3.1.0 + '@libp2p/peer-collections': 7.0.10 + '@libp2p/peer-id': 6.0.4 + '@libp2p/peer-record': 9.0.5 + '@multiformats/multiaddr': 13.0.1 + interface-datastore: 9.0.2 it-all: 3.0.9 main-event: 1.0.1 mortice: 3.3.1 - multiformats: 13.3.7 - protons-runtime: 5.5.0 + multiformats: 13.4.2 + protons-runtime: 5.6.0 uint8arraylist: 2.4.8 uint8arrays: 5.1.0 - '@libp2p/prometheus-metrics@4.3.29': + '@libp2p/prometheus-metrics@5.0.10': dependencies: - '@libp2p/interface': 2.10.5 - it-foreach: 2.1.4 - it-stream-types: 2.0.2 + '@libp2p/interface': 3.1.0 prom-client: 15.1.3 - uint8arraylist: 2.4.8 - '@libp2p/pubsub@10.1.17': + '@libp2p/tcp@11.0.10': dependencies: - '@libp2p/crypto': 5.1.7 - '@libp2p/interface': 2.10.5 - '@libp2p/interface-internal': 2.3.18 - '@libp2p/peer-collections': 6.0.34 - '@libp2p/peer-id': 5.1.8 - '@libp2p/utils': 6.7.1 - it-length-prefixed: 10.0.1 - it-pipe: 3.0.1 - it-pushable: 3.2.3 - main-event: 1.0.1 - multiformats: 13.3.7 - p-queue: 8.1.0 - uint8arraylist: 2.4.8 - uint8arrays: 5.1.0 - - '@libp2p/tcp@10.1.18': - dependencies: - '@libp2p/interface': 2.10.5 - '@libp2p/utils': 6.7.1 - '@multiformats/multiaddr': 12.5.1 - '@multiformats/multiaddr-matcher': 2.0.1 - '@types/sinon': 17.0.4 + '@libp2p/interface': 3.1.0 + '@libp2p/utils': 7.0.10 + '@multiformats/multiaddr': 13.0.1 + '@multiformats/multiaddr-matcher': 3.0.1 + '@types/sinon': 20.0.0 main-event: 1.0.1 - p-defer: 4.0.1 - p-event: 6.0.1 + p-event: 7.1.0 progress-events: 1.0.1 - race-event: 1.3.0 - stream-to-it: 1.0.1 + uint8arraylist: 2.4.8 - '@libp2p/utils@6.7.1': + '@libp2p/utils@7.0.10': dependencies: '@chainsafe/is-ip': 2.1.0 '@chainsafe/netmask': 2.0.0 - '@libp2p/crypto': 5.1.7 - '@libp2p/interface': 2.10.5 - '@libp2p/logger': 5.1.21 - '@multiformats/multiaddr': 12.5.1 + '@libp2p/crypto': 5.1.13 + '@libp2p/interface': 3.1.0 + '@libp2p/logger': 6.2.2 + '@multiformats/multiaddr': 13.0.1 '@sindresorhus/fnv1a': 3.1.0 any-signal: 4.1.1 - delay: 6.0.0 - get-iterator: 2.0.1 + cborg: 4.5.8 + delay: 7.0.0 is-loopback-addr: 2.0.2 - is-plain-obj: 4.1.0 - it-foreach: 2.1.4 + it-length-prefixed: 10.0.1 it-pipe: 3.0.1 it-pushable: 3.2.3 it-stream-types: 2.0.2 main-event: 1.0.1 netmask: 2.0.2 p-defer: 4.0.1 - race-event: 1.3.0 - race-signal: 1.1.3 + p-event: 7.1.0 + race-signal: 2.0.0 + uint8-varint: 2.0.4 uint8arraylist: 2.4.8 uint8arrays: 5.1.0 @@ -9124,21 +9086,14 @@ snapshots: progress-events: 1.0.1 uint8arrays: 5.1.0 - '@multiformats/mafmt@12.1.6': - dependencies: - '@multiformats/multiaddr': 12.5.1 - - '@multiformats/multiaddr-matcher@2.0.1': + '@multiformats/multiaddr-matcher@3.0.1': dependencies: - '@multiformats/multiaddr': 12.5.1 + '@multiformats/multiaddr': 13.0.1 - '@multiformats/multiaddr@12.5.1': + '@multiformats/multiaddr@13.0.1': dependencies: '@chainsafe/is-ip': 2.1.0 - '@chainsafe/netmask': 2.0.0 - '@multiformats/dns': 1.0.6 - abort-error: 1.0.1 - multiformats: 13.3.7 + multiformats: 13.4.2 uint8-varint: 2.0.4 uint8arrays: 5.1.0 @@ -9188,25 +9143,33 @@ snapshots: '@tybys/wasm-util': 0.9.0 optional: true - '@noble/ciphers@1.2.1': {} + '@noble/ciphers@1.3.0': {} + + '@noble/ciphers@2.1.1': {} '@noble/curves@1.4.2': dependencies: '@noble/hashes': 1.4.0 - '@noble/curves@1.9.2': + '@noble/curves@1.9.0': dependencies: '@noble/hashes': 1.8.0 + '@noble/curves@2.0.1': + dependencies: + '@noble/hashes': 2.0.1 + '@noble/hashes@1.1.2': {} '@noble/hashes@1.4.0': {} '@noble/hashes@1.8.0': {} + '@noble/hashes@2.0.1': {} + '@noble/secp256k1@1.7.1': {} - '@noble/secp256k1@2.2.3': {} + '@noble/secp256k1@3.0.0': {} '@node-rs/crc32-android-arm-eabi@1.10.6': optional: true @@ -9691,7 +9654,9 @@ snapshots: '@scure/base@1.1.8': {} - '@scure/base@1.2.4': {} + '@scure/base@1.2.6': {} + + '@scure/base@2.0.0': {} '@scure/bip32@1.4.0': dependencies: @@ -9699,11 +9664,22 @@ snapshots: '@noble/hashes': 1.4.0 '@scure/base': 1.1.8 + '@scure/bip32@1.7.0': + dependencies: + '@noble/curves': 1.9.0 + '@noble/hashes': 1.8.0 + '@scure/base': 1.2.6 + '@scure/bip39@1.3.0': dependencies: '@noble/hashes': 1.4.0 '@scure/base': 1.1.8 + '@scure/bip39@1.6.0': + dependencies: + '@noble/hashes': 1.8.0 + '@scure/base': 1.2.6 + '@sec-ant/readable-stream@0.4.1': {} '@sigstore/bundle@4.0.0': @@ -9930,7 +9906,7 @@ snapshots: '@types/retry@0.12.2': {} - '@types/sinon@17.0.4': + '@types/sinon@20.0.0': dependencies: '@types/sinonjs__fake-timers': 6.0.2 @@ -9974,20 +9950,6 @@ snapshots: - utf-8-validate - vite - '@vitest/browser-playwright@4.0.7(playwright@1.58.2)(vite@6.1.6(@types/node@24.10.1)(yaml@2.8.2))(vitest@4.0.7)': - dependencies: - '@vitest/browser': 4.0.7(vite@6.1.6(@types/node@24.10.1)(yaml@2.8.2))(vitest@4.0.7) - '@vitest/mocker': 4.0.7(vite@6.1.6(@types/node@24.10.1)(yaml@2.8.2)) - playwright: 1.58.2 - tinyrainbow: 3.0.3 - vitest: 4.0.7(@types/node@24.10.1)(@vitest/browser-playwright@4.0.7)(jsdom@23.0.1)(yaml@2.8.2) - transitivePeerDependencies: - - bufferutil - - msw - - utf-8-validate - - vite - optional: true - '@vitest/browser@4.0.7(vite@6.1.6(@types/node@24.10.1)(yaml@2.8.2))(vitest@4.0.7)': dependencies: '@vitest/mocker': 4.0.7(vite@6.1.6(@types/node@24.10.1)(yaml@2.8.2)) @@ -10143,6 +10105,15 @@ snapshots: module-error: 1.0.2 queue-microtask: 1.2.3 + abstract-level@3.1.1: + dependencies: + buffer: 6.0.3 + is-buffer: 2.0.5 + level-supports: 6.2.0 + level-transcoder: 1.0.1 + maybe-combine-errors: 1.0.0 + module-error: 1.0.2 + abstract-logging@2.0.1: {} acorn-walk@8.2.0: {} @@ -10391,6 +10362,10 @@ snapshots: module-error: 1.0.2 run-parallel-limit: 1.1.0 + browser-level@3.0.0: + dependencies: + abstract-level: 3.1.1 + browser-resolve@2.0.0: dependencies: resolve: 1.22.10 @@ -10545,6 +10520,8 @@ snapshots: catering@2.1.1: {} + cborg@4.5.8: {} + chai@6.2.0: {} chalk@2.4.2: @@ -10581,6 +10558,13 @@ snapshots: napi-macros: 2.2.2 node-gyp-build: 4.5.0 + classic-level@3.0.0: + dependencies: + abstract-level: 3.1.1 + module-error: 1.0.2 + napi-macros: 2.2.2 + node-gyp-build: 4.5.0 + clean-stack@2.2.0: {} cli-cursor@4.0.0: @@ -10824,41 +10808,40 @@ snapshots: whatwg-mimetype: 4.0.0 whatwg-url: 14.0.0 - datastore-core@10.0.2: + datastore-core@11.0.2: dependencies: - '@libp2p/logger': 5.1.21 - interface-datastore: 8.3.1 - interface-store: 6.0.2 + '@libp2p/logger': 6.2.2 + interface-datastore: 9.0.2 + interface-store: 7.0.1 it-drain: 3.0.10 it-filter: 3.1.4 it-map: 3.1.4 it-merge: 3.0.12 it-pipe: 3.0.1 - it-pushable: 3.2.3 it-sort: 3.0.9 it-take: 3.0.9 - datastore-fs@10.0.6: + datastore-fs@11.0.2: dependencies: - datastore-core: 10.0.2 - interface-datastore: 8.3.1 - interface-store: 6.0.2 + datastore-core: 11.0.2 + interface-datastore: 9.0.2 + interface-store: 7.0.1 it-glob: 3.0.4 it-map: 3.1.4 it-parallel-batch: 3.0.9 race-signal: 2.0.0 steno: 4.0.2 - datastore-level@11.0.4: + datastore-level@12.0.2: dependencies: - datastore-core: 10.0.2 - interface-datastore: 8.3.1 - interface-store: 6.0.2 + datastore-core: 11.0.2 + interface-datastore: 9.0.2 + interface-store: 7.0.1 it-filter: 3.1.4 it-map: 3.1.4 it-sort: 3.0.9 it-take: 3.0.9 - level: 8.0.1 + level: 10.0.0 race-signal: 2.0.0 de-indent@1.0.2: {} @@ -10901,7 +10884,10 @@ snapshots: has-property-descriptors: 1.0.1 object-keys: 1.1.1 - delay@6.0.0: {} + delay@7.0.0: + dependencies: + random-int: 3.1.0 + unlimited-timeout: 0.1.0 delayed-stream@1.0.0: {} @@ -11137,6 +11123,14 @@ snapshots: '@scure/bip32': 1.4.0 '@scure/bip39': 1.3.0 + ethereum-cryptography@3.2.0: + dependencies: + '@noble/ciphers': 1.3.0 + '@noble/curves': 1.9.0 + '@noble/hashes': 1.8.0 + '@scure/bip32': 1.7.0 + '@scure/bip39': 1.6.0 + ethers@5.7.2: dependencies: '@ethersproject/abi': 5.7.0 @@ -11501,8 +11495,6 @@ snapshots: hasown: 2.0.2 math-intrinsics: 1.1.0 - get-iterator@2.0.1: {} - get-proto@1.0.1: dependencies: dunder-proto: 1.0.1 @@ -11837,12 +11829,12 @@ snapshots: through: 2.3.8 wrap-ansi: 8.1.0 - interface-datastore@8.3.1: + interface-datastore@9.0.2: dependencies: - interface-store: 6.0.2 + interface-store: 7.0.1 uint8arrays: 5.1.0 - interface-store@6.0.2: {} + interface-store@7.0.1: {} internal-slot@1.0.5: dependencies: @@ -12026,36 +12018,16 @@ snapshots: it-batch@3.0.9: {} - it-byte-stream@2.0.3: - dependencies: - abort-error: 1.0.1 - it-queueless-pushable: 2.0.2 - it-stream-types: 2.0.2 - race-signal: 1.1.3 - uint8arraylist: 2.4.8 - it-drain@3.0.10: {} it-filter@3.1.4: dependencies: it-peekable: 3.0.1 - it-foreach@2.1.4: - dependencies: - it-peekable: 3.0.1 - it-glob@3.0.4: dependencies: fast-glob: 3.3.3 - it-length-prefixed-stream@2.0.3: - dependencies: - abort-error: 1.0.1 - it-byte-stream: 2.0.3 - it-stream-types: 2.0.2 - uint8-varint: 2.0.4 - uint8arraylist: 2.4.8 - it-length-prefixed@10.0.1: dependencies: it-reader: 6.0.4 @@ -12064,14 +12036,6 @@ snapshots: uint8arraylist: 2.4.8 uint8arrays: 5.1.0 - it-length-prefixed@9.1.1: - dependencies: - it-reader: 6.0.4 - it-stream-types: 2.0.2 - uint8-varint: 2.0.4 - uint8arraylist: 2.4.8 - uint8arrays: 5.1.0 - it-map@3.1.4: dependencies: it-peekable: 3.0.1 @@ -12080,11 +12044,6 @@ snapshots: dependencies: it-queueless-pushable: 2.0.2 - it-pair@2.0.6: - dependencies: - it-stream-types: 2.0.2 - p-defer: 4.0.1 - it-parallel-batch@3.0.9: dependencies: it-batch: 3.0.9 @@ -12101,13 +12060,6 @@ snapshots: it-pushable: 3.2.3 it-stream-types: 2.0.2 - it-protobuf-stream@2.0.3: - dependencies: - abort-error: 1.0.1 - it-length-prefixed-stream: 2.0.3 - it-stream-types: 2.0.2 - uint8arraylist: 2.4.8 - it-pushable@3.2.3: dependencies: p-defer: 4.0.1 @@ -12252,11 +12204,19 @@ snapshots: level-supports@4.0.1: {} + level-supports@6.2.0: {} + level-transcoder@1.0.1: dependencies: buffer: 6.0.3 module-error: 1.0.2 + level@10.0.0: + dependencies: + abstract-level: 3.1.1 + browser-level: 3.0.0 + classic-level: 3.0.0 + level@8.0.1: dependencies: abstract-level: 1.0.4 @@ -12283,35 +12243,34 @@ snapshots: transitivePeerDependencies: - supports-color - libp2p@2.9.0: + libp2p@3.1.3: dependencies: '@chainsafe/is-ip': 2.1.0 '@chainsafe/netmask': 2.0.0 - '@libp2p/crypto': 5.1.7 - '@libp2p/interface': 2.10.5 - '@libp2p/interface-internal': 2.3.18 - '@libp2p/logger': 5.1.21 - '@libp2p/multistream-select': 6.0.28 - '@libp2p/peer-collections': 6.0.34 - '@libp2p/peer-id': 5.1.8 - '@libp2p/peer-store': 11.2.6 - '@libp2p/utils': 6.7.1 + '@libp2p/crypto': 5.1.13 + '@libp2p/interface': 3.1.0 + '@libp2p/interface-internal': 3.0.10 + '@libp2p/logger': 6.2.2 + '@libp2p/multistream-select': 7.0.10 + '@libp2p/peer-collections': 7.0.10 + '@libp2p/peer-id': 6.0.4 + '@libp2p/peer-store': 12.0.10 + '@libp2p/utils': 7.0.10 '@multiformats/dns': 1.0.6 - '@multiformats/multiaddr': 12.5.1 - '@multiformats/multiaddr-matcher': 2.0.1 + '@multiformats/multiaddr': 13.0.1 + '@multiformats/multiaddr-matcher': 3.0.1 any-signal: 4.1.1 - datastore-core: 10.0.2 - interface-datastore: 8.3.1 - it-byte-stream: 2.0.3 + datastore-core: 11.0.2 + interface-datastore: 9.0.2 it-merge: 3.0.12 it-parallel: 3.0.13 main-event: 1.0.1 - multiformats: 13.3.7 + multiformats: 13.4.2 p-defer: 4.0.1 - p-retry: 6.2.1 + p-event: 7.1.0 + p-retry: 7.1.1 progress-events: 1.0.1 - race-event: 1.3.0 - race-signal: 1.1.3 + race-signal: 2.0.0 uint8arrays: 5.1.0 light-my-request@6.0.0: @@ -12399,6 +12358,8 @@ snapshots: lru-cache@11.2.4: {} + lru-cache@11.2.5: {} + lru-cache@5.1.1: dependencies: yallist: 3.1.1 @@ -12472,6 +12433,8 @@ snapshots: math-intrinsics@1.1.0: {} + maybe-combine-errors@1.0.0: {} + mcl-wasm@0.7.9: {} md5.js@1.3.5: @@ -12645,9 +12608,7 @@ snapshots: dns-packet: 5.6.1 thunky: 1.1.0 - multiformats@11.0.2: {} - - multiformats@13.3.7: {} + multiformats@13.4.2: {} mute-stream@1.0.0: {} @@ -12937,9 +12898,9 @@ snapshots: p-defer@4.0.1: {} - p-event@6.0.1: + p-event@7.1.0: dependencies: - p-timeout: 6.1.2 + p-timeout: 7.0.1 p-limit@2.3.0: dependencies: @@ -12989,11 +12950,9 @@ snapshots: p-reduce@3.0.0: {} - p-retry@6.2.1: + p-retry@7.1.1: dependencies: - '@types/retry': 0.12.2 is-network-error: 1.1.0 - retry: 0.13.1 p-timeout@6.1.2: {} @@ -13178,22 +13137,12 @@ snapshots: playwright-core@1.56.1: {} - playwright-core@1.58.2: - optional: true - playwright@1.56.1: dependencies: playwright-core: 1.56.1 optionalDependencies: fsevents: 2.3.2 - playwright@1.58.2: - dependencies: - playwright-core: 1.58.2 - optionalDependencies: - fsevents: 2.3.2 - optional: true - pngjs@7.0.0: {} postcss-selector-parser@7.1.1: @@ -13277,6 +13226,12 @@ snapshots: uint8arraylist: 2.4.8 uint8arrays: 5.1.0 + protons-runtime@5.6.0: + dependencies: + uint8-varint: 2.0.4 + uint8arraylist: 2.4.8 + uint8arrays: 5.1.0 + proxy-from-env@1.1.0: {} psl@1.9.0: {} @@ -13325,6 +13280,8 @@ snapshots: race-signal@2.0.0: {} + random-int@3.1.0: {} + randombytes@2.1.0: dependencies: safe-buffer: 5.2.1 @@ -13411,8 +13368,6 @@ snapshots: retry@0.12.0: {} - retry@0.13.1: {} - reusify@1.0.4: {} rewiremock@3.14.5: @@ -13796,10 +13751,6 @@ snapshots: readable-stream: 3.6.2 xtend: 4.0.2 - stream-to-it@1.0.1: - dependencies: - it-stream-types: 2.0.2 - strict-event-emitter-types@2.0.0: {} string-argv@0.3.2: {} @@ -13898,6 +13849,8 @@ snapshots: transitivePeerDependencies: - supports-color + supports-color@10.2.2: {} + supports-color@5.5.0: dependencies: has-flag: 3.0.0 @@ -13910,8 +13863,6 @@ snapshots: dependencies: has-flag: 4.0.0 - supports-color@9.4.0: {} - supports-preserve-symlinks-flag@1.0.0: {} symbol-tree@3.2.4: {} @@ -14121,7 +14072,7 @@ snapshots: uint8arrays@5.1.0: dependencies: - multiformats: 13.3.7 + multiformats: 13.4.2 unbox-primitive@1.0.2: dependencies: @@ -14168,6 +14119,8 @@ snapshots: universalify@2.0.0: {} + unlimited-timeout@0.1.0: {} + upath@2.0.1: {} uri-js@4.4.1: @@ -14305,7 +14258,7 @@ snapshots: why-is-node-running: 2.3.0 optionalDependencies: '@types/node': 24.10.1 - '@vitest/browser-playwright': 4.0.7(playwright@1.58.2)(vite@6.1.6(@types/node@24.10.1)(yaml@2.8.2))(vitest@4.0.7) + '@vitest/browser-playwright': 4.0.7(playwright@1.56.1)(vite@6.1.6(@types/node@24.10.1)(yaml@2.8.2))(vitest@4.0.7) jsdom: 23.0.1 transitivePeerDependencies: - jiti @@ -14343,10 +14296,10 @@ snapshots: dependencies: defaults: 1.0.3 - weald@1.0.4: + weald@1.1.1: dependencies: ms: 3.0.0-canary.1 - supports-color: 9.4.0 + supports-color: 10.2.2 web3-core@4.0.3(encoding@0.1.13): dependencies: diff --git a/tsconfig.build.json b/tsconfig.build.json index fc17ee92a43c..bcdd12f83369 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -1,7 +1,7 @@ { "compilerOptions": { - "target": "es2022", - "lib": ["es2022", "dom"], + "target": "es2023", + "lib": ["es2023", "dom"], "module": "nodenext", "moduleResolution": "nodenext", diff --git a/tsconfig.e2e.json b/tsconfig.e2e.json index 14cdc4bf044b..71983027398f 100644 --- a/tsconfig.e2e.json +++ b/tsconfig.e2e.json @@ -1,7 +1,7 @@ { "compilerOptions": { - "target": "es2019", - "lib": ["es2020", "esnext.bigint", "es2020.string", "es2020.symbol.wellknown"], + "target": "es2023", + "lib": ["es2023", "dom"], "module": "esnext", "moduleResolution": "node", diff --git a/types/it-pipe/index.d.ts b/types/it-pipe/index.d.ts deleted file mode 100644 index 0abe013aad78..000000000000 --- a/types/it-pipe/index.d.ts +++ /dev/null @@ -1,120 +0,0 @@ -// Overwrite types until this issue is fixed https://github.com/alanshaw/it-pipe/issues/4 -// Original types contain -// ``` -// export function pipe (f1: any, ...rest: any[]): any -// ``` -// Which act as a @ts-ignore compromising type safety - -declare module "it-pipe" { - type Source = AsyncIterable; - type Sink = (source: TSource) => TReturn; - type Duplex = { - sink: Sink; - source: TSource; - }; - - export function pipe(f1: A | (() => A), f2: (source: A) => B): B; - - export function pipe(f1: A | (() => A), f2: (source: A) => B, f3: (source: B) => C): C; - - export function pipe( - f1: A | (() => A), - f2: (source: A) => B, - f3: (source: B) => C, - f4: (source: C) => D - ): D; - - export function pipe( - f1: A | (() => A), - f2: (source: A) => B, - f3: (source: B) => C, - f4: (source: C) => D, - f5: (source: D) => E - ): E; - - export function pipe( - f1: A | (() => A), - f2: (source: A) => B, - f3: (source: B) => C, - f4: (source: C) => D, - f5: (source: D) => E, - f6: (source: E) => F - ): F; - - export function pipe( - f1: A | (() => A), - f2: (source: A) => B, - f3: (source: B) => C, - f4: (source: C) => D, - f5: (source: D) => E, - f6: (source: E) => F, - f7: (source: F) => G - ): G; - - export function pipe( - f1: A | (() => A), - f2: (source: A) => B, - f3: (source: B) => C, - f4: (source: C) => D, - f5: (source: D) => E, - f6: (source: E) => F, - f7: (source: F) => G, - f8: (source: G) => H - ): H; - - export function pipe( - f1: A | (() => A), - f2: (source: A) => B, - f3: (source: B) => C, - f4: (source: C) => D, - f5: (source: D) => E, - f6: (source: E) => F, - f7: (source: F) => G, - f8: (source: G) => H, - f9: (source: H) => I - ): I; - - // First argument is array - - export function pipe(f1: A[], f2: (source: AsyncIterable) => B): B; - - export function pipe(f1: A[], f2: (source: AsyncIterable) => B, f3: (source: B) => C): C; - - export function pipe( - f1: A[], - f2: (source: AsyncIterable) => B, - f3: (source: B) => C, - f4: (source: C) => D - ): D; - - export function pipe( - f1: A[], - f2: (source: AsyncIterable) => B, - f3: (source: B) => C, - f4: (source: C) => D, - f5: (source: D) => E - ): E; - - // First argument is duplex - - export function pipe(f1: Duplex, f2: (source: AsyncIterable) => B): B; - - export function pipe(f1: Duplex, f2: (source: AsyncIterable) => B, f3: (source: B) => C): C; - - export function pipe( - f1: Duplex, - f2: (source: AsyncIterable) => B, - f3: (source: B) => C, - f4: (source: C) => D - ): D; - - export function pipe( - f1: Duplex, - f2: (source: AsyncIterable) => B, - f3: (source: B) => C, - f4: (source: C) => D, - f5: (source: D) => E - ): E; - - export default pipe; -} From afd446235e52427e4b6982fa44fc0ddc012157e6 Mon Sep 17 00:00:00 2001 From: bing Date: Wed, 25 Feb 2026 19:21:18 +0800 Subject: [PATCH 09/63] ci: remove bun tests (#8959) **Motivation** as previously discussed, remove bun tests entirely. Extracted from #8900 cc @nazarhussain --- .github/workflows/test-bun.yml | 40 ---------------------------------- 1 file changed, 40 deletions(-) delete mode 100644 .github/workflows/test-bun.yml diff --git a/.github/workflows/test-bun.yml b/.github/workflows/test-bun.yml deleted file mode 100644 index bb2158fd744a..000000000000 --- a/.github/workflows/test-bun.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Bun Tests -# only one can run at a time -concurrency: - # If PR, cancel prev commits. head_ref = source branch name on pull_request, null if push - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -on: - push: - # We intentionally don't run push on feature branches. See PR for rational. - branches: [unstable, stable] - pull_request: - workflow_dispatch: - -jobs: - unit-tests-bun: - name: Unit Tests (Bun) - runs-on: buildjet-4vcpu-ubuntu-2204 - steps: - - uses: actions/checkout@v4 - - name: Install pnpm before setup node - uses: pnpm/action-setup@v4 - - name: Setup Node - uses: actions/setup-node@v6 - with: - node-version: 24 - cache: pnpm - - uses: oven-sh/setup-bun@v2 - with: - bun-version: latest - - name: Install pnpm - run: bun install -g npm:pnpm - - name: Install - run: pnpm install --frozen-lockfile - - name: Build - run: pnpm build - - name: Unit Tests - # These packages are not testable yet in Bun because of `@chainsafe/blst` dependency. - run: excluded=(beacon-node prover light-client cli); for pkg in packages/*/; do [[ ! " ${excluded[@]} " =~ " $(basename "$pkg") " ]] && echo "Testing $(basename "$pkg")" && (cd "$pkg" && bun run --bun test:unit); done - shell: bash From 6eb55025ce5182668e68ed86641aa5f34ecc09c2 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Wed, 25 Feb 2026 19:49:36 +0000 Subject: [PATCH 10/63] fix: libp2p v3 unknown peers (#8960) Squash merged https://github.com/ChainSafe/lodestar/pull/8958 and reverted https://github.com/ChainSafe/lodestar/pull/8955. Deployed to `feat1` --- packages/beacon-node/src/constants/network.ts | 1 - .../beacon-node/src/network/libp2p/index.ts | 32 +++- .../src/network/peers/peerManager.ts | 65 +------- .../e2e/network/peers/peerManager.test.ts | 154 ------------------ 4 files changed, 29 insertions(+), 223 deletions(-) diff --git a/packages/beacon-node/src/constants/network.ts b/packages/beacon-node/src/constants/network.ts index e26bbe69ee6b..c786bf25a3bc 100644 --- a/packages/beacon-node/src/constants/network.ts +++ b/packages/beacon-node/src/constants/network.ts @@ -30,5 +30,4 @@ export const GOODBYE_KNOWN_CODES: Record = { export enum Libp2pEvent { connectionOpen = "connection:open", connectionClose = "connection:close", - peerIdentify = "peer:identify", } diff --git a/packages/beacon-node/src/network/libp2p/index.ts b/packages/beacon-node/src/network/libp2p/index.ts index 9aa53d6fd10b..179b48f297eb 100644 --- a/packages/beacon-node/src/network/libp2p/index.ts +++ b/packages/beacon-node/src/network/libp2p/index.ts @@ -73,6 +73,30 @@ export async function createNodeJsLibp2p( noiseCrypto.chaCha20Poly1305Encrypt = asCrypto.chaCha20Poly1305Encrypt; } + const libp2pMetrics = nodeJsLibp2pOpts.metrics + ? (components: LodestarComponents) => { + const metrics = prometheusMetrics({ + collectDefaultMetrics: false, + preserveExistingMetrics: true, + registry: nodeJsLibp2pOpts.metricsRegistry, + })(components); + + // Work around identify EOF race: + // `trackProtocolStream` attaches a `message` listener immediately after protocol + // negotiation. For `/ipfs/id/1.0.0`, identify() adds its own reader later and can + // miss the first response frame when metrics listener drains events first. + const originalTrackProtocolStream = metrics.trackProtocolStream.bind(metrics); + metrics.trackProtocolStream = ((stream) => { + if (stream.protocol === "/ipfs/id/1.0.0") { + return; + } + originalTrackProtocolStream(stream); + }) as typeof metrics.trackProtocolStream; + + return metrics; + } + : undefined; + return createLibp2p({ privateKey, nodeInfo: { @@ -101,13 +125,7 @@ export async function createNodeJsLibp2p( ], streamMuxers: [mplex({disconnectThreshold})], peerDiscovery, - metrics: nodeJsLibp2pOpts.metrics - ? prometheusMetrics({ - collectDefaultMetrics: false, - preserveExistingMetrics: true, - registry: nodeJsLibp2pOpts.metricsRegistry, - }) - : undefined, + metrics: libp2pMetrics, connectionManager: { // dialer config maxParallelDials: 100, diff --git a/packages/beacon-node/src/network/peers/peerManager.ts b/packages/beacon-node/src/network/peers/peerManager.ts index 86f19c979a6d..ca8c8892dd6d 100644 --- a/packages/beacon-node/src/network/peers/peerManager.ts +++ b/packages/beacon-node/src/network/peers/peerManager.ts @@ -1,4 +1,4 @@ -import {Connection, type IdentifyResult, PeerId, PrivateKey} from "@libp2p/interface"; +import {Connection, PeerId, PrivateKey} from "@libp2p/interface"; import {BitArray} from "@chainsafe/ssz"; import {BeaconConfig} from "@lodestar/config"; import {LoggerNode} from "@lodestar/logger/node"; @@ -162,9 +162,6 @@ export class PeerManager { // A single map of connected peers with all necessary data to handle PINGs, STATUS, and metrics private connectedPeers: Map; - /** Track one in-flight identify call per peer/connection id */ - private readonly identifyInProgress = new Map(); - private opts: PeerManagerOpts; private intervals: NodeJS.Timeout[] = []; @@ -195,7 +192,6 @@ export class PeerManager { this.libp2p.services.components.events.addEventListener(Libp2pEvent.connectionOpen, this.onLibp2pPeerConnect); this.libp2p.services.components.events.addEventListener(Libp2pEvent.connectionClose, this.onLibp2pPeerDisconnect); - this.libp2p.services.components.events.addEventListener(Libp2pEvent.peerIdentify, this.onPeerIdentify); this.networkEventBus.on(NetworkEvent.reqRespRequest, this.onRequest); this.lastStatus = this.statusCache.get(); @@ -239,7 +235,6 @@ export class PeerManager { Libp2pEvent.connectionClose, this.onLibp2pPeerDisconnect ); - this.libp2p.services.components.events.removeEventListener(Libp2pEvent.peerIdentify, this.onPeerIdentify); this.networkEventBus.off(NetworkEvent.reqRespRequest, this.onRequest); for (const interval of this.intervals) clearInterval(interval); } @@ -490,25 +485,7 @@ export class PeerManager { // peers that close identify right after connection open or turn out to be // irrelevant. if (peerData?.agentVersion === null) { - const peerIdStr = peer.toString(); - const connection = getConnection(this.libp2p, peerIdStr); - if (!connection || connection.status !== "open") { - this.logger.debug("Peer has no open connection for identify", {peerId: prettyPrintPeerId(peer)}); - return; - } - - const identifyKey = connection.id; - if (this.identifyInProgress.get(peerIdStr) === identifyKey) { - return; - } - - this.identifyInProgress.set(peerIdStr, identifyKey); - void this.identifyPeer(peerIdStr, prettyPrintPeerId(peer), connection, identifyKey).finally(() => { - // Clear only if this identify attempt is still the active one for this peer - if (this.identifyInProgress.get(peerIdStr) === identifyKey) { - this.identifyInProgress.delete(peerIdStr); - } - }); + void this.identifyPeer(peer.toString(), prettyPrintPeerId(peer), getConnection(this.libp2p, peer.toString())); } } } @@ -867,7 +844,6 @@ export class PeerManager { // remove the ping and status timer for the peer this.connectedPeers.delete(peerIdStr); - this.identifyInProgress.delete(peerIdStr); this.logger.verbose(logMessage, logContext); this.networkEventBus.emit(NetworkEvent.peerDisconnected, {peer: peerIdStr}); @@ -885,47 +861,14 @@ export class PeerManager { } } - /** - * Consume successful identify results from libp2p events. - * This captures agentVersion from identify-push or successful inbound/outbound identify, - * even if our explicit identify request failed earlier. - */ - private onPeerIdentify = (evt: CustomEvent): void => { - const {peerId, agentVersion} = evt.detail; - if (!agentVersion) return; - - const peerIdStr = peerId.toString(); - const peerData = this.connectedPeers.get(peerIdStr); - if (!peerData) return; - - peerData.agentVersion = agentVersion; - peerData.agentClient = getKnownClientFromAgentVersion(agentVersion); - this.identifyInProgress.delete(peerIdStr); - }; - - private async identifyPeer( - peerIdStr: string, - peerIdPretty: string, - connection: Connection, - identifyKey: string - ): Promise { - if (this.identifyInProgress.get(peerIdStr) !== identifyKey) { - return; - } - - if (connection.status !== "open") { + private async identifyPeer(peerIdStr: string, peerIdPretty: string, connection?: Connection): Promise { + if (!connection || connection.status !== "open") { this.logger.debug("Peer has no open connection for identify", {peerId: peerIdPretty}); return; } try { const result = await this.libp2p.services.identify.identify(connection); - - // A newer identify attempt may have superseded this one (e.g. reconnect). - if (this.identifyInProgress.get(peerIdStr) !== identifyKey) { - return; - } - const agentVersion = result.agentVersion; if (agentVersion) { const connectedPeerData = this.connectedPeers.get(peerIdStr); diff --git a/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts b/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts index 2cde954688f6..deecd6e6ee25 100644 --- a/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts +++ b/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts @@ -342,158 +342,4 @@ describe("network / peers / PeerManager", () => { expect(peerData?.agentVersion).toBe("Nimbus/v25.0.0"); expect(peerData?.agentClient).toBe(ClientKind.Nimbus); }); - - it("Should deduplicate in-flight identify requests for the same connection", async () => { - const {libp2p, peerManager, statusCache, networkEventBus} = await mockModules(); - - let resolveIdentify!: (value: {agentVersion: string}) => void; - const identifyPromise = new Promise<{agentVersion: string}>((resolve) => { - resolveIdentify = resolve; - }); - - vi.spyOn(libp2p.services.identify, "identify").mockImplementation( - () => identifyPromise as ReturnType - ); - - const connection = { - id: "connection-1", - direction: "inbound", - status: "open", - remotePeer: peerId1, - close: async () => {}, - abort: () => {}, - } as unknown as Connection; - - getConnectionsMap(libp2p).set(peerId1.toString(), {key: peerId1, value: [connection]}); - await peerManager["onLibp2pPeerConnect"](new CustomEvent("evt", {detail: connection})); - - const remoteStatus = statusCache.get(); - networkEventBus.emit(NetworkEvent.reqRespRequest, { - request: {method: ReqRespMethod.Status, body: remoteStatus}, - peer: peerId1, - peerClient: "Unknown", - }); - networkEventBus.emit(NetworkEvent.reqRespRequest, { - request: {method: ReqRespMethod.Status, body: remoteStatus}, - peer: peerId1, - peerClient: "Unknown", - }); - - await sleep(0); - expect(libp2p.services.identify.identify).toHaveBeenCalledTimes(1); - - resolveIdentify({agentVersion: "Prysm/v6.0.0"}); - await sleep(0); - - const peerData = peerManager["connectedPeers"].get(peerId1.toString()); - expect(peerData?.agentVersion).toBe("Prysm/v6.0.0"); - expect(peerData?.agentClient).toBe(ClientKind.Prysm); - }); - - it("Should allow a new identify attempt after reconnect and ignore stale previous result", async () => { - const {libp2p, peerManager, statusCache, networkEventBus} = await mockModules(); - - let resolveFirstIdentify!: (value: {agentVersion: string}) => void; - const firstIdentifyPromise = new Promise<{agentVersion: string}>((resolve) => { - resolveFirstIdentify = resolve; - }); - - vi.spyOn(libp2p.services.identify, "identify") - .mockImplementationOnce(() => firstIdentifyPromise as ReturnType) - .mockImplementationOnce( - () => Promise.resolve({agentVersion: "Teku/v24.9.0"}) as ReturnType - ); - - const connection1 = { - id: "connection-1", - direction: "inbound", - status: "open", - remotePeer: peerId1, - close: async () => {}, - abort: () => {}, - } as unknown as Connection; - - getConnectionsMap(libp2p).set(peerId1.toString(), {key: peerId1, value: [connection1]}); - await peerManager["onLibp2pPeerConnect"](new CustomEvent("evt", {detail: connection1})); - - const remoteStatus = statusCache.get(); - networkEventBus.emit(NetworkEvent.reqRespRequest, { - request: {method: ReqRespMethod.Status, body: remoteStatus}, - peer: peerId1, - peerClient: "Unknown", - }); - await sleep(0); - - const closedConnection1 = {...connection1, status: "closed"} as Connection; - getConnectionsMap(libp2p).set(peerId1.toString(), {key: peerId1, value: [closedConnection1]}); - await peerManager["onLibp2pPeerDisconnect"](new CustomEvent("evt", {detail: closedConnection1})); - - const connection2 = { - id: "connection-2", - direction: "inbound", - status: "open", - remotePeer: peerId1, - close: async () => {}, - abort: () => {}, - } as unknown as Connection; - getConnectionsMap(libp2p).set(peerId1.toString(), {key: peerId1, value: [connection2]}); - await peerManager["onLibp2pPeerConnect"](new CustomEvent("evt", {detail: connection2})); - - networkEventBus.emit(NetworkEvent.reqRespRequest, { - request: {method: ReqRespMethod.Status, body: remoteStatus}, - peer: peerId1, - peerClient: "Unknown", - }); - await sleep(0); - - expect(libp2p.services.identify.identify).toHaveBeenCalledTimes(2); - - // Resolve old identify last; it must not overwrite new connection's identify result. - resolveFirstIdentify({agentVersion: "Lighthouse/v6.0.1"}); - await sleep(0); - - const peerData = peerManager["connectedPeers"].get(peerId1.toString()); - expect(peerData?.agentVersion).toBe("Teku/v24.9.0"); - expect(peerData?.agentClient).toBe(ClientKind.Teku); - }); - - it("Should update agentVersion via peer:identify event even if explicit identify fails", async () => { - const {libp2p, peerManager, statusCache, networkEventBus} = await mockModules(); - - vi.spyOn(libp2p.services.identify, "identify").mockRejectedValue(new Error("Unexpected EOF")); - - const connection = { - id: "connection-1", - direction: "inbound", - status: "open", - remotePeer: peerId1, - close: async () => {}, - abort: () => {}, - } as unknown as Connection; - - getConnectionsMap(libp2p).set(peerId1.toString(), {key: peerId1, value: [connection]}); - await peerManager["onLibp2pPeerConnect"](new CustomEvent("evt", {detail: connection})); - - const remoteStatus = statusCache.get(); - networkEventBus.emit(NetworkEvent.reqRespRequest, { - request: {method: ReqRespMethod.Status, body: remoteStatus}, - peer: peerId1, - peerClient: "Unknown", - }); - await sleep(0); - - libp2p.services.components.events.dispatchEvent( - new CustomEvent("peer:identify", { - detail: { - peerId: peerId1, - agentVersion: "Lighthouse/v6.0.1", - }, - }) - ); - await sleep(0); - - const peerData = peerManager["connectedPeers"].get(peerId1.toString()); - expect(peerData?.agentVersion).toBe("Lighthouse/v6.0.1"); - expect(peerData?.agentClient).toBe(ClientKind.Lighthouse); - }); }); From 31b0dd08730f253be08289e18ba6ee5ef696827d Mon Sep 17 00:00:00 2001 From: Lodekeeper <258435968+lodekeeper@users.noreply.github.com> Date: Wed, 25 Feb 2026 21:13:12 +0000 Subject: [PATCH 11/63] chore: add .claude project configuration with skills and permissions (#8961) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Sets up shared [Claude Code](https://code.claude.com/) configuration for the Lodestar project. This lets any team member using Claude Code get a curated set of permissions and Lodestar-specific skills out of the box. ## Changes ### `.claude/settings.json` — Permission Allowlist Pre-approves common dev commands so Claude Code doesn't prompt for confirmation on every: - Build/lint/test: `pnpm build`, `pnpm lint`, `pnpm check-types`, `pnpm test:unit`, `pnpm vitest` - Git operations: `git diff`, `git log`, `git branch`, `git fetch`, `git checkout`, `git merge`, `git worktree` - GitHub CLI: `gh pr`, `gh issue`, `gh api`, `gh run`, `gh release` - Read-only: `grep`, `find`, `cat`, `head`, `tail`, `ls`, `curl -s http://localhost:` Denies: `rm -rf /`, `sudo` ### `.claude/skills/` — Agent Skills Three skills that Claude Code auto-discovers based on task context: | Skill | What it does | |-------|-------------| | **kurtosis-devnet** | Run multi-client devnets — setup, config reference, debugging, fork transitions, assertoor testing, example topologies | | **release-notes** | Draft release notes — style guide, changelog analysis framework, operator-facing prioritization, Discord formatting | | **local-mainnet-debug** | Debug with real mainnet peers — checkpoint sync + engineMock, metrics scraping, A/B testing, libp2p instrumentation | ### `.gitignore` Update Changed from blanket `.claude/` ignore to selective: - ✅ Tracked: `settings.json`, `skills/`, `README.md` - 🚫 Ignored: `settings.local.json`, `plans/`, `CLAUDE.local.md` This lets personal/local config stay gitignored while sharing project-level config with the team. ### Shared Plugins (External) Cross-project plugins (ethereum-rnd, consensus-clients) live in [ChainSafe/lodestar-claude-plugins](https://github.com/ChainSafe/lodestar-claude-plugins) marketplace. Instructions in `.claude/README.md`. --- > This PR was authored by Lodekeeper (AI assistant). Skills adapted from battle-tested workflows used during EPBS devnet validation, v1.36-v1.40 releases, and libp2p debugging. --------- Co-authored-by: lodekeeper Co-authored-by: Nico Flaig Co-authored-by: Cayman --- .claude/README.md | 44 ++++ .claude/settings.json | 67 +++++ .claude/skills/kurtosis-devnet/SKILL.md | 245 ++++++++++++++++++ .../references/config-reference.md | 198 ++++++++++++++ .claude/skills/local-mainnet-debug/SKILL.md | 220 ++++++++++++++++ .claude/skills/release-notes/SKILL.md | 74 ++++++ .../references/changelog-analysis.md | 79 ++++++ .../release-notes/references/style-guide.md | 82 ++++++ .gitignore | 5 +- CLAUDE.md | 41 +++ 10 files changed, 1054 insertions(+), 1 deletion(-) create mode 100644 .claude/README.md create mode 100644 .claude/settings.json create mode 100644 .claude/skills/kurtosis-devnet/SKILL.md create mode 100644 .claude/skills/kurtosis-devnet/references/config-reference.md create mode 100644 .claude/skills/local-mainnet-debug/SKILL.md create mode 100644 .claude/skills/release-notes/SKILL.md create mode 100644 .claude/skills/release-notes/references/changelog-analysis.md create mode 100644 .claude/skills/release-notes/references/style-guide.md create mode 100644 CLAUDE.md diff --git a/.claude/README.md b/.claude/README.md new file mode 100644 index 000000000000..f9627a68316b --- /dev/null +++ b/.claude/README.md @@ -0,0 +1,44 @@ +# Claude Code Configuration + +This directory contains shared [Claude Code](https://code.claude.com/) configuration for the Lodestar project. See also [`AGENTS.md`](../AGENTS.md) at the repo root for project conventions and architecture guidance. + +## Structure + +``` +.claude/ +├── settings.json # Shared project settings (permissions allowlist) +├── settings.local.json # Personal settings (gitignored) +├── skills/ # Agent skills — Claude auto-discovers these +│ ├── kurtosis-devnet/ # Multi-client devnet testing +│ ├── release-notes/ # Release note drafting +│ └── local-mainnet-debug/ # Debug with real mainnet peers +└── README.md # This file +``` + +## Skills + +Skills are automatically loaded by Claude Code based on task context. You can also invoke them manually: + +| Skill | When to use | +| ----------------------- | -------------------------------------------------------------------------- | +| **kurtosis-devnet** | Spinning up local testnets, cross-client interop testing, fork transitions | +| **release-notes** | Drafting release notes for GitHub and Discord | +| **local-mainnet-debug** | Debugging networking/peer issues against real mainnet peers | + +## Shared Plugins + +For cross-project Ethereum development resources (consensus specs, client cross-reference), install from the shared marketplace: + +```bash +claude plugin marketplace add ChainSafe/lodestar-claude-plugins +claude plugin install ethereum-rnd +claude plugin install consensus-clients +``` + +## Personal Configuration + +Add personal settings to `.claude/settings.local.json` (gitignored). This is useful for: + +- Personal API keys or tool preferences +- Local MCP server configurations +- Additional permission rules for your workflow diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 000000000000..684ea9bb07df --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,67 @@ +{ + "$schema": "https://json.schemastore.org/claude-code-settings.json", + "permissions": { + "allow": [ + "Bash(pnpm build)", + "Bash(pnpm --filter * build)", + "Bash(pnpm lint)", + "Bash(pnpm lint:fix)", + "Bash(pnpm check-types)", + "Bash(pnpm --filter * check-types)", + "Bash(pnpm test:unit)", + "Bash(pnpm vitest *)", + "Bash(pnpm download-spec-tests)", + "Bash(pnpm test:spec)", + "Bash(pnpm docs:lint)", + "Bash(pnpm docs:lint:fix)", + "Bash(git status *)", + "Bash(git diff *)", + "Bash(git log *)", + "Bash(git branch *)", + "Bash(git show *)", + "Bash(git fetch *)", + "Bash(gh pr view *)", + "Bash(gh pr list *)", + "Bash(gh pr diff *)", + "Bash(gh pr checks *)", + "Bash(gh issue view *)", + "Bash(gh issue list *)", + "Bash(gh run view *)", + "Bash(gh run list *)", + "Bash(gh search *)", + "Bash(grep *)", + "Bash(cat *)", + "Bash(head *)", + "Bash(tail *)", + "Bash(wc *)", + "Bash(ls *)", + "Bash(curl -s http://localhost:*)" + ], + "deny": [ + "Bash(rm -rf / *)", + "Bash(sudo *)" + ] + }, + "enabledPlugins": { + "ethereum-rnd@lodestar-claude-plugins": true, + "consensus-clients@lodestar-claude-plugins": true, + "eth-rnd-archive@lodestar-claude-plugins": true, + "claude-md-management@claude-plugins-official": true, + "code-review@claude-plugins-official": true, + "code-simplifier@claude-plugins-official": true, + "commit-commands@claude-plugins-official": true, + "context7@claude-plugins-official": true, + "feature-dev@claude-plugins-official": true, + "pr-review-toolkit@claude-plugins-official": true, + "superpowers@claude-plugins-official": true, + "typescript-lsp@claude-plugins-official": true + }, + "extraKnownMarketplaces": { + "lodestar-claude-plugins": { + "source": { + "source": "github", + "repo": "ChainSafe/lodestar-claude-plugins" + } + } + } +} diff --git a/.claude/skills/kurtosis-devnet/SKILL.md b/.claude/skills/kurtosis-devnet/SKILL.md new file mode 100644 index 000000000000..b19c6ee82a46 --- /dev/null +++ b/.claude/skills/kurtosis-devnet/SKILL.md @@ -0,0 +1,245 @@ +--- +name: kurtosis-devnet +description: Run Ethereum multi-client devnets using Kurtosis and the ethpandaops/ethereum-package. Use for spinning up local testnets, validating cross-client interop, testing fork transitions, running assertoor checks, debugging CL/EL client interactions, or verifying new feature implementations across multiple consensus and execution clients. +--- + +# Kurtosis Devnet + +Run Ethereum consensus/execution client devnets via [Kurtosis](https://github.com/kurtosis-tech/kurtosis) + [ethereum-package](https://github.com/ethpandaops/ethereum-package). + +## Prerequisites + +- [Kurtosis CLI](https://docs.kurtosis.com/install/) installed +- Docker running with sufficient resources (8GB+ RAM recommended for multi-client devnets) + +## Quick Start + +```bash +# Start devnet from config +# Always use --image-download always to ensure external images are up to date +kurtosis run github.com/ethpandaops/ethereum-package \ + --enclave \ + --args-file network_params.yaml \ + --image-download always + +# List enclaves +kurtosis enclave ls + +# Inspect services +kurtosis enclave inspect + +# View logs +kurtosis service logs +kurtosis service logs --follow + +# Cleanup +kurtosis enclave rm -f +# Or clean all enclaves +kurtosis clean -a +``` + +## Config File (`network_params.yaml`) + +See `references/config-reference.md` for the full config structure. + +Key sections: + +- `participants`: list of CL+EL client pairs with images, flags, validator counts +- `network_params`: fork epochs, slot time, network-level settings +- `additional_services`: dora (explorer), assertoor (testing), prometheus, grafana +- `assertoor_params`: automated chain health checks +- `port_publisher`: expose CL/EL ports to host + +## Building Custom Client Images + +When testing local Lodestar branches, build a Docker image first: + +```bash +# Fast build (recommended for iteration) +cd ~/lodestar && docker build -t lodestar:custom -f Dockerfile.dev . + +# Production build (slower, for final validation only) +cd ~/lodestar && docker build -t lodestar:custom . + +# Then reference in config: +# cl_image: lodestar:custom +``` + +**Always use `Dockerfile.dev` for iterative development.** It caches dependency layers and rebuilds in seconds vs minutes for the production Dockerfile. Only use the production `Dockerfile` for final validation or debugging build issues. + +## Service Naming Convention + +Kurtosis names services as: `{role}-{index}-{cl_type}-{el_type}` + +Examples: + +- `cl-1-lodestar-reth` — first CL node (Lodestar with Reth EL) +- `el-1-reth-lodestar` — corresponding EL node +- `vc-1-lodestar-reth` — validator client + +## Accessing Services + +After `kurtosis enclave inspect `, find mapped ports: + +```bash +# CL beacon API (find actual port from inspect output) +curl http://127.0.0.1:/eth/v1/node/syncing + +# Or use port_publisher for predictable ports: +# port_publisher: +# cl: +# enabled: true +# public_port_start: 33000 # cl-1=33000, cl-2=33005, etc. +# el: +# enabled: true +# public_port_start: 32000 +``` + +Port publisher assigns sequential ports (step of 5 per service). + +## Assertoor (Automated Testing) + +Add to config: + +```yaml +additional_services: + - assertoor + +assertoor_params: + run_stability_check: true # chain stability, finality, no reorgs + run_block_proposal_check: true # every client pair proposes a block +``` + +Check results via the assertoor web UI (port shown in `kurtosis enclave inspect`). + +## Common Devnet Patterns + +### Fork Transition Testing + +```yaml +network_params: + electra_fork_epoch: 0 + fulu_fork_epoch: 1 # fork at epoch 1 (slot 32) + seconds_per_slot: 6 # faster for testing +``` + +### Mixed-Client Topology (Cross-Client Interop) + +```yaml +participants: + - cl_type: lodestar + el_type: reth + count: 2 + validator_count: 128 + - cl_type: lighthouse + el_type: geth + count: 2 + validator_count: 128 +``` + +### Observer Nodes (No Validators) + +```yaml +- cl_type: lodestar + cl_image: lodestar:custom + el_type: reth + count: 1 + validator_count: 0 # observer-only +``` + +### Supernode Mode + +Set `supernode: true` to run beacon+validator in a single process (faster startup, simpler topology): + +```yaml +- cl_type: lodestar + el_type: reth + supernode: true + validator_count: 128 +``` + +### Extra CL/VC Params + +```yaml +cl_extra_params: + - --targetPeers=8 + - --logLevel=debug +vc_extra_params: + - --suggestedFeeRecipient=0x... +``` + +## Monitoring & Debugging + +```bash +# Stream logs from a specific service +kurtosis service logs cl-1-lodestar-reth --follow + +# Save all CL logs for analysis +for svc in $(kurtosis enclave inspect 2>/dev/null | grep -oE 'cl-[0-9]+-[^[:space:]]+'); do + kurtosis service logs $svc > "/tmp/${svc}.log" 2>&1 +done + +# Dora explorer (if enabled) — find port via inspect output + +# Check chain finality +curl -s http://127.0.0.1:/eth/v1/beacon/states/head/finality_checkpoints | jq + +# Check peer count +curl -s http://127.0.0.1:/eth/v1/node/peers | jq '.data | length' + +# Check sync status +curl -s http://127.0.0.1:/eth/v1/node/syncing | jq +``` + +### Wait for Finality + +Finality typically takes 2-3 epochs after genesis. With `seconds_per_slot: 6` and 32 slots/epoch: + +- 1 epoch ≈ 192s (3.2 min) +- First finalization ≈ epoch 3-4 boundary (≈10-13 min) + +Monitor: + +```bash +curl -s http:///eth/v1/beacon/states/head/finality_checkpoints | jq '.data.finalized.epoch' +``` + +### Acceptance Criteria Pattern + +For interop validation, define acceptance criteria **before** running the soak test: + +```bash +# Example: check for zero errors across all CL nodes +for svc in cl-1-lodestar-reth cl-2-lodestar-reth; do + echo "=== $svc ===" + kurtosis service logs $svc 2>&1 | grep -c "ERROR" || echo "0 errors" +done +``` + +## Troubleshooting + +| Issue | Fix | +| ------------------------ | ------------------------------------------------------------------------- | +| Peers not connecting | Increase `--targetPeers`, check `directPeers` config | +| No finality | Need ≥2/3 validators attesting; check VC logs | +| "discv5 has no boot enr" | Harmless startup warning, ignore | +| Port conflicts | Change `public_port_start` or stop conflicting enclaves | +| Image not found | Ensure Docker image is built locally or available in registry | +| Slow startup | Use `Dockerfile.dev` for local builds; reduce validator count | +| Docker bind error | Use non-default port ranges; `kurtosis clean -a` between runs | +| Stale enclave state | Always `kurtosis clean -a` before relaunching (not `docker system prune`) | + +## Tips + +- **Start small:** 2+2 nodes for iteration, scale up for final validation +- **Test both roles:** Validator nodes show different bugs than observers — always check both +- **Multiple soak passes:** First clean run may miss edge cases. Run extended soaks (hours). +- **`kurtosis clean -a`** between runs — never use `docker system prune` (it can remove Kurtosis engine images) +- **Debug logs:** Set `global_log_level: "debug"` for verbose output from all nodes +- **Per-participant log levels:** Use `cl_log_level` / `el_log_level` on individual entries for targeted debugging + +--- + +## Self-Maintenance + +If any commands, file paths, URLs, or configurations in this skill are outdated or no longer work, update this SKILL.md with the correct information after completing your current task. Skills should stay accurate and self-healing — fix what you find broken. diff --git a/.claude/skills/kurtosis-devnet/references/config-reference.md b/.claude/skills/kurtosis-devnet/references/config-reference.md new file mode 100644 index 000000000000..ea85f8e6936a --- /dev/null +++ b/.claude/skills/kurtosis-devnet/references/config-reference.md @@ -0,0 +1,198 @@ +# Kurtosis ethereum-package Config Reference + +Full documentation: https://github.com/ethpandaops/ethereum-package + +## Participant Fields + +```yaml +participants: + - el_type: reth|geth|nethermind|besu|erigon + el_image: # e.g., ghcr.io/paradigmxyz/reth:latest + el_extra_params: [] # extra CLI flags for EL + el_log_level: "" # per-participant EL log level + cl_type: lodestar|lighthouse|prysm|teku|nimbus|grandine + cl_image: # e.g., lodestar:custom or chainsafe/lodestar:latest + cl_extra_params: [] # extra CLI flags for CL beacon + cl_log_level: "" # per-participant CL log level + vc_type: # defaults to cl_type + vc_image: # custom VC image (if different from CL) + vc_extra_params: [] # extra CLI flags for VC + supernode: false # combine beacon+vc in single process + count: 1 # number of instances with this config + validator_count: 64 # validators assigned to this participant +``` + +## Network Parameters + +```yaml +network_params: + # Fork epochs (0 = from genesis, null = disabled) + electra_fork_epoch: 0 + fulu_fork_epoch: 1 + + # Timing + seconds_per_slot: 12 # default 12, use 6 for faster devnets + slots_per_epoch: 32 # default 32 + + # Network + network_id: "3151908" + deposit_contract_address: "0x..." + + # Validator + num_validator_keys_per_node: 64 + preregistered_validator_keys_mnemonic: "..." +``` + +## Additional Services + +```yaml +additional_services: + - dora # chain explorer UI + - assertoor # automated testing + - prometheus # metrics collection + - grafana # metrics dashboards + - blob_spammer # blob transaction generator + - el_forkmon # fork monitor + - beacon_metrics_gazer + - blockscout # block explorer with contract verification +``` + +## Assertoor Parameters + +```yaml +assertoor_params: + run_stability_check: true # chain health, finality, reorgs + run_block_proposal_check: true # every client pair proposes a block + run_transaction_test: false # transaction lifecycle + run_blob_transaction_test: false # blob tx test + run_opcodes_transaction_test: false + tests: [] # custom test configs +``` + +## Port Publisher + +```yaml +port_publisher: + el: + enabled: true + public_port_start: 32000 # EL HTTP/WS + cl: + enabled: true + public_port_start: 33000 # CL beacon API + vc: + enabled: false + public_port_start: 34000 +``` + +Ports increment by 5 per service instance: + +- cl-1: 33000, cl-2: 33005, cl-3: 33010, etc. +- el-1: 32000, el-2: 32005, el-3: 32010, etc. + +## Global Settings + +```yaml +global_log_level: info|debug|warn|error # default: info +``` + +## Example Configs + +### Minimal 2-Node Devnet + +```yaml +participants: + - el_type: geth + cl_type: lodestar + count: 1 + validator_count: 128 + - el_type: reth + cl_type: lighthouse + count: 1 + validator_count: 128 + +network_params: + electra_fork_epoch: 0 + fulu_fork_epoch: 1 + seconds_per_slot: 6 + +additional_services: + - dora + - assertoor + +assertoor_params: + run_stability_check: true + run_block_proposal_check: true +``` + +### Lodestar-Only with Custom Image + +```yaml +participants: + - el_type: reth + cl_type: lodestar + cl_image: lodestar:custom + supernode: true + count: 4 + validator_count: 64 + +network_params: + electra_fork_epoch: 0 + fulu_fork_epoch: 0 + seconds_per_slot: 6 + +additional_services: + - dora + - prometheus + - grafana + +port_publisher: + cl: + enabled: true + public_port_start: 33000 +``` + +### Full Multi-Client Interop (6 clients) + +```yaml +participants: + - cl_type: lodestar + el_type: reth + count: 1 + validator_count: 64 + - cl_type: lighthouse + el_type: geth + count: 1 + validator_count: 64 + - cl_type: prysm + el_type: geth + count: 1 + validator_count: 64 + - cl_type: teku + el_type: besu + count: 1 + validator_count: 64 + - cl_type: nimbus + el_type: nethermind + count: 1 + validator_count: 64 + - cl_type: grandine + el_type: reth + count: 1 + validator_count: 64 + +network_params: + electra_fork_epoch: 0 + fulu_fork_epoch: 1 + seconds_per_slot: 6 + +additional_services: + - dora + - assertoor + - prometheus + - grafana + - el_forkmon + +assertoor_params: + run_stability_check: true + run_block_proposal_check: true +``` diff --git a/.claude/skills/local-mainnet-debug/SKILL.md b/.claude/skills/local-mainnet-debug/SKILL.md new file mode 100644 index 000000000000..9b9202923f9a --- /dev/null +++ b/.claude/skills/local-mainnet-debug/SKILL.md @@ -0,0 +1,220 @@ +--- +name: local-mainnet-debug +description: Debug Lodestar beacon node issues by running a local mainnet node with checkpoint sync and engineMock. Use for investigating networking bugs, peer discovery issues, identify failures, metrics anomalies, or any behavior that needs real-world peer interactions without a full execution client. +--- + +# Local Mainnet Debugging + +Run a local Lodestar beacon node against mainnet peers using checkpoint sync and engineMock to reproduce and debug networking, peer discovery, and protocol-level issues in a real-world environment. + +## Quick Start + +```bash +cd ~/lodestar # or worktree directory + +# Basic run — connects to mainnet peers, no EL needed +./lodestar beacon \ + --network mainnet \ + --dataDir /tmp/lodestar-debug \ + --rest false \ + --metrics \ + --execution.engineMock \ + --port 19771 \ + --logLevel debug \ + --checkpointSyncUrl https://beaconstate-mainnet.chainsafe.io \ + --forceCheckpointSync + +# Time-boxed run (e.g., 2 minutes) +timeout 120 ./lodestar beacon \ + --network mainnet \ + --dataDir /tmp/lodestar-debug \ + --rest false \ + --metrics \ + --execution.engineMock \ + --port 19771 \ + --logLevel debug \ + --checkpointSyncUrl https://beaconstate-mainnet.chainsafe.io \ + --forceCheckpointSync +``` + +## Key Parameters + +| Parameter | Purpose | Notes | +| ------------------------ | --------------------------------------- | ------------------------------------------------------ | +| `--network mainnet` | Connect to real mainnet peers | Use `holesky` for testnet | +| `--execution.engineMock` | Skip EL requirement | Node won't validate execution payloads | +| `--rest false` | Disable REST API | Reduces noise, avoids port conflicts | +| `--metrics` | Enable Prometheus metrics | Scrape at `http://localhost:8008/metrics` | +| `--port 19771` | Custom P2P port | Avoid conflicts with other instances | +| `--logLevel debug` | Verbose logging | Use `trace` for maximum detail | +| `--checkpointSyncUrl` | Checkpoint sync endpoint | `https://beaconstate-mainnet.chainsafe.io` for mainnet | +| `--forceCheckpointSync` | Force checkpoint sync even if DB exists | Clean start each run | + +## Checkpoint Sync Endpoints + +```bash +# Mainnet +--checkpointSyncUrl https://beaconstate-mainnet.chainsafe.io +--checkpointSyncUrl https://mainnet-checkpoint-sync.stakely.io +--checkpointSyncUrl https://sync-mainnet.beaconcha.in + +# Holesky +--checkpointSyncUrl https://beaconstate-holesky.chainsafe.io +--checkpointSyncUrl https://holesky.beaconstate.ethstaker.cc +``` + +## Metrics Scraping + +```bash +# One-shot metric grab +curl -s http://localhost:8008/metrics | grep + +# Periodic sampling (every 30s) +while true; do + echo "=== $(date -u +%H:%M:%S) ===" + curl -s http://localhost:8008/metrics | grep -E 'lodestar_peers_by_client|peer_count' + sleep 30 +done + +# Key metrics for peer debugging +curl -s http://localhost:8008/metrics | grep -E \ + 'lodestar_peers_by_client|libp2p_identify|peer_count|connected_peers' +``` + +## Debugging Techniques + +### 1. A/B Testing with Code Changes + +The most powerful technique — test a hypothesis by comparing control vs modified runs. + +```bash +# Control: capture baseline (2 min) +timeout 120 ./lodestar beacon [flags] 2>&1 | tee /tmp/control.log & +# Sample metrics during run +for i in 1 2 3 4; do sleep 30; curl -s http://localhost:8008/metrics > /tmp/control-$i.metrics; done + +# Test: apply change, repeat +timeout 120 ./lodestar beacon [flags] 2>&1 | tee /tmp/test.log & +for i in 1 2 3 4; do sleep 30; curl -s http://localhost:8008/metrics > /tmp/test-$i.metrics; done + +# Compare +diff <(grep pattern /tmp/control-4.metrics) <(grep pattern /tmp/test-4.metrics) +``` + +### 2. Instrument libp2p Internals (Monkeypatching) + +For deep protocol debugging, add temporary instrumentation to `node_modules`: + +```bash +# Find the file to patch +find node_modules -path '*libp2p*identify*' -name '*.js' | head -20 + +# Key files: +# - node_modules/@libp2p/identify/dist/src/identify.js +# - node_modules/@chainsafe/libp2p-yamux/dist/src/stream.js (yamux streams) +# - node_modules/@libp2p/mplex/dist/src/mplex.js (mplex streams) +``` + +**Important:** Always remove monkeypatches before committing. Use `pnpm install` to restore. + +### 3. Log Analysis + +```bash +# Count specific errors +grep -c "Error setting agentVersion" /tmp/run.log + +# Track identify success/failure over time +grep -E "identify (success|error|timeout)" /tmp/run.log | head -50 + +# Extract peer connection events +grep -E "peer:(connect|disconnect|identify)" /tmp/run.log +``` + +### 4. Stream-Level Debugging + +For protocol stream issues (identify, ping, metadata): + +``` +Trace stream lifecycle: +1. Stream opened (protocol, direction, connection ID) +2. MSS negotiation (success/failure) +3. Data read/write (first frame timing) +4. Stream close (who closed, when) +``` + +Key locations for instrumentation: + +- `@libp2p/identify`: identify.js → `_identify()` method +- Stream open/close: yamux or mplex stream.js +- Protocol negotiation: `@libp2p/multistream-select` + +## Common Issues & Diagnostic Approaches + +### Unknown Peers (identify failures) + +**Symptoms:** High ratio of "Unknown" in `lodestar_peers_by_client` metric. + +**Diagnostic approach:** + +1. Check `lodestar_peers_by_client` for Unknown ratio +2. Enable debug logs, grep for "Error setting agentVersion" +3. Instrument identify stream to check stream state before `pb.read()` +4. A/B test: disable suspect components → if Unknown drops to 0, found the cause + +### Slow Peer Discovery + +**Symptoms:** Low peer count after startup. + +**Diagnostic approach:** + +1. Check `libp2p_dialer_pending_dial_count` and `libp2p_connections` +2. Verify port is accessible: `lsof -i :` +3. Check discv5 logs for bootstrap ENR issues +4. Monitor `lodestar_discovery_find_node_query_*` metrics + +### High Disconnect Rate + +**Symptoms:** Peers connecting and quickly disconnecting. + +**Diagnostic approach:** + +1. Check `libp2p_connections` vs `libp2p_protocol_streams` +2. Grep logs for disconnect reasons +3. Check if peers are being pruned by score: `lodestar_gossip_peer_score_*` +4. Monitor `lodestar_peers_by_direction` for inbound/outbound balance + +## Port Conflicts + +```bash +# Check if port is in use +lsof -i :19771 +lsof -i :8008 + +# Use different ports +--port 19772 --metrics.port 8009 +``` + +## Cleanup + +Always use `--dataDir /tmp/lodestar-debug` when running debug sessions to avoid touching the default mainnet database. This isolates debug data from any production node on the same machine. + +```bash +# Remove isolated debug data directory after testing +rm -r /tmp/lodestar-debug +``` + +## Tips + +- **Short runs are fine.** 60-120 seconds is enough to connect to 15-30 peers and observe identify behavior. +- **engineMock means no execution validation.** The node will sync headers but won't validate blocks. Fine for networking/peer debugging. +- **Custom port avoids conflicts** with any production nodes on the same machine. +- **Always use `--forceCheckpointSync`** to ensure a clean start. Stale DB state can mask issues. +- **Metrics lag behind logs.** After stopping the node, the last metrics scrape reflects final state. Periodic sampling during the run gives time-series data. +- **Debug builds not needed.** Standard `pnpm build` output is sufficient; libp2p debug info comes from log level and instrumentation. +- **Use `--network holesky`** when you need a smaller, faster-syncing network for testing. + +--- + +## Self-Maintenance + +If any commands, file paths, URLs, or configurations in this skill are outdated or no longer work, update this SKILL.md with the correct information after completing your current task. Skills should stay accurate and self-healing — fix what you find broken. diff --git a/.claude/skills/release-notes/SKILL.md b/.claude/skills/release-notes/SKILL.md new file mode 100644 index 000000000000..52b4c6adf524 --- /dev/null +++ b/.claude/skills/release-notes/SKILL.md @@ -0,0 +1,74 @@ +--- +name: release-notes +description: > + Write Lodestar release notes and Discord announcements for new versions. + Use when cutting a new release, drafting announcement text, or updating + the GitHub release body. Covers header text, Discord announcement, + and changelog formatting. Requires access to the Lodestar repo and + GitHub CLI for changelog generation. +disable-model-invocation: true +--- + +# Release Notes + +Write release notes for Lodestar versions by analyzing the changelog, identifying +operator-facing changes, and producing both GitHub release header text and a Discord +announcement. + +## Quick Start + +1. Read `references/style-guide.md` for tone, structure, and past examples +2. Read `references/changelog-analysis.md` for how to categorize and prioritize changes +3. Generate the changelog diff between the previous stable and new version +4. Draft the GitHub release header and Discord announcement +5. Output both as a single markdown file for review + +## Workflow + +### 1. Generate Changelog + +```bash +# Get commits between last stable and new version +git log .. --oneline --no-merges + +# Or use the auto-generated changelog from the RC +gh release view --json body -q '.body' +``` + +### 2. Categorize Changes + +Read `references/changelog-analysis.md` for the prioritization framework. +Focus on what matters to **node operators**, not internal refactors. + +### 3. Write Header Text + +The header text goes at the top of the GitHub release body, before the `# Changelog` section. +Follow the structure in `references/style-guide.md`. + +### 4. Write Discord Announcement + +Shorter version of the header text for the `#lodestar-announcements` channel. +See `references/style-guide.md` for Discord-specific formatting rules. + +### 5. Output + +Produce a single markdown file with two sections: + +- `## GitHub Release Header` — full header text +- `## Discord Announcement` — shorter announcement + +## Key Rules + +- **Operator-first**: Every highlight should answer "why should I upgrade?" +- **Breaking changes up front**: Node.js version drops, flag changes, migration steps +- **PR numbers**: Always link PRs as `(#XXXX)` +- **Recommendation level**: Always state mandatory / recommended / optional +- **Don't over-detail**: The full changelog is linked below the header +- **3-6 highlights**: Enough to convey the release but not overwhelming +- **Group related PRs**: Combine related changes into single highlights + +--- + +## Self-Maintenance + +If any commands, file paths, URLs, or configurations in this skill are outdated or no longer work, update this SKILL.md with the correct information after completing your current task. Skills should stay accurate and self-healing — fix what you find broken. diff --git a/.claude/skills/release-notes/references/changelog-analysis.md b/.claude/skills/release-notes/references/changelog-analysis.md new file mode 100644 index 000000000000..46e4d9137377 --- /dev/null +++ b/.claude/skills/release-notes/references/changelog-analysis.md @@ -0,0 +1,79 @@ +# Changelog Analysis Framework + +## How to Prioritize Changes + +When analyzing a changelog for the release header, categorize each commit: + +### Tier 1: Must Mention (Operator-Facing Impact) + +- **Breaking changes**: Dropped runtime versions, removed flags, changed defaults +- **Security fixes**: Vulnerability patches, dependency CVEs +- **Critical bug fixes**: Crash fixes, data corruption, consensus failures +- **New required flags**: Config changes operators need to make +- **Migration steps**: Things operators must do when upgrading + +### Tier 2: Should Mention (Significant Improvements) + +- **Performance improvements**: Memory reduction, faster processing, lower CPU +- **New features operators care about**: APIs, CLI flags, monitoring endpoints +- **Stability fixes**: OOM prevention, crash-loop fixes, sync improvements +- **PeerDAS/networking changes**: Custody, peering, gossip improvements + +### Tier 3: Brief Mention or Skip (Internal) + +- **Refactors**: Code cleanup, type changes (skip unless perf impact) +- **Test improvements**: CI, E2E, flaky test fixes (skip) +- **Documentation**: Only mention if user-facing docs changed +- **Dependency bumps**: Only mention if security-related +- **Maintenance**: Build system, workflows (skip) + +## Grouping Related Changes + +Combine related PRs into single highlights: + +### Example: Memory improvements + +Instead of listing 3 separate PRs: + +- Remove in-memory state caches (#8813) +- Replace bigint-buffer (#8789) +- Signature sets use indices (#8803) + +Write one highlight: + +> **Reduced memory usage** — In-memory state caches removed, bigint-buffer replaced with a +> lighter alternative, and signature verification now uses validator indices. (#8813, #8789, #8803) + +### Example: PeerDAS improvements + +Instead of listing separately: + +- Block import after half columns (#8818) +- Backpressure fix (#8885) + +Write: + +> **PeerDAS supernode stability** — Blocks import after receiving half the columns, and a new +> write queue backpressure mechanism prevents OOM during finalized sync. (#8818, #8885) + +## Generating the Changelog Section + +The full changelog below the header is auto-generated. Use the RC release body as a starting +point — it's generated by the release workflow from conventional commits. + +Ensure the `Full Changelog` comparison link is correct: + +``` +[Full Changelog](https://github.com/ChainSafe/lodestar/compare/...) +``` + +## Checklist Before Finalizing + +- [ ] Recommendation level stated (mandatory/recommended/optional) +- [ ] Breaking changes called out prominently +- [ ] 3-6 highlights covering the most impactful changes +- [ ] PR numbers referenced for each highlight +- [ ] Migration steps included if any (with code blocks) +- [ ] Link to full changelog at the bottom +- [ ] Discord version fits under 2000 chars +- [ ] No internal/CI/test changes in highlights diff --git a/.claude/skills/release-notes/references/style-guide.md b/.claude/skills/release-notes/references/style-guide.md new file mode 100644 index 000000000000..ee9d1ff1819f --- /dev/null +++ b/.claude/skills/release-notes/references/style-guide.md @@ -0,0 +1,82 @@ +# Release Notes Style Guide + +## Tone & Voice + +- Friendly, professional, operator-focused +- Open with a casual greeting: "Good day Lodestar operators!", "Hey Lodestar users!", "Happy [month]!" +- State version and recommendation level immediately: "We've just released vX.Y.Z and **recommend** users update" +- Recommendation levels: **mandatory** (breaking/security), **recommended** (features/perf), maintenance (minor) +- Close with a link to full changelog and optionally a seasonal/fun sign-off + +## GitHub Release Header Structure + +```markdown +[Greeting]. We've just released vX.Y.Z, a **[level]** upgrade for all [mainnet and testnet] users. +[1-2 sentence summary of the theme of this release.] + +[Optional: Breaking change / migration callout in bold] + +### Highlights + +- **[Feature/Fix name]** — 1-2 sentence description of what changed and why operators care. ([#XXXX]) +- **[Feature/Fix name]** — ... ([#XXXX]) +- [3-6 highlights total, ordered by operator impact] + +[Optional: Additional context paragraphs for complex changes like migrations] + +For the full changelog, please see: https://github.com/ChainSafe/lodestar/releases/vX.Y.Z +``` + +### Formatting Rules + +- Use markdown headers, bold, bullet lists +- PR references: `(#8813)` or `(#8813, #8789)` for related PRs +- Keep highlights to 3-6 items — enough to convey the release but not overwhelming +- Each highlight should be one concept (combine related PRs into one bullet) +- Don't list every PR — that's what the changelog is for + +## Discord Announcement Structure + +``` +[Greeting]. We've just released vX.Y.Z and **[level]** users update [reason]. + +[Breaking change if any] + +Key highlights: +[emoji] [highlight 1] +[emoji] [highlight 2] +[emoji] [highlight 3] + +For the full changelog: +``` + +### Discord-Specific Rules + +- No markdown tables — use bullet lists +- Wrap links in `<>` to suppress embeds (except the main release link) +- Use emoji bullets for visual scanning: 🧠 ⚡ 🔧 🌐 📡 🛡️ 🎯 +- Keep under 2000 chars (Discord message limit) +- No headers — use **bold** or emoji for emphasis + +## Past Examples (Distilled Patterns) + +### Major Feature Release (e.g., mainnet fork activation) + +- "**Mandatory** for all node operators" +- Detailed config changes (new flags, default values) +- Compatibility notes for validators +- Signed off with enthusiasm + emoji + +### Performance Release + +- "**Highly recommended** if you've seen performance declines" +- Runtime changes highlighted (e.g., Node.js version bump) +- Build-from-source migration note +- Holiday sign-off + +### Maintenance Release + +- "**Recommend** users update for latest features and best performance" +- Package manager migration with commands +- Breaking: deprecated feature removal +- Affected-user-specific note (e.g., DVT operators) diff --git a/.gitignore b/.gitignore index 4edee34dcdb4..6abb0efe9c48 100644 --- a/.gitignore +++ b/.gitignore @@ -29,7 +29,10 @@ validators .vscode/launch.json !.vscode/settings.json .vscode/tasks.json -.claude/ +# Claude Code — ignore local/personal config, keep shared project config +.claude/settings.local.json +.claude/plans/ +.claude/CLAUDE.local.md # Tests artifacts packages/*/spec-tests* diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000000..535f1fb86767 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,41 @@ +# CLAUDE.md + +See [AGENTS.md](./AGENTS.md) for the full project guide — build commands, code style, architecture patterns, testing, and PR guidelines. + +## Quick Reference + +```bash +# Build +pnpm build + +# Lint (biome) +pnpm lint +pnpm lint:fix + +# Type check +pnpm check-types + +# Unit tests +pnpm test:unit + +# Run specific test +pnpm vitest run --project unit test/unit/path/to/test.test.ts + +# Spec tests (download first) +pnpm download-spec-tests +pnpm test:spec + +# Docs lint (markdown) +pnpm docs:lint +pnpm docs:lint:fix +``` + +## Key Conventions + +- **Default branch:** `unstable` (all PRs target this) +- **Import extensions:** Always use `.js` for relative imports (even for `.ts` files) +- **No default exports:** Named exports only +- **Metrics:** Prometheus naming, always suffix with units (`_seconds`, `_bytes`, `_total`) +- **Fork order:** phase0 → altair → bellatrix → capella → deneb → electra → fulu → gloas +- **Commit style:** [Conventional Commits](https://www.conventionalcommits.org/) (`feat:`, `fix:`, `refactor:`, etc.) +- **AI disclosure:** Required in PR descriptions when AI-assisted From 4e115ed25d8583461fae18ccb6272b2b548118bc Mon Sep 17 00:00:00 2001 From: NC <17676176+ensi321@users.noreply.github.com> Date: Wed, 25 Feb 2026 13:39:19 -0800 Subject: [PATCH 12/63] feat: add gloas block import pipeline (#8940) In Gloas, beacon blocks are valid immediately upon receipt (builder may not reveal payload). Execution payloads arrive separately and require data availability (all sampled columns) before being valid. This PR adds: - BlockInputNoData class - immediately complete block input (no DA requirements) - DAType.NoData and DataAvailabilityStatus.NotRequired - new enum values for ePBS - ExecutionStatus.PayloadSeparated - fork choice status for blocks without embedded payloads --------- Co-authored-by: Claude Opus 4.5 --- .../beacon-node/src/api/impl/debug/index.ts | 1 + .../src/chain/blocks/blockInput/blockInput.ts | 71 ++++++++++++++++++- .../src/chain/blocks/blockInput/types.ts | 1 + .../blocks/verifyBlocksDataAvailability.ts | 3 + .../blocks/verifyBlocksExecutionPayloads.ts | 11 ++- .../chain/seenCache/seenGossipBlockInput.ts | 23 ++++-- .../src/network/processor/gossipHandlers.ts | 10 ++- .../chain/seenCache/seenBlockInput.test.ts | 9 +-- .../fork-choice/src/forkChoice/forkChoice.ts | 10 +-- .../fork-choice/src/protoArray/interface.ts | 12 ++++ .../fork-choice/src/protoArray/protoArray.ts | 6 ++ .../src/block/externalData.ts | 2 + 12 files changed, 136 insertions(+), 23 deletions(-) diff --git a/packages/beacon-node/src/api/impl/debug/index.ts b/packages/beacon-node/src/api/impl/debug/index.ts index 04546d7a60aa..48bd0cd447ff 100644 --- a/packages/beacon-node/src/api/impl/debug/index.ts +++ b/packages/beacon-node/src/api/impl/debug/index.ts @@ -43,6 +43,7 @@ export function getDebugApi({ validity: (() => { switch (node.executionStatus) { case ExecutionStatus.Valid: + case ExecutionStatus.PayloadSeparated: return "valid"; case ExecutionStatus.Invalid: return "invalid"; diff --git a/packages/beacon-node/src/chain/blocks/blockInput/blockInput.ts b/packages/beacon-node/src/chain/blocks/blockInput/blockInput.ts index be8a4609b12d..875b55b1ac12 100644 --- a/packages/beacon-node/src/chain/blocks/blockInput/blockInput.ts +++ b/packages/beacon-node/src/chain/blocks/blockInput/blockInput.ts @@ -1,5 +1,5 @@ -import {ForkName, ForkPostFulu, ForkPreDeneb, ForkPreGloas, NUMBER_OF_COLUMNS} from "@lodestar/params"; -import {BeaconBlockBody, BlobIndex, ColumnIndex, SignedBeaconBlock, Slot, deneb, fulu} from "@lodestar/types"; +import {ForkName, ForkPostFulu, ForkPostGloas, ForkPreDeneb, ForkPreGloas, NUMBER_OF_COLUMNS} from "@lodestar/params"; +import {BeaconBlockBody, BlobIndex, ColumnIndex, SignedBeaconBlock, Slot, deneb, fulu, gloas} from "@lodestar/types"; import {byteArrayEquals, fromHex, prettyBytes, toRootHex, withTimeout} from "@lodestar/utils"; import {VersionedHashes} from "../../../execution/index.js"; import {kzgCommitmentToVersionedHash} from "../../../util/blobs.js"; @@ -24,7 +24,7 @@ import { SourceMeta, } from "./types.js"; -export type BlockInput = BlockInputPreData | BlockInputBlobs | BlockInputColumns; +export type BlockInput = BlockInputPreData | BlockInputBlobs | BlockInputColumns | BlockInputNoData; export function isBlockInputPreDeneb(blockInput: IBlockInput): blockInput is BlockInputPreData { return blockInput.type === DAType.PreData; @@ -37,6 +37,10 @@ export function isBlockInputColumns(blockInput: IBlockInput): blockInput is Bloc return blockInput.type === DAType.Columns; } +export function isBlockInputNoData(blockInput: IBlockInput): blockInput is BlockInputNoData { + return blockInput.type === DAType.NoData; +} + function createPromise(): PromiseParts { let resolve!: (value: T) => void; let reject!: (e: Error) => void; @@ -903,3 +907,64 @@ export class BlockInputColumns extends AbstractBlockInput; + source: SourceMeta; + timeCompleteSec: number; +}; + +export class BlockInputNoData extends AbstractBlockInput { + type = DAType.NoData as const; + + state: BlockInputNoDataState; + + private constructor(init: BlockInputInit, state: BlockInputNoDataState) { + super(init); + this.state = state; + this.dataPromise.resolve(null); + this.blockPromise.resolve(state.block); + } + + static createFromBlock(props: AddBlock & CreateBlockInputMeta): BlockInputNoData { + const init: BlockInputInit = { + daOutOfRange: props.daOutOfRange, + timeCreated: props.seenTimestampSec, + forkName: props.forkName, + slot: props.block.message.slot, + blockRootHex: props.blockRootHex, + parentRootHex: toRootHex(props.block.message.parentRoot), + }; + const state: BlockInputNoDataState = { + hasBlock: true, + hasAllData: true, + block: props.block, + source: { + source: props.source, + seenTimestampSec: props.seenTimestampSec, + peerIdStr: props.peerIdStr, + }, + timeCompleteSec: props.seenTimestampSec, + }; + return new BlockInputNoData(init, state); + } + + addBlock(_: AddBlock, opts = {throwOnDuplicateAdd: true}): void { + if (opts.throwOnDuplicateAdd) { + throw new BlockInputError( + { + code: BlockInputErrorCode.INVALID_CONSTRUCTION, + blockRoot: this.blockRootHex, + }, + "Cannot addBlock to BlockInputNoData - block already exists" + ); + } + } + + getBlobKzgCommitments(): deneb.BlobKzgCommitments { + return (this.state.block.message.body as gloas.BeaconBlockBody).signedExecutionPayloadBid.message + .blobKzgCommitments; + } +} diff --git a/packages/beacon-node/src/chain/blocks/blockInput/types.ts b/packages/beacon-node/src/chain/blocks/blockInput/types.ts index 252457113321..587ad83c70e8 100644 --- a/packages/beacon-node/src/chain/blocks/blockInput/types.ts +++ b/packages/beacon-node/src/chain/blocks/blockInput/types.ts @@ -6,6 +6,7 @@ export enum DAType { PreData = "pre-data", Blobs = "blobs", Columns = "columns", + NoData = "no-data", } export type DAData = null | deneb.BlobSidecars | fulu.DataColumnSidecars; diff --git a/packages/beacon-node/src/chain/blocks/verifyBlocksDataAvailability.ts b/packages/beacon-node/src/chain/blocks/verifyBlocksDataAvailability.ts index 470158b1aef5..fdf6369c0327 100644 --- a/packages/beacon-node/src/chain/blocks/verifyBlocksDataAvailability.ts +++ b/packages/beacon-node/src/chain/blocks/verifyBlocksDataAvailability.ts @@ -29,6 +29,9 @@ export async function verifyBlocksDataAvailability( const availableTime = Math.max(0, Math.max(...blocks.map((blockInput) => blockInput.getTimeComplete()))); const dataAvailabilityStatuses: DataAvailabilityStatus[] = blocks.map((blockInput) => { + if (blockInput.type === DAType.NoData) { + return DataAvailabilityStatus.NotRequired; + } if (blockInput.type === DAType.PreData) { return DataAvailabilityStatus.PreData; } diff --git a/packages/beacon-node/src/chain/blocks/verifyBlocksExecutionPayloads.ts b/packages/beacon-node/src/chain/blocks/verifyBlocksExecutionPayloads.ts index 2869cb12e793..7682de88af12 100644 --- a/packages/beacon-node/src/chain/blocks/verifyBlocksExecutionPayloads.ts +++ b/packages/beacon-node/src/chain/blocks/verifyBlocksExecutionPayloads.ts @@ -21,7 +21,7 @@ import {Metrics} from "../../metrics/metrics.js"; import {IClock} from "../../util/clock.js"; import {BlockError, BlockErrorCode} from "../errors/index.js"; import {BlockProcessOpts} from "../options.js"; -import {isBlockInputBlobs, isBlockInputColumns} from "./blockInput/blockInput.js"; +import {isBlockInputBlobs, isBlockInputColumns, isBlockInputNoData} from "./blockInput/blockInput.js"; import {IBlockInput} from "./blockInput/types.js"; import {ImportBlockOpts} from "./types.js"; @@ -51,7 +51,8 @@ type VerifyBlockExecutionResponse = | VerifyExecutionErrorResponse | {executionStatus: ExecutionStatus.Valid; lvhResponse: LVHValidResponse; execError: null} | {executionStatus: ExecutionStatus.Syncing; lvhResponse?: LVHValidResponse; execError: null} - | {executionStatus: ExecutionStatus.PreMerge; lvhResponse: undefined; execError: null}; + | {executionStatus: ExecutionStatus.PreMerge; lvhResponse: undefined; execError: null} + | {executionStatus: ExecutionStatus.PayloadSeparated; lvhResponse: undefined; execError: null}; /** * Verifies 1 or more execution payloads from a linear sequence of blocks. @@ -148,6 +149,12 @@ export async function verifyBlockExecutionPayload( preState0: CachedBeaconStateAllForks ): Promise { const block = blockInput.getBlock(); + + // Gloas block doesn't have execution payload. Return right away + if (isBlockInputNoData(blockInput)) { + return {executionStatus: ExecutionStatus.PayloadSeparated, lvhResponse: undefined, execError: null}; + } + /** Not null if execution is enabled */ const executionPayloadEnabled = isExecutionStateType(preState0) && diff --git a/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts b/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts index 6b7a38a93e97..a11f2e2a03b0 100644 --- a/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts +++ b/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts @@ -3,6 +3,7 @@ import {CheckpointWithHex} from "@lodestar/fork-choice"; import { ForkName, ForkPostFulu, + ForkPostGloas, ForkPreGloas, SLOTS_PER_EPOCH, isForkPostDeneb, @@ -20,6 +21,7 @@ import { BlockInput, BlockInputBlobs, BlockInputColumns, + BlockInputNoData, BlockInputPreData, BlockWithSource, DAType, @@ -179,12 +181,19 @@ export class SeenBlockInput { if (!blockInput) { const {forkName, daOutOfRange} = this.buildCommonProps(block.message.slot); - // TODO GLOAS: Implement if (isForkPostGloas(forkName)) { - throw Error("Not implemented"); - } - // Pre-deneb - if (!isForkPostDeneb(forkName)) { + // Post-gloas + blockInput = BlockInputNoData.createFromBlock({ + block: block as SignedBeaconBlock, + blockRootHex, + daOutOfRange, + forkName, + source, + seenTimestampSec, + peerIdStr, + }); + } else if (!isForkPostDeneb(forkName)) { + // Pre-deneb blockInput = BlockInputPreData.createFromBlock({ block, blockRootHex, @@ -194,8 +203,8 @@ export class SeenBlockInput { seenTimestampSec, peerIdStr, }); - // Fulu Only } else if (isForkPostFulu(forkName)) { + // Fulu Only blockInput = BlockInputColumns.createFromBlock({ block: block as SignedBeaconBlock, blockRootHex, @@ -207,8 +216,8 @@ export class SeenBlockInput { seenTimestampSec, peerIdStr, }); - // Deneb and Electra } else { + // Deneb and Electra blockInput = BlockInputBlobs.createFromBlock({ block: block as SignedBeaconBlock, blockRootHex, diff --git a/packages/beacon-node/src/network/processor/gossipHandlers.ts b/packages/beacon-node/src/network/processor/gossipHandlers.ts index e196c90cc8ea..3aac4ed945f6 100644 --- a/packages/beacon-node/src/network/processor/gossipHandlers.ts +++ b/packages/beacon-node/src/network/processor/gossipHandlers.ts @@ -2,6 +2,7 @@ import {routes} from "@lodestar/api"; import {BeaconConfig, ChainForkConfig} from "@lodestar/config"; import { ForkName, + ForkPostDeneb, ForkPostElectra, ForkPreElectra, ForkSeq, @@ -70,6 +71,7 @@ import {validateGossipPayloadAttestationMessage} from "../../chain/validation/pa import {OpSource} from "../../chain/validatorMonitor.js"; import {Metrics} from "../../metrics/index.js"; import {kzgCommitmentToVersionedHash} from "../../util/blobs.js"; +import {getBlobKzgCommitments} from "../../util/dataColumns.ts"; import {INetworkCore} from "../core/index.js"; import {NetworkEventBus} from "../events.js"; import { @@ -417,9 +419,11 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand chain.getBlobsTracker.triggerGetBlobs(blockInput); } else { metrics?.blockInputFetchStats.totalDataAvailableBlockInputs.inc(); - metrics?.blockInputFetchStats.totalDataAvailableBlockInputBlobs.inc( - (signedBlock.message as deneb.BeaconBlock).body.blobKzgCommitments.length - ); + const blobCount = getBlobKzgCommitments( + blockInput.forkName, + signedBlock as SignedBeaconBlock + ).length; + metrics?.blockInputFetchStats.totalDataAvailableBlockInputBlobs.inc(blobCount); } chain diff --git a/packages/beacon-node/test/unit/chain/seenCache/seenBlockInput.test.ts b/packages/beacon-node/test/unit/chain/seenCache/seenBlockInput.test.ts index 1d82643f17ef..0c7465554f9a 100644 --- a/packages/beacon-node/test/unit/chain/seenCache/seenBlockInput.test.ts +++ b/packages/beacon-node/test/unit/chain/seenCache/seenBlockInput.test.ts @@ -1,10 +1,10 @@ import {generateKeyPair} from "@libp2p/crypto/keys"; import {beforeEach, describe, expect, it} from "vitest"; -import {ForkName, ForkPostFulu, ForkPreGloas} from "@lodestar/params"; +import {ForkName} from "@lodestar/params"; import {signedBlockToSignedHeader} from "@lodestar/state-transition"; -import {SignedBeaconBlock} from "@lodestar/types"; import {toRootHex} from "@lodestar/utils"; import { + BlockInputPreData, BlockInputSource, IBlockInput, isBlockInputBlobs, @@ -308,9 +308,10 @@ describe("SeenBlockInputCache", async () => { source: BlockInputSource.gossip, seenTimestampSec: Date.now() / 1000, }); + expect(isBlockInputPreDeneb(blockInput)).toBeTruthy(); expect(() => - blockInput.addBlock({ - block: block as SignedBeaconBlock, + (blockInput as BlockInputPreData).addBlock({ + block, blockRootHex: rootHex, source: BlockInputSource.gossip, seenTimestampSec: Date.now() / 1000, diff --git a/packages/fork-choice/src/forkChoice/forkChoice.ts b/packages/fork-choice/src/forkChoice/forkChoice.ts index 1d87d6608c40..9e0548e1ae93 100644 --- a/packages/fork-choice/src/forkChoice/forkChoice.ts +++ b/packages/fork-choice/src/forkChoice/forkChoice.ts @@ -743,11 +743,13 @@ export class ForkChoice implements IForkChoice { unrealizedFinalizedEpoch: unrealizedFinalizedCheckpoint.epoch, unrealizedFinalizedRoot: unrealizedFinalizedCheckpoint.rootHex, + // TODO GLOAS: Need to update this when we are merging nc/epbs-fc. Need to define `getPostGloasExecStatus` + // to make sure execution status of post-gloas blocks is ExecutionStatus.PayloadSeparated ...(isExecutionBlockBodyType(block.body) && isExecutionStateType(state) && isExecutionEnabled(state, block) ? { executionPayloadBlockHash: toRootHex(block.body.executionPayload.blockHash), executionPayloadNumber: block.body.executionPayload.blockNumber, - executionStatus: this.getPostMergeExecStatus(executionStatus), + executionStatus: this.getPreGloasExecStatus(executionStatus), dataAvailabilityStatus, } : { @@ -1240,12 +1242,12 @@ export class ForkChoice implements IForkChoice { return dataAvailabilityStatus; } - private getPostMergeExecStatus( + private getPreGloasExecStatus( executionStatus: MaybeValidExecutionStatus ): ExecutionStatus.Valid | ExecutionStatus.Syncing { - if (executionStatus === ExecutionStatus.PreMerge) + if (executionStatus === ExecutionStatus.PreMerge || executionStatus === ExecutionStatus.PayloadSeparated) throw Error( - `Invalid post-merge execution status: expected: ${ExecutionStatus.Syncing} or ${ExecutionStatus.Valid} , got ${executionStatus}` + `Invalid post-merge execution status: expected: ${ExecutionStatus.Syncing} or ${ExecutionStatus.Valid}, got ${executionStatus}` ); return executionStatus; } diff --git a/packages/fork-choice/src/protoArray/interface.ts b/packages/fork-choice/src/protoArray/interface.ts index 9c476b29a372..480ac7ffe04f 100644 --- a/packages/fork-choice/src/protoArray/interface.ts +++ b/packages/fork-choice/src/protoArray/interface.ts @@ -17,11 +17,23 @@ export const NULL_VOTE_INDEX = 0xffffffff; */ export type VoteIndex = number; +/** + * Execution status of a block in fork choice. + * + * - Valid: Execution payload verified as valid by the EL + * - Syncing: EL is syncing, payload validity unknown (optimistic sync) + * - PreMerge: Block is from before The Merge, no execution payload exists + * - Invalid: Execution payload was invalidated by the EL (post-import status) + * - PayloadSeparated: Gloas beacon block without embedded execution payload. + * The execution payload arrives separately via SignedExecutionPayloadEnvelope. + * Gloas blocks WITH execution payload (FULL variant) use Valid/Invalid/Syncing. + */ export enum ExecutionStatus { Valid = "Valid", Syncing = "Syncing", PreMerge = "PreMerge", Invalid = "Invalid", + PayloadSeparated = "PayloadSeparated", } export type LVHValidResponse = { diff --git a/packages/fork-choice/src/protoArray/protoArray.ts b/packages/fork-choice/src/protoArray/protoArray.ts index 50a994d46476..a006de102b2f 100644 --- a/packages/fork-choice/src/protoArray/protoArray.ts +++ b/packages/fork-choice/src/protoArray/protoArray.ts @@ -317,6 +317,12 @@ export class ProtoArray { if (node.executionStatus === ExecutionStatus.PreMerge || node.executionStatus === ExecutionStatus.Valid) { break; } + // If PayloadSeparated, that means the node is either PENDING or EMPTY, there could be + // some ancestor still has syncing status. + if (node.executionStatus === ExecutionStatus.PayloadSeparated) { + nodeIndex = node.parent; + continue; + } this.validateNodeByIndex(nodeIndex); nodeIndex = node.parent; } diff --git a/packages/state-transition/src/block/externalData.ts b/packages/state-transition/src/block/externalData.ts index affe2a3b7fc6..7a02b8a6fa8a 100644 --- a/packages/state-transition/src/block/externalData.ts +++ b/packages/state-transition/src/block/externalData.ts @@ -18,6 +18,8 @@ export enum DataAvailabilityStatus { /* validator activities can't be performed on out of range data */ OutOfRange = "OutOfRange", Available = "Available", + /* Gloas: beacon blocks have no DA requirement, execution payload is separate */ + NotRequired = "NotRequired", } export interface BlockExternalData { From 167a02a7b98a993c090abdd4d6c8eb9532ca220e Mon Sep 17 00:00:00 2001 From: Lodekeeper <258435968+lodekeeper@users.noreply.github.com> Date: Mon, 2 Mar 2026 09:55:44 +0000 Subject: [PATCH 13/63] fix: use inclusive boundary for gossip clock disparity checks (#8968) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description Fix off-by-one in gossip clock disparity boundary checks. The clock used strict less-than (`<`) instead of less-than-or-equal (`<=`) when comparing time differences against `MAXIMUM_GOSSIP_CLOCK_DISPARITY`, causing messages arriving at exactly the disparity boundary to be incorrectly rejected. ### Spec Reference The [consensus spec](https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/p2p-interface.md#beacon_block) defines the future slot check as: ```python block_time_ms = compute_time_at_slot_ms(state, block.slot) if current_time_ms + MAXIMUM_GOSSIP_CLOCK_DISPARITY < block_time_ms: raise GossipIgnore("block is from a future slot") ``` The spec uses strict `<` for the **rejection** condition, meaning a message should be **accepted** when: ``` current_time_ms + MAXIMUM_GOSSIP_CLOCK_DISPARITY >= block_time_ms ``` Which is equivalent to: ``` block_time_ms - current_time_ms <= MAXIMUM_GOSSIP_CLOCK_DISPARITY ``` ### The Bug Our clock was checking `nextSlotTime - Date.now() < MAXIMUM_GOSSIP_CLOCK_DISPARITY` (strict `<`), but the spec requires `<=` (inclusive). At exactly the boundary (e.g. exactly 500ms before the next slot), the clock would reject the message as a future slot when the spec says it should be accepted. This was caught by the [gossip validation spec tests](https://github.com/ethereum/consensus-specs/tree/dev/tests/minimal/phase0/networking/gossip_beacon_block) which include a test pair that validates the exact boundary: - `gossip_beacon_block__valid_within_clock_disparity`: `current_time_ms: 5500`, block at slot 1 (6000ms) → gap = 500ms → **expected: valid** - `gossip_beacon_block__ignore_future_slot`: `current_time_ms: 5499`, same block → gap = 501ms → **expected: ignore** ### Changes Changed `<` to `<=` in three places in `Clock`: 1. `currentSlotWithGossipDisparity` getter 2. `isCurrentSlotGivenGossipDisparity` — next slot boundary check 3. `isCurrentSlotGivenGossipDisparity` — previous slot boundary check ### Practical Impact Minimal — this only affects messages arriving at the exact millisecond boundary of the disparity window. But it aligns our implementation with the spec. --- *This PR was authored with AI assistance (Lodekeeper 🌟).* --------- Co-authored-by: lodekeeper --- packages/beacon-node/src/util/clock.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/beacon-node/src/util/clock.ts b/packages/beacon-node/src/util/clock.ts index bd530ef61d4a..52ade3e79079 100644 --- a/packages/beacon-node/src/util/clock.ts +++ b/packages/beacon-node/src/util/clock.ts @@ -92,15 +92,17 @@ export class Clock extends EventEmitter implements IClock { } return slot; } - /** * If it's too close to next slot given MAXIMUM_GOSSIP_CLOCK_DISPARITY, return currentSlot + 1. * Otherwise return currentSlot + * + * Spec: phase0/p2p-interface.md - gossip validation uses `current_time + MAXIMUM_GOSSIP_CLOCK_DISPARITY < message_time` + * to reject future messages (strict `<`), so the boundary (exactly equal) is accepted, hence `<=` here. */ get currentSlotWithGossipDisparity(): Slot { const currentSlot = this.currentSlot; const nextSlotTime = computeTimeAtSlot(this.config, currentSlot + 1, this.genesisTime) * 1000; - return nextSlotTime - Date.now() < this.config.MAXIMUM_GOSSIP_CLOCK_DISPARITY ? currentSlot + 1 : currentSlot; + return nextSlotTime - Date.now() <= this.config.MAXIMUM_GOSSIP_CLOCK_DISPARITY ? currentSlot + 1 : currentSlot; } get currentEpoch(): Epoch { @@ -121,6 +123,9 @@ export class Clock extends EventEmitter implements IClock { /** * Check if a slot is current slot given MAXIMUM_GOSSIP_CLOCK_DISPARITY. + * + * Uses `<=` for disparity checks because the spec rejects with strict `<` + * (phase0/p2p-interface.md), meaning the boundary (exactly equal) is accepted. */ isCurrentSlotGivenGossipDisparity(slot: Slot): boolean { const currentSlot = this.currentSlot; @@ -129,12 +134,12 @@ export class Clock extends EventEmitter implements IClock { } const nextSlotTime = computeTimeAtSlot(this.config, currentSlot + 1, this.genesisTime) * 1000; // we're too close to next slot, accept next slot - if (nextSlotTime - Date.now() < this.config.MAXIMUM_GOSSIP_CLOCK_DISPARITY) { + if (nextSlotTime - Date.now() <= this.config.MAXIMUM_GOSSIP_CLOCK_DISPARITY) { return slot === currentSlot + 1; } const currentSlotTime = computeTimeAtSlot(this.config, currentSlot, this.genesisTime) * 1000; // we've just passed the current slot, accept previous slot - if (Date.now() - currentSlotTime < this.config.MAXIMUM_GOSSIP_CLOCK_DISPARITY) { + if (Date.now() - currentSlotTime <= this.config.MAXIMUM_GOSSIP_CLOCK_DISPARITY) { return slot === currentSlot - 1; } return false; From f2caa915abb14c0d496e2753ed0f09d3c0fa502d Mon Sep 17 00:00:00 2001 From: Lodekeeper <258435968+lodekeeper@users.noreply.github.com> Date: Mon, 2 Mar 2026 15:29:08 +0000 Subject: [PATCH 14/63] chore: bump libp2p deps and fix peer-collections type mismatch (#8970) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem Build fails in `packages/beacon-node/src/network/libp2p/index.ts` with 3 TypeScript errors: ``` Type 'PeerMap<...>' is not assignable to type 'PeerMap<...>'. Types have separate declarations of a private property 'map'. ``` Root cause: `@libp2p/peer-collections` has two versions in the dependency tree (7.0.10 and 7.0.11). The `PeerMap` class has private properties that make the two versions structurally incompatible at the TypeScript level. ## Fix Bump all `@libp2p/*` packages to their latest patch versions, which converge on `@libp2p/peer-collections@7.0.11`: | Package | Old | New | |---------|-----|-----| | `@libp2p/bootstrap` | 12.0.11 | 12.0.12 | | `@libp2p/gossipsub` | 15.0.12 | 15.0.13 | | `@libp2p/identify` | 4.0.10 | 4.0.11 | | `@libp2p/mdns` | 12.0.11 | 12.0.12 | | `@libp2p/mplex` | 12.0.11 | 12.0.12 | | `@libp2p/prometheus-metrics` | 5.0.10 | 5.0.11 | | `@libp2p/tcp` | 11.0.10 | 11.0.11 | | `@libp2p/interface-internal` | 3.0.10 | 3.0.11 | | `@libp2p/utils` | 7.0.10 | 7.0.11 | Also excludes `@libp2p/*` from pnpm `minimumReleaseAge` to prevent future resolution delays when new patch versions are published. ## Verification - `pnpm build` ✅ — all packages compile cleanly - `pnpm lint` ✅ — no new warnings - Only 1 version of `@libp2p/peer-collections` (7.0.11) in lockfile --------- Co-authored-by: lodekeeper --- packages/beacon-node/package.json | 18 +- .../beacon-node/src/network/libp2p/index.ts | 32 +- packages/reqresp/package.json | 2 +- pnpm-lock.yaml | 507 ++++++++---------- pnpm-workspace.yaml | 3 + 5 files changed, 257 insertions(+), 305 deletions(-) diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index 65834918361b..cc12095a465e 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -124,16 +124,16 @@ "@fastify/cors": "^10.0.1", "@fastify/swagger": "^9.0.0", "@fastify/swagger-ui": "^5.0.1", - "@libp2p/bootstrap": "^12.0.11", + "@libp2p/bootstrap": "^12.0.12", "@libp2p/crypto": "^5.1.13", - "@libp2p/gossipsub": "^15.0.12", - "@libp2p/identify": "^4.0.10", + "@libp2p/gossipsub": "^15.0.13", + "@libp2p/identify": "^4.0.11", "@libp2p/interface": "^3.1.0", - "@libp2p/mdns": "^12.0.11", - "@libp2p/mplex": "^12.0.11", + "@libp2p/mdns": "^12.0.12", + "@libp2p/mplex": "^12.0.12", "@libp2p/peer-id": "^6.0.4", - "@libp2p/prometheus-metrics": "^5.0.10", - "@libp2p/tcp": "^11.0.10", + "@libp2p/prometheus-metrics": "^5.0.11", + "@libp2p/tcp": "^11.0.11", "@lodestar/api": "workspace:^", "@lodestar/config": "workspace:^", "@lodestar/db": "workspace:^", @@ -165,9 +165,9 @@ }, "devDependencies": { "@chainsafe/swap-or-not-shuffle": "^1.2.1", - "@libp2p/interface-internal": "^3.0.10", + "@libp2p/interface-internal": "^3.0.11", "@libp2p/logger": "^6.2.2", - "@libp2p/utils": "^7.0.10", + "@libp2p/utils": "^7.0.11", "@lodestar/spec-test-util": "workspace:^", "@types/js-yaml": "^4.0.5", "@types/qs": "^6.9.7", diff --git a/packages/beacon-node/src/network/libp2p/index.ts b/packages/beacon-node/src/network/libp2p/index.ts index 179b48f297eb..9aa53d6fd10b 100644 --- a/packages/beacon-node/src/network/libp2p/index.ts +++ b/packages/beacon-node/src/network/libp2p/index.ts @@ -73,30 +73,6 @@ export async function createNodeJsLibp2p( noiseCrypto.chaCha20Poly1305Encrypt = asCrypto.chaCha20Poly1305Encrypt; } - const libp2pMetrics = nodeJsLibp2pOpts.metrics - ? (components: LodestarComponents) => { - const metrics = prometheusMetrics({ - collectDefaultMetrics: false, - preserveExistingMetrics: true, - registry: nodeJsLibp2pOpts.metricsRegistry, - })(components); - - // Work around identify EOF race: - // `trackProtocolStream` attaches a `message` listener immediately after protocol - // negotiation. For `/ipfs/id/1.0.0`, identify() adds its own reader later and can - // miss the first response frame when metrics listener drains events first. - const originalTrackProtocolStream = metrics.trackProtocolStream.bind(metrics); - metrics.trackProtocolStream = ((stream) => { - if (stream.protocol === "/ipfs/id/1.0.0") { - return; - } - originalTrackProtocolStream(stream); - }) as typeof metrics.trackProtocolStream; - - return metrics; - } - : undefined; - return createLibp2p({ privateKey, nodeInfo: { @@ -125,7 +101,13 @@ export async function createNodeJsLibp2p( ], streamMuxers: [mplex({disconnectThreshold})], peerDiscovery, - metrics: libp2pMetrics, + metrics: nodeJsLibp2pOpts.metrics + ? prometheusMetrics({ + collectDefaultMetrics: false, + preserveExistingMetrics: true, + registry: nodeJsLibp2pOpts.metricsRegistry, + }) + : undefined, connectionManager: { // dialer config maxParallelDials: 100, diff --git a/packages/reqresp/package.json b/packages/reqresp/package.json index f72fabd5a7f3..b336b40ef0e5 100644 --- a/packages/reqresp/package.json +++ b/packages/reqresp/package.json @@ -46,7 +46,7 @@ "dependencies": { "@chainsafe/fast-crc32c": "^4.2.0", "@libp2p/interface": "^3.1.0", - "@libp2p/utils": "^7.0.10", + "@libp2p/utils": "^7.0.11", "@lodestar/config": "workspace:^", "@lodestar/params": "workspace:^", "@lodestar/utils": "workspace:^", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d2b20b93e3a2..0a47b573c52a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -49,7 +49,7 @@ importers: version: 1.0.0(@biomejs/biome@2.2.2) '@lerna-lite/cli': specifier: ^4.9.4 - version: 4.9.4(@lerna-lite/exec@4.10.3)(@lerna-lite/publish@4.9.4)(@lerna-lite/run@4.9.4)(@lerna-lite/version@4.9.4)(@types/node@24.10.1) + version: 4.10.3(@lerna-lite/exec@4.10.3)(@lerna-lite/publish@4.9.4)(@lerna-lite/run@4.9.4)(@lerna-lite/version@4.9.4)(@types/node@24.10.1) '@lerna-lite/exec': specifier: ^4.9.4 version: 4.10.3(@lerna-lite/publish@4.9.4)(@lerna-lite/run@4.9.4)(@lerna-lite/version@4.9.4)(@types/node@24.10.1) @@ -262,35 +262,35 @@ importers: specifier: ^5.0.1 version: 5.0.1 '@libp2p/bootstrap': - specifier: ^12.0.11 - version: 12.0.11 + specifier: ^12.0.12 + version: 12.0.12 '@libp2p/crypto': specifier: ^5.1.13 version: 5.1.13 '@libp2p/gossipsub': - specifier: ^15.0.12 - version: 15.0.12 + specifier: ^15.0.13 + version: 15.0.13 '@libp2p/identify': - specifier: ^4.0.10 - version: 4.0.10 + specifier: ^4.0.11 + version: 4.0.11 '@libp2p/interface': specifier: ^3.1.0 version: 3.1.0 '@libp2p/mdns': - specifier: ^12.0.11 - version: 12.0.11 + specifier: ^12.0.12 + version: 12.0.12 '@libp2p/mplex': - specifier: ^12.0.11 - version: 12.0.11 + specifier: ^12.0.12 + version: 12.0.12 '@libp2p/peer-id': specifier: ^6.0.4 version: 6.0.4 '@libp2p/prometheus-metrics': - specifier: ^5.0.10 - version: 5.0.10 + specifier: ^5.0.11 + version: 5.0.11 '@libp2p/tcp': - specifier: ^11.0.10 - version: 11.0.10 + specifier: ^11.0.11 + version: 11.0.11 '@lodestar/api': specifier: workspace:^ version: link:../api @@ -380,14 +380,14 @@ importers: specifier: ^1.2.1 version: 1.2.1 '@libp2p/interface-internal': - specifier: ^3.0.10 - version: 3.0.10 + specifier: ^3.0.11 + version: 3.0.11 '@libp2p/logger': specifier: ^6.2.2 version: 6.2.2 '@libp2p/utils': - specifier: ^7.0.10 - version: 7.0.10 + specifier: ^7.0.11 + version: 7.0.11 '@lodestar/spec-test-util': specifier: workspace:^ version: link:../spec-test-util @@ -890,8 +890,8 @@ importers: specifier: ^3.1.0 version: 3.1.0 '@libp2p/utils': - specifier: ^7.0.10 - version: 7.0.10 + specifier: ^7.0.11 + version: 7.0.11 '@lodestar/config': specifier: workspace:^ version: link:../config @@ -955,7 +955,7 @@ importers: version: 0.7.0 vitest: specifier: 'catalog:' - version: 4.0.7(@types/node@24.10.1)(@vitest/browser-playwright@4.0.7)(jsdom@23.0.1)(yaml@2.8.2) + version: 4.0.7(@types/node@25.3.2)(@vitest/browser-playwright@4.0.7)(jsdom@23.0.1)(yaml@2.8.2) packages/state-transition: dependencies: @@ -1022,7 +1022,7 @@ importers: version: 0.2.4 vitest: specifier: 'catalog:' - version: 4.0.7(@types/node@24.10.1)(@vitest/browser-playwright@4.0.7)(jsdom@23.0.1)(yaml@2.8.2) + version: 4.0.7(@types/node@25.3.2)(@vitest/browser-playwright@4.0.7)(jsdom@23.0.1)(yaml@2.8.2) devDependencies: '@types/tmp': specifier: ^0.2.3 @@ -2190,21 +2190,21 @@ packages: resolution: {integrity: sha512-hRFGB47RWHNKZkh9t+ka34kNlv5DQwrIDBvjI+mmzZqZK7UAtPRsJoE+MlQHVlQApDFwUPiSTdxY9UXzJrYFdQ==} engines: {node: ^20.17.0 || >=22.9.0} - '@libp2p/bootstrap@12.0.11': - resolution: {integrity: sha512-ZIG8QKS+4w7ugK7a1ftdopjIA+NvOPKUq7JY1OsRxaiLdCdxgghPTiNIbinYsVv5iHULBnFZe4o5l+5L7+Hssw==} + '@libp2p/bootstrap@12.0.12': + resolution: {integrity: sha512-OFVBDm2UO5J/0olQKypj9VjhQOs4SCdVXqCEdItOrZeahsgB6XeVS+eHlbAle87jzh6r6gO/yVMGCEeJeRiMHQ==} '@libp2p/crypto@5.1.13': resolution: {integrity: sha512-8NN9cQP3jDn+p9+QE9ByiEoZ2lemDFf/unTgiKmS3JF93ph240EUVdbCyyEgOMfykzb0okTM4gzvwfx9osJebQ==} - '@libp2p/gossipsub@15.0.12': - resolution: {integrity: sha512-jhyzujQeH/o+WMS9L+m785ygF8W5++Lay6pgnfkgHj1qrtc/IJtMIgsASZaoTTQY+yC6Zy2EL4r5dJ131mBVNQ==} + '@libp2p/gossipsub@15.0.13': + resolution: {integrity: sha512-cqDaJ8wdJ0n6sbhk1xe5dZIFfAvB5laaorXPIw7jWhkG+WA9h02INgbUPz+YkMhBJdsNwa5gKRMjVU3sdaaAPA==} engines: {npm: '>=8.7.0'} - '@libp2p/identify@4.0.10': - resolution: {integrity: sha512-DROyV+bZIlz9czCCHJdeVtm1+hEOKUigJHyTzzA/cuwwyvtm8Dco8F+VRYcrwpafuVtjv7yN7CskN4oIys56jw==} + '@libp2p/identify@4.0.11': + resolution: {integrity: sha512-DfQcNVkRROtSChTxoTlqBWKO9+ph4G8vv4Z7CtiN2wFALVudssIUcPwCeTdj10fcjL0NN/rckf09c9HJHt3/rw==} - '@libp2p/interface-internal@3.0.10': - resolution: {integrity: sha512-Gd/eQAoAlXqeCRJ6wOwcnTQ/SDe95bQow8osY8zq0nbfFBu26aChQHjAd+CjcCADJRh+Sd+7+dYG7BrhpxGt1A==} + '@libp2p/interface-internal@3.0.11': + resolution: {integrity: sha512-tByDMIf3LGOEgmSxxe/zamKxYIWHD/2pLWntOdtcZ/fHB7Wnhk2HvOlPgD0nPVqIDWHwFfmS6uajGRWOtS150w==} '@libp2p/interface@3.1.0': resolution: {integrity: sha512-RE7/XyvC47fQBe1cHxhMvepYKa5bFCUyFrrpj8PuM0E7JtzxU7F+Du5j4VXbg2yLDcToe0+j8mB7jvwE2AThYw==} @@ -2212,17 +2212,17 @@ packages: '@libp2p/logger@6.2.2': resolution: {integrity: sha512-XtanXDT+TuMuZoCK760HGV1AmJsZbwAw5AiRUxWDbsZPwAroYq64nb41AHRu9Gyc0TK9YD+p72+5+FIxbw0hzw==} - '@libp2p/mdns@12.0.11': - resolution: {integrity: sha512-OB6am5A21Yc5c7KBZONQhTao4BHRDc3MurZ1qHzqU4FQidi719cNRw4ac6TVk4dcdtOYx+1ef8pvvLX+57hXAQ==} + '@libp2p/mdns@12.0.12': + resolution: {integrity: sha512-Tu4FaLrguteEaD+MUwi9fFXxfjVm306czSEmZWxnoGjPc3+iVG0B4c4VVNunWHedhMktJALNKHw6FoM9PfQ/uQ==} - '@libp2p/mplex@12.0.11': - resolution: {integrity: sha512-jD77lX3FkgHM4FdznF5G2aN8G6BoQrPZPTdox56KvFOUjUKs04KoiC58kuSHVuEo/25gBTpjTLnYJN07ixC3vg==} + '@libp2p/mplex@12.0.12': + resolution: {integrity: sha512-xGGOR4tS1NBeLVzvNXVewEVIYz5y2DkuOXgUUrtufSRZqBS5Pyy+mgchn6qR2X73Tmx5AzD783T7RlhSsAhnVg==} - '@libp2p/multistream-select@7.0.10': - resolution: {integrity: sha512-6RAFctqWzwQ/qPaN3CxoueSs1b7pBVMZ+0n6G0kcsqVBj0wc4eB+dcJyUNrTV1NGgMCAl6tVAGztZaE8XZc9lw==} + '@libp2p/multistream-select@7.0.11': + resolution: {integrity: sha512-zoIeiA/jl+/pwLAU4hDo1l4YVxr/ZMh1IOYpnCz6VwRBh9+2uuUFjbLRanQdP4hGQXdg0L6BsPChxgcag5MJrg==} - '@libp2p/peer-collections@7.0.10': - resolution: {integrity: sha512-OvlSY5N3J6q8U+EbTrQGbW8zdyOa3y7nz9Y3IbuE55tIiMd7pwm1U3Lknfb6IPkOWkHNfQDfCGGfGVQcMRodvQ==} + '@libp2p/peer-collections@7.0.11': + resolution: {integrity: sha512-LJ+3lew9FcwFpL0zA4ZYhUHcFR+mekr8mHPHo/6YM9lY4feIBekZPidzkpKHNY8nn2SCXJb2e47ruqjy+whijg==} '@libp2p/peer-id@6.0.4': resolution: {integrity: sha512-Z3xK0lwwKn4bPg3ozEpPr1HxsRi2CxZdghOL+MXoFah/8uhJJHxHFA8A/jxtKn4BB8xkk6F8R5vKNIS05yaCYw==} @@ -2230,17 +2230,17 @@ packages: '@libp2p/peer-record@9.0.5': resolution: {integrity: sha512-disk23OO00yD52O4VmItbDkjJZ/YZJsKbMsqNgVhr+D3PcM+KRpu9VVbiCnN5Tzn9XvFEHhrMJY7BPE+rvT5MQ==} - '@libp2p/peer-store@12.0.10': - resolution: {integrity: sha512-fe/6m0vXny9pvCyaSjg2GisdSVgxtHYZtp6op1WNm8dBvYqRXLuqSYi0QGEbLtSDSL4SeE8BKZyadyk/tYAqfg==} + '@libp2p/peer-store@12.0.11': + resolution: {integrity: sha512-vOoUPv1UqFWFyRrheh31AQgUxooNPwdJz5riklXsv6d/BpCwIQjGLsTi9WCQ/S7QdPVG71hfBDWtfUg8dO8hLA==} - '@libp2p/prometheus-metrics@5.0.10': - resolution: {integrity: sha512-93VjiHcu/D5DaTbwV2zikkR65WXBa3HEFNWWUjSJzhHCNYt8518r2nH5zmQVPyCNNK+lnXc/R7HkLbRAq/6mIw==} + '@libp2p/prometheus-metrics@5.0.11': + resolution: {integrity: sha512-sVB7GrUZ5UlfSaXR1QsivBKU+REUHNUrBrPAo/N1Lv9UdKnru4p49HEgIzA80IMW10RnM/xJqwR26tTznNyusw==} - '@libp2p/tcp@11.0.10': - resolution: {integrity: sha512-vp1XvbRUU6JyVZMDfrr8UX+xs1sybT2r3PFoN5m07r3GSrMMPOKpWN2HkhT2pCBZWJG6ADQOy5+K0tBRE782oA==} + '@libp2p/tcp@11.0.11': + resolution: {integrity: sha512-6Tg8+0isJCZ/ci6+IjDuRQ6NVovT0xxqFoICtnU3u1Tu6mSZvEbebUZjGNrq7kAN9EcXYiEBwI/fRhXQtmFQGg==} - '@libp2p/utils@7.0.10': - resolution: {integrity: sha512-+mzD+7yLMoZ8+34y/iS9d1CnwHjJJ/qEsao9FckHf9T9tnVXEyLLu9TpzBCcGRm4fUK/QCSHK2AcZH50kkAFkw==} + '@libp2p/utils@7.0.11': + resolution: {integrity: sha512-M7jAWD9V0Hb7FYsPgzbmNxNwp3P8U9hmdllVhyz26rhVAZxoDdhSf0G+9Hey7DfZvMNxyMER/h/Sojqk0rYhPg==} '@lukeed/ms@2.0.2': resolution: {integrity: sha512-9I2Zn6+NJLfaGoz9jN3lpwDgAYvfGeNYdbAIjJOqzs4Tpc+VU3Jqq4IofSUBKajiDS8k9fZIg18/z13mpk1bsA==} @@ -3118,6 +3118,9 @@ packages: '@types/node@24.10.1': resolution: {integrity: sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ==} + '@types/node@25.3.2': + resolution: {integrity: sha512-RpV6r/ij22zRRdyBPcxDeKAzH43phWVKEjL2iksqo1Vz3CuBUrgmPpPhALKiRfU7OMCmeeO9vECBMsV0hMTG8Q==} + '@types/normalize-package-data@2.4.4': resolution: {integrity: sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA==} @@ -3146,8 +3149,8 @@ packages: '@types/sinon@20.0.0': resolution: {integrity: sha512-etYGUC6IEevDGSWvR9WrECRA01ucR2/Oi9XMBUAdV0g4bLkNf4HlZWGiGlDOq5lgwXRwcV+PSeKgFcW4QzzYOg==} - '@types/sinonjs__fake-timers@6.0.2': - resolution: {integrity: sha512-dIPoZ3g5gcx9zZEszaxLSVTvMReD3xxyyDnQUjA6IYDG9Ba2AV0otMPs+77sG9ojB4Qr2N2Vk5RnKeuA0X/0bg==} + '@types/sinonjs__fake-timers@15.0.1': + resolution: {integrity: sha512-Ko2tjWJq8oozHzHV+reuvS5KYIRAokHnGbDwGh/J64LntgpbuylF74ipEL24HCyRjf9FOlBiBHWBR1RlVKsI1w==} '@types/through@0.0.30': resolution: {integrity: sha512-FvnCJljyxhPM3gkRgWmxmDZyAQSiBQQWLI0A0VFL0K7W1oRUrPJSqNO0NvTnLkBcotdlp3lKvaT0JrnyRDkzOg==} @@ -3382,9 +3385,6 @@ packages: resolution: {integrity: sha512-iADenERppdC+A2YKbOXXB2WUeABLaM6qnpZ70kZbPZ1cZMMJ7eF+3CaYm+/PhBizgkzlvssC7QuHS30oOiQYWA==} engines: {node: '>=16.0.0', npm: '>=7.0.0'} - aproba@2.0.0: - resolution: {integrity: sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==} - aproba@2.1.0: resolution: {integrity: sha512-tLIEcj5GuR2RSTnxNKdkK0dJ/GrC7P38sUkiDmDuHfsHmbagTFAxDVIBltoklXEVIQ/f14IL8IMJ5pn9Hez1Ew==} @@ -3857,10 +3857,6 @@ packages: resolution: {integrity: sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==} engines: {node: '>=4.8'} - cross-spawn@7.0.3: - resolution: {integrity: sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==} - engines: {node: '>= 8'} - cross-spawn@7.0.6: resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} engines: {node: '>= 8'} @@ -3918,14 +3914,6 @@ packages: resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==} engines: {node: '>=10'} - dedent@1.7.0: - resolution: {integrity: sha512-HGFtf8yhuhGhqO07SV79tRp+br4MnbdjeVxotpn1QBl30pcLLCQjX5b2295ll0fv8RKDKsmWYrl05usHM9CewQ==} - peerDependencies: - babel-plugin-macros: ^3.1.0 - peerDependenciesMeta: - babel-plugin-macros: - optional: true - dedent@1.7.1: resolution: {integrity: sha512-9JmrhGZpOlEgOLdQgSm0zxFaYoQon408V1v49aqTWuXENVlnCuY9JBZcXZiCsZQWDjTm5Qf/nIvAy77mXDAjEg==} peerDependencies: @@ -4319,15 +4307,6 @@ packages: debug: optional: true - follow-redirects@1.15.6: - resolution: {integrity: sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==} - engines: {node: '>=4.0'} - peerDependencies: - debug: '*' - peerDependenciesMeta: - debug: - optional: true - for-each@0.3.3: resolution: {integrity: sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==} @@ -4339,10 +4318,6 @@ packages: resolution: {integrity: sha512-jqdObeR2rxZZbPSGL+3VckHMYtu+f9//KXBsVny6JSX/pa38Fy+bGjuG8eW/H6USNQWhLi8Num++cU2yOCNz4A==} engines: {node: '>= 0.12'} - form-data@4.0.0: - resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==} - engines: {node: '>= 6'} - form-data@4.0.5: resolution: {integrity: sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==} engines: {node: '>= 6'} @@ -4351,10 +4326,6 @@ packages: resolution: {integrity: sha512-CM3GuJ57US06mlpQ47YcunuUZ9jpm8Vx+P2CGt2j7HpgkKZO/DJYQ0Bobim8G6PFQmK5lOqOOdUXboU+h73A4g==} deprecated: 'ACTION REQUIRED: SWITCH TO v3 - v1 and v2 are VULNERABLE! v1 is DEPRECATED FOR OVER 2 YEARS! Use formidable@latest or try formidable-mini for fresh projects' - fs-extra@11.3.2: - resolution: {integrity: sha512-Xr9F6z6up6Ws+NjzMCZc6WXg2YFRlrLP9NQDO3VQrWrfiojdhS56TzueT88ze0uBdCTwEIhQ3ptnmKeWGFAe0A==} - engines: {node: '>=14.14'} - fs-extra@11.3.3: resolution: {integrity: sha512-VWSRii4t0AFm6ixFFmLLx1t7wS1gh+ckoa84aOeapGum0h+EZd1EhEumSB+ZdDLnEPuucsVB9oB7cxJHap6Afg==} engines: {node: '>=14.14'} @@ -4419,9 +4390,6 @@ packages: get-func-name@2.0.2: resolution: {integrity: sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==} - get-intrinsic@1.2.2: - resolution: {integrity: sha512-0gSo4ml/0j98Y3lngkFEot/zhiCeWsbYIlZ+uZOVgzLyLaUw7wxUL+nCTP0XJvJg1AXulJRI3UJi8GsbDuxdGA==} - get-intrinsic@1.3.0: resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==} engines: {node: '>= 0.4'} @@ -4493,9 +4461,6 @@ packages: resolution: {integrity: sha512-sFdI5LyBiNTHjRd7cGPWapiHWMOXKyuBNX/cWJ3NfzrZQVa8GI/8cofCl74AOVqq9W5kNmguTIzJ/1s2gyI9wA==} engines: {node: '>= 0.4'} - gopd@1.0.1: - resolution: {integrity: sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==} - gopd@1.2.0: resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} engines: {node: '>= 0.4'} @@ -4536,18 +4501,10 @@ packages: resolution: {integrity: sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==} engines: {node: '>= 0.4'} - has-symbols@1.0.3: - resolution: {integrity: sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==} - engines: {node: '>= 0.4'} - has-symbols@1.1.0: resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==} engines: {node: '>= 0.4'} - has-tostringtag@1.0.0: - resolution: {integrity: sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==} - engines: {node: '>= 0.4'} - has-tostringtag@1.0.2: resolution: {integrity: sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==} engines: {node: '>= 0.4'} @@ -4727,9 +4684,6 @@ packages: resolution: {integrity: sha512-XXADHxXmvT9+CRxhXg56LJovE+bmWnEWB78LB83VZTprKTmaC5QfruXocxzTZ2Kl0DNwKuBdlIhjL8LeY8Sf8Q==} engines: {node: '>= 12'} - ip@2.0.1: - resolution: {integrity: sha512-lJUL9imLTNi1ZfXT+DU6rBBdbiKGBuay9B6xGSPVjUeQwaH1RIGqef8RZkUtHioLmSNpPR5M4HVKJGm1j8FWVQ==} - ipaddr.js@2.2.0: resolution: {integrity: sha512-Ag3wB2o37wslZS19hZqorUnrnzSkpOVy+IiiDEiTqNubEYpYuHWIf6K4psgN2ZWKExS4xhVCrRVfb/wfW8fWJA==} engines: {node: '>= 10'} @@ -5197,14 +5151,6 @@ packages: lru-cache@10.4.3: resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} - lru-cache@11.0.1: - resolution: {integrity: sha512-CgeuL5uom6j/ZVrg7G/+1IXqRY8JXX4Hghfy5YE0EhoYQWvndP1kufu58cmZLNIDKnRhZrXfdS9urVWx98AipQ==} - engines: {node: 20 || >=22} - - lru-cache@11.2.4: - resolution: {integrity: sha512-B5Y16Jr9LB9dHVkh6ZevG+vAbOsNOYCX+sXvFWFu7B3Iz5mijW3zdbMyhsh8ANd2mSWBYdJgnqi+mL7/LrOPYg==} - engines: {node: 20 || >=22} - lru-cache@11.2.5: resolution: {integrity: sha512-vFrFJkWtJvJnD5hg+hJvVE8Lh/TcMzKnTgCWmtBipwI5yLX/iX+5UB2tfuyODF5E7k9xEzMdYgGqaSb1c0c5Yw==} engines: {node: 20 || >=22} @@ -6002,9 +5948,6 @@ packages: protocols@2.0.1: resolution: {integrity: sha512-/XJ368cyBJ7fzLMwLKv1e4vLxOju2MNAIokcr7meSaNcVbWz/CPcW22cP04mwxOErdA5mwjA8Q6w/cdAQxVn7Q==} - protons-runtime@5.5.0: - resolution: {integrity: sha512-EsALjF9QsrEk6gbCx3lmfHxVN0ah7nG3cY7GySD4xf4g8cr7g543zB88Foh897Sr1RQJ9yDCUsoT1i1H/cVUFA==} - protons-runtime@5.6.0: resolution: {integrity: sha512-/Kde+sB9DsMFrddJT/UZWe6XqvL7SL5dbag/DBCElFKhkwDj7XKt53S+mzLyaDP5OqS0wXjV5SA572uWDaT0Hg==} @@ -6381,10 +6324,6 @@ packages: resolution: {integrity: sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==} engines: {node: '>= 14'} - socks@2.7.1: - resolution: {integrity: sha512-7maUZy1N7uo6+WVEX6psASxtNlKaNVMlGQKkG/63nEDdLOWNbiUMoLK7X4uYoLhQstau72mLgfEWcXcwsaHbYQ==} - engines: {node: '>= 10.13.0', npm: '>= 3.0.0'} - socks@2.8.7: resolution: {integrity: sha512-HLpt+uLy/pxB+bum/9DzAgiKS8CX1EvbWxI4zlmgGCExImLdiad2iCwXT5Z4c9c3Eq8rP2318mPW2c+QbtjK8A==} engines: {node: '>= 10.0.0', npm: '>= 3.0.0'} @@ -6803,6 +6742,9 @@ packages: undici-types@7.16.0: resolution: {integrity: sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==} + undici-types@7.18.2: + resolution: {integrity: sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==} + undici@5.29.0: resolution: {integrity: sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg==} engines: {node: '>=14.0'} @@ -7318,11 +7260,6 @@ packages: resolution: {integrity: sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==} engines: {node: '>= 6'} - yaml@2.7.0: - resolution: {integrity: sha512-+hSoy/QHluxmC9kCIJyL/uyFmLmc+e5CFR5Wa+bpIhIj85LVb9ZH2nVnqrHoSvKogwODv0ClqZkmiSSaIH5LTA==} - engines: {node: '>= 14'} - hasBin: true - yaml@2.8.2: resolution: {integrity: sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A==} engines: {node: '>= 14.6'} @@ -7355,10 +7292,6 @@ packages: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} - yocto-queue@1.0.0: - resolution: {integrity: sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==} - engines: {node: '>=12.20'} - yocto-queue@1.2.2: resolution: {integrity: sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==} engines: {node: '>=12.20'} @@ -7613,7 +7546,7 @@ snapshots: debug: 4.4.3 glob: 10.4.5 log-symbols: 7.0.0 - yaml: 2.7.0 + yaml: 2.8.2 yargs: 17.7.2 transitivePeerDependencies: - encoding @@ -7632,7 +7565,7 @@ snapshots: dependencies: '@chainsafe/bls-hd-key': 0.3.0 '@noble/hashes': 1.8.0 - '@scure/bip39': 1.3.0 + '@scure/bip39': 1.6.0 '@chainsafe/bls-keystore@3.1.0': dependencies: @@ -7742,7 +7675,7 @@ snapshots: '@libp2p/crypto': 5.1.13 '@libp2p/interface': 3.1.0 '@libp2p/peer-id': 6.0.4 - '@libp2p/utils': 7.0.10 + '@libp2p/utils': 7.0.11 '@noble/ciphers': 2.1.1 '@noble/curves': 2.0.1 '@noble/hashes': 2.0.1 @@ -8456,7 +8389,7 @@ snapshots: fastify-plugin: 5.0.1 openapi-types: 12.1.3 rfdc: 1.4.1 - yaml: 2.7.0 + yaml: 2.8.2 '@fastify/swagger@9.0.0': dependencies: @@ -8464,7 +8397,7 @@ snapshots: json-schema-resolver: 2.0.0 openapi-types: 12.1.3 rfdc: 1.4.1 - yaml: 2.7.0 + yaml: 2.8.2 transitivePeerDependencies: - supports-color @@ -8586,7 +8519,7 @@ snapshots: '@lerna-lite/core': 4.9.4(@types/node@24.10.1) '@lerna-lite/init': 4.9.4(@types/node@24.10.1) '@lerna-lite/npmlog': 4.9.4 - dedent: 1.7.0 + dedent: 1.7.1 dotenv: 17.2.3 import-local: 3.2.0 load-json-file: 7.0.1 @@ -8642,9 +8575,9 @@ snapshots: '@npmcli/run-script': 10.0.3 ci-info: 4.3.1 config-chain: 1.1.13 - dedent: 1.7.0 + dedent: 1.7.1 execa: 9.6.1 - fs-extra: 11.3.2 + fs-extra: 11.3.3 glob-parent: 6.0.2 json5: 2.2.3 lilconfig: 3.1.3 @@ -8698,7 +8631,7 @@ snapshots: '@lerna-lite/init@4.9.4(@types/node@24.10.1)': dependencies: '@lerna-lite/core': 4.9.4(@types/node@24.10.1) - fs-extra: 11.3.2 + fs-extra: 11.3.3 p-map: 7.0.4 write-json-file: 7.0.0 transitivePeerDependencies: @@ -8741,7 +8674,7 @@ snapshots: dependencies: '@lerna-lite/core': 4.9.4(@types/node@24.10.1) '@lerna-lite/npmlog': 4.9.4 - fs-extra: 11.3.2 + fs-extra: 11.3.3 upath: 2.0.1 transitivePeerDependencies: - '@types/node' @@ -8759,7 +8692,7 @@ snapshots: byte-size: 9.0.1 ci-info: 4.3.1 columnify: 1.6.0 - fs-extra: 11.3.2 + fs-extra: 11.3.3 has-unicode: 2.0.1 libnpmaccess: 10.0.3 libnpmpublish: 11.1.3 @@ -8792,7 +8725,7 @@ snapshots: '@lerna-lite/core': 4.9.4(@types/node@24.10.1) '@lerna-lite/npmlog': 4.9.4 '@lerna-lite/profiler': 4.9.4(@types/node@24.10.1) - fs-extra: 11.3.2 + fs-extra: 11.3.3 p-map: 7.0.4 tinyrainbow: 3.0.3 transitivePeerDependencies: @@ -8818,8 +8751,8 @@ snapshots: conventional-changelog-writer: 8.2.0 conventional-commits-parser: 6.2.1 conventional-recommended-bump: 11.2.0 - dedent: 1.7.0 - fs-extra: 11.3.2 + dedent: 1.7.1 + fs-extra: 11.3.3 git-url-parse: 16.1.0 is-stream: 4.0.1 load-json-file: 7.0.1 @@ -8847,10 +8780,10 @@ snapshots: - conventional-commits-filter - supports-color - '@libp2p/bootstrap@12.0.11': + '@libp2p/bootstrap@12.0.12': dependencies: '@libp2p/interface': 3.1.0 - '@libp2p/interface-internal': 3.0.10 + '@libp2p/interface-internal': 3.0.11 '@libp2p/peer-id': 6.0.4 '@multiformats/multiaddr': 13.0.1 '@multiformats/multiaddr-matcher': 3.0.1 @@ -8866,31 +8799,31 @@ snapshots: uint8arraylist: 2.4.8 uint8arrays: 5.1.0 - '@libp2p/gossipsub@15.0.12': + '@libp2p/gossipsub@15.0.13': dependencies: '@libp2p/crypto': 5.1.13 '@libp2p/interface': 3.1.0 - '@libp2p/interface-internal': 3.0.10 + '@libp2p/interface-internal': 3.0.11 '@libp2p/peer-id': 6.0.4 - '@libp2p/utils': 7.0.10 + '@libp2p/utils': 7.0.11 '@multiformats/multiaddr': 13.0.1 denque: 2.1.0 it-length-prefixed: 10.0.1 it-pipe: 3.0.1 it-pushable: 3.2.3 multiformats: 13.4.2 - protons-runtime: 5.5.0 + protons-runtime: 5.6.0 uint8arraylist: 2.4.8 uint8arrays: 5.1.0 - '@libp2p/identify@4.0.10': + '@libp2p/identify@4.0.11': dependencies: '@libp2p/crypto': 5.1.13 '@libp2p/interface': 3.1.0 - '@libp2p/interface-internal': 3.0.10 + '@libp2p/interface-internal': 3.0.11 '@libp2p/peer-id': 6.0.4 '@libp2p/peer-record': 9.0.5 - '@libp2p/utils': 7.0.10 + '@libp2p/utils': 7.0.11 '@multiformats/multiaddr': 13.0.1 '@multiformats/multiaddr-matcher': 3.0.1 it-drain: 3.0.10 @@ -8900,10 +8833,10 @@ snapshots: uint8arraylist: 2.4.8 uint8arrays: 5.1.0 - '@libp2p/interface-internal@3.0.10': + '@libp2p/interface-internal@3.0.11': dependencies: '@libp2p/interface': 3.1.0 - '@libp2p/peer-collections': 7.0.10 + '@libp2p/peer-collections': 7.0.11 '@multiformats/multiaddr': 13.0.1 progress-events: 1.0.1 @@ -8924,40 +8857,40 @@ snapshots: multiformats: 13.4.2 weald: 1.1.1 - '@libp2p/mdns@12.0.11': + '@libp2p/mdns@12.0.12': dependencies: '@libp2p/interface': 3.1.0 - '@libp2p/interface-internal': 3.0.10 + '@libp2p/interface-internal': 3.0.11 '@libp2p/peer-id': 6.0.4 - '@libp2p/utils': 7.0.10 + '@libp2p/utils': 7.0.11 '@multiformats/multiaddr': 13.0.1 '@types/multicast-dns': 7.2.4 dns-packet: 5.6.1 main-event: 1.0.1 multicast-dns: 7.2.5 - '@libp2p/mplex@12.0.11': + '@libp2p/mplex@12.0.12': dependencies: '@libp2p/interface': 3.1.0 - '@libp2p/utils': 7.0.10 + '@libp2p/utils': 7.0.11 it-pushable: 3.2.3 uint8-varint: 2.0.4 uint8arraylist: 2.4.8 uint8arrays: 5.1.0 - '@libp2p/multistream-select@7.0.10': + '@libp2p/multistream-select@7.0.11': dependencies: '@libp2p/interface': 3.1.0 - '@libp2p/utils': 7.0.10 + '@libp2p/utils': 7.0.11 it-length-prefixed: 10.0.1 uint8arraylist: 2.4.8 uint8arrays: 5.1.0 - '@libp2p/peer-collections@7.0.10': + '@libp2p/peer-collections@7.0.11': dependencies: '@libp2p/interface': 3.1.0 '@libp2p/peer-id': 6.0.4 - '@libp2p/utils': 7.0.10 + '@libp2p/utils': 7.0.11 multiformats: 13.4.2 '@libp2p/peer-id@6.0.4': @@ -8979,11 +8912,11 @@ snapshots: uint8arraylist: 2.4.8 uint8arrays: 5.1.0 - '@libp2p/peer-store@12.0.10': + '@libp2p/peer-store@12.0.11': dependencies: '@libp2p/crypto': 5.1.13 '@libp2p/interface': 3.1.0 - '@libp2p/peer-collections': 7.0.10 + '@libp2p/peer-collections': 7.0.11 '@libp2p/peer-id': 6.0.4 '@libp2p/peer-record': 9.0.5 '@multiformats/multiaddr': 13.0.1 @@ -8996,15 +8929,15 @@ snapshots: uint8arraylist: 2.4.8 uint8arrays: 5.1.0 - '@libp2p/prometheus-metrics@5.0.10': + '@libp2p/prometheus-metrics@5.0.11': dependencies: '@libp2p/interface': 3.1.0 prom-client: 15.1.3 - '@libp2p/tcp@11.0.10': + '@libp2p/tcp@11.0.11': dependencies: '@libp2p/interface': 3.1.0 - '@libp2p/utils': 7.0.10 + '@libp2p/utils': 7.0.11 '@multiformats/multiaddr': 13.0.1 '@multiformats/multiaddr-matcher': 3.0.1 '@types/sinon': 20.0.0 @@ -9013,7 +8946,7 @@ snapshots: progress-events: 1.0.1 uint8arraylist: 2.4.8 - '@libp2p/utils@7.0.10': + '@libp2p/utils@7.0.11': dependencies: '@chainsafe/is-ip': 2.1.0 '@chainsafe/netmask': 2.0.0 @@ -9250,7 +9183,7 @@ snapshots: agent-base: 7.1.3 http-proxy-agent: 7.0.0 https-proxy-agent: 7.0.6 - lru-cache: 11.2.4 + lru-cache: 11.2.5 socks-proxy-agent: 8.0.5 transitivePeerDependencies: - supports-color @@ -9273,8 +9206,8 @@ snapshots: common-ancestor-path: 1.0.1 hosted-git-info: 9.0.2 json-stringify-nice: 1.1.4 - lru-cache: 11.2.4 - minimatch: 10.0.3 + lru-cache: 11.2.5 + minimatch: 10.1.1 nopt: 9.0.0 npm-install-checks: 8.0.0 npm-package-arg: 13.0.2 @@ -9306,7 +9239,7 @@ snapshots: dependencies: '@npmcli/promise-spawn': 9.0.1 ini: 6.0.0 - lru-cache: 11.2.4 + lru-cache: 11.2.5 npm-pick-manifest: 11.0.3 proc-log: 6.1.0 promise-retry: 2.0.1 @@ -9323,7 +9256,7 @@ snapshots: '@npmcli/name-from-folder': 4.0.0 '@npmcli/package-json': 7.0.4 glob: 13.0.0 - minimatch: 10.0.3 + minimatch: 10.1.1 '@npmcli/metavuln-calculator@9.0.3': dependencies: @@ -9618,7 +9551,7 @@ snapshots: ajv: 8.13.0 ajv-draft-04: 1.0.0(ajv@8.13.0) ajv-formats: 3.0.1(ajv@8.13.0) - fs-extra: 11.3.2 + fs-extra: 11.3.3 import-lazy: 4.0.0 jju: 1.4.0 resolve: 1.22.10 @@ -9879,6 +9812,11 @@ snapshots: dependencies: undici-types: 7.16.0 + '@types/node@25.3.2': + dependencies: + undici-types: 7.18.2 + optional: true + '@types/normalize-package-data@2.4.4': {} '@types/parse-path@7.1.0': @@ -9908,9 +9846,9 @@ snapshots: '@types/sinon@20.0.0': dependencies: - '@types/sinonjs__fake-timers': 6.0.2 + '@types/sinonjs__fake-timers': 15.0.1 - '@types/sinonjs__fake-timers@6.0.2': {} + '@types/sinonjs__fake-timers@15.0.1': {} '@types/through@0.0.30': dependencies: @@ -9950,6 +9888,20 @@ snapshots: - utf-8-validate - vite + '@vitest/browser-playwright@4.0.7(playwright@1.56.1)(vite@6.1.6(@types/node@25.3.2)(yaml@2.8.2))(vitest@4.0.7)': + dependencies: + '@vitest/browser': 4.0.7(vite@6.1.6(@types/node@25.3.2)(yaml@2.8.2))(vitest@4.0.7) + '@vitest/mocker': 4.0.7(vite@6.1.6(@types/node@25.3.2)(yaml@2.8.2)) + playwright: 1.56.1 + tinyrainbow: 3.0.3 + vitest: 4.0.7(@types/node@25.3.2)(@vitest/browser-playwright@4.0.7)(jsdom@23.0.1)(yaml@2.8.2) + transitivePeerDependencies: + - bufferutil + - msw + - utf-8-validate + - vite + optional: true + '@vitest/browser@4.0.7(vite@6.1.6(@types/node@24.10.1)(yaml@2.8.2))(vitest@4.0.7)': dependencies: '@vitest/mocker': 4.0.7(vite@6.1.6(@types/node@24.10.1)(yaml@2.8.2)) @@ -9967,6 +9919,24 @@ snapshots: - utf-8-validate - vite + '@vitest/browser@4.0.7(vite@6.1.6(@types/node@25.3.2)(yaml@2.8.2))(vitest@4.0.7)': + dependencies: + '@vitest/mocker': 4.0.7(vite@6.1.6(@types/node@25.3.2)(yaml@2.8.2)) + '@vitest/utils': 4.0.7 + magic-string: 0.30.21 + pixelmatch: 7.1.0 + pngjs: 7.0.0 + sirv: 3.0.2 + tinyrainbow: 3.0.3 + vitest: 4.0.7(@types/node@25.3.2)(@vitest/browser-playwright@4.0.7)(jsdom@23.0.1)(yaml@2.8.2) + ws: 8.18.3 + transitivePeerDependencies: + - bufferutil + - msw + - utf-8-validate + - vite + optional: true + '@vitest/coverage-v8@4.0.7(@vitest/browser@4.0.7(vite@6.1.6(@types/node@24.10.1)(yaml@2.8.2))(vitest@4.0.7))(vitest@4.0.7)': dependencies: '@bcoe/v8-coverage': 1.0.2 @@ -10003,6 +9973,14 @@ snapshots: optionalDependencies: vite: 6.1.6(@types/node@24.10.1)(yaml@2.8.2) + '@vitest/mocker@4.0.7(vite@6.1.6(@types/node@25.3.2)(yaml@2.8.2))': + dependencies: + '@vitest/spy': 4.0.7 + estree-walker: 3.0.3 + magic-string: 0.30.21 + optionalDependencies: + vite: 6.1.6(@types/node@25.3.2)(yaml@2.8.2) + '@vitest/pretty-format@2.1.8': dependencies: tinyrainbow: 1.2.0 @@ -10202,8 +10180,6 @@ snapshots: any-signal@4.1.1: {} - aproba@2.0.0: {} - aproba@2.1.0: {} are-we-there-yet@3.0.0: @@ -10232,7 +10208,7 @@ snapshots: call-bind: 1.0.5 define-properties: 1.2.1 es-abstract: 1.22.3 - get-intrinsic: 1.2.2 + get-intrinsic: 1.3.0 is-array-buffer: 3.0.2 is-shared-array-buffer: 1.0.2 @@ -10472,7 +10448,7 @@ snapshots: '@npmcli/fs': 5.0.0 fs-minipass: 3.0.1 glob: 13.0.0 - lru-cache: 11.2.4 + lru-cache: 11.2.5 minipass: 7.1.2 minipass-collect: 2.0.1 minipass-flush: 1.0.5 @@ -10501,7 +10477,7 @@ snapshots: call-bind@1.0.5: dependencies: function-bind: 1.1.2 - get-intrinsic: 1.2.2 + get-intrinsic: 1.3.0 set-function-length: 1.2.0 call-bound@1.0.4: @@ -10765,12 +10741,6 @@ snapshots: shebang-command: 1.2.0 which: 1.3.1 - cross-spawn@7.0.3: - dependencies: - path-key: 3.1.1 - shebang-command: 2.0.0 - which: 2.0.2 - cross-spawn@7.0.6: dependencies: path-key: 3.1.1 @@ -10856,8 +10826,6 @@ snapshots: dependencies: mimic-response: 3.1.0 - dedent@1.7.0: {} - dedent@1.7.1: {} deepmerge-ts@7.1.5: {} @@ -10873,7 +10841,7 @@ snapshots: define-data-property@1.1.1: dependencies: get-intrinsic: 1.3.0 - gopd: 1.0.1 + gopd: 1.2.0 has-property-descriptors: 1.0.1 define-lazy-prop@2.0.0: {} @@ -11007,13 +10975,13 @@ snapshots: es-set-tostringtag: 2.1.0 es-to-primitive: 1.2.1 function.prototype.name: 1.1.6 - get-intrinsic: 1.2.2 + get-intrinsic: 1.3.0 get-symbol-description: 1.0.0 globalthis: 1.0.3 - gopd: 1.0.1 + gopd: 1.2.0 has-property-descriptors: 1.0.1 has-proto: 1.0.1 - has-symbols: 1.0.3 + has-symbols: 1.1.0 hasown: 2.0.2 internal-slot: 1.0.5 is-array-buffer: 3.0.2 @@ -11360,17 +11328,13 @@ snapshots: optionalDependencies: debug: 4.4.3 - follow-redirects@1.15.6(debug@4.4.3): - optionalDependencies: - debug: 4.4.3 - for-each@0.3.3: dependencies: is-callable: 1.2.7 foreground-child@3.1.1: dependencies: - cross-spawn: 7.0.3 + cross-spawn: 7.0.6 signal-exit: 4.1.0 form-data@2.5.5: @@ -11382,12 +11346,6 @@ snapshots: mime-types: 2.1.35 safe-buffer: 5.2.1 - form-data@4.0.0: - dependencies: - asynckit: 0.4.0 - combined-stream: 1.0.8 - mime-types: 2.1.35 - form-data@4.0.5: dependencies: asynckit: 0.4.0 @@ -11403,12 +11361,6 @@ snapshots: once: 1.4.0 qs: 6.14.1 - fs-extra@11.3.2: - dependencies: - graceful-fs: 4.2.11 - jsonfile: 6.1.0 - universalify: 2.0.0 - fs-extra@11.3.3: dependencies: graceful-fs: 4.2.11 @@ -11452,7 +11404,7 @@ snapshots: gauge@4.0.4: dependencies: - aproba: 2.0.0 + aproba: 2.1.0 color-support: 1.1.3 console-control-strings: 1.1.0 has-unicode: 2.0.1 @@ -11475,13 +11427,6 @@ snapshots: get-func-name@2.0.2: {} - get-intrinsic@1.2.2: - dependencies: - function-bind: 1.1.2 - has-proto: 1.0.1 - has-symbols: 1.0.3 - hasown: 2.0.2 - get-intrinsic@1.3.0: dependencies: call-bind-apply-helpers: 1.0.2 @@ -11512,7 +11457,7 @@ snapshots: get-symbol-description@1.0.0: dependencies: call-bind: 1.0.5 - get-intrinsic: 1.2.2 + get-intrinsic: 1.3.0 git-up@8.1.1: dependencies: @@ -11544,7 +11489,7 @@ snapshots: dependencies: foreground-child: 3.1.1 jackspeak: 4.0.1 - minimatch: 10.0.3 + minimatch: 10.1.1 minipass: 7.1.2 package-json-from-dist: 1.0.0 path-scurry: 2.0.0 @@ -11593,10 +11538,6 @@ snapshots: dependencies: define-properties: 1.2.1 - gopd@1.0.1: - dependencies: - get-intrinsic: 1.2.2 - gopd@1.2.0: {} got@11.8.6: @@ -11640,14 +11581,8 @@ snapshots: has-proto@1.0.1: {} - has-symbols@1.0.3: {} - has-symbols@1.1.0: {} - has-tostringtag@1.0.0: - dependencies: - has-symbols: 1.1.0 - has-tostringtag@1.0.2: dependencies: has-symbols: 1.1.0 @@ -11697,7 +11632,7 @@ snapshots: hosted-git-info@9.0.2: dependencies: - lru-cache: 11.2.4 + lru-cache: 11.2.5 html-encoding-sniffer@4.0.0: dependencies: @@ -11733,7 +11668,7 @@ snapshots: http-proxy@1.18.1(debug@4.4.3): dependencies: eventemitter3: 4.0.7 - follow-redirects: 1.15.6(debug@4.4.3) + follow-redirects: 1.15.11(debug@4.4.3) requires-port: 1.0.0 transitivePeerDependencies: - debug @@ -11779,7 +11714,7 @@ snapshots: ignore-walk@8.0.0: dependencies: - minimatch: 10.0.3 + minimatch: 10.1.1 import-lazy@4.0.0: {} @@ -11838,14 +11773,12 @@ snapshots: internal-slot@1.0.5: dependencies: - get-intrinsic: 1.2.2 + get-intrinsic: 1.3.0 has: 1.0.3 side-channel: 1.1.0 ip-address@10.1.0: {} - ip@2.0.1: {} - ipaddr.js@2.2.0: {} is-arguments@1.1.0: @@ -11855,7 +11788,7 @@ snapshots: is-array-buffer@3.0.2: dependencies: call-bind: 1.0.5 - get-intrinsic: 1.2.2 + get-intrinsic: 1.3.0 is-typed-array: 1.1.12 is-arrayish@0.2.1: {} @@ -11961,7 +11894,7 @@ snapshots: is-symbol@1.0.4: dependencies: - has-symbols: 1.0.3 + has-symbols: 1.1.0 is-typed-array@1.1.12: dependencies: @@ -12124,7 +12057,7 @@ snapshots: cssstyle: 3.0.0 data-urls: 5.0.0 decimal.js: 10.4.3 - form-data: 4.0.0 + form-data: 4.0.5 html-encoding-sniffer: 4.0.0 http-proxy-agent: 7.0.0 https-proxy-agent: 7.0.6 @@ -12249,13 +12182,13 @@ snapshots: '@chainsafe/netmask': 2.0.0 '@libp2p/crypto': 5.1.13 '@libp2p/interface': 3.1.0 - '@libp2p/interface-internal': 3.0.10 + '@libp2p/interface-internal': 3.0.11 '@libp2p/logger': 6.2.2 - '@libp2p/multistream-select': 7.0.10 - '@libp2p/peer-collections': 7.0.10 + '@libp2p/multistream-select': 7.0.11 + '@libp2p/peer-collections': 7.0.11 '@libp2p/peer-id': 6.0.4 - '@libp2p/peer-store': 12.0.10 - '@libp2p/utils': 7.0.10 + '@libp2p/peer-store': 12.0.11 + '@libp2p/utils': 7.0.11 '@multiformats/dns': 1.0.6 '@multiformats/multiaddr': 13.0.1 '@multiformats/multiaddr-matcher': 3.0.1 @@ -12354,10 +12287,6 @@ snapshots: lru-cache@10.4.3: {} - lru-cache@11.0.1: {} - - lru-cache@11.2.4: {} - lru-cache@11.2.5: {} lru-cache@5.1.1: @@ -12849,7 +12778,7 @@ snapshots: dependencies: call-bind: 1.0.5 define-properties: 1.2.1 - has-symbols: 1.0.3 + has-symbols: 1.1.0 object-keys: 1.1.1 obliterator@2.0.4: {} @@ -12912,7 +12841,7 @@ snapshots: p-limit@4.0.0: dependencies: - yocto-queue: 1.0.0 + yocto-queue: 1.2.2 p-limit@7.2.0: dependencies: @@ -13056,7 +12985,7 @@ snapshots: path-scurry@2.0.0: dependencies: - lru-cache: 11.0.1 + lru-cache: 11.2.5 minipass: 7.1.2 path-to-regexp@6.3.0: {} @@ -13220,12 +13149,6 @@ snapshots: protocols@2.0.1: {} - protons-runtime@5.5.0: - dependencies: - uint8-varint: 2.0.4 - uint8arraylist: 2.4.8 - uint8arrays: 5.1.0 - protons-runtime@5.6.0: dependencies: uint8-varint: 2.0.4 @@ -13464,8 +13387,8 @@ snapshots: safe-array-concat@1.0.1: dependencies: call-bind: 1.0.5 - get-intrinsic: 1.2.2 - has-symbols: 1.0.3 + get-intrinsic: 1.3.0 + has-symbols: 1.1.0 isarray: 2.0.5 safe-buffer@5.1.2: {} @@ -13475,7 +13398,7 @@ snapshots: safe-regex-test@1.0.0: dependencies: call-bind: 1.0.5 - get-intrinsic: 1.2.2 + get-intrinsic: 1.3.0 is-regex: 1.1.4 safe-regex2@4.0.0: @@ -13524,8 +13447,8 @@ snapshots: dependencies: define-data-property: 1.1.1 function-bind: 1.1.2 - get-intrinsic: 1.2.2 - gopd: 1.0.1 + get-intrinsic: 1.3.0 + gopd: 1.2.0 has-property-descriptors: 1.0.1 set-function-name@2.0.1: @@ -13638,7 +13561,7 @@ snapshots: dependencies: agent-base: 6.0.2 debug: 4.4.3 - socks: 2.7.1 + socks: 2.8.7 transitivePeerDependencies: - supports-color @@ -13650,11 +13573,6 @@ snapshots: transitivePeerDependencies: - supports-color - socks@2.7.1: - dependencies: - ip: 2.0.1 - smart-buffer: 4.2.0 - socks@2.8.7: dependencies: ip-address: 10.1.0 @@ -14026,7 +13944,7 @@ snapshots: typed-array-buffer@1.0.0: dependencies: call-bind: 1.0.5 - get-intrinsic: 1.2.2 + get-intrinsic: 1.3.0 is-typed-array: 1.1.12 typed-array-byte-length@1.0.0: @@ -14078,13 +13996,16 @@ snapshots: dependencies: call-bind: 1.0.5 has-bigints: 1.0.2 - has-symbols: 1.0.3 + has-symbols: 1.1.0 which-boxed-primitive: 1.0.2 undici-types@6.21.0: {} undici-types@7.16.0: {} + undici-types@7.18.2: + optional: true + undici@5.29.0: dependencies: '@fastify/busboy': 2.1.1 @@ -14227,6 +14148,16 @@ snapshots: fsevents: 2.3.3 yaml: 2.8.2 + vite@6.1.6(@types/node@25.3.2)(yaml@2.8.2): + dependencies: + esbuild: 0.24.2 + postcss: 8.5.6 + rollup: 4.52.5 + optionalDependencies: + '@types/node': 25.3.2 + fsevents: 2.3.3 + yaml: 2.8.2 + vitest-when@0.9.0(@vitest/expect@4.0.7)(vitest@4.0.7): dependencies: pretty-format: 30.2.0 @@ -14274,6 +14205,46 @@ snapshots: - tsx - yaml + vitest@4.0.7(@types/node@25.3.2)(@vitest/browser-playwright@4.0.7)(jsdom@23.0.1)(yaml@2.8.2): + dependencies: + '@vitest/expect': 4.0.7 + '@vitest/mocker': 4.0.7(vite@6.1.6(@types/node@25.3.2)(yaml@2.8.2)) + '@vitest/pretty-format': 4.0.7 + '@vitest/runner': 4.0.7 + '@vitest/snapshot': 4.0.7 + '@vitest/spy': 4.0.7 + '@vitest/utils': 4.0.7 + debug: 4.4.3 + es-module-lexer: 1.7.0 + expect-type: 1.2.2 + magic-string: 0.30.21 + pathe: 2.0.3 + picomatch: 4.0.3 + std-env: 3.10.0 + tinybench: 2.9.0 + tinyexec: 0.3.2 + tinyglobby: 0.2.15 + tinyrainbow: 3.0.3 + vite: 6.1.6(@types/node@25.3.2)(yaml@2.8.2) + why-is-node-running: 2.3.0 + optionalDependencies: + '@types/node': 25.3.2 + '@vitest/browser-playwright': 4.0.7(playwright@1.56.1)(vite@6.1.6(@types/node@25.3.2)(yaml@2.8.2))(vitest@4.0.7) + jsdom: 23.0.1 + transitivePeerDependencies: + - jiti + - less + - lightningcss + - msw + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + - tsx + - yaml + vm-browserify@1.1.2: {} vscode-uri@3.0.8: {} @@ -14542,8 +14513,8 @@ snapshots: available-typed-arrays: 1.0.5 call-bind: 1.0.5 for-each: 0.3.3 - gopd: 1.0.1 - has-tostringtag: 1.0.0 + gopd: 1.2.0 + has-tostringtag: 1.0.2 which@1.3.1: dependencies: @@ -14703,8 +14674,6 @@ snapshots: yaml@1.10.2: {} - yaml@2.7.0: {} - yaml@2.8.2: {} yargs-parser@21.1.1: {} @@ -14739,8 +14708,6 @@ snapshots: yocto-queue@0.1.0: {} - yocto-queue@1.0.0: {} - yocto-queue@1.2.2: {} yoctocolors-cjs@2.1.3: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 65e7c6f24ca9..066840a788f0 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -5,6 +5,9 @@ linkWorkspacePackages: true minimumReleaseAge: 2880 +minimumReleaseAgeExclude: + - "@libp2p/*" + nodeLinker: isolated catalog: From bb33751bfdcaf4a4b24f7c9118ff0bf2790d1aed Mon Sep 17 00:00:00 2001 From: Lodekeeper <258435968+lodekeeper@users.noreply.github.com> Date: Tue, 3 Mar 2026 00:56:36 +0000 Subject: [PATCH 15/63] chore: bump libp2p to 3.1.4 (#8973) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Bump `libp2p` from `3.1.3` to `3.1.4` in Lodestar packages that pin it directly, and align reqresp peer range with the same line. ## What changed - `package.json` - `libp2p`: `3.1.3` -> `3.1.4` - `packages/beacon-node/package.json` - `libp2p`: `3.1.3` -> `3.1.4` - `packages/reqresp/package.json` - `libp2p`: `3.1.3` -> `3.1.4` - `peerDependencies.libp2p`: `^3.1.3` -> `^3.1.4` - `pnpm-lock.yaml` - refreshed after rebasing/merging latest `unstable` and conflict resolution ## Validation - `pnpm build` ✅ - `pnpm lint` ✅ (existing warning-only suppression in light-client test file) AI assistance: OpenClaw/Codex --------- Co-authored-by: lodekeeper Co-authored-by: Nico Flaig --- package.json | 2 +- packages/beacon-node/package.json | 2 +- packages/reqresp/package.json | 4 +- pnpm-lock.yaml | 435 ++++++++++++++++++++---------- 4 files changed, 292 insertions(+), 151 deletions(-) diff --git a/package.json b/package.json index 1c5eeee965b1..a46a30be7eb3 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,7 @@ "https-browserify": "^1.0.0", "inquirer": "^9.1.5", "jsdom": "^23.0.1", - "libp2p": "3.1.3", + "libp2p": "3.1.4", "node-gyp": "^9.4.0", "npm-run-all": "^4.1.5", "path-browserify": "^1.0.1", diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index cc12095a465e..6264540b3627 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -154,7 +154,7 @@ "fastify": "^5.7.4", "interface-datastore": "^9.0.2", "jwt-simple": "0.5.6", - "libp2p": "3.1.3", + "libp2p": "3.1.4", "multiformats": "^13.4.2", "prom-client": "^15.1.0", "qs": "^6.11.1", diff --git a/packages/reqresp/package.json b/packages/reqresp/package.json index b336b40ef0e5..8e1dbb5a14d6 100644 --- a/packages/reqresp/package.json +++ b/packages/reqresp/package.json @@ -62,10 +62,10 @@ "@libp2p/peer-id": "^6.0.4", "@lodestar/logger": "workspace:^", "@lodestar/types": "workspace:^", - "libp2p": "3.1.3" + "libp2p": "3.1.4" }, "peerDependencies": { - "libp2p": "^3.1.3" + "libp2p": "^3.1.4" }, "keywords": [ "ethereum", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0a47b573c52a..45128f8438f2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -49,7 +49,7 @@ importers: version: 1.0.0(@biomejs/biome@2.2.2) '@lerna-lite/cli': specifier: ^4.9.4 - version: 4.10.3(@lerna-lite/exec@4.10.3)(@lerna-lite/publish@4.9.4)(@lerna-lite/run@4.9.4)(@lerna-lite/version@4.9.4)(@types/node@24.10.1) + version: 4.9.4(@lerna-lite/exec@4.10.3)(@lerna-lite/publish@4.9.4)(@lerna-lite/run@4.9.4)(@lerna-lite/version@4.9.4)(@types/node@24.10.1) '@lerna-lite/exec': specifier: ^4.9.4 version: 4.10.3(@lerna-lite/publish@4.9.4)(@lerna-lite/run@4.9.4)(@lerna-lite/version@4.9.4)(@types/node@24.10.1) @@ -105,8 +105,8 @@ importers: specifier: ^23.0.1 version: 23.0.1 libp2p: - specifier: 3.1.3 - version: 3.1.3 + specifier: 3.1.4 + version: 3.1.4 node-gyp: specifier: ^9.4.0 version: 9.4.1 @@ -352,8 +352,8 @@ importers: specifier: 0.5.6 version: 0.5.6 libp2p: - specifier: 3.1.3 - version: 3.1.3 + specifier: 3.1.4 + version: 3.1.4 multiformats: specifier: ^13.4.2 version: 13.4.2 @@ -933,8 +933,8 @@ importers: specifier: workspace:^ version: link:../types libp2p: - specifier: 3.1.3 - version: 3.1.3 + specifier: 3.1.4 + version: 3.1.4 packages/spec-test-util: dependencies: @@ -955,7 +955,7 @@ importers: version: 0.7.0 vitest: specifier: 'catalog:' - version: 4.0.7(@types/node@25.3.2)(@vitest/browser-playwright@4.0.7)(jsdom@23.0.1)(yaml@2.8.2) + version: 4.0.7(@types/node@25.3.3)(@vitest/browser-playwright@4.0.7)(jsdom@23.0.1)(yaml@2.8.2) packages/state-transition: dependencies: @@ -1022,7 +1022,7 @@ importers: version: 0.2.4 vitest: specifier: 'catalog:' - version: 4.0.7(@types/node@25.3.2)(@vitest/browser-playwright@4.0.7)(jsdom@23.0.1)(yaml@2.8.2) + version: 4.0.7(@types/node@25.3.3)(@vitest/browser-playwright@4.0.7)(jsdom@23.0.1)(yaml@2.8.2) devDependencies: '@types/tmp': specifier: ^0.2.3 @@ -1644,6 +1644,10 @@ packages: '@dabh/diagnostics@2.0.2': resolution: {integrity: sha512-+A1YivoVDNNVCdfozHSR8v/jyuuLTMXwjWuxPFlFlUapXoGc+Gj9mDlTDDfrwl7rXCl2tNZ0kE8sIBO6YOn96Q==} + '@dnsquery/dns-packet@6.1.1': + resolution: {integrity: sha512-WXTuFvL3G+74SchFAtz3FgIYVOe196ycvGsMgvSH/8Goptb1qpIQtIuM4SOK9G9lhMWYpHxnXyy544ZhluFOew==} + engines: {node: '>=6'} + '@electron/get@2.0.3': resolution: {integrity: sha512-Qkzpg2s9GnVV2I2BjRksUi43U5e6+zaQMcjoJy0C+C5oxaKl+fmckGDQFtRpZpZV0NQekuZZ+tGz7EA9TVnQtQ==} engines: {node: '>=12'} @@ -2089,8 +2093,8 @@ packages: '@jridgewell/trace-mapping@0.3.9': resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==} - '@leichtgewicht/ip-codec@2.0.4': - resolution: {integrity: sha512-Hcv+nVC0kZnQ3tD9GVu5xSMR4VVYOteQIr/hwFPVEvPdlXqgGEuRjiheChHgdM+JyqdgNcmzZOX/tnl0JOiI7A==} + '@leichtgewicht/ip-codec@2.0.5': + resolution: {integrity: sha512-Vo+PSpZG2/fmgmiNzYK9qWRh8h/CHrwD0mo1h1DzL4yzHNSfWYujGTYsWGreD000gcgmZ7K4Ys6Tx9TxtsKdDw==} '@lerna-lite/cli@4.10.3': resolution: {integrity: sha512-cRiBwXYuQNjHk0KnbMnfcj9T6ncpkr3iUP1EWwbaXQvDlGO6KzuGqm17LUng1tbR5KzqAex8ih3sZvz9paHFZQ==} @@ -2259,8 +2263,8 @@ packages: '@microsoft/tsdoc@0.15.1': resolution: {integrity: sha512-4aErSrCR/On/e5G2hDP0wjooqDdauzEbIq8hIkIe5pXV0rtWJZvdCEKL0ykZxex+IxIwBp0eGeV48hQN07dXtw==} - '@multiformats/dns@1.0.6': - resolution: {integrity: sha512-nt/5UqjMPtyvkG9BQYdJ4GfLK3nMqGpFZOzf4hAmIa0sJh2LlS9YKXZ4FgwBDsaHvzZqR/rUFIywIc7pkHNNuw==} + '@multiformats/dns@1.0.13': + resolution: {integrity: sha512-yr4bxtA3MbvJ+2461kYIYMsiiZj/FIqKI64hE4SdvWJUdWF9EtZLar38juf20Sf5tguXKFUruluswAO6JsjS2w==} '@multiformats/multiaddr-matcher@3.0.1': resolution: {integrity: sha512-jvjwzCPysVTQ53F4KqwmcqZw73BqHMk0UUZrMP9P4OtJ/YHrfs122ikTqhVA2upe0P/Qz9l8HVlhEifVYB2q9A==} @@ -3118,8 +3122,8 @@ packages: '@types/node@24.10.1': resolution: {integrity: sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ==} - '@types/node@25.3.2': - resolution: {integrity: sha512-RpV6r/ij22zRRdyBPcxDeKAzH43phWVKEjL2iksqo1Vz3CuBUrgmPpPhALKiRfU7OMCmeeO9vECBMsV0hMTG8Q==} + '@types/node@25.3.3': + resolution: {integrity: sha512-DpzbrH7wIcBaJibpKo9nnSQL0MTRdnWttGyE5haGwK86xgMOkFLp7vEyfQPGLOJh5wNYiJ3V9PmUMDhV9u8kkQ==} '@types/normalize-package-data@2.4.4': resolution: {integrity: sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA==} @@ -3385,6 +3389,13 @@ packages: resolution: {integrity: sha512-iADenERppdC+A2YKbOXXB2WUeABLaM6qnpZ70kZbPZ1cZMMJ7eF+3CaYm+/PhBizgkzlvssC7QuHS30oOiQYWA==} engines: {node: '>=16.0.0', npm: '>=7.0.0'} + any-signal@4.2.0: + resolution: {integrity: sha512-LndMvYuAPf4rC195lk7oSFuHOYFpOszIYrNYv0gHAvz+aEhE9qPZLhmrIz5pXP2BSsPOXvsuHDXEGaiQhIh9wA==} + engines: {node: '>=16.0.0', npm: '>=7.0.0'} + + aproba@2.0.0: + resolution: {integrity: sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==} + aproba@2.1.0: resolution: {integrity: sha512-tLIEcj5GuR2RSTnxNKdkK0dJ/GrC7P38sUkiDmDuHfsHmbagTFAxDVIBltoklXEVIQ/f14IL8IMJ5pn9Hez1Ew==} @@ -3857,6 +3868,10 @@ packages: resolution: {integrity: sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==} engines: {node: '>=4.8'} + cross-spawn@7.0.3: + resolution: {integrity: sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==} + engines: {node: '>= 8'} + cross-spawn@7.0.6: resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} engines: {node: '>= 8'} @@ -3914,6 +3929,14 @@ packages: resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==} engines: {node: '>=10'} + dedent@1.7.0: + resolution: {integrity: sha512-HGFtf8yhuhGhqO07SV79tRp+br4MnbdjeVxotpn1QBl30pcLLCQjX5b2295ll0fv8RKDKsmWYrl05usHM9CewQ==} + peerDependencies: + babel-plugin-macros: ^3.1.0 + peerDependenciesMeta: + babel-plugin-macros: + optional: true + dedent@1.7.1: resolution: {integrity: sha512-9JmrhGZpOlEgOLdQgSm0zxFaYoQon408V1v49aqTWuXENVlnCuY9JBZcXZiCsZQWDjTm5Qf/nIvAy77mXDAjEg==} peerDependencies: @@ -4161,6 +4184,9 @@ packages: eventemitter3@5.0.1: resolution: {integrity: sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==} + eventemitter3@5.0.4: + resolution: {integrity: sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==} + events@1.1.1: resolution: {integrity: sha512-kEcvvCBByWXGnZy6JUlgAp2gBIUjfCAV6P6TgT1/aaQKcmuAEC4OZTV1I4EWQLz2gxZw76atuVyvHhTxvi0Flw==} engines: {node: '>=0.4.x'} @@ -4307,6 +4333,15 @@ packages: debug: optional: true + follow-redirects@1.15.6: + resolution: {integrity: sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==} + engines: {node: '>=4.0'} + peerDependencies: + debug: '*' + peerDependenciesMeta: + debug: + optional: true + for-each@0.3.3: resolution: {integrity: sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==} @@ -4318,6 +4353,10 @@ packages: resolution: {integrity: sha512-jqdObeR2rxZZbPSGL+3VckHMYtu+f9//KXBsVny6JSX/pa38Fy+bGjuG8eW/H6USNQWhLi8Num++cU2yOCNz4A==} engines: {node: '>= 0.12'} + form-data@4.0.0: + resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==} + engines: {node: '>= 6'} + form-data@4.0.5: resolution: {integrity: sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==} engines: {node: '>= 6'} @@ -4326,6 +4365,10 @@ packages: resolution: {integrity: sha512-CM3GuJ57US06mlpQ47YcunuUZ9jpm8Vx+P2CGt2j7HpgkKZO/DJYQ0Bobim8G6PFQmK5lOqOOdUXboU+h73A4g==} deprecated: 'ACTION REQUIRED: SWITCH TO v3 - v1 and v2 are VULNERABLE! v1 is DEPRECATED FOR OVER 2 YEARS! Use formidable@latest or try formidable-mini for fresh projects' + fs-extra@11.3.2: + resolution: {integrity: sha512-Xr9F6z6up6Ws+NjzMCZc6WXg2YFRlrLP9NQDO3VQrWrfiojdhS56TzueT88ze0uBdCTwEIhQ3ptnmKeWGFAe0A==} + engines: {node: '>=14.14'} + fs-extra@11.3.3: resolution: {integrity: sha512-VWSRii4t0AFm6ixFFmLLx1t7wS1gh+ckoa84aOeapGum0h+EZd1EhEumSB+ZdDLnEPuucsVB9oB7cxJHap6Afg==} engines: {node: '>=14.14'} @@ -4390,6 +4433,9 @@ packages: get-func-name@2.0.2: resolution: {integrity: sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==} + get-intrinsic@1.2.2: + resolution: {integrity: sha512-0gSo4ml/0j98Y3lngkFEot/zhiCeWsbYIlZ+uZOVgzLyLaUw7wxUL+nCTP0XJvJg1AXulJRI3UJi8GsbDuxdGA==} + get-intrinsic@1.3.0: resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==} engines: {node: '>= 0.4'} @@ -4461,6 +4507,9 @@ packages: resolution: {integrity: sha512-sFdI5LyBiNTHjRd7cGPWapiHWMOXKyuBNX/cWJ3NfzrZQVa8GI/8cofCl74AOVqq9W5kNmguTIzJ/1s2gyI9wA==} engines: {node: '>= 0.4'} + gopd@1.0.1: + resolution: {integrity: sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==} + gopd@1.2.0: resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} engines: {node: '>= 0.4'} @@ -4501,10 +4550,18 @@ packages: resolution: {integrity: sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==} engines: {node: '>= 0.4'} + has-symbols@1.0.3: + resolution: {integrity: sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==} + engines: {node: '>= 0.4'} + has-symbols@1.1.0: resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==} engines: {node: '>= 0.4'} + has-tostringtag@1.0.0: + resolution: {integrity: sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==} + engines: {node: '>= 0.4'} + has-tostringtag@1.0.2: resolution: {integrity: sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==} engines: {node: '>= 0.4'} @@ -4684,6 +4741,9 @@ packages: resolution: {integrity: sha512-XXADHxXmvT9+CRxhXg56LJovE+bmWnEWB78LB83VZTprKTmaC5QfruXocxzTZ2Kl0DNwKuBdlIhjL8LeY8Sf8Q==} engines: {node: '>= 12'} + ip@2.0.1: + resolution: {integrity: sha512-lJUL9imLTNi1ZfXT+DU6rBBdbiKGBuay9B6xGSPVjUeQwaH1RIGqef8RZkUtHioLmSNpPR5M4HVKJGm1j8FWVQ==} + ipaddr.js@2.2.0: resolution: {integrity: sha512-Ag3wB2o37wslZS19hZqorUnrnzSkpOVy+IiiDEiTqNubEYpYuHWIf6K4psgN2ZWKExS4xhVCrRVfb/wfW8fWJA==} engines: {node: '>= 10'} @@ -4729,8 +4789,8 @@ packages: engines: {node: '>=8'} hasBin: true - is-electron@2.2.0: - resolution: {integrity: sha512-SpMppC2XR3YdxSzczXReBjqs2zGscWQpBIKqwXYBFic0ERaxNVgwLCHwOLZeESfdJQjX0RDvrJ1lBXX2ij+G1Q==} + is-electron@2.2.2: + resolution: {integrity: sha512-FO/Rhvz5tuw4MCWkpMzHFKWD2LsfHzIb7i6MdPYZ/KW7AlxawyLkqdy+jPZP1WubqEADE3O4FUENlJHDfQASRg==} is-extglob@2.1.1: resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==} @@ -4772,8 +4832,8 @@ packages: resolution: {integrity: sha512-dqJvarLawXsFbNDeJW7zAz8ItJ9cd28YufuuFzh0G8pNHjJMnY08Dv7sYX2uF5UpQOwieAeOExEYAWWfu7ZZUA==} engines: {node: '>= 0.4'} - is-network-error@1.1.0: - resolution: {integrity: sha512-tUdRRAnhT+OtCZR/LxZelH/C7QtjtFrTu5tXCA8pl55eTUElUHT+GPYV8MBMBvea/j+NxQqVt3LbWMRir7Gx9g==} + is-network-error@1.3.1: + resolution: {integrity: sha512-6QCxa49rQbmUWLfk0nuGqzql9U8uaV2H6279bRErPBHe/109hCzsLUBUHfbEtvLIHBd6hyXbgedBSHevm43Edw==} engines: {node: '>=16'} is-number-object@1.0.7: @@ -4927,11 +4987,11 @@ packages: it-pushable@3.2.3: resolution: {integrity: sha512-gzYnXYK8Y5t5b/BnJUr7glfQLO4U5vyb05gPx/TyTw+4Bv1zM9gFk4YsOrnulWefMewlphCjKkakFvj1y99Tcg==} - it-queue@1.1.0: - resolution: {integrity: sha512-aK9unJRIaJc9qiv53LByhF7/I2AuD7Ro4oLfLieVLL9QXNvRx++ANMpv8yCp2UO0KAtBuf70GOxSYb6ElFVRpQ==} + it-queue@1.1.1: + resolution: {integrity: sha512-yeYCV22WF1QDyb3ylw+g3TGEdkmnoHUH2mc12QoGOQuxW4XP1V7Zd3BfsEF1iq2IFBwIK7wCPUcRLTAQVeZ3SQ==} - it-queueless-pushable@2.0.2: - resolution: {integrity: sha512-2BqIt7XvDdgEgudLAdJkdseAwbVSBc0yAd8yPVHrll4eBuJPWIj9+8C3OIxzEKwhswLtd3bi+yLrzgw9gCyxMA==} + it-queueless-pushable@2.0.3: + resolution: {integrity: sha512-USa5EzTvmQswOcVE7+o6qsj2o2G+6KHCxSogPOs23sGYkDWFidhqVO7dAvv6ve/Z+Q+nvxpEa9rrRo6VEK7w4Q==} it-reader@6.0.4: resolution: {integrity: sha512-XCWifEcNFFjjBHtor4Sfaj8rcpt+FkY0L6WdhD578SCDhV4VUm7fCkF3dv5a+fTcfQqvN9BsxBTvWbYO6iCjTg==} @@ -5077,8 +5137,8 @@ packages: resolution: {integrity: sha512-NVPTth/71cfbdYHqypcO9Lt5WFGTzFEcx81lWd7GDJIgZ95ERdYHGUfCtFejHCyqodKsQkNEx2JCkMpreDty/A==} engines: {node: ^20.17.0 || >=22.9.0} - libp2p@3.1.3: - resolution: {integrity: sha512-Jgl6Km1PfFTKR7krDNDxuuxQ6ya3D6VHFOi/XYJA539F62PmbxOQLd+nqbqozwB9BgJVTxaXRVmGTKo7dyrdQw==} + libp2p@3.1.4: + resolution: {integrity: sha512-GiF4ZGFuldDHHEE12R8R0Pvv8mRBmiksgSgMds+zXRmcu08X1fpXfVV/YknuWrLteOIxDdgY2E3Ur6d6XmKWJA==} light-my-request@6.0.0: resolution: {integrity: sha512-kFkFXrmKCL0EEeOmJybMH5amWFd+AFvlvMlvFTRxCUwbhfapZqDmeLMPoWihntnYY6JpoQDE9k+vOzObF1fDqg==} @@ -5151,6 +5211,14 @@ packages: lru-cache@10.4.3: resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} + lru-cache@11.0.1: + resolution: {integrity: sha512-CgeuL5uom6j/ZVrg7G/+1IXqRY8JXX4Hghfy5YE0EhoYQWvndP1kufu58cmZLNIDKnRhZrXfdS9urVWx98AipQ==} + engines: {node: 20 || >=22} + + lru-cache@11.2.4: + resolution: {integrity: sha512-B5Y16Jr9LB9dHVkh6ZevG+vAbOsNOYCX+sXvFWFu7B3Iz5mijW3zdbMyhsh8ANd2mSWBYdJgnqi+mL7/LrOPYg==} + engines: {node: 20 || >=22} + lru-cache@11.2.5: resolution: {integrity: sha512-vFrFJkWtJvJnD5hg+hJvVE8Lh/TcMzKnTgCWmtBipwI5yLX/iX+5UB2tfuyODF5E7k9xEzMdYgGqaSb1c0c5Yw==} engines: {node: 20 || >=22} @@ -5385,9 +5453,9 @@ packages: ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} - ms@3.0.0-canary.1: - resolution: {integrity: sha512-kh8ARjh8rMN7Du2igDRO9QJnqCb2xYTJxyQYK7vJJS4TvLLmsbyhiKpSW+t+y26gyOyMd0riphX0GeWKU3ky5g==} - engines: {node: '>=12.13'} + ms@3.0.0-canary.202508261828: + resolution: {integrity: sha512-NotsCoUCIUkojWCzQff4ttdCfIPoA1UGZsyQbi7KmqkNRfKCrvga8JJi2PknHymHOuor0cJSn/ylj52Cbt2IrQ==} + engines: {node: '>=18'} muggle-string@0.4.1: resolution: {integrity: sha512-VNTrAak/KhO2i8dqqnqnAHOa3cYBwXEZe9h+D5h/1ZqFSTEFHdM65lR7RoIqq3tBBYavsOXV84NoHXZ0AkPyqQ==} @@ -5657,14 +5725,14 @@ packages: resolution: {integrity: sha512-HkPfFklpZQPUKBFXzKFB6ihLriIHxnmuQdK9WmLDwe4hf2PdhhfWT/FJa+pc3bA1ywvKXtedxIRmd4Y7BTXE4w==} engines: {node: '>=12'} - p-queue@8.1.0: - resolution: {integrity: sha512-mxLDbbGIBEXTJL0zEx8JIylaj3xQ7Z/7eEVjcF9fJX4DBiH9oqe+oahYnlKKxm0Ci9TlWTyhSHgygxMxjIB2jw==} - engines: {node: '>=18'} - p-queue@9.0.1: resolution: {integrity: sha512-RhBdVhSwJb7Ocn3e8ULk4NMwBEuOxe+1zcgphUy9c2e5aR/xbEsdVXxHJ3lynw6Qiqu7OINEyHlZkiblEpaq7w==} engines: {node: '>=20'} + p-queue@9.1.0: + resolution: {integrity: sha512-O/ZPaXuQV29uSLbxWBGGZO1mCQXV2BLIwUr59JUU9SoH76mnYvtms7aafH/isNSNGwuEfP6W/4xD0/TJXxrizw==} + engines: {node: '>=20'} + p-reduce@3.0.0: resolution: {integrity: sha512-xsrIUgI0Kn6iyDYm9StOpOeK29XM1aboGji26+QEortiFST1hGZaUQOLhtEbqHErPpGW/aSz6allwK2qcptp0Q==} engines: {node: '>=12'} @@ -5673,10 +5741,6 @@ packages: resolution: {integrity: sha512-J5ApzjyRkkf601HpEeykoiCvzHQjWxPAHhyjFcEUP2SWq0+35NKh8TLhpLw+Dkq5TZBFvUM6UigdE9hIVYTl5w==} engines: {node: '>=20'} - p-timeout@6.1.2: - resolution: {integrity: sha512-UbD77BuZ9Bc9aABo74gfXhNvzC9Tx7SxtHSh1fxvx3jTLLYvmVhiQZZrJzqqU0jKbN32kb5VOKiLEQI/3bIjgQ==} - engines: {node: '>=14.16'} - p-timeout@7.0.1: resolution: {integrity: sha512-AxTM2wDGORHGEkPCt8yqxOTMgpfbEHqF51f/5fJCmwFC3C/zNcGT63SymH2ttOAaiIws2zVg4+izQCjrakcwHg==} engines: {node: '>=20'} @@ -6002,11 +6066,8 @@ packages: resolution: {integrity: sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==} engines: {node: '>=10'} - race-event@1.3.0: - resolution: {integrity: sha512-kaLm7axfOnahIqD3jQ4l1e471FIFcEGebXEnhxyLscuUzV8C94xVHtWEqDDXxll7+yu/6lW0w1Ff4HbtvHvOHg==} - - race-signal@1.1.3: - resolution: {integrity: sha512-Mt2NznMgepLfORijhQMncE26IhkmjEphig+/1fKC0OtaKwys/gpvpmswSjoN01SS+VO951mj0L4VIDXdXsjnfA==} + race-event@1.6.1: + resolution: {integrity: sha512-vi7WH5g5KoTFpu2mme/HqZiWH14XSOtg5rfp6raBskBHl7wnmy3F/biAIyY5MsK+BHWhoPhxtZ1Y2R7OHHaWyQ==} race-signal@2.0.0: resolution: {integrity: sha512-P31bLhE4ByBX/70QDXMutxnqgwrF1WUXea1O8DXuviAgkdbQ1iQMQotNgzJIBC9yUSn08u/acZrMUhgw7w6GpA==} @@ -6324,6 +6385,10 @@ packages: resolution: {integrity: sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==} engines: {node: '>= 14'} + socks@2.7.1: + resolution: {integrity: sha512-7maUZy1N7uo6+WVEX6psASxtNlKaNVMlGQKkG/63nEDdLOWNbiUMoLK7X4uYoLhQstau72mLgfEWcXcwsaHbYQ==} + engines: {node: '>= 10.13.0', npm: '>= 3.0.0'} + socks@2.8.7: resolution: {integrity: sha512-HLpt+uLy/pxB+bum/9DzAgiKS8CX1EvbWxI4zlmgGCExImLdiad2iCwXT5Z4c9c3Eq8rP2318mPW2c+QbtjK8A==} engines: {node: '>= 10.0.0', npm: '>= 3.0.0'} @@ -6809,6 +6874,9 @@ packages: url@0.11.0: resolution: {integrity: sha512-kbailJa29QrtXnxgq+DdCEGlbTeYM2eJUxsz6vjZavrCYPMIFHMKQmSKYAIuUK2i7hgPm28a8piX5NTUtM/LKQ==} + utf8-codec@1.0.0: + resolution: {integrity: sha512-S/QSLezp3qvG4ld5PUfXiH7mCFxLKjSVZRFkB3DOjgwHuJPFDkInAXc/anf7BAbHt/D38ozDzL+QMZ6/7gsI6w==} + util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} @@ -7260,6 +7328,11 @@ packages: resolution: {integrity: sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==} engines: {node: '>= 6'} + yaml@2.7.0: + resolution: {integrity: sha512-+hSoy/QHluxmC9kCIJyL/uyFmLmc+e5CFR5Wa+bpIhIj85LVb9ZH2nVnqrHoSvKogwODv0ClqZkmiSSaIH5LTA==} + engines: {node: '>= 14'} + hasBin: true + yaml@2.8.2: resolution: {integrity: sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A==} engines: {node: '>= 14.6'} @@ -7292,6 +7365,10 @@ packages: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} + yocto-queue@1.0.0: + resolution: {integrity: sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==} + engines: {node: '>=12.20'} + yocto-queue@1.2.2: resolution: {integrity: sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==} engines: {node: '>=12.20'} @@ -7546,7 +7623,7 @@ snapshots: debug: 4.4.3 glob: 10.4.5 log-symbols: 7.0.0 - yaml: 2.8.2 + yaml: 2.7.0 yargs: 17.7.2 transitivePeerDependencies: - encoding @@ -7565,7 +7642,7 @@ snapshots: dependencies: '@chainsafe/bls-hd-key': 0.3.0 '@noble/hashes': 1.8.0 - '@scure/bip39': 1.6.0 + '@scure/bip39': 1.3.0 '@chainsafe/bls-keystore@3.1.0': dependencies: @@ -7846,6 +7923,11 @@ snapshots: enabled: 2.0.0 kuler: 2.0.0 + '@dnsquery/dns-packet@6.1.1': + dependencies: + '@leichtgewicht/ip-codec': 2.0.5 + utf8-codec: 1.0.0 + '@electron/get@2.0.3': dependencies: debug: 4.4.3 @@ -8389,7 +8471,7 @@ snapshots: fastify-plugin: 5.0.1 openapi-types: 12.1.3 rfdc: 1.4.1 - yaml: 2.8.2 + yaml: 2.7.0 '@fastify/swagger@9.0.0': dependencies: @@ -8397,7 +8479,7 @@ snapshots: json-schema-resolver: 2.0.0 openapi-types: 12.1.3 rfdc: 1.4.1 - yaml: 2.8.2 + yaml: 2.7.0 transitivePeerDependencies: - supports-color @@ -8492,7 +8574,7 @@ snapshots: '@jridgewell/resolve-uri': 3.1.1 '@jridgewell/sourcemap-codec': 1.5.5 - '@leichtgewicht/ip-codec@2.0.4': {} + '@leichtgewicht/ip-codec@2.0.5': {} '@lerna-lite/cli@4.10.3(@lerna-lite/exec@4.10.3)(@lerna-lite/publish@4.9.4)(@lerna-lite/run@4.9.4)(@lerna-lite/version@4.9.4)(@types/node@24.10.1)': dependencies: @@ -8519,7 +8601,7 @@ snapshots: '@lerna-lite/core': 4.9.4(@types/node@24.10.1) '@lerna-lite/init': 4.9.4(@types/node@24.10.1) '@lerna-lite/npmlog': 4.9.4 - dedent: 1.7.1 + dedent: 1.7.0 dotenv: 17.2.3 import-local: 3.2.0 load-json-file: 7.0.1 @@ -8575,9 +8657,9 @@ snapshots: '@npmcli/run-script': 10.0.3 ci-info: 4.3.1 config-chain: 1.1.13 - dedent: 1.7.1 + dedent: 1.7.0 execa: 9.6.1 - fs-extra: 11.3.3 + fs-extra: 11.3.2 glob-parent: 6.0.2 json5: 2.2.3 lilconfig: 3.1.3 @@ -8631,7 +8713,7 @@ snapshots: '@lerna-lite/init@4.9.4(@types/node@24.10.1)': dependencies: '@lerna-lite/core': 4.9.4(@types/node@24.10.1) - fs-extra: 11.3.3 + fs-extra: 11.3.2 p-map: 7.0.4 write-json-file: 7.0.0 transitivePeerDependencies: @@ -8674,7 +8756,7 @@ snapshots: dependencies: '@lerna-lite/core': 4.9.4(@types/node@24.10.1) '@lerna-lite/npmlog': 4.9.4 - fs-extra: 11.3.3 + fs-extra: 11.3.2 upath: 2.0.1 transitivePeerDependencies: - '@types/node' @@ -8692,7 +8774,7 @@ snapshots: byte-size: 9.0.1 ci-info: 4.3.1 columnify: 1.6.0 - fs-extra: 11.3.3 + fs-extra: 11.3.2 has-unicode: 2.0.1 libnpmaccess: 10.0.3 libnpmpublish: 11.1.3 @@ -8725,7 +8807,7 @@ snapshots: '@lerna-lite/core': 4.9.4(@types/node@24.10.1) '@lerna-lite/npmlog': 4.9.4 '@lerna-lite/profiler': 4.9.4(@types/node@24.10.1) - fs-extra: 11.3.3 + fs-extra: 11.3.2 p-map: 7.0.4 tinyrainbow: 3.0.3 transitivePeerDependencies: @@ -8751,8 +8833,8 @@ snapshots: conventional-changelog-writer: 8.2.0 conventional-commits-parser: 6.2.1 conventional-recommended-bump: 11.2.0 - dedent: 1.7.1 - fs-extra: 11.3.3 + dedent: 1.7.0 + fs-extra: 11.3.2 git-url-parse: 16.1.0 is-stream: 4.0.1 load-json-file: 7.0.1 @@ -8842,7 +8924,7 @@ snapshots: '@libp2p/interface@3.1.0': dependencies: - '@multiformats/dns': 1.0.6 + '@multiformats/dns': 1.0.13 '@multiformats/multiaddr': 13.0.1 main-event: 1.0.1 multiformats: 13.4.2 @@ -8955,7 +9037,7 @@ snapshots: '@libp2p/logger': 6.2.2 '@multiformats/multiaddr': 13.0.1 '@sindresorhus/fnv1a': 3.1.0 - any-signal: 4.1.1 + any-signal: 4.2.0 cborg: 4.5.8 delay: 7.0.0 is-loopback-addr: 2.0.2 @@ -9009,13 +9091,12 @@ snapshots: '@microsoft/tsdoc@0.15.1': {} - '@multiformats/dns@1.0.6': + '@multiformats/dns@1.0.13': dependencies: - '@types/dns-packet': 5.6.5 - buffer: 6.0.3 - dns-packet: 5.6.1 + '@dnsquery/dns-packet': 6.1.1 + '@libp2p/interface': 3.1.0 hashlru: 2.3.0 - p-queue: 8.1.0 + p-queue: 9.1.0 progress-events: 1.0.1 uint8arrays: 5.1.0 @@ -9183,7 +9264,7 @@ snapshots: agent-base: 7.1.3 http-proxy-agent: 7.0.0 https-proxy-agent: 7.0.6 - lru-cache: 11.2.5 + lru-cache: 11.2.4 socks-proxy-agent: 8.0.5 transitivePeerDependencies: - supports-color @@ -9206,8 +9287,8 @@ snapshots: common-ancestor-path: 1.0.1 hosted-git-info: 9.0.2 json-stringify-nice: 1.1.4 - lru-cache: 11.2.5 - minimatch: 10.1.1 + lru-cache: 11.2.4 + minimatch: 10.0.3 nopt: 9.0.0 npm-install-checks: 8.0.0 npm-package-arg: 13.0.2 @@ -9239,7 +9320,7 @@ snapshots: dependencies: '@npmcli/promise-spawn': 9.0.1 ini: 6.0.0 - lru-cache: 11.2.5 + lru-cache: 11.2.4 npm-pick-manifest: 11.0.3 proc-log: 6.1.0 promise-retry: 2.0.1 @@ -9256,7 +9337,7 @@ snapshots: '@npmcli/name-from-folder': 4.0.0 '@npmcli/package-json': 7.0.4 glob: 13.0.0 - minimatch: 10.1.1 + minimatch: 10.0.3 '@npmcli/metavuln-calculator@9.0.3': dependencies: @@ -9551,7 +9632,7 @@ snapshots: ajv: 8.13.0 ajv-draft-04: 1.0.0(ajv@8.13.0) ajv-formats: 3.0.1(ajv@8.13.0) - fs-extra: 11.3.3 + fs-extra: 11.3.2 import-lazy: 4.0.0 jju: 1.4.0 resolve: 1.22.10 @@ -9770,7 +9851,7 @@ snapshots: '@types/dns-packet@5.6.5': dependencies: - '@types/node': 24.10.1 + '@types/node': 25.3.3 '@types/estree@1.0.8': {} @@ -9798,7 +9879,7 @@ snapshots: '@types/multicast-dns@7.2.4': dependencies: '@types/dns-packet': 5.6.5 - '@types/node': 24.10.1 + '@types/node': 25.3.3 '@types/node@18.15.13': {} @@ -9812,10 +9893,9 @@ snapshots: dependencies: undici-types: 7.16.0 - '@types/node@25.3.2': + '@types/node@25.3.3': dependencies: undici-types: 7.18.2 - optional: true '@types/normalize-package-data@2.4.4': {} @@ -9888,13 +9968,13 @@ snapshots: - utf-8-validate - vite - '@vitest/browser-playwright@4.0.7(playwright@1.56.1)(vite@6.1.6(@types/node@25.3.2)(yaml@2.8.2))(vitest@4.0.7)': + '@vitest/browser-playwright@4.0.7(playwright@1.56.1)(vite@6.1.6(@types/node@25.3.3)(yaml@2.8.2))(vitest@4.0.7)': dependencies: - '@vitest/browser': 4.0.7(vite@6.1.6(@types/node@25.3.2)(yaml@2.8.2))(vitest@4.0.7) - '@vitest/mocker': 4.0.7(vite@6.1.6(@types/node@25.3.2)(yaml@2.8.2)) + '@vitest/browser': 4.0.7(vite@6.1.6(@types/node@25.3.3)(yaml@2.8.2))(vitest@4.0.7) + '@vitest/mocker': 4.0.7(vite@6.1.6(@types/node@25.3.3)(yaml@2.8.2)) playwright: 1.56.1 tinyrainbow: 3.0.3 - vitest: 4.0.7(@types/node@25.3.2)(@vitest/browser-playwright@4.0.7)(jsdom@23.0.1)(yaml@2.8.2) + vitest: 4.0.7(@types/node@25.3.3)(@vitest/browser-playwright@4.0.7)(jsdom@23.0.1)(yaml@2.8.2) transitivePeerDependencies: - bufferutil - msw @@ -9919,16 +9999,16 @@ snapshots: - utf-8-validate - vite - '@vitest/browser@4.0.7(vite@6.1.6(@types/node@25.3.2)(yaml@2.8.2))(vitest@4.0.7)': + '@vitest/browser@4.0.7(vite@6.1.6(@types/node@25.3.3)(yaml@2.8.2))(vitest@4.0.7)': dependencies: - '@vitest/mocker': 4.0.7(vite@6.1.6(@types/node@25.3.2)(yaml@2.8.2)) + '@vitest/mocker': 4.0.7(vite@6.1.6(@types/node@25.3.3)(yaml@2.8.2)) '@vitest/utils': 4.0.7 magic-string: 0.30.21 pixelmatch: 7.1.0 pngjs: 7.0.0 sirv: 3.0.2 tinyrainbow: 3.0.3 - vitest: 4.0.7(@types/node@25.3.2)(@vitest/browser-playwright@4.0.7)(jsdom@23.0.1)(yaml@2.8.2) + vitest: 4.0.7(@types/node@25.3.3)(@vitest/browser-playwright@4.0.7)(jsdom@23.0.1)(yaml@2.8.2) ws: 8.18.3 transitivePeerDependencies: - bufferutil @@ -9973,13 +10053,13 @@ snapshots: optionalDependencies: vite: 6.1.6(@types/node@24.10.1)(yaml@2.8.2) - '@vitest/mocker@4.0.7(vite@6.1.6(@types/node@25.3.2)(yaml@2.8.2))': + '@vitest/mocker@4.0.7(vite@6.1.6(@types/node@25.3.3)(yaml@2.8.2))': dependencies: '@vitest/spy': 4.0.7 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 6.1.6(@types/node@25.3.2)(yaml@2.8.2) + vite: 6.1.6(@types/node@25.3.3)(yaml@2.8.2) '@vitest/pretty-format@2.1.8': dependencies: @@ -10180,6 +10260,10 @@ snapshots: any-signal@4.1.1: {} + any-signal@4.2.0: {} + + aproba@2.0.0: {} + aproba@2.1.0: {} are-we-there-yet@3.0.0: @@ -10208,7 +10292,7 @@ snapshots: call-bind: 1.0.5 define-properties: 1.2.1 es-abstract: 1.22.3 - get-intrinsic: 1.3.0 + get-intrinsic: 1.2.2 is-array-buffer: 3.0.2 is-shared-array-buffer: 1.0.2 @@ -10448,7 +10532,7 @@ snapshots: '@npmcli/fs': 5.0.0 fs-minipass: 3.0.1 glob: 13.0.0 - lru-cache: 11.2.5 + lru-cache: 11.2.4 minipass: 7.1.2 minipass-collect: 2.0.1 minipass-flush: 1.0.5 @@ -10477,7 +10561,7 @@ snapshots: call-bind@1.0.5: dependencies: function-bind: 1.1.2 - get-intrinsic: 1.3.0 + get-intrinsic: 1.2.2 set-function-length: 1.2.0 call-bound@1.0.4: @@ -10741,6 +10825,12 @@ snapshots: shebang-command: 1.2.0 which: 1.3.1 + cross-spawn@7.0.3: + dependencies: + path-key: 3.1.1 + shebang-command: 2.0.0 + which: 2.0.2 + cross-spawn@7.0.6: dependencies: path-key: 3.1.1 @@ -10826,6 +10916,8 @@ snapshots: dependencies: mimic-response: 3.1.0 + dedent@1.7.0: {} + dedent@1.7.1: {} deepmerge-ts@7.1.5: {} @@ -10841,7 +10933,7 @@ snapshots: define-data-property@1.1.1: dependencies: get-intrinsic: 1.3.0 - gopd: 1.2.0 + gopd: 1.0.1 has-property-descriptors: 1.0.1 define-lazy-prop@2.0.0: {} @@ -10894,7 +10986,7 @@ snapshots: dns-packet@5.6.1: dependencies: - '@leichtgewicht/ip-codec': 2.0.4 + '@leichtgewicht/ip-codec': 2.0.5 domain-browser@1.2.0: {} @@ -10975,13 +11067,13 @@ snapshots: es-set-tostringtag: 2.1.0 es-to-primitive: 1.2.1 function.prototype.name: 1.1.6 - get-intrinsic: 1.3.0 + get-intrinsic: 1.2.2 get-symbol-description: 1.0.0 globalthis: 1.0.3 - gopd: 1.2.0 + gopd: 1.0.1 has-property-descriptors: 1.0.1 has-proto: 1.0.1 - has-symbols: 1.1.0 + has-symbols: 1.0.3 hasown: 2.0.2 internal-slot: 1.0.5 is-array-buffer: 3.0.2 @@ -11154,6 +11246,8 @@ snapshots: eventemitter3@5.0.1: {} + eventemitter3@5.0.4: {} + events@1.1.1: {} events@3.3.0: {} @@ -11328,13 +11422,17 @@ snapshots: optionalDependencies: debug: 4.4.3 + follow-redirects@1.15.6(debug@4.4.3): + optionalDependencies: + debug: 4.4.3 + for-each@0.3.3: dependencies: is-callable: 1.2.7 foreground-child@3.1.1: dependencies: - cross-spawn: 7.0.6 + cross-spawn: 7.0.3 signal-exit: 4.1.0 form-data@2.5.5: @@ -11346,6 +11444,12 @@ snapshots: mime-types: 2.1.35 safe-buffer: 5.2.1 + form-data@4.0.0: + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + mime-types: 2.1.35 + form-data@4.0.5: dependencies: asynckit: 0.4.0 @@ -11361,6 +11465,12 @@ snapshots: once: 1.4.0 qs: 6.14.1 + fs-extra@11.3.2: + dependencies: + graceful-fs: 4.2.11 + jsonfile: 6.1.0 + universalify: 2.0.0 + fs-extra@11.3.3: dependencies: graceful-fs: 4.2.11 @@ -11404,7 +11514,7 @@ snapshots: gauge@4.0.4: dependencies: - aproba: 2.1.0 + aproba: 2.0.0 color-support: 1.1.3 console-control-strings: 1.1.0 has-unicode: 2.0.1 @@ -11427,6 +11537,13 @@ snapshots: get-func-name@2.0.2: {} + get-intrinsic@1.2.2: + dependencies: + function-bind: 1.1.2 + has-proto: 1.0.1 + has-symbols: 1.0.3 + hasown: 2.0.2 + get-intrinsic@1.3.0: dependencies: call-bind-apply-helpers: 1.0.2 @@ -11457,7 +11574,7 @@ snapshots: get-symbol-description@1.0.0: dependencies: call-bind: 1.0.5 - get-intrinsic: 1.3.0 + get-intrinsic: 1.2.2 git-up@8.1.1: dependencies: @@ -11489,7 +11606,7 @@ snapshots: dependencies: foreground-child: 3.1.1 jackspeak: 4.0.1 - minimatch: 10.1.1 + minimatch: 10.0.3 minipass: 7.1.2 package-json-from-dist: 1.0.0 path-scurry: 2.0.0 @@ -11538,6 +11655,10 @@ snapshots: dependencies: define-properties: 1.2.1 + gopd@1.0.1: + dependencies: + get-intrinsic: 1.2.2 + gopd@1.2.0: {} got@11.8.6: @@ -11581,8 +11702,14 @@ snapshots: has-proto@1.0.1: {} + has-symbols@1.0.3: {} + has-symbols@1.1.0: {} + has-tostringtag@1.0.0: + dependencies: + has-symbols: 1.1.0 + has-tostringtag@1.0.2: dependencies: has-symbols: 1.1.0 @@ -11632,7 +11759,7 @@ snapshots: hosted-git-info@9.0.2: dependencies: - lru-cache: 11.2.5 + lru-cache: 11.2.4 html-encoding-sniffer@4.0.0: dependencies: @@ -11668,7 +11795,7 @@ snapshots: http-proxy@1.18.1(debug@4.4.3): dependencies: eventemitter3: 4.0.7 - follow-redirects: 1.15.11(debug@4.4.3) + follow-redirects: 1.15.6(debug@4.4.3) requires-port: 1.0.0 transitivePeerDependencies: - debug @@ -11714,7 +11841,7 @@ snapshots: ignore-walk@8.0.0: dependencies: - minimatch: 10.1.1 + minimatch: 10.0.3 import-lazy@4.0.0: {} @@ -11773,12 +11900,14 @@ snapshots: internal-slot@1.0.5: dependencies: - get-intrinsic: 1.3.0 + get-intrinsic: 1.2.2 has: 1.0.3 side-channel: 1.1.0 ip-address@10.1.0: {} + ip@2.0.1: {} + ipaddr.js@2.2.0: {} is-arguments@1.1.0: @@ -11788,7 +11917,7 @@ snapshots: is-array-buffer@3.0.2: dependencies: call-bind: 1.0.5 - get-intrinsic: 1.3.0 + get-intrinsic: 1.2.2 is-typed-array: 1.1.12 is-arrayish@0.2.1: {} @@ -11818,7 +11947,7 @@ snapshots: is-docker@2.2.1: {} - is-electron@2.2.0: {} + is-electron@2.2.2: {} is-extglob@2.1.1: {} @@ -11853,7 +11982,7 @@ snapshots: is-negative-zero@2.0.2: {} - is-network-error@1.1.0: {} + is-network-error@1.3.1: {} is-number-object@1.0.7: dependencies: @@ -11894,7 +12023,7 @@ snapshots: is-symbol@1.0.4: dependencies: - has-symbols: 1.1.0 + has-symbols: 1.0.3 is-typed-array@1.1.12: dependencies: @@ -11975,7 +12104,7 @@ snapshots: it-merge@3.0.12: dependencies: - it-queueless-pushable: 2.0.2 + it-queueless-pushable: 2.0.3 it-parallel-batch@3.0.9: dependencies: @@ -11997,19 +12126,19 @@ snapshots: dependencies: p-defer: 4.0.1 - it-queue@1.1.0: + it-queue@1.1.1: dependencies: abort-error: 1.0.1 it-pushable: 3.2.3 main-event: 1.0.1 - race-event: 1.3.0 - race-signal: 1.1.3 + race-event: 1.6.1 + race-signal: 2.0.0 - it-queueless-pushable@2.0.2: + it-queueless-pushable@2.0.3: dependencies: abort-error: 1.0.1 p-defer: 4.0.1 - race-signal: 1.1.3 + race-signal: 2.0.0 it-reader@6.0.4: dependencies: @@ -12057,7 +12186,7 @@ snapshots: cssstyle: 3.0.0 data-urls: 5.0.0 decimal.js: 10.4.3 - form-data: 4.0.5 + form-data: 4.0.0 html-encoding-sniffer: 4.0.0 http-proxy-agent: 7.0.0 https-proxy-agent: 7.0.6 @@ -12176,7 +12305,7 @@ snapshots: transitivePeerDependencies: - supports-color - libp2p@3.1.3: + libp2p@3.1.4: dependencies: '@chainsafe/is-ip': 2.1.0 '@chainsafe/netmask': 2.0.0 @@ -12189,10 +12318,10 @@ snapshots: '@libp2p/peer-id': 6.0.4 '@libp2p/peer-store': 12.0.11 '@libp2p/utils': 7.0.11 - '@multiformats/dns': 1.0.6 + '@multiformats/dns': 1.0.13 '@multiformats/multiaddr': 13.0.1 '@multiformats/multiaddr-matcher': 3.0.1 - any-signal: 4.1.1 + any-signal: 4.2.0 datastore-core: 11.0.2 interface-datastore: 9.0.2 it-merge: 3.0.12 @@ -12287,6 +12416,10 @@ snapshots: lru-cache@10.4.3: {} + lru-cache@11.0.1: {} + + lru-cache@11.2.4: {} + lru-cache@11.2.5: {} lru-cache@5.1.1: @@ -12521,14 +12654,14 @@ snapshots: mortice@3.3.1: dependencies: abort-error: 1.0.1 - it-queue: 1.1.0 + it-queue: 1.1.1 main-event: 1.0.1 mrmime@2.0.0: {} ms@2.1.3: {} - ms@3.0.0-canary.1: {} + ms@3.0.0-canary.202508261828: {} muggle-string@0.4.1: {} @@ -12778,7 +12911,7 @@ snapshots: dependencies: call-bind: 1.0.5 define-properties: 1.2.1 - has-symbols: 1.1.0 + has-symbols: 1.0.3 object-keys: 1.1.1 obliterator@2.0.4: {} @@ -12841,7 +12974,7 @@ snapshots: p-limit@4.0.0: dependencies: - yocto-queue: 1.2.2 + yocto-queue: 1.0.0 p-limit@7.2.0: dependencies: @@ -12867,23 +13000,21 @@ snapshots: p-pipe@4.0.0: {} - p-queue@8.1.0: + p-queue@9.0.1: dependencies: eventemitter3: 5.0.1 - p-timeout: 6.1.2 + p-timeout: 7.0.1 - p-queue@9.0.1: + p-queue@9.1.0: dependencies: - eventemitter3: 5.0.1 + eventemitter3: 5.0.4 p-timeout: 7.0.1 p-reduce@3.0.0: {} p-retry@7.1.1: dependencies: - is-network-error: 1.1.0 - - p-timeout@6.1.2: {} + is-network-error: 1.3.1 p-timeout@7.0.1: {} @@ -12985,7 +13116,7 @@ snapshots: path-scurry@2.0.0: dependencies: - lru-cache: 11.2.5 + lru-cache: 11.0.1 minipass: 7.1.2 path-to-regexp@6.3.0: {} @@ -13197,9 +13328,9 @@ snapshots: quick-lru@5.1.1: {} - race-event@1.3.0: {} - - race-signal@1.1.3: {} + race-event@1.6.1: + dependencies: + abort-error: 1.0.1 race-signal@2.0.0: {} @@ -13387,8 +13518,8 @@ snapshots: safe-array-concat@1.0.1: dependencies: call-bind: 1.0.5 - get-intrinsic: 1.3.0 - has-symbols: 1.1.0 + get-intrinsic: 1.2.2 + has-symbols: 1.0.3 isarray: 2.0.5 safe-buffer@5.1.2: {} @@ -13398,7 +13529,7 @@ snapshots: safe-regex-test@1.0.0: dependencies: call-bind: 1.0.5 - get-intrinsic: 1.3.0 + get-intrinsic: 1.2.2 is-regex: 1.1.4 safe-regex2@4.0.0: @@ -13447,8 +13578,8 @@ snapshots: dependencies: define-data-property: 1.1.1 function-bind: 1.1.2 - get-intrinsic: 1.3.0 - gopd: 1.2.0 + get-intrinsic: 1.2.2 + gopd: 1.0.1 has-property-descriptors: 1.0.1 set-function-name@2.0.1: @@ -13561,7 +13692,7 @@ snapshots: dependencies: agent-base: 6.0.2 debug: 4.4.3 - socks: 2.8.7 + socks: 2.7.1 transitivePeerDependencies: - supports-color @@ -13573,6 +13704,11 @@ snapshots: transitivePeerDependencies: - supports-color + socks@2.7.1: + dependencies: + ip: 2.0.1 + smart-buffer: 4.2.0 + socks@2.8.7: dependencies: ip-address: 10.1.0 @@ -13944,7 +14080,7 @@ snapshots: typed-array-buffer@1.0.0: dependencies: call-bind: 1.0.5 - get-intrinsic: 1.3.0 + get-intrinsic: 1.2.2 is-typed-array: 1.1.12 typed-array-byte-length@1.0.0: @@ -13996,15 +14132,14 @@ snapshots: dependencies: call-bind: 1.0.5 has-bigints: 1.0.2 - has-symbols: 1.1.0 + has-symbols: 1.0.3 which-boxed-primitive: 1.0.2 undici-types@6.21.0: {} undici-types@7.16.0: {} - undici-types@7.18.2: - optional: true + undici-types@7.18.2: {} undici@5.29.0: dependencies: @@ -14063,6 +14198,8 @@ snapshots: punycode: 1.3.2 querystring: 0.2.0 + utf8-codec@1.0.0: {} + util-deprecate@1.0.2: {} util@0.10.3: @@ -14148,13 +14285,13 @@ snapshots: fsevents: 2.3.3 yaml: 2.8.2 - vite@6.1.6(@types/node@25.3.2)(yaml@2.8.2): + vite@6.1.6(@types/node@25.3.3)(yaml@2.8.2): dependencies: esbuild: 0.24.2 postcss: 8.5.6 rollup: 4.52.5 optionalDependencies: - '@types/node': 25.3.2 + '@types/node': 25.3.3 fsevents: 2.3.3 yaml: 2.8.2 @@ -14205,10 +14342,10 @@ snapshots: - tsx - yaml - vitest@4.0.7(@types/node@25.3.2)(@vitest/browser-playwright@4.0.7)(jsdom@23.0.1)(yaml@2.8.2): + vitest@4.0.7(@types/node@25.3.3)(@vitest/browser-playwright@4.0.7)(jsdom@23.0.1)(yaml@2.8.2): dependencies: '@vitest/expect': 4.0.7 - '@vitest/mocker': 4.0.7(vite@6.1.6(@types/node@25.3.2)(yaml@2.8.2)) + '@vitest/mocker': 4.0.7(vite@6.1.6(@types/node@25.3.3)(yaml@2.8.2)) '@vitest/pretty-format': 4.0.7 '@vitest/runner': 4.0.7 '@vitest/snapshot': 4.0.7 @@ -14225,11 +14362,11 @@ snapshots: tinyexec: 0.3.2 tinyglobby: 0.2.15 tinyrainbow: 3.0.3 - vite: 6.1.6(@types/node@25.3.2)(yaml@2.8.2) + vite: 6.1.6(@types/node@25.3.3)(yaml@2.8.2) why-is-node-running: 2.3.0 optionalDependencies: - '@types/node': 25.3.2 - '@vitest/browser-playwright': 4.0.7(playwright@1.56.1)(vite@6.1.6(@types/node@25.3.2)(yaml@2.8.2))(vitest@4.0.7) + '@types/node': 25.3.3 + '@vitest/browser-playwright': 4.0.7(playwright@1.56.1)(vite@6.1.6(@types/node@25.3.3)(yaml@2.8.2))(vitest@4.0.7) jsdom: 23.0.1 transitivePeerDependencies: - jiti @@ -14269,7 +14406,7 @@ snapshots: weald@1.1.1: dependencies: - ms: 3.0.0-canary.1 + ms: 3.0.0-canary.202508261828 supports-color: 10.2.2 web3-core@4.0.3(encoding@0.1.13): @@ -14498,7 +14635,7 @@ snapshots: wherearewe@2.0.1: dependencies: - is-electron: 2.2.0 + is-electron: 2.2.2 which-boxed-primitive@1.0.2: dependencies: @@ -14513,8 +14650,8 @@ snapshots: available-typed-arrays: 1.0.5 call-bind: 1.0.5 for-each: 0.3.3 - gopd: 1.2.0 - has-tostringtag: 1.0.2 + gopd: 1.0.1 + has-tostringtag: 1.0.0 which@1.3.1: dependencies: @@ -14674,6 +14811,8 @@ snapshots: yaml@1.10.2: {} + yaml@2.7.0: {} + yaml@2.8.2: {} yargs-parser@21.1.1: {} @@ -14708,6 +14847,8 @@ snapshots: yocto-queue@0.1.0: {} + yocto-queue@1.0.0: {} + yocto-queue@1.2.2: {} yoctocolors-cjs@2.1.3: {} From 192806a314cdae6f8335173bf017683e9084444f Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Tue, 3 Mar 2026 12:32:03 +0000 Subject: [PATCH 16/63] feat: integrate gossip validation spec tests (#8965) see https://github.com/ethereum/consensus-specs/pull/4902 --- .../src/chain/validation/attestation.ts | 2 +- .../src/chain/validation/attesterSlashing.ts | 9 + .../beacon-node/src/chain/validation/block.ts | 2 +- .../test/spec/presets/networking.test.ts | 68 ++- .../test/spec/utils/gossipValidation.ts | 549 ++++++++++++++++++ 5 files changed, 605 insertions(+), 25 deletions(-) create mode 100644 packages/beacon-node/test/spec/utils/gossipValidation.ts diff --git a/packages/beacon-node/src/chain/validation/attestation.ts b/packages/beacon-node/src/chain/validation/attestation.ts index 9a2a62a11432..eee57c92d378 100644 --- a/packages/beacon-node/src/chain/validation/attestation.ts +++ b/packages/beacon-node/src/chain/validation/attestation.ts @@ -186,7 +186,7 @@ export async function validateGossipAttestationsSameAttData( chain.seenAttesters.add(targetEpoch, validatorIndex); } else { step0ResultOrErrors[oldIndex] = { - err: new AttestationError(GossipAction.IGNORE, { + err: new AttestationError(GossipAction.REJECT, { code: AttestationErrorCode.INVALID_SIGNATURE, }), }; diff --git a/packages/beacon-node/src/chain/validation/attesterSlashing.ts b/packages/beacon-node/src/chain/validation/attesterSlashing.ts index 3ddfac6166de..4fd27a1cdff3 100644 --- a/packages/beacon-node/src/chain/validation/attesterSlashing.ts +++ b/packages/beacon-node/src/chain/validation/attesterSlashing.ts @@ -2,6 +2,7 @@ import { assertValidAttesterSlashing, getAttesterSlashableIndices, getAttesterSlashingSignatureSets, + isSlashableValidator, } from "@lodestar/state-transition"; import {AttesterSlashing} from "@lodestar/types"; import {AttesterSlashingError, AttesterSlashingErrorCode, GossipAction} from "../errors/index.js"; @@ -58,6 +59,14 @@ export async function validateAttesterSlashing( }); } + const currentEpoch = state.epochCtx.epoch; + if (!intersectingIndices.some((index) => isSlashableValidator(state.validators.getReadonly(index), currentEpoch))) { + throw new AttesterSlashingError(GossipAction.REJECT, { + code: AttesterSlashingErrorCode.INVALID, + error: Error("AttesterSlashing has no slashable validators"), + }); + } + const signatureSets = getAttesterSlashingSignatureSets(chain.config, state.slot, attesterSlashing); if (!(await chain.bls.verifySignatureSets(signatureSets, {batchable: true, priority: prioritizeBls}))) { throw new AttesterSlashingError(GossipAction.REJECT, { diff --git a/packages/beacon-node/src/chain/validation/block.ts b/packages/beacon-node/src/chain/validation/block.ts index 73e6e2944097..dd699caf25fa 100644 --- a/packages/beacon-node/src/chain/validation/block.ts +++ b/packages/beacon-node/src/chain/validation/block.ts @@ -104,7 +104,7 @@ export async function validateGossipBlock( // [REJECT] The block is from a higher slot than its parent. if (parentBlock.slot >= blockSlot) { - throw new BlockGossipError(GossipAction.IGNORE, { + throw new BlockGossipError(GossipAction.REJECT, { code: BlockErrorCode.NOT_LATER_THAN_PARENT, parentSlot: parentBlock.slot, slot: blockSlot, diff --git a/packages/beacon-node/test/spec/presets/networking.test.ts b/packages/beacon-node/test/spec/presets/networking.test.ts index eb5c550f143c..577794bbb224 100644 --- a/packages/beacon-node/test/spec/presets/networking.test.ts +++ b/packages/beacon-node/test/spec/presets/networking.test.ts @@ -1,12 +1,14 @@ +import fs from "node:fs"; import path from "node:path"; +import {expect, it} from "vitest"; import {config} from "@lodestar/config/default"; import {ACTIVE_PRESET} from "@lodestar/params"; -import {InputType} from "@lodestar/spec-test-util"; -import {bigIntToBytes} from "@lodestar/utils"; +import {bigIntToBytes, loadYaml} from "@lodestar/utils"; import {computeColumnsForCustodyGroup, getCustodyGroups} from "../../../src/util/dataColumns.js"; import {ethereumConsensusSpecsTests} from "../specTestVersioning.js"; -import {specTestIterator} from "../utils/specTestIterator.js"; -import {RunnerType, TestRunnerFn} from "../utils/types.js"; +import {isGossipValidationHandler, runGossipValidationTest} from "../utils/gossipValidation.js"; +import {readdirSyncSpec, specTestIterator} from "../utils/specTestIterator.js"; +import {RunnerType, TestRunnerCustom} from "../utils/types.js"; type ComputeColumnForCustodyGroupInput = { custody_group: number; @@ -28,30 +30,50 @@ const networkingFns: Record = { }, }; -const networking: TestRunnerFn = (_fork, testName) => { - return { - testFunction: (testcase) => { - const networkingFn = networkingFns[testName]; - if (networkingFn === undefined) { - throw Error(`No networkingFn for ${testName}`); - } - - return networkingFn(testcase.meta); - }, - options: { - inputTypes: {meta: InputType.YAML}, - getExpected: (testCase) => testCase.meta.result.map(Number), - // Do not manually skip tests here, do it in packages/beacon-node/test/spec/presets/index.test.ts - }, - }; -}; - type NetworkingTestCase = { meta: { result: number[]; }; }; +function loadNetworkingTestMeta(testCaseDir: string): NetworkingTestCase["meta"] { + return loadYaml(fs.readFileSync(path.join(testCaseDir, "meta.yaml"), "utf8")); +} + +function runNetworkingFnTests(testHandler: string, testSuite: string, testSuiteDirpath: string): void { + const networkingFn = networkingFns[testHandler]; + if (networkingFn === undefined) { + throw Error(`No networkingFn for ${testHandler}`); + } + + for (const testCaseName of readdirSyncSpec(testSuiteDirpath)) { + const testCaseDir = path.join(testSuiteDirpath, testCaseName); + it(testCaseName, () => { + const meta = loadNetworkingTestMeta(testCaseDir); + const actual = networkingFn(meta); + expect(actual).toEqualWithMessage( + meta.result.map(Number), + `Unexpected networking result for ${testHandler}/${testSuite}/${testCaseName}` + ); + }); + } +} + +const networking: TestRunnerCustom = (fork, testHandler, testSuite, testSuiteDirpath) => { + if (isGossipValidationHandler(testHandler)) { + for (const testCaseName of readdirSyncSpec(testSuiteDirpath)) { + const testCaseDir = path.join(testSuiteDirpath, testCaseName); + it(testCaseName, async () => { + await runGossipValidationTest(fork, testHandler, testCaseDir); + }, 30_000); + } + } else if (networkingFns[testHandler] !== undefined) { + runNetworkingFnTests(testHandler, testSuite, testSuiteDirpath); + } else { + throw new Error(`No runner for networking handler ${testHandler}`); + } +}; + specTestIterator(path.join(ethereumConsensusSpecsTests.outputDir, "tests", ACTIVE_PRESET), { - networking: {type: RunnerType.default, fn: networking}, + networking: {type: RunnerType.custom, fn: networking}, }); diff --git a/packages/beacon-node/test/spec/utils/gossipValidation.ts b/packages/beacon-node/test/spec/utils/gossipValidation.ts new file mode 100644 index 000000000000..01680f2ffe0b --- /dev/null +++ b/packages/beacon-node/test/spec/utils/gossipValidation.ts @@ -0,0 +1,549 @@ +import {EventEmitter} from "node:events"; +import fs from "node:fs"; +import path from "node:path"; +import {generateKeyPair} from "@libp2p/crypto/keys"; +import snappy from "snappy"; +import {expect} from "vitest"; +import {createBeaconConfig} from "@lodestar/config"; +import {ForkName} from "@lodestar/params"; +import { + BeaconStateAllForks, + computeEpochAtSlot, + computeStartSlotAtEpoch, + createCachedBeaconState, + createPubkeyCache, + isExecutionStateType, + syncPubkeys, +} from "@lodestar/state-transition"; +import {RootHex, sszTypesFor} from "@lodestar/types"; +import {fromHex, loadYaml, toHex, toRootHex} from "@lodestar/utils"; +import {BlockInputPreData, BlockInputSource} from "../../../src/chain/blocks/blockInput/index.js"; +import {AttestationImportOpt, BlobSidecarValidation} from "../../../src/chain/blocks/types.js"; +import {GossipAction, GossipActionError} from "../../../src/chain/errors/gossipValidation.js"; +import {BeaconChain, ChainEvent} from "../../../src/chain/index.js"; +import {defaultChainOptions} from "../../../src/chain/options.js"; +import {validateGossipAggregateAndProof} from "../../../src/chain/validation/aggregateAndProof.js"; +import {GossipAttestation, validateGossipAttestationsSameAttData} from "../../../src/chain/validation/attestation.js"; +import {validateGossipAttesterSlashing} from "../../../src/chain/validation/attesterSlashing.js"; +import {validateGossipBlock} from "../../../src/chain/validation/block.js"; +import {validateGossipProposerSlashing} from "../../../src/chain/validation/proposerSlashing.js"; +import {validateGossipVoluntaryExit} from "../../../src/chain/validation/voluntaryExit.js"; +import {ZERO_HASH_HEX} from "../../../src/constants/constants.js"; +import {ExecutionEngineMockBackend} from "../../../src/execution/engine/mock.js"; +import {getExecutionEngineFromBackend} from "../../../src/execution/index.js"; +import {GossipType} from "../../../src/network/gossip/interface.js"; +import type {IClock} from "../../../src/util/clock.js"; +import {getBeaconAttestationGossipIndex, getSlotFromBeaconAttestationSerialized} from "../../../src/util/sszBytes.js"; +import {getMockedBeaconDb} from "../../mocks/mockedBeaconDb.js"; +import {getConfig} from "../../utils/config.js"; +import {testLogger} from "../../utils/logger.js"; +import {assertCorrectProgressiveBalances} from "../config.js"; + +/** + * A test clock that models gossip clock disparity from a millisecond timestamp. + * Unlike ClockStopped which returns exact slot values, this clock computes + * currentSlotWithGossipDisparity correctly for spec conformance tests. + */ +class GossipTestClock extends EventEmitter implements IClock { + genesisTime: number; + private currentTimeMs: number; + private secondsPerSlot: number; + private maxDisparityMs: number; + + constructor(genesisTimeSec: number, secondsPerSlot: number, maxDisparityMs: number) { + super(); + this.genesisTime = genesisTimeSec; + this.currentTimeMs = genesisTimeSec * 1000; + this.secondsPerSlot = secondsPerSlot; + this.maxDisparityMs = maxDisparityMs; + } + + get currentSlot(): number { + return Math.floor((this.currentTimeMs / 1000 - this.genesisTime) / this.secondsPerSlot); + } + + get currentSlotWithGossipDisparity(): number { + // Model: if we're within maxDisparityMs of next slot, return next slot + // Spec: current_time_ms + MAXIMUM_GOSSIP_CLOCK_DISPARITY >= block_time_ms + // This means: nextSlotTimeMs - currentTimeMs <= maxDisparityMs + const slot = this.currentSlot; + const nextSlotTimeMs = (this.genesisTime + (slot + 1) * this.secondsPerSlot) * 1000; + if (nextSlotTimeMs - this.currentTimeMs <= this.maxDisparityMs) { + return slot + 1; + } + return slot; + } + + get currentEpoch(): number { + return computeEpochAtSlot(this.currentSlot); + } + + slotWithFutureTolerance(toleranceSec: number): number { + return Math.floor((this.currentTimeMs / 1000 + toleranceSec - this.genesisTime) / this.secondsPerSlot); + } + + slotWithPastTolerance(toleranceSec: number): number { + return Math.floor((this.currentTimeMs / 1000 - toleranceSec - this.genesisTime) / this.secondsPerSlot); + } + + isCurrentSlotGivenGossipDisparity(slot: number): boolean { + const current = this.currentSlot; + if (slot === current) return true; + const nextSlotTimeMs = (this.genesisTime + (current + 1) * this.secondsPerSlot) * 1000; + if (nextSlotTimeMs - this.currentTimeMs <= this.maxDisparityMs) { + return slot === current + 1; + } + const currentSlotTimeMs = (this.genesisTime + current * this.secondsPerSlot) * 1000; + if (this.currentTimeMs - currentSlotTimeMs <= this.maxDisparityMs) { + return slot === current - 1; + } + return false; + } + + async waitForSlot(): Promise { + // Not used in tests + } + + secFromSlot(slot: number, toSec?: number): number { + const slotTimeSec = this.genesisTime + slot * this.secondsPerSlot; + return (toSec ?? this.currentTimeMs / 1000) - slotTimeSec; + } + + msFromSlot(slot: number, toMs?: number): number { + const slotTimeMs = (this.genesisTime + slot * this.secondsPerSlot) * 1000; + return (toMs ?? this.currentTimeMs) - slotTimeMs; + } + + /** Set the current time in milliseconds since genesis */ + setCurrentTimeMs(ms: number): void { + this.currentTimeMs = this.genesisTime * 1000 + ms; + } + + /** Also support setSlot for block import phases */ + setSlot(slot: number): void { + this.currentTimeMs = (this.genesisTime + slot * this.secondsPerSlot) * 1000; + } +} + +interface MetaYaml { + topic: GossipType; + blocks?: {block: string; failed?: boolean}[]; + finalized_checkpoint?: {epoch: number; root?: string; block?: string}; + current_time_ms?: number; + messages: { + offset_ms?: number; + subnet_id?: number; + message: string; + expected: "valid" | "ignore" | "reject"; + reason?: string; + }[]; +} + +const gossipTopicByHandler = { + gossip_beacon_block: GossipType.beacon_block, + gossip_beacon_aggregate_and_proof: GossipType.beacon_aggregate_and_proof, + gossip_beacon_attestation: GossipType.beacon_attestation, + gossip_proposer_slashing: GossipType.proposer_slashing, + gossip_attester_slashing: GossipType.attester_slashing, + gossip_voluntary_exit: GossipType.voluntary_exit, +} as const satisfies Record; + +export function isGossipValidationHandler(topicHandler: string): topicHandler is keyof typeof gossipTopicByHandler { + return topicHandler in gossipTopicByHandler; +} + +function getGossipTopic(topicHandler: string): GossipType { + if (!isGossipValidationHandler(topicHandler)) { + throw Error(`Unsupported gossip test handler ${topicHandler}`); + } + return gossipTopicByHandler[topicHandler]; +} + +function loadMeta(testCaseDir: string): MetaYaml { + const raw = fs.readFileSync(path.join(testCaseDir, "meta.yaml"), "utf8"); + return loadYaml(raw); +} + +function loadSszSnappy(testCaseDir: string, name: string): Uint8Array { + const compressed = fs.readFileSync(path.join(testCaseDir, `${name}.ssz_snappy`)); + const decompressed = snappy.uncompressSync(compressed); + return typeof decompressed === "string" ? Buffer.from(decompressed) : decompressed; +} + +function loadState(testCaseDir: string, fork: ForkName): BeaconStateAllForks { + const bytes = loadSszSnappy(testCaseDir, "state"); + return sszTypesFor(fork).BeaconState.deserializeToViewDU(bytes); +} + +type FinalizedCheckpoint = {epoch: number; rootHex: RootHex}; + +function loadBlockRootHex(testCaseDir: string, fork: ForkName, name: string): RootHex { + const signedBlock = sszTypesFor(fork).SignedBeaconBlock.deserialize(loadSszSnappy(testCaseDir, name)); + return toHex(sszTypesFor(fork).BeaconBlock.hashTreeRoot(signedBlock.message)); +} + +function resolveFinalizedCheckpoint( + meta: MetaYaml, + testCaseDir: string, + fork: ForkName, + blockRootsByName: Map +): FinalizedCheckpoint | null { + const cp = meta.finalized_checkpoint; + if (!cp) return null; + + let rootHex: RootHex | null = null; + if (cp.root) { + rootHex = toRootHex(fromHex(cp.root)); + } + if (cp.block) { + const blockRootHex = blockRootsByName.get(cp.block) ?? loadBlockRootHex(testCaseDir, fork, cp.block); + blockRootsByName.set(cp.block, blockRootHex); + if (rootHex !== null && rootHex !== blockRootHex) { + throw new Error(`finalized_checkpoint.root does not match root of ${cp.block}`); + } + rootHex = blockRootHex; + } + + if (rootHex === null) { + throw new Error("finalized_checkpoint must include either root or block"); + } + + if (cp.epoch == null) { + throw new Error("finalized_checkpoint must include an epoch"); + } + return {epoch: Number(cp.epoch), rootHex}; +} + +function setFinalizedCheckpoint(chain: BeaconChain, checkpoint: FinalizedCheckpoint): void { + const checkpointWithHex = { + epoch: checkpoint.epoch, + root: fromHex(checkpoint.rootHex), + rootHex: checkpoint.rootHex, + }; + + const forkChoice = chain.forkChoice as unknown as { + fcStore: { + finalizedCheckpoint: typeof checkpointWithHex; + unrealizedFinalizedCheckpoint: typeof checkpointWithHex; + }; + protoArray: { + finalizedEpoch: number; + finalizedRoot: RootHex; + }; + updateHead?: () => unknown; + }; + + forkChoice.fcStore.finalizedCheckpoint = checkpointWithHex; + forkChoice.fcStore.unrealizedFinalizedCheckpoint = checkpointWithHex; + forkChoice.protoArray.finalizedEpoch = checkpoint.epoch; + forkChoice.protoArray.finalizedRoot = checkpoint.rootHex; + forkChoice.updateHead?.(); +} + +function isDescendantAtFinalizedCheckpoint( + chain: BeaconChain, + blockRootHex: RootHex, + checkpoint: FinalizedCheckpoint +): boolean { + try { + const finalizedSlot = computeStartSlotAtEpoch(checkpoint.epoch); + return chain.forkChoice.getAncestor(blockRootHex, finalizedSlot) === checkpoint.rootHex; + } catch { + return false; + } +} + +function mapErrorToResult(e: unknown): "valid" | "ignore" | "reject" { + if (e instanceof GossipActionError) { + return e.action === GossipAction.IGNORE ? "ignore" : "reject"; + } + // Some validation paths throw raw errors instead of GossipActionError + // (e.g., validator index out of range → TypeError on undefined access). + if (e instanceof TypeError || e instanceof RangeError || e instanceof Error) { + return "reject"; + } + throw e; +} + +export async function runGossipValidationTest( + fork: ForkName, + topicHandler: string, + testCaseDir: string +): Promise { + const meta = loadMeta(testCaseDir); + const topic = getGossipTopic(topicHandler); + if (meta.topic !== topic) { + throw Error(`Gossip test topic mismatch for ${topicHandler}: expected ${topic}, got ${meta.topic}`); + } + + const anchorState = loadState(testCaseDir, fork); + const config = getConfig(fork); + const beaconConfig = createBeaconConfig(config, anchorState.genesisValidatorsRoot); + + const genesisTimeSec = Number(anchorState.genesisTime); + const clock = new GossipTestClock( + genesisTimeSec, + beaconConfig.SLOT_DURATION_MS / 1000, + beaconConfig.MAXIMUM_GOSSIP_CLOCK_DISPARITY + ); + + const controller = new AbortController(); + const executionEngineBackend = new ExecutionEngineMockBackend({ + onlyPredefinedResponses: false, + genesisBlockHash: isExecutionStateType(anchorState) + ? toHex(anchorState.latestExecutionPayloadHeader.blockHash) + : ZERO_HASH_HEX, + }); + const executionEngine = getExecutionEngineFromBackend(executionEngineBackend, { + signal: controller.signal, + logger: testLogger("executionEngine"), + }); + + const pubkeyCache = createPubkeyCache(); + syncPubkeys(pubkeyCache, anchorState.validators.getAllReadonlyValues()); + const cachedState = createCachedBeaconState( + anchorState, + {config: beaconConfig, pubkeyCache}, + {skipSyncPubkeys: true} + ); + + const chain = new BeaconChain( + { + ...defaultChainOptions, + // Disable non-spec maxSkipSlots check for conformance tests + maxSkipSlots: undefined, + blsVerifyAllMainThread: true, + disableArchiveOnCheckpoint: true, + disableLightClientServerOnImportBlockHead: true, + disableOnBlockError: true, + disablePrepareNextSlot: true, + assertCorrectProgressiveBalances, + proposerBoost: true, + proposerBoostReorg: true, + }, + { + privateKey: await generateKeyPair("secp256k1"), + config: beaconConfig, + pubkeyCache, + db: getMockedBeaconDb(), + dataDir: ".", + dbName: ",", + logger: testLogger("spec-gossip"), + processShutdownCallback: () => {}, + clock, + metrics: null, + validatorMonitor: null, + anchorState: cachedState, + isAnchorStateFinalized: true, + executionEngine, + executionBuilder: undefined, + } + ); + + chain.emitter.removeAllListeners(ChainEvent.forkChoiceFinalized); + + try { + const blockRootsByName = new Map(); + + if (meta.blocks) { + for (const blockEntry of meta.blocks) { + const signedBlock = sszTypesFor(fork).SignedBeaconBlock.deserialize( + loadSszSnappy(testCaseDir, blockEntry.block) + ); + const slot = signedBlock.message.slot; + const blockRootHex = toHex(beaconConfig.getForkTypes(slot).BeaconBlock.hashTreeRoot(signedBlock.message)); + blockRootsByName.set(blockEntry.block, blockRootHex); + + if (blockEntry.failed) continue; + + // Skip genesis block — it's already the anchor state + if (slot === 0) continue; + + clock.setSlot(slot); + chain.forkChoice.updateTime(slot); + + const blockImport = BlockInputPreData.createFromBlock({ + forkName: fork, + block: signedBlock, + blockRootHex, + source: BlockInputSource.gossip, + seenTimestampSec: 0, + daOutOfRange: false, + }); + + await chain.processBlock(blockImport, { + seenTimestampSec: 0, + validBlobSidecars: BlobSidecarValidation.Full, + importAttestations: AttestationImportOpt.Force, + validSignatures: false, + }); + } + } + + const finalizedCheckpoint = resolveFinalizedCheckpoint(meta, testCaseDir, fork, blockRootsByName); + if (finalizedCheckpoint) { + setFinalizedCheckpoint(chain, finalizedCheckpoint); + } + + const failedBlockRoots = new Set( + (meta.blocks ?? []) + .filter((blockEntry) => blockEntry.failed === true) + .map((blockEntry) => { + const rootHex = blockRootsByName.get(blockEntry.block); + if (!rootHex) throw new Error(`Missing cached root for block ${blockEntry.block}`); + return rootHex; + }) + ); + + const baseCurrentTimeMs = Number(meta.current_time_ms ?? 0); + for (const message of meta.messages) { + const messageTimeMs = baseCurrentTimeMs + Number(message.offset_ms ?? 0); + clock.setCurrentTimeMs(messageTimeMs); + + let result: "valid" | "ignore" | "reject"; + try { + await validateMessageForTopic(chain, fork, topic, testCaseDir, message, failedBlockRoots, finalizedCheckpoint); + result = "valid"; + } catch (e) { + result = mapErrorToResult(e); + } + + expect(result).toEqualWithMessage( + message.expected, + `Unexpected gossip result for ${topicHandler}/${path.basename(testCaseDir)}/${message.message}` + ); + } + } finally { + controller.abort(); + await chain.close(); + } +} + +async function validateMessageForTopic( + chain: BeaconChain, + fork: ForkName, + topic: GossipType, + testCaseDir: string, + message: MetaYaml["messages"][number], + failedBlockRoots: Set, + finalizedCheckpoint: FinalizedCheckpoint | null +): Promise { + const bytes = rejectOnInvalidSerializedBytes(() => loadSszSnappy(testCaseDir, message.message)); + + switch (topic) { + case GossipType.beacon_block: { + const signedBlock = rejectOnInvalidSerializedBytes(() => sszTypesFor(fork).SignedBeaconBlock.deserialize(bytes)); + const parentRootHex = toRootHex(signedBlock.message.parentRoot); + + if (failedBlockRoots.has(parentRootHex)) { + throw new GossipActionError(GossipAction.REJECT, {code: "SPEC_PARENT_BLOCK_FAILED"}); + } + + if ( + finalizedCheckpoint !== null && + !isDescendantAtFinalizedCheckpoint(chain, parentRootHex, finalizedCheckpoint) + ) { + throw new GossipActionError(GossipAction.REJECT, {code: "SPEC_FINALIZED_NOT_ANCESTOR"}); + } + + await validateGossipBlock(chain.config, chain, signedBlock, fork); + chain.seenBlockProposers.add(signedBlock.message.slot, signedBlock.message.proposerIndex); + break; + } + + case GossipType.beacon_aggregate_and_proof: { + const aggregate = rejectOnInvalidSerializedBytes(() => + sszTypesFor(fork).SignedAggregateAndProof.deserialize(bytes) + ); + const beaconBlockRootHex = toRootHex(aggregate.message.aggregate.data.beaconBlockRoot); + + if (failedBlockRoots.has(beaconBlockRootHex)) { + throw new GossipActionError(GossipAction.REJECT, {code: "SPEC_BLOCK_FAILED_VALIDATION"}); + } + + if ( + finalizedCheckpoint !== null && + !isDescendantAtFinalizedCheckpoint(chain, beaconBlockRootHex, finalizedCheckpoint) + ) { + throw new GossipActionError(GossipAction.IGNORE, {code: "SPEC_FINALIZED_NOT_ANCESTOR"}); + } + + await validateGossipAggregateAndProof(fork, chain, aggregate, bytes); + break; + } + + case GossipType.beacon_attestation: { + const attestation = rejectOnInvalidSerializedBytes(() => sszTypesFor(fork).Attestation.deserialize(bytes)); + const beaconBlockRootHex = toRootHex(attestation.data.beaconBlockRoot); + + if (failedBlockRoots.has(beaconBlockRootHex)) { + throw new GossipActionError(GossipAction.REJECT, {code: "SPEC_BLOCK_FAILED_VALIDATION"}); + } + + if ( + finalizedCheckpoint !== null && + !isDescendantAtFinalizedCheckpoint(chain, beaconBlockRootHex, finalizedCheckpoint) + ) { + throw new GossipActionError(GossipAction.IGNORE, {code: "SPEC_FINALIZED_NOT_ANCESTOR"}); + } + + const attDataBase64 = getBeaconAttestationGossipIndex(fork, bytes); + const attSlot = getSlotFromBeaconAttestationSerialized(fork, bytes); + if (attDataBase64 == null || attSlot == null) { + throw new GossipActionError(GossipAction.REJECT, {code: "SPEC_INVALID_ATTESTATION_SERIALIZATION"}); + } + + const gossipAttestation: GossipAttestation = { + attestation: null, + serializedData: bytes, + attSlot, + attDataBase64, + subnet: Number(message.subnet_id ?? 0), + }; + + const batchResult = await validateGossipAttestationsSameAttData(fork, chain, [gossipAttestation]); + const first = batchResult.results[0]; + if (first?.err) throw first.err; + break; + } + + case GossipType.proposer_slashing: { + const slashing = rejectOnInvalidSerializedBytes(() => sszTypesFor(fork).ProposerSlashing.deserialize(bytes)); + await validateGossipProposerSlashing(chain, slashing); + // Mirror gossip handler: insert into opPool so duplicate detection works + chain.opPool.insertProposerSlashing(slashing); + break; + } + + case GossipType.attester_slashing: { + const slashing = rejectOnInvalidSerializedBytes(() => sszTypesFor(fork).AttesterSlashing.deserialize(bytes)); + await validateGossipAttesterSlashing(chain, slashing); + // Mirror gossip handler: insert into opPool + fork choice + chain.opPool.insertAttesterSlashing(fork, slashing); + chain.forkChoice.onAttesterSlashing(slashing); + break; + } + + case GossipType.voluntary_exit: { + const exit = rejectOnInvalidSerializedBytes(() => sszTypesFor(fork).SignedVoluntaryExit.deserialize(bytes)); + await validateGossipVoluntaryExit(chain, exit); + // Mirror gossip handler: insert into opPool so duplicate detection works + chain.opPool.insertVoluntaryExit(exit); + break; + } + + default: + throw new Error(`Unknown gossip topic: ${topic}`); + } +} + +function rejectOnInvalidSerializedBytes(fn: () => T): T { + try { + return fn(); + } catch (e) { + if (e instanceof Error) { + throw new GossipActionError(GossipAction.REJECT, {code: "SPEC_INVALID_SERIALIZED_BYTES"}); + } + throw e; + } +} From 09945f65899e09e4da151ebea109b2414a8218fe Mon Sep 17 00:00:00 2001 From: NC <17676176+ensi321@users.noreply.github.com> Date: Tue, 3 Mar 2026 16:41:53 -0800 Subject: [PATCH 17/63] feat: implement epbs fork choice (#8739) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Gloas Fork Choice Implementation (ePBS) This PR implements most of the [Gloas fork choice specification](https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.0/specs/gloas/fork-choice.md). Blocks can now exist in multiple payload status variants (PENDING, EMPTY, FULL), allowing the beacon chain to progress before execution payloads arrive. For high level concept please read https://devlog.lodestar.casa/gloas-fork-choice ## Key Concept: Payload Status Variants **New enum** (`packages/fork-choice/src/protoArray/interface.ts:27-32`): ```typescript enum PayloadStatus { PENDING = 0, // Canonical variant, created when beacon block arrives EMPTY = 1, // Block without execution payload FULL = 2 // Block with execution payload } ``` - **Pre-Gloas**: Only FULL exists (payload embedded in block) - **Gloas**: PENDING + EMPTY created initially, FULL added when payload arrives --- ## Major Changes ### 1. Multi-Variant Node Storage **ProtoArray indices structure** (`packages/fork-choice/src/protoArray/protoArray.ts:41-50`): ```typescript // Before: indices = Map // After: indices = Map // Array: [PENDING_INDEX, EMPTY_INDEX, FULL_INDEX] ``` **Pre-Gloas**: `variants[0]` = FULL index (single variant) **Gloas**: `variants[PayloadStatus.PENDING/EMPTY/FULL]` = respective indices ### 2. Block Creation (`onBlock`) **Pre-Gloas** (`packages/fork-choice/src/protoArray/protoArray.ts:428-451`): - Creates single FULL node - Parent = parent block's FULL **Gloas** (`packages/fork-choice/src/protoArray/protoArray.ts:352-428`): - Creates PENDING + EMPTY nodes - PENDING parent = parent block's EMPTY or FULL (inter-block edge, determined by `parentBlockHash`) - EMPTY parent = own PENDING (intra-block edge) - Initializes PTC votes **Fork transition**: First Gloas block's PENDING correctly points to last Fulu block's FULL. ### 3. Payload Arrival (`onExecutionPayload`) When execution payload arrives (`packages/fork-choice/src/protoArray/protoArray.ts:519-585`): - Creates FULL variant as sibling to EMPTY - FULL parent = own PENDING (intra-block edge) - Updates bestChild pointers ### 4. PTC (Payload Timeliness Committee) **New voting mechanism** (`packages/fork-choice/src/protoArray/protoArray.ts:122-151, 753-823`): ```typescript ptcVote = Map // PTC_SIZE votes per block notifyPtcMessage(blockRoot, ptcIndices[], payloadPresent) isPayloadTimely(blockRoot) // true if >50% voted yes AND FULL exists ``` Used by `shouldExtendPayload()` to determine FULL vs EMPTY preference. ### 5. Parent Payload Status Determination **`getParentPayloadStatus()`** (`packages/fork-choice/src/protoArray/protoArray.ts:173-203`): ```typescript // Compare parentBlockHash from child's bid with parent's execution hash if (block.parentBlockHash == parent.executionPayloadBlockHash) return FULL // Child extends parent with payload else return EMPTY // Child extends parent without payload ``` ### 6. Head Selection Changes **`findHead()`** (`packages/fork-choice/src/protoArray/protoArray.ts:882-946`): - Now returns **ProtoNode** instead of `RootHex` - Starts from justified checkpoint's default variant - May return PENDING/EMPTY/FULL variant **`getAncestor()`** (`packages/fork-choice/src/protoArray/protoArray.ts:1327-1401`): - Now returns **ProtoNode** instead of `RootHex` - Determines correct parent variant based on `parentBlockHash` ### 7. EMPTY vs FULL Tiebreaker For comparing variants of **same block** from slot n-1 or n (`packages/fork-choice/src/protoArray/protoArray.ts:854-874, 1132-1299`): ```typescript getPayloadStatusTiebreaker(node, currentSlot, proposerBoostRoot): if (node.payloadStatus == PENDING) return 0 if (node.slot + 1 != currentSlot) return node.payloadStatus // For slot n-1: use should_extend_payload() logic if (node.payloadStatus == EMPTY) return 1 return shouldExtendPayload(node.blockRoot, proposerBoostRoot) ? 2 : 0 ``` **`shouldExtendPayload()`** returns true if: 1. Payload is timely (PTC >50%), OR 2. No proposer boost, OR 3. Proposer boost parent ≠ this block, OR 4. Proposer boost extends FULL parent ### 8. Vote Tracking (Epoch → Slot) **Changed to slot-based tracking** (`packages/fork-choice/src/forkChoice/forkChoice.ts:99-126`): ```typescript // Before: voteNextEpochs: Epoch[] // After: voteNextSlots: Slot[] addLatestMessage(validatorIndex, nextSlot, nextRoot, nextPayloadStatus) // Queued attestations now track payload status per validator queuedAttestations: Map>> ``` **Attestation interpretation** (Gloas) (`packages/fork-choice/src/forkChoice/forkChoice.ts:836-876`): - `slot == block.slot` → vote PENDING - `slot > block.slot && index == 0` → vote EMPTY - `slot > block.slot && index == 1` → vote FULL **Note**: `voteCurrentIndices` and `voteNextIndices` now point to **exact variant node indices**, not just block indices. ### 9. Tree Traversal (Omit Variants) All tree walking methods **filter to default variants only** to avoid noise from EMPTY/FULL siblings (`packages/fork-choice/src/protoArray/protoArray.ts:1430-1632`): ```typescript private isDefaultVariant = (node: ProtoNode): boolean => { return node.payloadStatus === this.getDefaultVariant(node.blockRoot); }; ``` **Affected methods**: - `iterateAncestorNodes()` - Only yields PENDING (Gloas) or FULL (pre-Gloas) - `getAllAncestorNodes()`, `getAllNonAncestorNodes()` - Filter to default variants - `forwardIterateDescendants()` - Uses minimum variant index **Why**: EMPTY/FULL are payload variants, not distinct branches. Including them would make chains 3x longer with redundant entries. Tree operations care about **beacon block relationships**, not payload status. ### 10. Pruning Pruning now (`packages/fork-choice/src/protoArray/protoArray.ts:964-1022`): - Removes all variants of finalized blocks - Adjusts all variant indices in the Map - Cleans up PTC votes for pruned blocks ### 11. Fork Choice API Changes - Renamed getBlock() → getBlockDefaultStatus() throughout the codebase for calls that don't specify a variant - Updated getBlock() signature to require explicit PayloadStatus parameter: getBlock(blockRoot, payloadStatus) - Added new getBlockHexAndBlockHash() method to find blocks matching both beacon block root and execution payload block ha --- ## Node Relationships ``` Pre-Gloas: Block A [FULL] → Block B [FULL] → Block C [FULL] Gloas: Block A [PENDING] → Block B [PENDING] → Block C [PENDING] ↓ ↓ ↓ [EMPTY] [EMPTY] [EMPTY] ↓ ↓ ↓ [FULL] [FULL] [FULL] Inter-block: PENDING → parent's EMPTY/FULL (based on parentBlockHash) Intra-block: EMPTY/FULL → own PENDING ``` --- ## Testing New test file `packages/fork-choice/test/unit/protoArray/gloas.test.ts` covers: - Pre-Gloas backward compatibility - Gloas block creation (PENDING + EMPTY) - Fork transition (Fulu → Gloas) - `onExecutionPayload()` (FULL creation) - PTC voting and timeliness - Parent relationships - EMPTY vs FULL tiebreaker --- ## Outstanding Items - Proposer boost --------- Co-authored-by: twoeths <10568965+twoeths@users.noreply.github.com> Co-authored-by: Tuyen Nguyen Co-authored-by: Claude Sonnet 4.6 --- .../src/api/impl/beacon/blocks/index.ts | 5 +- .../src/api/impl/validator/index.ts | 4 +- .../archiveStore/utils/updateBackfillRange.ts | 2 +- .../src/chain/blocks/importBlock.ts | 37 +- .../chain/blocks/verifyBlocksSanityChecks.ts | 9 +- packages/beacon-node/src/chain/chain.ts | 12 +- packages/beacon-node/src/chain/emitter.ts | 6 +- .../chain/errors/executionPayloadEnvelope.ts | 8 +- .../beacon-node/src/chain/forkChoice/index.ts | 60 +- .../opPools/aggregatedAttestationPool.ts | 2 +- .../chain/produceBlock/produceBlockBody.ts | 3 +- .../beacon-node/src/chain/regen/queued.ts | 9 +- packages/beacon-node/src/chain/regen/regen.ts | 10 +- .../src/chain/validation/aggregateAndProof.ts | 2 +- .../src/chain/validation/attestation.ts | 4 +- .../src/chain/validation/blobSidecar.ts | 4 +- .../beacon-node/src/chain/validation/block.ts | 11 +- .../src/chain/validation/dataColumnSidecar.ts | 2 +- .../chain/validation/executionPayloadBid.ts | 3 +- .../validation/executionPayloadEnvelope.ts | 8 +- .../validation/payloadAttestationMessage.ts | 3 +- .../src/network/processor/gossipHandlers.ts | 5 + .../reqresp/handlers/blobSidecarsByRoot.ts | 2 +- .../handlers/dataColumnSidecarsByRoot.ts | 2 +- .../test/mocks/mockedBeaconChain.ts | 3 + .../opPools/aggregatedAttestationPool.test.ts | 39 +- .../test/spec/presets/fork_choice.test.ts | 2 +- .../test/spec/utils/gossipValidation.ts | 2 +- .../opPools/aggregatedAttestationPool.test.ts | 2 +- .../api/impl/validator/produceBlockV3.test.ts | 4 +- .../blocks/verifyBlocksSanityChecks.test.ts | 24 +- .../unit/chain/forkChoice/forkChoice.test.ts | 8 +- .../chain/seenCache/seenBlockInput.test.ts | 3 + .../test/unit/chain/validation/block.test.ts | 56 +- packages/beacon-node/test/utils/state.ts | 6 +- .../test/utils/validationData/attestation.ts | 20 +- packages/fork-choice/src/forkChoice/errors.ts | 9 +- .../fork-choice/src/forkChoice/forkChoice.ts | 469 ++++++-- .../fork-choice/src/forkChoice/interface.ts | 71 +- packages/fork-choice/src/forkChoice/store.ts | 72 +- packages/fork-choice/src/index.ts | 12 +- .../src/protoArray/computeDeltas.ts | 6 + packages/fork-choice/src/protoArray/errors.ts | 8 +- .../fork-choice/src/protoArray/interface.ts | 39 +- .../fork-choice/src/protoArray/protoArray.ts | 1007 +++++++++++++++-- .../test/perf/forkChoice/updateHead.test.ts | 8 +- .../fork-choice/test/perf/forkChoice/util.ts | 44 +- .../test/unit/forkChoice/forkChoice.test.ts | 56 +- .../unit/forkChoice/getProposerHead.test.ts | 44 +- .../shouldOverrideForkChoiceUpdate.test.ts | 46 +- .../protoArray/executionStatusUpdates.test.ts | 24 +- .../unit/protoArray/getCommonAncestor.test.ts | 21 +- .../test/unit/protoArray/gloas.test.ts | 625 ++++++++++ .../test/unit/protoArray/protoArray.test.ts | 23 +- 54 files changed, 2530 insertions(+), 436 deletions(-) create mode 100644 packages/fork-choice/test/unit/protoArray/gloas.test.ts diff --git a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts index 37d34f6c3345..af41e4d33fbc 100644 --- a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts @@ -1,5 +1,6 @@ import {routes} from "@lodestar/api"; import {ApiError, ApplicationMethods} from "@lodestar/api/server"; +import {PayloadStatus} from "@lodestar/fork-choice"; import { BUILDER_INDEX_SELF_BUILD, ForkPostBellatrix, @@ -205,7 +206,7 @@ export function getBeaconBlockApi({ case routes.beacon.BroadcastValidation.consensus: { // check if this beacon node produced the block else run validations if (!blockLocallyProduced) { - const parentBlock = chain.forkChoice.getBlock(signedBlock.message.parentRoot); + const parentBlock = chain.forkChoice.getBlockDefaultStatus(signedBlock.message.parentRoot); if (parentBlock === null) { chain.emitter.emit(ChainEvent.unknownParent, { blockInput: blockForImport, @@ -650,7 +651,7 @@ export function getBeaconBlockApi({ } // TODO GLOAS: review checks, do we want to implement `broadcast_validation`? - const block = chain.forkChoice.getBlockHex(blockRootHex); + const block = chain.forkChoice.getBlockHex(blockRootHex, PayloadStatus.EMPTY); if (block === null) { throw new ApiError(404, `Block not found for beacon block root ${blockRootHex}`); } diff --git a/packages/beacon-node/src/api/impl/validator/index.ts b/packages/beacon-node/src/api/impl/validator/index.ts index 023ca07fd46e..3312ebc7cae3 100644 --- a/packages/beacon-node/src/api/impl/validator/index.ts +++ b/packages/beacon-node/src/api/impl/validator/index.ts @@ -389,7 +389,7 @@ export function getValidatorApi( */ function notOnOptimisticBlockRoot(beaconBlockRoot: Root): void { - const protoBeaconBlock = chain.forkChoice.getBlock(beaconBlockRoot); + const protoBeaconBlock = chain.forkChoice.getBlockDefaultStatus(beaconBlockRoot); if (!protoBeaconBlock) { throw new ApiError(404, `Block not in forkChoice, beaconBlockRoot=${toRootHex(beaconBlockRoot)}`); } @@ -401,7 +401,7 @@ export function getValidatorApi( } function notOnOutOfRangeData(beaconBlockRoot: Root): void { - const protoBeaconBlock = chain.forkChoice.getBlock(beaconBlockRoot); + const protoBeaconBlock = chain.forkChoice.getBlockDefaultStatus(beaconBlockRoot); if (!protoBeaconBlock) { throw new ApiError(404, `Block not in forkChoice, beaconBlockRoot=${toRootHex(beaconBlockRoot)}`); } diff --git a/packages/beacon-node/src/chain/archiveStore/utils/updateBackfillRange.ts b/packages/beacon-node/src/chain/archiveStore/utils/updateBackfillRange.ts index 49ddc92b4941..1ae4d5540ace 100644 --- a/packages/beacon-node/src/chain/archiveStore/utils/updateBackfillRange.ts +++ b/packages/beacon-node/src/chain/archiveStore/utils/updateBackfillRange.ts @@ -20,7 +20,7 @@ export async function updateBackfillRange( try { // Mark the sequence in backfill db from finalized block's slot till anchor slot as // filled. - const finalizedBlockFC = chain.forkChoice.getBlockHex(finalized.rootHex); + const finalizedBlockFC = chain.forkChoice.getBlockHexDefaultStatus(finalized.rootHex); if (finalizedBlockFC && finalizedBlockFC.slot > chain.anchorStateLatestBlockSlot) { await db.backfilledRanges.put(finalizedBlockFC.slot, chain.anchorStateLatestBlockSlot); diff --git a/packages/beacon-node/src/chain/blocks/importBlock.ts b/packages/beacon-node/src/chain/blocks/importBlock.ts index 7225441df389..aaef371a480c 100644 --- a/packages/beacon-node/src/chain/blocks/importBlock.ts +++ b/packages/beacon-node/src/chain/blocks/importBlock.ts @@ -20,7 +20,7 @@ import { isStartSlotOfEpoch, isStateValidatorsNodesPopulated, } from "@lodestar/state-transition"; -import {Attestation, BeaconBlock, altair, capella, electra, phase0, ssz} from "@lodestar/types"; +import {Attestation, BeaconBlock, altair, capella, electra, isGloasBeaconBlock, phase0, ssz} from "@lodestar/types"; import {isErrorAborted, toRootHex} from "@lodestar/utils"; import {ZERO_HASH_HEX} from "../../constants/index.js"; import {callInNextEventLoop} from "../../util/eventLoop.js"; @@ -230,6 +230,32 @@ export async function importBlock( } } + // 4.5. Import payload attestations to fork choice (Gloas) + // + if (isGloasBeaconBlock(block.message)) { + for (const payloadAttestation of block.message.body.payloadAttestations) { + try { + // Extract PTC indices from aggregation bits + const ptcIndices: number[] = []; + for (let i = 0; i < payloadAttestation.aggregationBits.bitLen; i++) { + if (payloadAttestation.aggregationBits.get(i)) { + ptcIndices.push(i); + } + } + + if (ptcIndices.length > 0) { + this.forkChoice.notifyPtcMessages( + toRootHex(payloadAttestation.data.beaconBlockRoot), + ptcIndices, + payloadAttestation.data.payloadPresent + ); + } + } catch (e) { + this.logger.warn("Error processing PayloadAttestation from block", {slot: blockSlot}, e as Error); + } + } + } + // 5. Compute head. If new head, immediately stateCache.setHeadState() const oldHead = this.forkChoice.getHead(); @@ -345,7 +371,7 @@ export async function importBlock( // 3) Proposer boost reorg related flag is turned on (this is checked inside the function) // 4) Block meets the criteria of being re-orged out (this is also checked inside the function) const result = this.forkChoice.shouldOverrideForkChoiceUpdate( - blockSummary.blockRoot, + blockSummary, this.clock.secFromSlot(currentSlot), currentSlot ); @@ -441,7 +467,12 @@ export async function importBlock( this.metrics?.currentActiveValidators.set(activeValidatorsCount); this.metrics?.currentValidators.set({status: "active"}, activeValidatorsCount); - const parentBlockSummary = this.forkChoice.getBlock(checkpointState.latestBlockHeader.parentRoot); + const parentBlockSummary = isGloasBeaconBlock(block.message) + ? this.forkChoice.getBlockHexAndBlockHash( + toRootHex(checkpointState.latestBlockHeader.parentRoot), + toRootHex(block.message.body.signedExecutionPayloadBid.message.parentBlockHash) + ) + : this.forkChoice.getBlockDefaultStatus(checkpointState.latestBlockHeader.parentRoot); if (parentBlockSummary) { const justifiedCheckpoint = checkpointState.currentJustifiedCheckpoint; diff --git a/packages/beacon-node/src/chain/blocks/verifyBlocksSanityChecks.ts b/packages/beacon-node/src/chain/blocks/verifyBlocksSanityChecks.ts index 63e5c7b471c5..02d686818d2a 100644 --- a/packages/beacon-node/src/chain/blocks/verifyBlocksSanityChecks.ts +++ b/packages/beacon-node/src/chain/blocks/verifyBlocksSanityChecks.ts @@ -1,7 +1,7 @@ import {ChainForkConfig} from "@lodestar/config"; import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; import {computeStartSlotAtEpoch} from "@lodestar/state-transition"; -import {RootHex, Slot} from "@lodestar/types"; +import {RootHex, Slot, isGloasBeaconBlock} from "@lodestar/types"; import {toRootHex} from "@lodestar/utils"; import {IClock} from "../../util/clock.js"; import {BlockError, BlockErrorCode} from "../errors/index.js"; @@ -90,7 +90,12 @@ export function verifyBlocksSanityChecks( } else { // When importing a block segment, only the first NON-IGNORED block must be known to the fork-choice. const parentRoot = toRootHex(block.message.parentRoot); - parentBlock = chain.forkChoice.getBlockHex(parentRoot); + parentBlock = isGloasBeaconBlock(block.message) + ? chain.forkChoice.getBlockHexAndBlockHash( + parentRoot, + toRootHex(block.message.body.signedExecutionPayloadBid.message.parentBlockHash) + ) + : chain.forkChoice.getBlockHexDefaultStatus(parentRoot); if (!parentBlock) { throw new BlockError(block, {code: BlockErrorCode.PARENT_UNKNOWN, parentRoot}); } diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index a0a29bcc0c0e..5a063713008f 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -600,7 +600,7 @@ export class BeaconChain implements IBeaconChain { ): Promise<{state: CachedBeaconStateAllForks | Uint8Array; executionOptimistic: boolean; finalized: boolean} | null> { if (opts?.allowRegen) { const state = await this.regen.getState(stateRoot, RegenCaller.restApi); - const block = this.forkChoice.getBlock(state.latestBlockHeader.hashTreeRoot()); + const block = this.forkChoice.getBlockDefaultStatus(state.latestBlockHeader.hashTreeRoot()); const finalizedEpoch = this.forkChoice.getFinalizedCheckpoint().epoch; return { state, @@ -616,7 +616,7 @@ export class BeaconChain implements IBeaconChain { // TODO: This is very inneficient for debug requests of serialized content, since it deserializes to serialize again const cachedStateCtx = this.regen.getStateSync(stateRoot); if (cachedStateCtx) { - const block = this.forkChoice.getBlock(cachedStateCtx.latestBlockHeader.hashTreeRoot()); + const block = this.forkChoice.getBlockDefaultStatus(cachedStateCtx.latestBlockHeader.hashTreeRoot()); const finalizedEpoch = this.forkChoice.getFinalizedCheckpoint().epoch; return { state: cachedStateCtx, @@ -650,7 +650,7 @@ export class BeaconChain implements IBeaconChain { // finalized or justified checkpoint states maynot be available with PersistentCheckpointStateCache, use getCheckpointStateOrBytes() api to get Uint8Array const cachedStateCtx = this.regen.getCheckpointStateSync(checkpoint); if (cachedStateCtx) { - const block = this.forkChoice.getBlock(cachedStateCtx.latestBlockHeader.hashTreeRoot()); + const block = this.forkChoice.getBlockDefaultStatus(cachedStateCtx.latestBlockHeader.hashTreeRoot()); const finalizedEpoch = this.forkChoice.getFinalizedCheckpoint().epoch; return { state: cachedStateCtx, @@ -667,7 +667,7 @@ export class BeaconChain implements IBeaconChain { ): Promise<{state: CachedBeaconStateAllForks | Uint8Array; executionOptimistic: boolean; finalized: boolean} | null> { const cachedStateCtx = await this.regen.getCheckpointStateOrBytes(checkpoint); if (cachedStateCtx) { - const block = this.forkChoice.getBlock(checkpoint.root); + const block = this.forkChoice.getBlockDefaultStatus(checkpoint.root); const finalizedEpoch = this.forkChoice.getFinalizedCheckpoint().epoch; return { state: cachedStateCtx, @@ -711,7 +711,7 @@ export class BeaconChain implements IBeaconChain { async getBlockByRoot( root: string ): Promise<{block: SignedBeaconBlock; executionOptimistic: boolean; finalized: boolean} | null> { - const block = this.forkChoice.getBlockHex(root); + const block = this.forkChoice.getBlockHexDefaultStatus(root); if (block) { // Block found in fork-choice. // It may be in the block input cache, awaiting full DA reconstruction, check there first @@ -735,7 +735,7 @@ export class BeaconChain implements IBeaconChain { async getSerializedBlockByRoot( root: string ): Promise<{block: Uint8Array; executionOptimistic: boolean; finalized: boolean; slot: Slot} | null> { - const block = this.forkChoice.getBlockHex(root); + const block = this.forkChoice.getBlockHexDefaultStatus(root); if (block) { // Block found in fork-choice. // It may be in the block input cache, awaiting full DA reconstruction, check there first diff --git a/packages/beacon-node/src/chain/emitter.ts b/packages/beacon-node/src/chain/emitter.ts index e8be09536048..95a12ead032b 100644 --- a/packages/beacon-node/src/chain/emitter.ts +++ b/packages/beacon-node/src/chain/emitter.ts @@ -1,7 +1,7 @@ import {EventEmitter} from "node:events"; import {StrictEventEmitter} from "strict-event-emitter-types"; import {routes} from "@lodestar/api"; -import {CheckpointWithHex} from "@lodestar/fork-choice"; +import {CheckpointWithPayload} from "@lodestar/fork-choice"; import {CachedBeaconStateAllForks} from "@lodestar/state-transition"; import {DataColumnSidecars, RootHex, deneb, phase0} from "@lodestar/types"; import {PeerIdStr} from "../util/peerId.js"; @@ -83,8 +83,8 @@ export type ChainEventData = { export type IChainEvents = ApiEvents & { [ChainEvent.checkpoint]: (checkpoint: phase0.Checkpoint, state: CachedBeaconStateAllForks) => void; - [ChainEvent.forkChoiceJustified]: (checkpoint: CheckpointWithHex) => void; - [ChainEvent.forkChoiceFinalized]: (checkpoint: CheckpointWithHex) => void; + [ChainEvent.forkChoiceJustified]: (checkpoint: CheckpointWithPayload) => void; + [ChainEvent.forkChoiceFinalized]: (checkpoint: CheckpointWithPayload) => void; [ChainEvent.updateTargetCustodyGroupCount]: (targetGroupCount: number) => void; diff --git a/packages/beacon-node/src/chain/errors/executionPayloadEnvelope.ts b/packages/beacon-node/src/chain/errors/executionPayloadEnvelope.ts index 051f07e04ba1..af3a040a8d17 100644 --- a/packages/beacon-node/src/chain/errors/executionPayloadEnvelope.ts +++ b/packages/beacon-node/src/chain/errors/executionPayloadEnvelope.ts @@ -25,9 +25,13 @@ export type ExecutionPayloadEnvelopeErrorType = | { code: ExecutionPayloadEnvelopeErrorCode.BUILDER_INDEX_MISMATCH; envelopeBuilderIndex: BuilderIndex; - bidBuilderIndex: BuilderIndex; + bidBuilderIndex: BuilderIndex | null; + } + | { + code: ExecutionPayloadEnvelopeErrorCode.BLOCK_HASH_MISMATCH; + envelopeBlockHash: RootHex; + bidBlockHash: RootHex | null; } - | {code: ExecutionPayloadEnvelopeErrorCode.BLOCK_HASH_MISMATCH; envelopeBlockHash: RootHex; bidBlockHash: RootHex} | {code: ExecutionPayloadEnvelopeErrorCode.INVALID_SIGNATURE} | {code: ExecutionPayloadEnvelopeErrorCode.CACHE_FAIL; blockRoot: RootHex}; diff --git a/packages/beacon-node/src/chain/forkChoice/index.ts b/packages/beacon-node/src/chain/forkChoice/index.ts index 31548ddbf8f6..f37721a270cb 100644 --- a/packages/beacon-node/src/chain/forkChoice/index.ts +++ b/packages/beacon-node/src/chain/forkChoice/index.ts @@ -4,9 +4,11 @@ import { ForkChoice, ForkChoiceStore, JustifiedBalancesGetter, + PayloadStatus, ProtoArray, ProtoBlock, ForkChoiceOpts as RawForkChoiceOpts, + getCheckpointPayloadStatus, } from "@lodestar/fork-choice"; import {ZERO_HASH_HEX} from "@lodestar/params"; import { @@ -104,6 +106,14 @@ export function initializeForkChoiceFromFinalizedState( // production code use ForkChoice constructor directly const forkchoiceConstructor = opts.forkchoiceConstructor ?? ForkChoice; + const isForkPostGloas = (state as CachedBeaconStateGloas).latestBlockHash !== undefined; + + // Determine justified checkpoint payload status + const justifiedPayloadStatus = getCheckpointPayloadStatus(state, justifiedCheckpoint.epoch); + + // Determine finalized checkpoint payload status + const finalizedPayloadStatus = getCheckpointPayloadStatus(state, finalizedCheckpoint.epoch); + return new forkchoiceConstructor( config, @@ -113,6 +123,8 @@ export function initializeForkChoiceFromFinalizedState( finalizedCheckpoint, justifiedBalances, justifiedBalancesGetter, + justifiedPayloadStatus, + finalizedPayloadStatus, { onJustified: (cp) => emitter.emit(ChainEvent.forkChoiceJustified, cp), onFinalized: (cp) => emitter.emit(ChainEvent.forkChoiceFinalized, cp), @@ -145,15 +157,12 @@ export function initializeForkChoiceFromFinalizedState( : {executionPayloadBlockHash: null, executionStatus: ExecutionStatus.PreMerge}), dataAvailabilityStatus: DataAvailabilityStatus.PreData, - ...(computeEpochAtSlot(blockHeader.slot) < state.config.GLOAS_FORK_EPOCH - ? { - builderIndex: undefined, - blockHashHex: undefined, - } - : { - builderIndex: (state as CachedBeaconStateGloas).latestExecutionPayloadBid.builderIndex, - blockHashHex: toRootHex((state as CachedBeaconStateGloas).latestExecutionPayloadBid.blockHash), - }), + payloadStatus: isForkPostGloas ? PayloadStatus.PENDING : PayloadStatus.FULL, // TODO GLOAS: Post-gloas how do we know if the checkpoint payload is FULL or EMPTY? + builderIndex: isForkPostGloas ? (state as CachedBeaconStateGloas).latestExecutionPayloadBid.builderIndex : null, + blockHashFromBid: isForkPostGloas + ? toRootHex((state as CachedBeaconStateGloas).latestExecutionPayloadBid.blockHash) + : null, + parentBlockHash: isForkPostGloas ? toRootHex((state as CachedBeaconStateGloas).latestBlockHash) : null, }, currentSlot ), @@ -196,12 +205,22 @@ export function initializeForkChoiceFromUnfinalizedState( // this is not the justified state, but there is no other ways to get justified balances const justifiedBalances = getEffectiveBalanceIncrementsZeroInactive(unfinalizedState); + + const isForkPostGloas = (unfinalizedState as CachedBeaconStateGloas).latestBlockHash !== undefined; + + // For unfinalized state, use getCheckpointPayloadStatus to determine the correct status. + // It checks state.execution_payload_availability to determine EMPTY vs FULL. + const justifiedPayloadStatus = getCheckpointPayloadStatus(unfinalizedState, justifiedCheckpoint.epoch); + const finalizedPayloadStatus = getCheckpointPayloadStatus(unfinalizedState, finalizedCheckpoint.epoch); + const store = new ForkChoiceStore( currentSlot, justifiedCheckpoint, finalizedCheckpoint, justifiedBalances, justifiedBalancesGetter, + justifiedPayloadStatus, + finalizedPayloadStatus, { onJustified: (cp) => emitter.emit(ChainEvent.forkChoiceJustified, cp), onFinalized: (cp) => emitter.emit(ChainEvent.forkChoiceFinalized, cp), @@ -235,15 +254,14 @@ export function initializeForkChoiceFromUnfinalizedState( : {executionPayloadBlockHash: null, executionStatus: ExecutionStatus.PreMerge}), dataAvailabilityStatus: DataAvailabilityStatus.PreData, - ...(computeEpochAtSlot(blockHeader.slot) < unfinalizedState.config.GLOAS_FORK_EPOCH - ? { - builderIndex: undefined, - blockHashHex: undefined, - } - : { - builderIndex: (unfinalizedState as CachedBeaconStateGloas).latestExecutionPayloadBid.builderIndex, - blockHashHex: toRootHex((unfinalizedState as CachedBeaconStateGloas).latestExecutionPayloadBid.blockHash), - }), + payloadStatus: isForkPostGloas ? PayloadStatus.PENDING : PayloadStatus.FULL, // TODO GLOAS: Post-gloas how do we know if the checkpoint payload is FULL or EMPTY? + builderIndex: isForkPostGloas + ? (unfinalizedState as CachedBeaconStateGloas).latestExecutionPayloadBid.builderIndex + : null, + blockHashFromBid: isForkPostGloas + ? toRootHex((unfinalizedState as CachedBeaconStateGloas).latestExecutionPayloadBid.blockHash) + : null, + parentBlockHash: isForkPostGloas ? toRootHex((unfinalizedState as CachedBeaconStateGloas).latestBlockHash) : null, }; const parentSlot = blockHeader.slot - 1; @@ -285,9 +303,9 @@ export function initializeForkChoiceFromUnfinalizedState( }; const protoArray = ProtoArray.initialize(finalizedBlock, currentSlot); - protoArray.onBlock(justifiedBlock, currentSlot); - protoArray.onBlock(parentBlock, currentSlot); - protoArray.onBlock(headBlock, currentSlot); + protoArray.onBlock(justifiedBlock, currentSlot, null); + protoArray.onBlock(parentBlock, currentSlot, null); + protoArray.onBlock(headBlock, currentSlot, null); logger?.verbose("Initialized protoArray successfully", {...logCtx, length: protoArray.length()}); diff --git a/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts b/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts index 196b8d590fea..e19ec2dbe950 100644 --- a/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts +++ b/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts @@ -864,7 +864,7 @@ function isValidShuffling( // attestation's shuffling is the same as the current state's. // To account for skipped slots, find the first block at *or before* the pivot slot. const beaconBlockRootHex = blockRootHex; - const beaconBlock = forkChoice.getBlockHex(beaconBlockRootHex); + const beaconBlock = forkChoice.getBlockHexDefaultStatus(beaconBlockRootHex); if (!beaconBlock) { return InvalidAttestationData.BlockNotInForkChoice; } diff --git a/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts b/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts index c3eee555e734..9642c483b63b 100644 --- a/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts +++ b/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts @@ -723,8 +723,7 @@ export function getPayloadAttributesForSSE( let parentBlockNumber: number; if (isForkPostGloas(fork)) { - // TODO GLOAS: revisit this after fork choice changes are merged - const parentBlock = chain.forkChoice.getBlock(parentBlockRoot); + const parentBlock = chain.forkChoice.getBlockHexAndBlockHash(toRootHex(parentBlockRoot), toRootHex(parentHash)); if (parentBlock?.executionPayloadBlockHash == null) { throw Error(`Parent block not found in fork choice root=${toRootHex(parentBlockRoot)}`); } diff --git a/packages/beacon-node/src/chain/regen/queued.ts b/packages/beacon-node/src/chain/regen/queued.ts index e99b06e78577..04011d53cbd1 100644 --- a/packages/beacon-node/src/chain/regen/queued.ts +++ b/packages/beacon-node/src/chain/regen/queued.ts @@ -1,7 +1,7 @@ import {routes} from "@lodestar/api"; import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; import {CachedBeaconStateAllForks, computeEpochAtSlot} from "@lodestar/state-transition"; -import {BeaconBlock, Epoch, RootHex, Slot, phase0} from "@lodestar/types"; +import {BeaconBlock, Epoch, RootHex, Slot, isGloasBeaconBlock, phase0} from "@lodestar/types"; import {Logger, toRootHex} from "@lodestar/utils"; import {Metrics} from "../../metrics/index.js"; import {JobItemQueue} from "../../util/queue/index.js"; @@ -88,7 +88,12 @@ export class QueuedStateRegenerator implements IStateRegenerator { */ getPreStateSync(block: BeaconBlock): CachedBeaconStateAllForks | null { const parentRoot = toRootHex(block.parentRoot); - const parentBlock = this.forkChoice.getBlockHex(parentRoot); + const parentBlock = isGloasBeaconBlock(block) + ? this.forkChoice.getBlockHexAndBlockHash( + parentRoot, + toRootHex(block.body.signedExecutionPayloadBid.message.parentBlockHash) + ) + : this.forkChoice.getBlockHexDefaultStatus(parentRoot); if (!parentBlock) { throw new RegenError({ code: RegenErrorCode.BLOCK_NOT_IN_FORKCHOICE, diff --git a/packages/beacon-node/src/chain/regen/regen.ts b/packages/beacon-node/src/chain/regen/regen.ts index 7dfebb9af51d..6ad654280293 100644 --- a/packages/beacon-node/src/chain/regen/regen.ts +++ b/packages/beacon-node/src/chain/regen/regen.ts @@ -11,7 +11,7 @@ import { processSlots, stateTransition, } from "@lodestar/state-transition"; -import {BeaconBlock, RootHex, SignedBeaconBlock, Slot} from "@lodestar/types"; +import {BeaconBlock, RootHex, SignedBeaconBlock, Slot, isGloasBeaconBlock} from "@lodestar/types"; import {Logger, fromHex, toRootHex} from "@lodestar/utils"; import {IBeaconDb} from "../../db/index.js"; import {Metrics} from "../../metrics/index.js"; @@ -58,7 +58,13 @@ export class StateRegenerator implements IStateRegeneratorInternal { opts: StateRegenerationOpts, regenCaller: RegenCaller ): Promise { - const parentBlock = this.modules.forkChoice.getBlock(block.parentRoot); + const parentRoot = toRootHex(block.parentRoot); + const parentBlock = isGloasBeaconBlock(block) + ? this.modules.forkChoice.getBlockHexAndBlockHash( + parentRoot, + toRootHex(block.body.signedExecutionPayloadBid.message.parentBlockHash) + ) + : this.modules.forkChoice.getBlockHexDefaultStatus(parentRoot); if (!parentBlock) { throw new RegenError({ code: RegenErrorCode.BLOCK_NOT_IN_FORKCHOICE, diff --git a/packages/beacon-node/src/chain/validation/aggregateAndProof.ts b/packages/beacon-node/src/chain/validation/aggregateAndProof.ts index 162788f6caf1..6956e62ed392 100644 --- a/packages/beacon-node/src/chain/validation/aggregateAndProof.ts +++ b/packages/beacon-node/src/chain/validation/aggregateAndProof.ts @@ -81,7 +81,7 @@ async function validateAggregateAndProof( }); } // [REJECT] `aggregate.data.index == 0` if `block.slot == aggregate.data.slot`. - const block = chain.forkChoice.getBlock(attData.beaconBlockRoot); + const block = chain.forkChoice.getBlockDefaultStatus(attData.beaconBlockRoot); // If block is unknown, we don't handle it here. It will throw error later on at `verifyHeadBlockAndTargetRoot()` if (block !== null && block.slot === attData.slot && attData.index !== 0) { diff --git a/packages/beacon-node/src/chain/validation/attestation.ts b/packages/beacon-node/src/chain/validation/attestation.ts index eee57c92d378..046c482d01ee 100644 --- a/packages/beacon-node/src/chain/validation/attestation.ts +++ b/packages/beacon-node/src/chain/validation/attestation.ts @@ -307,7 +307,7 @@ async function validateAttestationNoSignatureCheck( } // [REJECT] `attestation.data.index == 0` if `block.slot == attestation.data.slot`. - const block = chain.forkChoice.getBlock(attData.beaconBlockRoot); + const block = chain.forkChoice.getBlockDefaultStatus(attData.beaconBlockRoot); // block being null will be handled by `verifyHeadBlockAndTargetRoot` if (block !== null && block.slot === attSlot && attData.index !== 0) { @@ -756,7 +756,7 @@ export function getAttestationDataSigningRoot(config: BeaconConfig, data: phase0 function verifyHeadBlockIsKnown(chain: IBeaconChain, beaconBlockRoot: Root): ProtoBlock { // TODO (LH): Enforce a maximum skip distance for unaggregated attestations. - const headBlock = chain.forkChoice.getBlock(beaconBlockRoot); + const headBlock = chain.forkChoice.getBlockDefaultStatus(beaconBlockRoot); if (headBlock === null) { throw new AttestationError(GossipAction.IGNORE, { code: AttestationErrorCode.UNKNOWN_OR_PREFINALIZED_BEACON_BLOCK_ROOT, diff --git a/packages/beacon-node/src/chain/validation/blobSidecar.ts b/packages/beacon-node/src/chain/validation/blobSidecar.ts index cbf27dcf03ae..0346d7d93b98 100644 --- a/packages/beacon-node/src/chain/validation/blobSidecar.ts +++ b/packages/beacon-node/src/chain/validation/blobSidecar.ts @@ -78,7 +78,7 @@ export async function validateGossipBlobSidecar( // already know this block. const blockRoot = ssz.phase0.BeaconBlockHeader.hashTreeRoot(blobSidecar.signedBlockHeader.message); const blockHex = toRootHex(blockRoot); - if (chain.forkChoice.getBlockHex(blockHex) !== null) { + if (chain.forkChoice.getBlockHexDefaultStatus(blockHex) !== null) { throw new BlobSidecarGossipError(GossipAction.IGNORE, {code: BlobSidecarErrorCode.ALREADY_KNOWN, root: blockHex}); } @@ -89,7 +89,7 @@ export async function validateGossipBlobSidecar( // gossip and non-gossip sources) (a client MAY queue blocks for processing once the parent block is // retrieved). const parentRoot = toRootHex(blobSidecar.signedBlockHeader.message.parentRoot); - const parentBlock = chain.forkChoice.getBlockHex(parentRoot); + const parentBlock = chain.forkChoice.getBlockHexDefaultStatus(parentRoot); if (parentBlock === null) { // If fork choice does *not* consider the parent to be a descendant of the finalized block, // then there are two more cases: diff --git a/packages/beacon-node/src/chain/validation/block.ts b/packages/beacon-node/src/chain/validation/block.ts index dd699caf25fa..f7addf9a5794 100644 --- a/packages/beacon-node/src/chain/validation/block.ts +++ b/packages/beacon-node/src/chain/validation/block.ts @@ -9,7 +9,7 @@ import { isExecutionEnabled, isExecutionStateType, } from "@lodestar/state-transition"; -import {SignedBeaconBlock, deneb, gloas} from "@lodestar/types"; +import {SignedBeaconBlock, deneb, gloas, isGloasBeaconBlock} from "@lodestar/types"; import {byteArrayEquals, sleep, toRootHex} from "@lodestar/utils"; import {BlockErrorCode, BlockGossipError, GossipAction} from "../errors/index.js"; import {IBeaconChain} from "../interface.js"; @@ -56,7 +56,7 @@ export async function validateGossipBlock( // check, we will load the parent and state from disk only to find out later that we // already know this block. const blockRoot = toRootHex(config.getForkTypes(blockSlot).BeaconBlock.hashTreeRoot(block)); - if (chain.forkChoice.getBlockHex(blockRoot) !== null) { + if (chain.forkChoice.getBlockHexDefaultStatus(blockRoot) !== null) { throw new BlockGossipError(GossipAction.IGNORE, {code: BlockErrorCode.ALREADY_KNOWN, root: blockRoot}); } @@ -72,7 +72,12 @@ export async function validateGossipBlock( // [REJECT] The current finalized_checkpoint is an ancestor of block -- i.e. // get_ancestor(store, block.parent_root, compute_start_slot_at_epoch(store.finalized_checkpoint.epoch)) == store.finalized_checkpoint.root const parentRoot = toRootHex(block.parentRoot); - const parentBlock = chain.forkChoice.getBlockHex(parentRoot); + const parentBlock = isGloasBeaconBlock(block) + ? chain.forkChoice.getBlockHexAndBlockHash( + parentRoot, + toRootHex(block.body.signedExecutionPayloadBid.message.parentBlockHash) + ) + : chain.forkChoice.getBlockHexDefaultStatus(parentRoot); if (parentBlock === null) { // If fork choice does *not* consider the parent to be a descendant of the finalized block, // then there are two more cases: diff --git a/packages/beacon-node/src/chain/validation/dataColumnSidecar.ts b/packages/beacon-node/src/chain/validation/dataColumnSidecar.ts index e10ce40f7a9d..0aa5fde20cae 100644 --- a/packages/beacon-node/src/chain/validation/dataColumnSidecar.ts +++ b/packages/beacon-node/src/chain/validation/dataColumnSidecar.ts @@ -73,7 +73,7 @@ export async function validateGossipDataColumnSidecar( // 6) [IGNORE] The sidecar's block's parent (defined by block_header.parent_root) has been seen (via gossip // or non-gossip sources) const parentRoot = toRootHex(blockHeader.parentRoot); - const parentBlock = chain.forkChoice.getBlockHex(parentRoot); + const parentBlock = chain.forkChoice.getBlockHexDefaultStatus(parentRoot); if (parentBlock === null) { // If fork choice does *not* consider the parent to be a descendant of the finalized block, // then there are two more cases: diff --git a/packages/beacon-node/src/chain/validation/executionPayloadBid.ts b/packages/beacon-node/src/chain/validation/executionPayloadBid.ts index 771f9d1e2842..58863dae5fc5 100644 --- a/packages/beacon-node/src/chain/validation/executionPayloadBid.ts +++ b/packages/beacon-node/src/chain/validation/executionPayloadBid.ts @@ -113,8 +113,7 @@ async function validateExecutionPayloadBid( // [IGNORE] `bid.parent_block_root` is the hash tree root of a known beacon // block in fork choice. - const block = chain.forkChoice.getBlock(bid.parentBlockRoot); - if (block === null) { + if (!chain.forkChoice.hasBlock(bid.parentBlockRoot)) { throw new ExecutionPayloadBidError(GossipAction.IGNORE, { code: ExecutionPayloadBidErrorCode.UNKNOWN_BLOCK_ROOT, parentBlockRoot: parentBlockRootHex, diff --git a/packages/beacon-node/src/chain/validation/executionPayloadEnvelope.ts b/packages/beacon-node/src/chain/validation/executionPayloadEnvelope.ts index d1935d1986c2..467a9676f2a3 100644 --- a/packages/beacon-node/src/chain/validation/executionPayloadEnvelope.ts +++ b/packages/beacon-node/src/chain/validation/executionPayloadEnvelope.ts @@ -37,7 +37,7 @@ async function validateExecutionPayloadEnvelope( // the block is retrieved). // TODO GLOAS: Need to review this, we should queue the envelope for later // processing if the block is not yet known, otherwise we would ignore it here - const block = chain.forkChoice.getBlock(envelope.beaconBlockRoot); + const block = chain.forkChoice.getBlockDefaultStatus(envelope.beaconBlockRoot); if (block === null) { throw new ExecutionPayloadEnvelopeError(GossipAction.IGNORE, { code: ExecutionPayloadEnvelopeErrorCode.BLOCK_ROOT_UNKNOWN, @@ -79,7 +79,7 @@ async function validateExecutionPayloadEnvelope( }); } - if (block.builderIndex === undefined || block.blockHashHex === undefined) { + if (block.builderIndex == null || block.blockHashFromBid == null) { // This indicates this block is a pre-gloas block which is wrong throw new ExecutionPayloadEnvelopeError(GossipAction.IGNORE, { code: ExecutionPayloadEnvelopeErrorCode.CACHE_FAIL, @@ -97,11 +97,11 @@ async function validateExecutionPayloadEnvelope( } // [REJECT] `payload.block_hash == bid.block_hash` - if (toRootHex(payload.blockHash) !== block.blockHashHex) { + if (toRootHex(payload.blockHash) !== block.blockHashFromBid) { throw new ExecutionPayloadEnvelopeError(GossipAction.REJECT, { code: ExecutionPayloadEnvelopeErrorCode.BLOCK_HASH_MISMATCH, envelopeBlockHash: toRootHex(payload.blockHash), - bidBlockHash: block.blockHashHex, + bidBlockHash: block.blockHashFromBid, }); } diff --git a/packages/beacon-node/src/chain/validation/payloadAttestationMessage.ts b/packages/beacon-node/src/chain/validation/payloadAttestationMessage.ts index 09f81c2d45d7..f4a25a4ed7fb 100644 --- a/packages/beacon-node/src/chain/validation/payloadAttestationMessage.ts +++ b/packages/beacon-node/src/chain/validation/payloadAttestationMessage.ts @@ -59,8 +59,7 @@ async function validatePayloadAttestationMessage( // [IGNORE] The message's block `data.beacon_block_root` has been seen (via // gossip or non-gossip sources) (a client MAY queue attestation for processing // once the block is retrieved. Note a client might want to request payload after). - const block = chain.forkChoice.getBlock(data.beaconBlockRoot); - if (block === null) { + if (!chain.forkChoice.hasBlock(data.beaconBlockRoot)) { throw new PayloadAttestationError(GossipAction.IGNORE, { code: PayloadAttestationErrorCode.UNKNOWN_BLOCK_ROOT, blockRoot: toRootHex(data.beaconBlockRoot), diff --git a/packages/beacon-node/src/network/processor/gossipHandlers.ts b/packages/beacon-node/src/network/processor/gossipHandlers.ts index 3aac4ed945f6..d681e49b9c80 100644 --- a/packages/beacon-node/src/network/processor/gossipHandlers.ts +++ b/packages/beacon-node/src/network/processor/gossipHandlers.ts @@ -856,6 +856,11 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand } catch (e) { logger.error("Error adding to payloadAttestation pool", {}, e as Error); } + chain.forkChoice.notifyPtcMessages( + toRootHex(payloadAttestationMessage.data.beaconBlockRoot), + [validationResult.validatorCommitteeIndex], + payloadAttestationMessage.data.payloadPresent + ); }, [GossipType.execution_payload_bid]: async ({ gossipData, diff --git a/packages/beacon-node/src/network/reqresp/handlers/blobSidecarsByRoot.ts b/packages/beacon-node/src/network/reqresp/handlers/blobSidecarsByRoot.ts index 751f9d4980ee..3113a21ea703 100644 --- a/packages/beacon-node/src/network/reqresp/handlers/blobSidecarsByRoot.ts +++ b/packages/beacon-node/src/network/reqresp/handlers/blobSidecarsByRoot.ts @@ -20,7 +20,7 @@ export async function* onBlobSidecarsByRoot( for (const blobIdentifier of requestBody) { const {blockRoot, index} = blobIdentifier; const blockRootHex = toRootHex(blockRoot); - const block = chain.forkChoice.getBlockHex(blockRootHex); + const block = chain.forkChoice.getBlockHexDefaultStatus(blockRootHex); // NOTE: Only support non-finalized blocks. // SPEC: Clients MUST support requesting blocks and sidecars since the latest finalized epoch. diff --git a/packages/beacon-node/src/network/reqresp/handlers/dataColumnSidecarsByRoot.ts b/packages/beacon-node/src/network/reqresp/handlers/dataColumnSidecarsByRoot.ts index 8e598ebe5a4d..aa51c73b1d97 100644 --- a/packages/beacon-node/src/network/reqresp/handlers/dataColumnSidecarsByRoot.ts +++ b/packages/beacon-node/src/network/reqresp/handlers/dataColumnSidecarsByRoot.ts @@ -34,7 +34,7 @@ export async function* onDataColumnSidecarsByRoot( } const blockRootHex = toRootHex(blockRoot); - const block = chain.forkChoice.getBlockHex(blockRootHex); + const block = chain.forkChoice.getBlockHexDefaultStatus(blockRootHex); // If the block is not in fork choice, it may be finalized. Attempt to find its slot in block archive const slot = block ? block.slot : await db.blockArchive.getSlotByRoot(blockRoot); diff --git a/packages/beacon-node/test/mocks/mockedBeaconChain.ts b/packages/beacon-node/test/mocks/mockedBeaconChain.ts index 9bbe71c9fea1..12d26de6f90b 100644 --- a/packages/beacon-node/test/mocks/mockedBeaconChain.ts +++ b/packages/beacon-node/test/mocks/mockedBeaconChain.ts @@ -52,6 +52,9 @@ vi.mock("@lodestar/fork-choice", async (importActual) => { getDependentRoot: vi.fn(), getBlockHex: vi.fn(), getBlock: vi.fn(), + getBlockDefaultStatus: vi.fn(), + getBlockHexDefaultStatus: vi.fn(), + getBlockHexAndBlockHash: vi.fn(), getAllAncestorBlocks: vi.fn(), getAllNonAncestorBlocks: vi.fn(), getAllAncestorAndNonAncestorBlocks: vi.fn(), diff --git a/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts b/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts index 9c04938cf492..785c92fc44ef 100644 --- a/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts +++ b/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts @@ -1,7 +1,7 @@ import {beforeAll, bench, describe} from "@chainsafe/benchmark"; import {BitArray, toHexString} from "@chainsafe/ssz"; import {createBeaconConfig, defaultChainConfig} from "@lodestar/config"; -import {ExecutionStatus, ForkChoice, IForkChoiceStore, ProtoArray} from "@lodestar/fork-choice"; +import {ExecutionStatus, ForkChoice, IForkChoiceStore, PayloadStatus, ProtoArray} from "@lodestar/fork-choice"; import {HISTORICAL_ROOTS_LIMIT, SLOTS_PER_EPOCH} from "@lodestar/params"; import { CachedBeaconStateAltair, @@ -68,6 +68,11 @@ describe(`getAttestationsForBlock vc=${vc}`, () => { timeliness: false, dataAvailabilityStatus: DataAvailabilityStatus.PreData, + + parentBlockHash: null, + payloadStatus: 2, // PayloadStatus.FULL + builderIndex: null, + blockHashFromBid: null, }, originalState.slot ); @@ -93,8 +98,14 @@ describe(`getAttestationsForBlock vc=${vc}`, () => { executionStatus: ExecutionStatus.PreMerge, timeliness: false, dataAvailabilityStatus: DataAvailabilityStatus.PreData, + + parentBlockHash: null, + payloadStatus: 2, // PayloadStatus.FULL + builderIndex: null, + blockHashFromBid: null, }, - slot + slot, + null ); } @@ -106,16 +117,32 @@ describe(`getAttestationsForBlock vc=${vc}`, () => { const fcStore: IForkChoiceStore = { currentSlot: originalState.slot, justified: { - checkpoint: {...justifiedCheckpoint, rootHex: toHexString(justifiedCheckpoint.root)}, + checkpoint: { + ...justifiedCheckpoint, + rootHex: toHexString(justifiedCheckpoint.root), + payloadStatus: PayloadStatus.FULL, + }, balances: originalState.epochCtx.effectiveBalanceIncrements, totalBalance, }, unrealizedJustified: { - checkpoint: {...justifiedCheckpoint, rootHex: toHexString(justifiedCheckpoint.root)}, + checkpoint: { + ...justifiedCheckpoint, + rootHex: toHexString(justifiedCheckpoint.root), + payloadStatus: PayloadStatus.FULL, + }, balances: originalState.epochCtx.effectiveBalanceIncrements, }, - finalizedCheckpoint: {...finalizedCheckpoint, rootHex: toHexString(finalizedCheckpoint.root)}, - unrealizedFinalizedCheckpoint: {...finalizedCheckpoint, rootHex: toHexString(finalizedCheckpoint.root)}, + finalizedCheckpoint: { + ...finalizedCheckpoint, + rootHex: toHexString(finalizedCheckpoint.root), + payloadStatus: PayloadStatus.FULL, + }, + unrealizedFinalizedCheckpoint: { + ...finalizedCheckpoint, + rootHex: toHexString(finalizedCheckpoint.root), + payloadStatus: PayloadStatus.FULL, + }, justifiedBalancesGetter: () => originalState.epochCtx.effectiveBalanceIncrements, equivocatingIndices: new Set(), }; diff --git a/packages/beacon-node/test/spec/presets/fork_choice.test.ts b/packages/beacon-node/test/spec/presets/fork_choice.test.ts index fd7d59f55bf9..8908296c6dd2 100644 --- a/packages/beacon-node/test/spec/presets/fork_choice.test.ts +++ b/packages/beacon-node/test/spec/presets/fork_choice.test.ts @@ -416,7 +416,7 @@ const forkChoiceTest = if (step.checks.should_override_forkchoice_update) { const currentSlot = Math.floor(tickTime / (config.SLOT_DURATION_MS / 1000)); const result = chain.forkChoice.shouldOverrideForkChoiceUpdate( - head.blockRoot, + head, tickTime % (config.SLOT_DURATION_MS / 1000), currentSlot ); diff --git a/packages/beacon-node/test/spec/utils/gossipValidation.ts b/packages/beacon-node/test/spec/utils/gossipValidation.ts index 01680f2ffe0b..fdea09cc23d8 100644 --- a/packages/beacon-node/test/spec/utils/gossipValidation.ts +++ b/packages/beacon-node/test/spec/utils/gossipValidation.ts @@ -247,7 +247,7 @@ function isDescendantAtFinalizedCheckpoint( ): boolean { try { const finalizedSlot = computeStartSlotAtEpoch(checkpoint.epoch); - return chain.forkChoice.getAncestor(blockRootHex, finalizedSlot) === checkpoint.rootHex; + return chain.forkChoice.getAncestor(blockRootHex, finalizedSlot).blockRoot === checkpoint.rootHex; } catch { return false; } diff --git a/packages/beacon-node/test/unit-minimal/chain/opPools/aggregatedAttestationPool.test.ts b/packages/beacon-node/test/unit-minimal/chain/opPools/aggregatedAttestationPool.test.ts index e9d58fa1145b..9ad1f66de838 100644 --- a/packages/beacon-node/test/unit-minimal/chain/opPools/aggregatedAttestationPool.test.ts +++ b/packages/beacon-node/test/unit-minimal/chain/opPools/aggregatedAttestationPool.test.ts @@ -246,7 +246,7 @@ describe("AggregatedAttestationPool - get packed attestations - Electra", () => } } - forkchoiceStub.getBlockHex.mockReturnValue(generateProtoBlock()); + forkchoiceStub.getBlockHexDefaultStatus.mockReturnValue(generateProtoBlock()); forkchoiceStub.getDependentRoot.mockReturnValue(ZERO_HASH_HEX); const shufflingCache = new ShufflingCache(); diff --git a/packages/beacon-node/test/unit/api/impl/validator/produceBlockV3.test.ts b/packages/beacon-node/test/unit/api/impl/validator/produceBlockV3.test.ts index 5c76c2599cc6..99ff85ff5da0 100644 --- a/packages/beacon-node/test/unit/api/impl/validator/produceBlockV3.test.ts +++ b/packages/beacon-node/test/unit/api/impl/validator/produceBlockV3.test.ts @@ -96,7 +96,7 @@ describe("api/validator - produceBlockV3", () => { blockRoot: toRootHex(fullBlock.parentRoot), } as ProtoBlock); modules.chain.getProposerHead.mockReturnValue({blockRoot: toRootHex(fullBlock.parentRoot)} as ProtoBlock); - modules.chain.forkChoice.getBlock.mockReturnValue(zeroProtoBlock); + modules.chain.forkChoice.getBlockDefaultStatus.mockReturnValue(zeroProtoBlock); modules.chain.produceCommonBlockBody.mockResolvedValue({ attestations: fullBlock.body.attestations, attesterSlashings: fullBlock.body.attesterSlashings, @@ -185,7 +185,7 @@ describe("api/validator - produceBlockV3", () => { modules.chain.getProposerHead.mockReturnValue(parentBlock); modules.chain.recomputeForkChoiceHead.mockReturnValue(parentBlock); - modules.chain.forkChoice.getBlock.mockReturnValue(parentBlock); + modules.chain.forkChoice.getBlockDefaultStatus.mockReturnValue(parentBlock); modules.chain.produceBlock.mockResolvedValue({ block: fullBlock, executionPayloadValue, diff --git a/packages/beacon-node/test/unit/chain/blocks/verifyBlocksSanityChecks.test.ts b/packages/beacon-node/test/unit/chain/blocks/verifyBlocksSanityChecks.test.ts index 48a83f18916b..f1edb03d9789 100644 --- a/packages/beacon-node/test/unit/chain/blocks/verifyBlocksSanityChecks.test.ts +++ b/packages/beacon-node/test/unit/chain/blocks/verifyBlocksSanityChecks.test.ts @@ -1,6 +1,6 @@ import {beforeEach, describe, expect, it} from "vitest"; import {config} from "@lodestar/config/default"; -import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; +import {IForkChoice, PayloadStatus, ProtoBlock} from "@lodestar/fork-choice"; import {computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {SignedBeaconBlock, Slot, ssz} from "@lodestar/types"; import {toHex, toRootHex} from "@lodestar/utils"; @@ -25,15 +25,20 @@ describe("chain / blocks / verifyBlocksSanityChecks", () => { block.message.slot = currentSlot; forkChoice = getMockedBeaconChain().forkChoice; - forkChoice.getFinalizedCheckpoint.mockReturnValue({epoch: 0, root: Buffer.alloc(32), rootHex: ""}); + forkChoice.getFinalizedCheckpoint.mockReturnValue({ + epoch: 0, + root: Buffer.alloc(32), + rootHex: "", + payloadStatus: PayloadStatus.FULL, + }); clock = new ClockStopped(currentSlot); modules = {config, forkChoice, clock, opts: {} as IChainOptions, blacklistedBlocks: new Map()}; // On first call, parentRoot is known - forkChoice.getBlockHex.mockReturnValue({} as ProtoBlock); + forkChoice.getBlockHexDefaultStatus.mockReturnValue({} as ProtoBlock); }); it("PARENT_UNKNOWN", () => { - forkChoice.getBlockHex.mockReturnValue(null); + forkChoice.getBlockHexDefaultStatus.mockReturnValue(null); expectThrowsLodestarError(() => verifyBlocksSanityChecks(modules, [block], {}), BlockErrorCode.PARENT_UNKNOWN); }); @@ -48,7 +53,12 @@ describe("chain / blocks / verifyBlocksSanityChecks", () => { }); it("WOULD_REVERT_FINALIZED_SLOT", () => { - forkChoice.getFinalizedCheckpoint.mockReturnValue({epoch: 5, root: Buffer.alloc(32), rootHex: ""}); + forkChoice.getFinalizedCheckpoint.mockReturnValue({ + epoch: 5, + root: Buffer.alloc(32), + rootHex: "", + payloadStatus: PayloadStatus.FULL, + }); expectThrowsLodestarError( () => verifyBlocksSanityChecks(modules, [block], {}), BlockErrorCode.WOULD_REVERT_FINALIZED_SLOT @@ -180,14 +190,14 @@ function getForkChoice(knownBlocks: SignedBeaconBlock[], finalizedEpoch = 0): IF } return { - getBlockHex(blockRoot) { + getBlockHexDefaultStatus(blockRoot) { return blocks.get(blockRoot) ?? null; }, hasBlockHex(blockRoot) { return blocks.has(blockRoot); }, getFinalizedCheckpoint() { - return {epoch: finalizedEpoch, root: Buffer.alloc(32), rootHex: ""}; + return {epoch: finalizedEpoch, root: Buffer.alloc(32), rootHex: "", payloadStatus: PayloadStatus.FULL}; }, } as Partial as IForkChoice; } diff --git a/packages/beacon-node/test/unit/chain/forkChoice/forkChoice.test.ts b/packages/beacon-node/test/unit/chain/forkChoice/forkChoice.test.ts index f35e7be05188..b9ca0f658226 100644 --- a/packages/beacon-node/test/unit/chain/forkChoice/forkChoice.test.ts +++ b/packages/beacon-node/test/unit/chain/forkChoice/forkChoice.test.ts @@ -202,8 +202,8 @@ describe("LodestarForkChoice", () => { forkChoice.onBlock(block28.message, state28, blockDelaySec, currentSlot, executionStatus, dataAvailabilityStatus); expect(forkChoice.getAllAncestorBlocks(hashBlock(block16.message))).toHaveLength(3); expect(forkChoice.getAllAncestorBlocks(hashBlock(block24.message))).toHaveLength(5); - expect(forkChoice.getBlockHex(hashBlock(block08.message))).not.toBeNull(); - expect(forkChoice.getBlockHex(hashBlock(block12.message))).not.toBeNull(); + expect(forkChoice.getBlockHexDefaultStatus(hashBlock(block08.message))).not.toBeNull(); + expect(forkChoice.getBlockHexDefaultStatus(hashBlock(block12.message))).not.toBeNull(); expect(forkChoice.hasBlockHex(hashBlock(block08.message))).toBe(true); expect(forkChoice.hasBlockHex(hashBlock(block12.message))).toBe(true); forkChoice.onBlock(block32.message, state32, blockDelaySec, currentSlot, executionStatus, dataAvailabilityStatus); @@ -213,8 +213,8 @@ describe("LodestarForkChoice", () => { "getAllAncestorBlocks should not return finalized block" ); expect(forkChoice.getAllAncestorBlocks(hashBlock(block24.message))).toHaveLength(2); - expect(forkChoice.getBlockHex(hashBlock(block08.message))).toBe(null); - expect(forkChoice.getBlockHex(hashBlock(block12.message))).toBe(null); + expect(forkChoice.getBlockHexDefaultStatus(hashBlock(block08.message))).toBe(null); + expect(forkChoice.getBlockHexDefaultStatus(hashBlock(block12.message))).toBe(null); expect(forkChoice.hasBlockHex(hashBlock(block08.message))).toBe(false); expect(forkChoice.hasBlockHex(hashBlock(block12.message))).toBe(false); }); diff --git a/packages/beacon-node/test/unit/chain/seenCache/seenBlockInput.test.ts b/packages/beacon-node/test/unit/chain/seenCache/seenBlockInput.test.ts index 0c7465554f9a..21f92dd32d4e 100644 --- a/packages/beacon-node/test/unit/chain/seenCache/seenBlockInput.test.ts +++ b/packages/beacon-node/test/unit/chain/seenCache/seenBlockInput.test.ts @@ -1,5 +1,6 @@ import {generateKeyPair} from "@libp2p/crypto/keys"; import {beforeEach, describe, expect, it} from "vitest"; +import {PayloadStatus} from "@lodestar/fork-choice"; import {ForkName} from "@lodestar/params"; import {signedBlockToSignedHeader} from "@lodestar/state-transition"; import {toRootHex} from "@lodestar/utils"; @@ -218,6 +219,7 @@ describe("SeenBlockInputCache", async () => { epoch: config.DENEB_FORK_EPOCH, root, rootHex, + payloadStatus: PayloadStatus.FULL, }); expect(cache.get(childRootHex)).toBeUndefined(); expect(cache.get(parentRootHex)).toBeUndefined(); @@ -228,6 +230,7 @@ describe("SeenBlockInputCache", async () => { epoch: config.CAPELLA_FORK_EPOCH, root, rootHex, + payloadStatus: PayloadStatus.FULL, }); expect(cache.get(childRootHex)).toBe(childBlockInput); expect(cache.get(parentRootHex)).toBe(parentBlockInput); diff --git a/packages/beacon-node/test/unit/chain/validation/block.test.ts b/packages/beacon-node/test/unit/chain/validation/block.test.ts index 29defe4c28fc..1f21051fdc97 100644 --- a/packages/beacon-node/test/unit/chain/validation/block.test.ts +++ b/packages/beacon-node/test/unit/chain/validation/block.test.ts @@ -1,7 +1,7 @@ import {Mock, Mocked, beforeEach, describe, it, vi} from "vitest"; import {createBeaconConfig, createChainForkConfig} from "@lodestar/config"; import {config as configDef} from "@lodestar/config/default"; -import {ProtoBlock} from "@lodestar/fork-choice"; +import {PayloadStatus, ProtoBlock} from "@lodestar/fork-choice"; import {ForkName, ForkPostDeneb, ForkPreFulu} from "@lodestar/params"; import {SignedBeaconBlock, ssz} from "@lodestar/types"; import {BlockErrorCode} from "../../../../src/chain/errors/index.js"; @@ -38,7 +38,7 @@ describe("gossip block validation", () => { chain = getMockedBeaconChain({config}); vi.spyOn(chain.clock, "currentSlotWithGossipDisparity", "get").mockReturnValue(clockSlot); forkChoice = chain.forkChoice; - forkChoice.getBlockHex.mockReturnValue(null); + forkChoice.getBlockHexDefaultStatus.mockReturnValue(null); chain.forkChoice = forkChoice; regen = chain.regen; @@ -46,7 +46,12 @@ describe("gossip block validation", () => { verifySignature = chain.bls.verifySignatureSets; verifySignature.mockResolvedValue(true); - forkChoice.getFinalizedCheckpoint.mockReturnValue({epoch: 0, root: ZERO_HASH, rootHex: ""}); + forkChoice.getFinalizedCheckpoint.mockReturnValue({ + epoch: 0, + root: ZERO_HASH, + rootHex: "", + payloadStatus: PayloadStatus.FULL, + }); // Reset seen cache ( @@ -70,7 +75,12 @@ describe("gossip block validation", () => { it("WOULD_REVERT_FINALIZED_SLOT", async () => { // Set finalized epoch to be greater than block's epoch - forkChoice.getFinalizedCheckpoint.mockReturnValue({epoch: Infinity, root: ZERO_HASH, rootHex: ""}); + forkChoice.getFinalizedCheckpoint.mockReturnValue({ + epoch: Infinity, + root: ZERO_HASH, + rootHex: "", + payloadStatus: PayloadStatus.FULL, + }); await expectRejectedWithLodestarError( validateGossipBlock(config, chain, job, ForkName.phase0), @@ -80,7 +90,7 @@ describe("gossip block validation", () => { it("ALREADY_KNOWN", async () => { // Make the fork choice return a block summary for the proposed block - forkChoice.getBlockHex.mockReturnValue({} as ProtoBlock); + forkChoice.getBlockHexDefaultStatus.mockReturnValue({} as ProtoBlock); await expectRejectedWithLodestarError( validateGossipBlock(config, chain, job, ForkName.phase0), @@ -100,9 +110,9 @@ describe("gossip block validation", () => { it("PARENT_UNKNOWN (fork-choice)", async () => { // Return not known for proposed block - forkChoice.getBlockHex.mockReturnValueOnce(null); + forkChoice.getBlockHexDefaultStatus.mockReturnValueOnce(null); // Return not known for parent block too - forkChoice.getBlockHex.mockReturnValueOnce(null); + forkChoice.getBlockHexDefaultStatus.mockReturnValueOnce(null); await expectRejectedWithLodestarError( validateGossipBlock(config, chain, job, ForkName.phase0), @@ -112,9 +122,9 @@ describe("gossip block validation", () => { it("TOO_MANY_SKIPPED_SLOTS", async () => { // Return not known for proposed block - forkChoice.getBlockHex.mockReturnValueOnce(null); + forkChoice.getBlockHexDefaultStatus.mockReturnValueOnce(null); // Return parent block with 1 slot way back than maxSkipSlots - forkChoice.getBlockHex.mockReturnValueOnce({slot: block.slot - (maxSkipSlots + 1)} as ProtoBlock); + forkChoice.getBlockHexDefaultStatus.mockReturnValueOnce({slot: block.slot - (maxSkipSlots + 1)} as ProtoBlock); await expectRejectedWithLodestarError( validateGossipBlock(config, chain, job, ForkName.phase0), @@ -124,9 +134,9 @@ describe("gossip block validation", () => { it("NOT_LATER_THAN_PARENT", async () => { // Return not known for proposed block - forkChoice.getBlockHex.mockReturnValueOnce(null); + forkChoice.getBlockHexDefaultStatus.mockReturnValueOnce(null); // Returned parent block is latter than proposed block - forkChoice.getBlockHex.mockReturnValueOnce({slot: clockSlot + 1} as ProtoBlock); + forkChoice.getBlockHexDefaultStatus.mockReturnValueOnce({slot: clockSlot + 1} as ProtoBlock); await expectRejectedWithLodestarError( validateGossipBlock(config, chain, job, ForkName.phase0), @@ -136,9 +146,9 @@ describe("gossip block validation", () => { it("PARENT_UNKNOWN (regen)", async () => { // Return not known for proposed block - forkChoice.getBlockHex.mockReturnValueOnce(null); + forkChoice.getBlockHexDefaultStatus.mockReturnValueOnce(null); // Returned parent block is latter than proposed block - forkChoice.getBlockHex.mockReturnValueOnce({slot: clockSlot - 1} as ProtoBlock); + forkChoice.getBlockHexDefaultStatus.mockReturnValueOnce({slot: clockSlot - 1} as ProtoBlock); // Regen not able to get the parent block state regen.getPreState.mockRejectedValue(undefined); @@ -150,9 +160,9 @@ describe("gossip block validation", () => { it("PROPOSAL_SIGNATURE_INVALID", async () => { // Return not known for proposed block - forkChoice.getBlockHex.mockReturnValueOnce(null); + forkChoice.getBlockHexDefaultStatus.mockReturnValueOnce(null); // Returned parent block is latter than proposed block - forkChoice.getBlockHex.mockReturnValueOnce({slot: clockSlot - 1} as ProtoBlock); + forkChoice.getBlockHexDefaultStatus.mockReturnValueOnce({slot: clockSlot - 1} as ProtoBlock); // Regen returns some state regen.getPreState.mockResolvedValue(generateCachedState()); // BLS signature verifier returns invalid @@ -166,9 +176,9 @@ describe("gossip block validation", () => { it("INCORRECT_PROPOSER", async () => { // Return not known for proposed block - forkChoice.getBlockHex.mockReturnValueOnce(null); + forkChoice.getBlockHexDefaultStatus.mockReturnValueOnce(null); // Returned parent block is latter than proposed block - forkChoice.getBlockHex.mockReturnValueOnce({slot: clockSlot - 1} as ProtoBlock); + forkChoice.getBlockHexDefaultStatus.mockReturnValueOnce({slot: clockSlot - 1} as ProtoBlock); // Regen returns some state const state = generateCachedState(); regen.getPreState.mockResolvedValue(state); @@ -185,9 +195,9 @@ describe("gossip block validation", () => { it("valid", async () => { // Return not known for proposed block - forkChoice.getBlockHex.mockReturnValueOnce(null); + forkChoice.getBlockHexDefaultStatus.mockReturnValueOnce(null); // Returned parent block is latter than proposed block - forkChoice.getBlockHex.mockReturnValueOnce({slot: clockSlot - 1} as ProtoBlock); + forkChoice.getBlockHexDefaultStatus.mockReturnValueOnce({slot: clockSlot - 1} as ProtoBlock); // Regen returns some state const state = generateCachedState(); regen.getPreState.mockResolvedValue(state); @@ -206,9 +216,9 @@ describe("gossip block validation", () => { () => new Uint8Array([0]) ); // Return not known for proposed block - forkChoice.getBlockHex.mockReturnValueOnce(null); + forkChoice.getBlockHexDefaultStatus.mockReturnValueOnce(null); // Returned parent block is latter than proposed block - forkChoice.getBlockHex.mockReturnValueOnce({slot: clockSlot - 1} as ProtoBlock); + forkChoice.getBlockHexDefaultStatus.mockReturnValueOnce({slot: clockSlot - 1} as ProtoBlock); // Regen returns some state const state = generateCachedState(); regen.getPreState.mockResolvedValue(state); @@ -232,9 +242,9 @@ describe("gossip block validation", () => { () => new Uint8Array([0]) ); // Return not known for proposed block - forkChoice.getBlockHex.mockReturnValueOnce(null); + forkChoice.getBlockHexDefaultStatus.mockReturnValueOnce(null); // Returned parent block is latter than proposed block - forkChoice.getBlockHex.mockReturnValueOnce({slot: clockSlot - 1} as ProtoBlock); + forkChoice.getBlockHexDefaultStatus.mockReturnValueOnce({slot: clockSlot - 1} as ProtoBlock); // Regen returns some state const state = generateCachedState(); regen.getPreState.mockResolvedValue(state); diff --git a/packages/beacon-node/test/utils/state.ts b/packages/beacon-node/test/utils/state.ts index a6aa3aa7741e..bab23523b29b 100644 --- a/packages/beacon-node/test/utils/state.ts +++ b/packages/beacon-node/test/utils/state.ts @@ -1,7 +1,7 @@ import {SecretKey} from "@chainsafe/blst"; import {ChainForkConfig, createBeaconConfig} from "@lodestar/config"; import {config as minimalConfig} from "@lodestar/config/default"; -import {ExecutionStatus, ProtoBlock} from "@lodestar/fork-choice"; +import {ExecutionStatus, PayloadStatus, ProtoBlock} from "@lodestar/fork-choice"; import {FAR_FUTURE_EPOCH, ForkName, ForkSeq, MAX_EFFECTIVE_BALANCE, SYNC_COMMITTEE_SIZE} from "@lodestar/params"; import { BeaconStateAllForks, @@ -175,4 +175,8 @@ export const zeroProtoBlock: ProtoBlock = { ...{executionPayloadBlockHash: null, executionStatus: ExecutionStatus.PreMerge}, dataAvailabilityStatus: DataAvailabilityStatus.PreData, + payloadStatus: PayloadStatus.FULL, + builderIndex: null, + blockHashFromBid: null, + parentBlockHash: null, }; diff --git a/packages/beacon-node/test/utils/validationData/attestation.ts b/packages/beacon-node/test/utils/validationData/attestation.ts index e626ef9a66e7..18175f596b19 100644 --- a/packages/beacon-node/test/utils/validationData/attestation.ts +++ b/packages/beacon-node/test/utils/validationData/attestation.ts @@ -76,8 +76,14 @@ export function getAttestationValidData(opts: AttestationValidDataOpts): { timeliness: false, - ...{executionPayloadBlockHash: null, executionStatus: ExecutionStatus.PreMerge}, + executionPayloadBlockHash: null, + executionStatus: ExecutionStatus.PreMerge, dataAvailabilityStatus: DataAvailabilityStatus.PreData, + + parentBlockHash: null, + payloadStatus: 2, // PayloadStatus.FULL + builderIndex: null, + blockHashFromBid: null, }; const shufflingCache = new ShufflingCache(null, null, {}, [ @@ -104,6 +110,18 @@ export function getAttestationValidData(opts: AttestationValidDataOpts): { if (rootHex !== toHexString(beaconBlockRoot)) return null; return headBlock; }, + getBlockDefaultStatus: (root) => { + if (!ssz.Root.equals(root, beaconBlockRoot)) return null; + return headBlock; + }, + getBlockHexDefaultStatus: (rootHex) => { + if (rootHex !== toHexString(beaconBlockRoot)) return null; + return headBlock; + }, + getBlockHexAndBlockHash: (rootHex) => { + if (rootHex !== toHexString(beaconBlockRoot)) return null; + return headBlock; + }, getDependentRoot: () => state.epochCtx.currentDecisionRoot, } as Partial as IForkChoice; diff --git a/packages/fork-choice/src/forkChoice/errors.ts b/packages/fork-choice/src/forkChoice/errors.ts index cd0e0cd25274..c79309b17547 100644 --- a/packages/fork-choice/src/forkChoice/errors.ts +++ b/packages/fork-choice/src/forkChoice/errors.ts @@ -9,7 +9,7 @@ export enum InvalidBlockCode { } export type InvalidBlock = - | {code: InvalidBlockCode.UNKNOWN_PARENT; root: RootHex} + | {code: InvalidBlockCode.UNKNOWN_PARENT; root: RootHex; hash: RootHex | null} | {code: InvalidBlockCode.FUTURE_SLOT; currentSlot: Slot; blockSlot: Slot} | {code: InvalidBlockCode.FINALIZED_SLOT; finalizedSlot: Slot; blockSlot: Slot} | {code: InvalidBlockCode.NOT_FINALIZED_DESCENDANT; finalizedRoot: RootHex; blockAncestor?: RootHex}; @@ -53,6 +53,10 @@ export enum InvalidAttestationCode { * Delay consideration in the fork choice until their slot is in the past. */ FUTURE_SLOT = "FUTURE_SLOT", + /** + * The attestation data index is invalid for a Gloas block (must be 0 or 1). + */ + INVALID_DATA_INDEX = "INVALID_DATA_INDEX", } export type InvalidAttestation = @@ -64,7 +68,8 @@ export type InvalidAttestation = | {code: InvalidAttestationCode.PAST_EPOCH; attestationEpoch: Epoch; currentEpoch: Epoch} | {code: InvalidAttestationCode.INVALID_TARGET; attestation: RootHex; local: RootHex} | {code: InvalidAttestationCode.ATTESTS_TO_FUTURE_BLOCK; block: Slot; attestation: Slot} - | {code: InvalidAttestationCode.FUTURE_SLOT; attestationSlot: Slot; latestPermissibleSlot: Slot}; + | {code: InvalidAttestationCode.FUTURE_SLOT; attestationSlot: Slot; latestPermissibleSlot: Slot} + | {code: InvalidAttestationCode.INVALID_DATA_INDEX; index: number}; export enum ForkChoiceErrorCode { INVALID_ATTESTATION = "FORKCHOICE_ERROR_INVALID_ATTESTATION", diff --git a/packages/fork-choice/src/forkChoice/forkChoice.ts b/packages/fork-choice/src/forkChoice/forkChoice.ts index 9e0548e1ae93..d280abbeead2 100644 --- a/packages/fork-choice/src/forkChoice/forkChoice.ts +++ b/packages/fork-choice/src/forkChoice/forkChoice.ts @@ -1,7 +1,8 @@ import {ChainForkConfig} from "@lodestar/config"; -import {SLOTS_PER_EPOCH, SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params"; +import {ForkSeq, SLOTS_PER_EPOCH, SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params"; import { CachedBeaconStateAllForks, + CachedBeaconStateGloas, DataAvailabilityStatus, EffectiveBalanceIncrements, ZERO_HASH, @@ -37,9 +38,11 @@ import { LVHExecResponse, MaybeValidExecutionStatus, NULL_VOTE_INDEX, + PayloadStatus, ProtoBlock, ProtoNode, VoteIndex, + isGloasBlock, } from "../protoArray/interface.js"; import {ProtoArray} from "../protoArray/protoArray.js"; import {ForkChoiceError, ForkChoiceErrorCode, InvalidAttestationCode, InvalidBlockCode} from "./errors.js"; @@ -51,7 +54,7 @@ import { NotReorgedReason, ShouldOverrideForkChoiceUpdateResult, } from "./interface.js"; -import {CheckpointWithHex, IForkChoiceStore, JustifiedBalances, toCheckpointWithHex} from "./store.js"; +import {CheckpointWithPayload, IForkChoiceStore, JustifiedBalances, toCheckpointWithPayload} from "./store.js"; export type ForkChoiceOpts = { proposerBoost?: boolean; @@ -71,7 +74,7 @@ export type UpdateAndGetHeadOpt = | {mode: UpdateHeadOpt.GetPredictedProposerHead; secFromSlot: number; slot: Slot}; // the initial vote epoch for all validators -const INIT_VOTE_EPOCH: Epoch = 0; +const INIT_VOTE_SLOT: Slot = 0; /** * Provides an implementation of "Ethereum Consensus -- Beacon Chain Fork Choice": @@ -94,18 +97,28 @@ export class ForkChoice implements IForkChoice { irrecoverableError?: Error; /** * Votes currently tracked in the protoArray. Instead of tracking a VoteTracker of currentIndex, nextIndex and epoch, - * we decompose the struct and track them in 3 separate arrays for performance reason. + * we decompose the struct and track them in separate arrays for performance reason. + * + * For Gloas (ePBS), LatestMessage tracks slot instead of epoch and includes payload_present flag. + * Spec: gloas/fork-choice.md#modified-latestmessage + * + * IMPORTANT: voteCurrentIndices and voteNextIndices point to the EXACT variant node index. + * The payload status is encoded in the node index itself (different variants have different indices). + * For example, if a validator votes for the EMPTY variant, voteNextIndices[i] points to that specific EMPTY node. */ private readonly voteCurrentIndices: VoteIndex[]; private readonly voteNextIndices: VoteIndex[]; - private readonly voteNextEpochs: Epoch[]; + private readonly voteNextSlots: Slot[]; /** * Attestations that arrived at the current slot and must be queued for later processing. * NOT currently tracked in the protoArray + * + * Modified for Gloas to track PayloadStatus per validator. + * Maps: Slot -> BlockRoot -> ValidatorIndex -> PayloadStatus */ - private readonly queuedAttestations: MapDef>> = new MapDef( - () => new MapDef(() => new Set()) + private readonly queuedAttestations: MapDef>> = new MapDef( + () => new MapDef(() => new Map()) ); /** @@ -150,13 +163,14 @@ export class ForkChoice implements IForkChoice { this.voteCurrentIndices = new Array(validatorCount).fill(NULL_VOTE_INDEX); this.voteNextIndices = new Array(validatorCount).fill(NULL_VOTE_INDEX); // when compute deltas, we ignore epoch if voteNextIndex is NULL_VOTE_INDEX anyway - this.voteNextEpochs = new Array(validatorCount).fill(INIT_VOTE_EPOCH); + + this.voteNextSlots = new Array(validatorCount).fill(0); this.head = this.updateHead(); this.balances = this.fcStore.justified.balances; metrics?.forkChoice.votes.addCollect(() => { - metrics.forkChoice.votes.set(this.voteNextEpochs.length); + metrics.forkChoice.votes.set(this.voteNextSlots.length); metrics.forkChoice.queuedAttestations.set(this.queuedAttestationsPreviousSlot); metrics.forkChoice.validatedAttestationDatas.set(this.validatedAttestationDatas.size); metrics.forkChoice.balancesLength.set(this.balances.length); @@ -177,7 +191,7 @@ export class ForkChoice implements IForkChoice { * * https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/fork-choice.md#get_ancestor */ - getAncestor(blockRoot: RootHex, ancestorSlot: Slot): RootHex { + getAncestor(blockRoot: RootHex, ancestorSlot: Slot): ProtoNode { return this.protoArray.getAncestor(blockRoot, ancestorSlot); } @@ -237,11 +251,10 @@ export class ForkChoice implements IForkChoice { // Return false otherwise. // Note when proposer boost reorg is disabled, it always returns false shouldOverrideForkChoiceUpdate( - blockRoot: RootHex, + headBlock: ProtoBlock, secFromSlot: number, currentSlot: Slot ): ShouldOverrideForkChoiceUpdateResult { - const headBlock = this.getBlockHex(blockRoot); if (headBlock === null) { // should not happen because this block just got imported. Fall back to no-reorg. return {shouldOverrideFcu: false, reason: NotReorgedReason.HeadBlockNotAvailable}; @@ -257,7 +270,10 @@ export class ForkChoice implements IForkChoice { return {shouldOverrideFcu: false, reason: NotReorgedReason.ProposerBoostReorgDisabled}; } - const parentBlock = this.protoArray.getBlock(headBlock.parentRoot); + const parentBlock = this.protoArray.getBlock( + headBlock.parentRoot, + this.protoArray.getParentPayloadStatus(headBlock) + ); const proposalSlot = headBlock.slot + 1; // No reorg if parentBlock isn't available @@ -282,7 +298,10 @@ export class ForkChoice implements IForkChoice { return {shouldOverrideFcu: false, reason: NotReorgedReason.ReorgMoreThanOneSlot}; } - this.logger?.verbose("Block is weak. Should override forkchoice update", {blockRoot, slot: currentSlot}); + this.logger?.verbose("Block is weak. Should override forkchoice update", { + blockRoot: headBlock.blockRoot, + slot: currentSlot, + }); return {shouldOverrideFcu: true, parentBlock}; } @@ -316,7 +335,7 @@ export class ForkChoice implements IForkChoice { } const blockRoot = headBlock.blockRoot; - const result = this.shouldOverrideForkChoiceUpdate(blockRoot, secFromSlot, currentSlot); + const result = this.shouldOverrideForkChoiceUpdate(headBlock, secFromSlot, currentSlot); if (result.shouldOverrideFcu) { this.logger?.verbose("Current head is weak. Predicting next block to be built on parent of head.", { @@ -363,7 +382,10 @@ export class ForkChoice implements IForkChoice { return {proposerHead, isHeadTimely, notReorgedReason: NotReorgedReason.ProposerBoostReorgDisabled}; } - const parentBlock = this.protoArray.getBlock(headBlock.parentRoot); + const parentBlock = this.protoArray.getBlock( + headBlock.parentRoot, + this.protoArray.getParentPayloadStatus(headBlock) + ); // No reorg if parentBlock isn't available if (parentBlock === undefined) { @@ -400,7 +422,7 @@ export class ForkChoice implements IForkChoice { slotsPerEpoch: SLOTS_PER_EPOCH, committeePercent: this.config.REORG_HEAD_WEIGHT_THRESHOLD, }); - const headNode = this.protoArray.getNode(headBlock.blockRoot); + const headNode = this.protoArray.getNode(headBlock.blockRoot, headBlock.payloadStatus); // If headNode is unavailable, give up reorg if (headNode === undefined || headNode.weight >= reorgThreshold) { return {proposerHead, isHeadTimely, notReorgedReason: NotReorgedReason.HeadBlockNotWeak}; @@ -412,7 +434,7 @@ export class ForkChoice implements IForkChoice { slotsPerEpoch: SLOTS_PER_EPOCH, committeePercent: this.config.REORG_PARENT_WEIGHT_THRESHOLD, }); - const parentNode = this.protoArray.getNode(parentBlock.blockRoot); + const parentNode = this.protoArray.getNode(parentBlock.blockRoot, parentBlock.payloadStatus); // If parentNode is unavailable, give up reorg if (parentNode === undefined || parentNode.weight <= parentThreshold) { return {proposerHead, isHeadTimely, notReorgedReason: NotReorgedReason.ParentBlockNotStrong}; @@ -509,23 +531,10 @@ export class ForkChoice implements IForkChoice { currentSlot, }); - const headRoot = this.protoArray.findHead(this.fcStore.justified.checkpoint.rootHex, currentSlot); - const headIndex = this.protoArray.indices.get(headRoot); - if (headIndex === undefined) { - throw new ForkChoiceError({ - code: ForkChoiceErrorCode.MISSING_PROTO_ARRAY_BLOCK, - root: headRoot, - }); - } - const headNode = this.protoArray.nodes[headIndex]; - if (headNode === undefined) { - throw new ForkChoiceError({ - code: ForkChoiceErrorCode.MISSING_PROTO_ARRAY_BLOCK, - root: headRoot, - }); - } + // findHead returns the ProtoNode representing the head + const head = this.protoArray.findHead(this.fcStore.justified.checkpoint.rootHex, currentSlot); - this.head = headNode; + this.head = head; return this.head; } @@ -548,11 +557,11 @@ export class ForkChoice implements IForkChoice { return this.protoArray.nodes; } - getFinalizedCheckpoint(): CheckpointWithHex { + getFinalizedCheckpoint(): CheckpointWithPayload { return this.fcStore.finalizedCheckpoint; } - getJustifiedCheckpoint(): CheckpointWithHex { + getJustifiedCheckpoint(): CheckpointWithPayload { return this.fcStore.justified.checkpoint; } @@ -585,14 +594,18 @@ export class ForkChoice implements IForkChoice { ): ProtoBlock { const {parentRoot, slot} = block; const parentRootHex = toRootHex(parentRoot); - // Parent block must be known - const parentBlock = this.protoArray.getBlock(parentRootHex); + // Parent block must be known because state_transition would have failed otherwise. + const parentHashHex = isGloasBeaconBlock(block) + ? toRootHex(block.body.signedExecutionPayloadBid.message.parentBlockHash) + : null; + const parentBlock = this.protoArray.getParent(parentRootHex, parentHashHex); if (!parentBlock) { throw new ForkChoiceError({ code: ForkChoiceErrorCode.INVALID_BLOCK, err: { code: InvalidBlockCode.UNKNOWN_PARENT, root: parentRootHex, + hash: parentHashHex, }, }); } @@ -627,15 +640,18 @@ export class ForkChoice implements IForkChoice { } // Check block is a descendant of the finalized block at the checkpoint finalized slot. - const blockAncestorRoot = this.getAncestor(parentRootHex, finalizedSlot); - const finalizedRoot = this.fcStore.finalizedCheckpoint.rootHex; - if (blockAncestorRoot !== finalizedRoot) { + const blockAncestorNode = this.getAncestor(parentRootHex, finalizedSlot); + const fcStoreFinalized = this.fcStore.finalizedCheckpoint; + if ( + blockAncestorNode.blockRoot !== fcStoreFinalized.rootHex || + blockAncestorNode.payloadStatus !== fcStoreFinalized.payloadStatus + ) { throw new ForkChoiceError({ code: ForkChoiceErrorCode.INVALID_BLOCK, err: { code: InvalidBlockCode.NOT_FINALIZED_DESCENDANT, - finalizedRoot, - blockAncestor: blockAncestorRoot, + finalizedRoot: fcStoreFinalized.rootHex, + blockAncestor: blockAncestorNode.blockRoot, }, }); } @@ -655,10 +671,15 @@ export class ForkChoice implements IForkChoice { this.proposerBoostRoot = blockRootHex; } - const justifiedCheckpoint = toCheckpointWithHex(state.currentJustifiedCheckpoint); - const finalizedCheckpoint = toCheckpointWithHex(state.finalizedCheckpoint); + // Get justified checkpoint with payload status for Gloas + const justifiedPayloadStatus = getCheckpointPayloadStatus(state, state.currentJustifiedCheckpoint.epoch); + const justifiedCheckpoint = toCheckpointWithPayload(state.currentJustifiedCheckpoint, justifiedPayloadStatus); const stateJustifiedEpoch = justifiedCheckpoint.epoch; + // Get finalized checkpoint with payload status for Gloas + const finalizedPayloadStatus = getCheckpointPayloadStatus(state, state.finalizedCheckpoint.epoch); + const finalizedCheckpoint = toCheckpointWithPayload(state.finalizedCheckpoint, finalizedPayloadStatus); + // Justified balances for `justifiedCheckpoint` are new to the fork-choice. Compute them on demand only if // the justified checkpoint changes this.updateCheckpoints(justifiedCheckpoint, finalizedCheckpoint, () => @@ -679,29 +700,57 @@ export class ForkChoice implements IForkChoice { // This is an optimization. It should reduce the amount of times we run // `process_justification_and_finalization` by approximately 1/3rd when the chain is // performing optimally. - let unrealizedJustifiedCheckpoint: CheckpointWithHex; - let unrealizedFinalizedCheckpoint: CheckpointWithHex; + let unrealizedJustifiedCheckpoint: CheckpointWithPayload; + let unrealizedFinalizedCheckpoint: CheckpointWithPayload; if (this.opts?.computeUnrealized) { if ( parentBlock.unrealizedJustifiedEpoch === blockEpoch && parentBlock.unrealizedFinalizedEpoch + 1 >= blockEpoch ) { // reuse from parent, happens at 1/3 last blocks of epoch as monitored in mainnet + // Get payload status for unrealized justified checkpoint + const unrealizedJustifiedPayloadStatus = getCheckpointPayloadStatus( + state, + parentBlock.unrealizedJustifiedEpoch + ); unrealizedJustifiedCheckpoint = { epoch: parentBlock.unrealizedJustifiedEpoch, root: fromHex(parentBlock.unrealizedJustifiedRoot), rootHex: parentBlock.unrealizedJustifiedRoot, + payloadStatus: unrealizedJustifiedPayloadStatus, }; + // Get payload status for unrealized finalized checkpoint + const unrealizedFinalizedPayloadStatus = getCheckpointPayloadStatus( + state, + parentBlock.unrealizedFinalizedEpoch + ); unrealizedFinalizedCheckpoint = { epoch: parentBlock.unrealizedFinalizedEpoch, root: fromHex(parentBlock.unrealizedFinalizedRoot), rootHex: parentBlock.unrealizedFinalizedRoot, + payloadStatus: unrealizedFinalizedPayloadStatus, }; } else { // compute new, happens 2/3 first blocks of epoch as monitored in mainnet const unrealized = computeUnrealizedCheckpoints(state); - unrealizedJustifiedCheckpoint = toCheckpointWithHex(unrealized.justifiedCheckpoint); - unrealizedFinalizedCheckpoint = toCheckpointWithHex(unrealized.finalizedCheckpoint); + // Get payload status for unrealized justified checkpoint + const unrealizedJustifiedPayloadStatus = getCheckpointPayloadStatus( + state, + unrealized.justifiedCheckpoint.epoch + ); + unrealizedJustifiedCheckpoint = toCheckpointWithPayload( + unrealized.justifiedCheckpoint, + unrealizedJustifiedPayloadStatus + ); + // Get payload status for unrealized finalized checkpoint + const unrealizedFinalizedPayloadStatus = getCheckpointPayloadStatus( + state, + unrealized.finalizedCheckpoint.epoch + ); + unrealizedFinalizedCheckpoint = toCheckpointWithPayload( + unrealized.finalizedCheckpoint, + unrealizedFinalizedPayloadStatus + ); } } else { unrealizedJustifiedCheckpoint = justifiedCheckpoint; @@ -743,32 +792,56 @@ export class ForkChoice implements IForkChoice { unrealizedFinalizedEpoch: unrealizedFinalizedCheckpoint.epoch, unrealizedFinalizedRoot: unrealizedFinalizedCheckpoint.rootHex, - // TODO GLOAS: Need to update this when we are merging nc/epbs-fc. Need to define `getPostGloasExecStatus` - // to make sure execution status of post-gloas blocks is ExecutionStatus.PayloadSeparated - ...(isExecutionBlockBodyType(block.body) && isExecutionStateType(state) && isExecutionEnabled(state, block) - ? { - executionPayloadBlockHash: toRootHex(block.body.executionPayload.blockHash), - executionPayloadNumber: block.body.executionPayload.blockNumber, - executionStatus: this.getPreGloasExecStatus(executionStatus), - dataAvailabilityStatus, - } - : { - executionPayloadBlockHash: null, - executionStatus: this.getPreMergeExecStatus(executionStatus), - dataAvailabilityStatus: this.getPreMergeDataStatus(dataAvailabilityStatus), - }), ...(isGloasBeaconBlock(block) ? { - builderIndex: block.body.signedExecutionPayloadBid.message.builderIndex, - blockHashHex: toRootHex(block.body.signedExecutionPayloadBid.message.blockHash), + executionPayloadBlockHash: toRootHex(block.body.signedExecutionPayloadBid.message.parentBlockHash), // post-gloas, we don't know payload hash until we import execution payload. Set to parent payload hash for now + executionPayloadNumber: (() => { + // Determine parent's execution payload number based on which variant the block extends + const parentBlockHashFromBid = toRootHex(block.body.signedExecutionPayloadBid.message.parentBlockHash); + + // If parent is pre-merge, return 0 + if (parentBlock.executionPayloadBlockHash === null) { + return 0; + } + + // If parent is pre-Gloas, it only has FULL variant + if (parentBlock.parentBlockHash === null) { + return parentBlock.executionPayloadNumber; + } + + // Parent is Gloas: get the variant that matches the parentBlockHash from bid + const parentVariant = this.getBlockHexAndBlockHash(parentRootHex, parentBlockHashFromBid); + if (parentVariant && parentVariant.executionPayloadBlockHash !== null) { + return parentVariant.executionPayloadNumber; + } + // Fallback to parent block's number (we know it's post-merge from check above) + return parentBlock.executionPayloadNumber; + })(), + executionStatus: this.getPostGloasExecStatus(executionStatus), + dataAvailabilityStatus, } - : { - builderIndex: undefined, - blockHashHex: undefined, - }), + : isExecutionBlockBodyType(block.body) && isExecutionStateType(state) && isExecutionEnabled(state, block) + ? { + executionPayloadBlockHash: toRootHex(block.body.executionPayload.blockHash), + executionPayloadNumber: block.body.executionPayload.blockNumber, + executionStatus: this.getPreGloasExecStatus(executionStatus), + dataAvailabilityStatus, + } + : { + executionPayloadBlockHash: null, + executionStatus: this.getPreMergeExecStatus(executionStatus), + dataAvailabilityStatus: this.getPreMergeDataStatus(dataAvailabilityStatus), + }), + + payloadStatus: isGloasBeaconBlock(block) ? PayloadStatus.PENDING : PayloadStatus.FULL, + builderIndex: isGloasBeaconBlock(block) ? block.body.signedExecutionPayloadBid.message.builderIndex : null, + blockHashFromBid: isGloasBeaconBlock(block) + ? toRootHex(block.body.signedExecutionPayloadBid.message.blockHash) + : null, + parentBlockHash: parentHashHex, }; - this.protoArray.onBlock(protoBlock, currentSlot); + this.protoArray.onBlock(protoBlock, currentSlot, this.proposerBoostRoot); return protoBlock; } @@ -815,10 +888,45 @@ export class ForkChoice implements IForkChoice { this.validateOnAttestation(attestation, slot, blockRootHex, targetEpoch, attDataRoot, forceImport); + // Pre-gloas: payload is always present + // Post-gloas: + // - always add weight to PENDING + // - if message.slot > block.slot, it also add weights to FULL or EMPTY + let payloadStatus: PayloadStatus; + + // We need to retrieve block to check if it's Gloas and to compare slot + // https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.1/specs/gloas/fork-choice.md#new-is_supporting_vote + const block = this.getBlockHexDefaultStatus(blockRootHex); + + if (block && isGloasBlock(block)) { + // Post-Gloas block: determine FULL/EMPTY/PENDING based on slot and committee index + // If slot > block.slot, we can determine FULL or EMPTY. Else always PENDING + if (slot > block.slot) { + if (attestationData.index === 1) { + payloadStatus = PayloadStatus.FULL; + } else if (attestationData.index === 0) { + payloadStatus = PayloadStatus.EMPTY; + } else { + throw new ForkChoiceError({ + code: ForkChoiceErrorCode.INVALID_ATTESTATION, + err: { + code: InvalidAttestationCode.INVALID_DATA_INDEX, + index: attestationData.index, + }, + }); + } + } else { + payloadStatus = PayloadStatus.PENDING; + } + } else { + // Pre-Gloas block or block not found: always FULL + payloadStatus = PayloadStatus.FULL; + } + if (slot < this.fcStore.currentSlot) { for (const validatorIndex of attestation.attestingIndices) { if (!this.fcStore.equivocatingIndices.has(validatorIndex)) { - this.addLatestMessage(validatorIndex, targetEpoch, blockRootHex); + this.addLatestMessage(validatorIndex, slot, blockRootHex, payloadStatus); } } } else { @@ -829,10 +937,10 @@ export class ForkChoice implements IForkChoice { // Delay consideration in the fork choice until their slot is in the past. // ``` const byRoot = this.queuedAttestations.getOrDefault(slot); - const validatorIndices = byRoot.getOrDefault(blockRootHex); + const validatorVotes = byRoot.getOrDefault(blockRootHex); for (const validatorIndex of attestation.attestingIndices) { if (!this.fcStore.equivocatingIndices.has(validatorIndex)) { - validatorIndices.add(validatorIndex); + validatorVotes.set(validatorIndex, payloadStatus); } } } @@ -851,6 +959,36 @@ export class ForkChoice implements IForkChoice { } } + /** + * Process a PTC (Payload Timeliness Committee) message + * Updates the PTC votes for multiple validators attesting to a block + * Spec: gloas/fork-choice.md#new-on_payload_attestation_message + */ + notifyPtcMessages(blockRoot: RootHex, ptcIndices: number[], payloadPresent: boolean): void { + this.protoArray.notifyPtcMessages(blockRoot, ptcIndices, payloadPresent); + } + + /** + * Notify fork choice that an execution payload has arrived (Gloas fork) + * Creates the FULL variant of a Gloas block when the payload becomes available + * Spec: gloas/fork-choice.md#new-on_execution_payload + */ + onExecutionPayload( + blockRoot: RootHex, + executionPayloadBlockHash: RootHex, + executionPayloadNumber: number, + executionPayloadStateRoot: RootHex + ): void { + this.protoArray.onExecutionPayload( + blockRoot, + this.fcStore.currentSlot, + executionPayloadBlockHash, + executionPayloadNumber, + executionPayloadStateRoot, + this.proposerBoostRoot + ); + } + /** * Call `onTick` for all slots between `fcStore.getCurrentSlot()` and the provided `currentSlot`. * This should only be called once per slot because: @@ -882,15 +1020,21 @@ export class ForkChoice implements IForkChoice { return this.hasBlockHex(toRootHex(blockRoot)); } /** Returns a `ProtoBlock` if the block is known **and** a descendant of the finalized root. */ - getBlock(blockRoot: Root): ProtoBlock | null { - return this.getBlockHex(toRootHex(blockRoot)); + getBlock(blockRoot: Root, payloadStatus: PayloadStatus): ProtoBlock | null { + return this.getBlockHex(toRootHex(blockRoot), payloadStatus); + } + + getBlockDefaultStatus(blockRoot: Root): ProtoBlock | null { + return this.getBlockHexDefaultStatus(toRootHex(blockRoot)); } /** * Returns `true` if the block is known **and** a descendant of the finalized root. + * Uses default variant (PENDING for Gloas, FULL for pre-Gloas). */ hasBlockHex(blockRoot: RootHex): boolean { - const node = this.protoArray.getNode(blockRoot); + const defaultStatus = this.protoArray.getDefaultVariant(blockRoot); + const node = defaultStatus !== undefined ? this.protoArray.getNode(blockRoot, defaultStatus) : undefined; if (node === undefined) { return false; } @@ -915,8 +1059,8 @@ export class ForkChoice implements IForkChoice { /** * Returns a MUTABLE `ProtoBlock` if the block is known **and** a descendant of the finalized root. */ - getBlockHex(blockRoot: RootHex): ProtoBlock | null { - const node = this.protoArray.getNode(blockRoot); + getBlockHex(blockRoot: RootHex, payloadStatus: PayloadStatus): ProtoBlock | null { + const node = this.protoArray.getNode(blockRoot, payloadStatus); if (!node) { return null; } @@ -930,23 +1074,49 @@ export class ForkChoice implements IForkChoice { }; } + /** + * Returns a `ProtoBlock` with the default variant for the given block root + * - Pre-Gloas blocks: returns FULL variant (only variant) + * - Gloas blocks: returns PENDING variant + * + * Use this when you need the canonical block reference regardless of payload status. + * For searching by execution payload hash and variant-specific info, use `getBlockHexAndBlockHash` instead. + */ + getBlockHexDefaultStatus(blockRoot: RootHex): ProtoBlock | null { + const defaultStatus = this.protoArray.getDefaultVariant(blockRoot); + if (defaultStatus === undefined) { + return null; + } + + return this.getBlockHex(blockRoot, defaultStatus); + } + + /** + * Returns EMPTY or FULL `ProtoBlock` that has matching block root and block hash + */ + getBlockHexAndBlockHash(blockRoot: RootHex, blockHash: RootHex): ProtoBlock | null { + return this.protoArray.getBlockHexAndBlockHash(blockRoot, blockHash); + } + getJustifiedBlock(): ProtoBlock { - const block = this.getBlockHex(this.fcStore.justified.checkpoint.rootHex); + const {rootHex, payloadStatus} = this.fcStore.justified.checkpoint; + const block = this.getBlockHex(rootHex, payloadStatus); if (!block) { throw new ForkChoiceError({ code: ForkChoiceErrorCode.MISSING_PROTO_ARRAY_BLOCK, - root: this.fcStore.justified.checkpoint.rootHex, + root: rootHex, }); } return block; } getFinalizedBlock(): ProtoBlock { - const block = this.getBlockHex(this.fcStore.finalizedCheckpoint.rootHex); + const {rootHex, payloadStatus} = this.fcStore.finalizedCheckpoint; + const block = this.getBlockHex(rootHex, payloadStatus); if (!block) { throw new ForkChoiceError({ code: ForkChoiceErrorCode.MISSING_PROTO_ARRAY_BLOCK, - root: this.fcStore.finalizedCheckpoint.rootHex, + root: rootHex, }); } return block; @@ -973,7 +1143,7 @@ export class ForkChoice implements IForkChoice { prune(finalizedRoot: RootHex): ProtoBlock[] { const prunedNodes = this.protoArray.maybePrune(finalizedRoot); const prunedCount = prunedNodes.length; - for (let i = 0; i < this.voteNextEpochs.length; i++) { + for (let i = 0; i < this.voteNextSlots.length; i++) { const currentIndex = this.voteCurrentIndices[i]; if (currentIndex !== NULL_VOTE_INDEX) { @@ -1076,17 +1246,24 @@ export class ForkChoice implements IForkChoice { return this.protoArray.nodes; } + // TODO GLOAS: this function is ambiguous, consumer should also provide payload, or it should accept a ProtoBlock instead + // also consumer may want PENDING or EMPTY only *forwardIterateDescendants(blockRoot: RootHex): IterableIterator { const rootsInChain = new Set([blockRoot]); - const blockIndex = this.protoArray.indices.get(blockRoot); - if (blockIndex === undefined) { + const blockVariants = this.protoArray.indices.get(blockRoot); + if (blockVariants === undefined) { throw new ForkChoiceError({ code: ForkChoiceErrorCode.MISSING_PROTO_ARRAY_BLOCK, root: blockRoot, }); } + // Find the minimum index among all variants to start iteration + const blockIndex = Array.isArray(blockVariants) + ? Math.min(...blockVariants.filter((idx) => idx !== undefined)) + : blockVariants; + for (let i = blockIndex + 1; i < this.protoArray.nodes.length; i++) { const node = this.protoArray.nodes[i]; if (rootsInChain.has(node.parentRoot)) { @@ -1116,8 +1293,8 @@ export class ForkChoice implements IForkChoice { /** Returns the distance of common ancestor of nodes to the max of the newNode and the prevNode. */ getCommonAncestorDepth(prevBlock: ProtoBlock, newBlock: ProtoBlock): AncestorResult { - const prevNode = this.protoArray.getNode(prevBlock.blockRoot); - const newNode = this.protoArray.getNode(newBlock.blockRoot); + const prevNode = this.protoArray.getNode(prevBlock.blockRoot, prevBlock.payloadStatus); + const newNode = this.protoArray.getNode(newBlock.blockRoot, newBlock.payloadStatus); if (!prevNode || !newNode) { return {code: AncestorStatus.BlockUnknown}; } @@ -1198,12 +1375,17 @@ export class ForkChoice implements IForkChoice { return block.parentRoot; } - block = - block.blockRoot === block.targetRoot - ? // For the first slot of the epoch, a block is it's own target - this.protoArray.getBlockReadonly(block.parentRoot) - : // else we can navigate much faster jumping to the target block - this.protoArray.getBlockReadonly(block.targetRoot); + // For the first slot of the epoch, a block is it's own target + const nextRoot = block.blockRoot === block.targetRoot ? block.parentRoot : block.targetRoot; + // Use default variant (PENDING for Gloas, FULL for pre-Gloas) + // For Gloas: we search for PENDING blocks because dependent root is determined by the block itself, + // not the payload. In state-transition, block parentage is independent of payload status, + // so linking by PENDING block in fork-choice is correct. + const defaultStatus = this.protoArray.getDefaultVariant(nextRoot); + if (defaultStatus === undefined) { + throw Error(`No block for root ${nextRoot}`); + } + block = this.protoArray.getBlockReadonly(nextRoot, defaultStatus); } throw Error(`Not found dependent root for block slot ${block.slot}, epoch difference ${epochDifference}`); @@ -1252,6 +1434,14 @@ export class ForkChoice implements IForkChoice { return executionStatus; } + private getPostGloasExecStatus(executionStatus: MaybeValidExecutionStatus): ExecutionStatus.PayloadSeparated { + if (executionStatus !== ExecutionStatus.PayloadSeparated) + throw Error( + `Invalid post-gloas execution status: expected: ${ExecutionStatus.PayloadSeparated}, got ${executionStatus}` + ); + return executionStatus; + } + /** * Why `getJustifiedBalances` getter? * - updateCheckpoints() is called in both on_block and on_tick. @@ -1269,12 +1459,12 @@ export class ForkChoice implements IForkChoice { * * **`on_tick`** * May need the justified balances of: - * - unrealizedJustified: Already available in `CheckpointHexWithBalance` + * - unrealizedJustified: Already available in `CheckpointWithPayloadAndBalance` * Since this balances are already available the getter is just `() => balances`, without cache interaction */ private updateCheckpoints( - justifiedCheckpoint: CheckpointWithHex, - finalizedCheckpoint: CheckpointWithHex, + justifiedCheckpoint: CheckpointWithPayload, + finalizedCheckpoint: CheckpointWithPayload, getJustifiedBalances: () => JustifiedBalances ): void { // Update justified checkpoint. @@ -1294,8 +1484,8 @@ export class ForkChoice implements IForkChoice { * Update unrealized checkpoints in store if necessary */ private updateUnrealizedCheckpoints( - unrealizedJustifiedCheckpoint: CheckpointWithHex, - unrealizedFinalizedCheckpoint: CheckpointWithHex, + unrealizedJustifiedCheckpoint: CheckpointWithPayload, + unrealizedFinalizedCheckpoint: CheckpointWithPayload, getJustifiedBalances: () => JustifiedBalances ): void { if (unrealizedJustifiedCheckpoint.epoch > this.fcStore.unrealizedJustified.checkpoint.epoch) { @@ -1414,7 +1604,9 @@ export class ForkChoice implements IForkChoice { // // Attestations must be for a known block. If the block is unknown, we simply drop the // attestation and do not delay consideration for later. - const block = this.protoArray.getBlock(beaconBlockRootHex); + // We don't care which variant it is, just need to find the block + const defaultStatus = this.protoArray.getDefaultVariant(beaconBlockRootHex); + const block = defaultStatus !== undefined ? this.protoArray.getBlock(beaconBlockRootHex, defaultStatus) : undefined; if (!block) { throw new ForkChoiceError({ code: ForkChoiceErrorCode.INVALID_ATTESTATION, @@ -1455,33 +1647,57 @@ export class ForkChoice implements IForkChoice { }); } + // For Gloas blocks, attestation index must be 0 or 1 + if (isGloasBlock(block) && attestationData.index !== 0 && attestationData.index !== 1) { + throw new ForkChoiceError({ + code: ForkChoiceErrorCode.INVALID_ATTESTATION, + err: { + code: InvalidAttestationCode.INVALID_DATA_INDEX, + index: attestationData.index, + }, + }); + } + this.validatedAttestationDatas.add(attDataRoot); } /** * Add a validator's latest message to the tracked votes. * Always sync voteCurrentIndices and voteNextIndices so that it'll not throw in computeDeltas() + * + * Modified for Gloas to accept slot and payloadPresent. + * Spec: gloas/fork-choice.md#modified-update_latest_messages + * + * For backward compatibility with Fulu (pre-Gloas): + * - Accepts both epoch-derived and slot parameters + * - payloadPresent defaults to true for Fulu (payloads embedded in blocks) */ - private addLatestMessage(validatorIndex: ValidatorIndex, nextEpoch: Epoch, nextRoot: RootHex): void { + private addLatestMessage( + validatorIndex: ValidatorIndex, + nextSlot: Slot, + nextRoot: RootHex, + nextPayloadStatus: PayloadStatus + ): void { // should not happen, attestation is validated before this step - const nextIndex = this.protoArray.indices.get(nextRoot); + // Get the node index for the voted block + const nextIndex = this.protoArray.getNodeIndexByRootAndStatus(nextRoot, nextPayloadStatus); if (nextIndex === undefined) { - throw new Error(`Could not find proto index for nextRoot ${nextRoot}`); + throw new Error(`Could not find proto index for nextRoot ${nextRoot} with payloadStatus ${nextPayloadStatus}`); } // ensure there is no undefined entries in Votes arrays - if (this.voteNextEpochs.length < validatorIndex + 1) { - for (let i = this.voteNextEpochs.length; i < validatorIndex + 1; i++) { - this.voteNextEpochs[i] = INIT_VOTE_EPOCH; + if (this.voteNextSlots.length < validatorIndex + 1) { + for (let i = this.voteNextSlots.length; i < validatorIndex + 1; i++) { + this.voteNextSlots[i] = INIT_VOTE_SLOT; this.voteCurrentIndices[i] = this.voteNextIndices[i] = NULL_VOTE_INDEX; } } - const existingNextEpoch = this.voteNextEpochs[validatorIndex]; - if (existingNextEpoch === INIT_VOTE_EPOCH || nextEpoch > existingNextEpoch) { + const existingNextSlot = this.voteNextSlots[validatorIndex]; + if (existingNextSlot === INIT_VOTE_SLOT || computeEpochAtSlot(nextSlot) > computeEpochAtSlot(existingNextSlot)) { // nextIndex is transfered to currentIndex in computeDeltas() this.voteNextIndices[validatorIndex] = nextIndex; - this.voteNextEpochs[validatorIndex] = nextEpoch; + this.voteNextSlots[validatorIndex] = nextSlot; } // else its an old vote, don't count it } @@ -1493,18 +1709,17 @@ export class ForkChoice implements IForkChoice { private processAttestationQueue(): void { const currentSlot = this.fcStore.currentSlot; for (const [slot, byRoot] of this.queuedAttestations.entries()) { - const targetEpoch = computeEpochAtSlot(slot); if (slot < currentSlot) { this.queuedAttestations.delete(slot); - for (const [blockRoot, validatorIndices] of byRoot.entries()) { + for (const [blockRoot, validatorVotes] of byRoot.entries()) { const blockRootHex = blockRoot; - for (const validatorIndex of validatorIndices) { + for (const [validatorIndex, payloadStatus] of validatorVotes.entries()) { // equivocatingIndices was checked in onAttestation - this.addLatestMessage(validatorIndex, targetEpoch, blockRootHex); + this.addLatestMessage(validatorIndex, slot, blockRootHex, payloadStatus); } if (slot === currentSlot - 1) { - this.queuedAttestationsPreviousSlot += validatorIndices.size; + this.queuedAttestationsPreviousSlot += validatorVotes.size; } } } else { @@ -1618,3 +1833,31 @@ export function getCommitteeFraction( const committeeWeight = Math.floor(justifiedTotalActiveBalanceByIncrement / config.slotsPerEpoch); return Math.floor((committeeWeight * config.committeePercent) / 100); } + +/** + * Get the payload status for a checkpoint. + * + * Pre-Gloas: always FULL (payload embedded in block) + * Gloas: determined by state.execution_payload_availability + * + * @param state - The state to check execution_payload_availability + * @param checkpointEpoch - The epoch of the checkpoint + */ +export function getCheckpointPayloadStatus(state: CachedBeaconStateAllForks, checkpointEpoch: number): PayloadStatus { + // Compute checkpoint slot first to determine the correct fork + const checkpointSlot = computeStartSlotAtEpoch(checkpointEpoch); + const fork = state.config.getForkSeq(checkpointSlot); + + // Pre-Gloas: always FULL + if (fork < ForkSeq.gloas) { + return PayloadStatus.FULL; + } + + // For Gloas, check state.execution_payload_availability + // - For non-skipped slots at checkpoint: returns false (EMPTY) since payload hasn't arrived yet + // - For skipped slots at checkpoint: returns the actual availability status from state + const gloasState = state as CachedBeaconStateGloas; + const payloadAvailable = gloasState.executionPayloadAvailability.get(checkpointSlot % SLOTS_PER_HISTORICAL_ROOT); + + return payloadAvailable ? PayloadStatus.FULL : PayloadStatus.EMPTY; +} diff --git a/packages/fork-choice/src/forkChoice/interface.ts b/packages/fork-choice/src/forkChoice/interface.ts index 32d9b6c68f78..e104f7a31709 100644 --- a/packages/fork-choice/src/forkChoice/interface.ts +++ b/packages/fork-choice/src/forkChoice/interface.ts @@ -4,9 +4,15 @@ import { EffectiveBalanceIncrements, } from "@lodestar/state-transition"; import {AttesterSlashing, BeaconBlock, Epoch, IndexedAttestation, Root, RootHex, Slot} from "@lodestar/types"; -import {LVHExecResponse, MaybeValidExecutionStatus, ProtoBlock, ProtoNode} from "../protoArray/interface.js"; +import { + LVHExecResponse, + MaybeValidExecutionStatus, + PayloadStatus, + ProtoBlock, + ProtoNode, +} from "../protoArray/interface.js"; import {UpdateAndGetHeadOpt} from "./forkChoice.js"; -import {CheckpointWithHex} from "./store.js"; +import {CheckpointWithHex, CheckpointWithPayload} from "./store.js"; export type CheckpointHex = { epoch: Epoch; @@ -18,12 +24,12 @@ export type CheckpointsWithHex = { finalizedCheckpoint: CheckpointWithHex; }; -export type CheckpointHexWithBalance = { - checkpoint: CheckpointWithHex; +export type CheckpointWithPayloadAndBalance = { + checkpoint: CheckpointWithPayload; balances: EffectiveBalanceIncrements; }; -export type CheckpointHexWithTotalBalance = CheckpointHexWithBalance & { +export type CheckpointWithPayloadAndTotalBalance = CheckpointWithPayloadAndBalance & { totalBalance: number; }; @@ -72,16 +78,18 @@ export interface IForkChoice { irrecoverableError?: Error; /** - * Returns the block root of an ancestor of `block_root` at the given `slot`. (Note: `slot` refers + * Returns the ancestor node of `block_root` at the given `slot`. (Note: `slot` refers * to the block that is *returned*, not the one that is supplied.) * * ## Specification * - * Equivalent to: + * Modified for Gloas to return ProtoNode instead of just root: + * https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.1/specs/gloas/fork-choice.md#modified-get_ancestor * - * https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/fork-choice.md#get_ancestor + * Pre-Gloas: Returns (root, PAYLOAD_STATUS_FULL) + * Gloas: Returns (root, payloadStatus) based on actual node state */ - getAncestor(blockRoot: RootHex, ancestorSlot: Slot): RootHex; + getAncestor(blockRoot: RootHex, ancestorSlot: Slot): ProtoNode; /** * Run the fork choice rule to determine the head. * @@ -104,7 +112,7 @@ export interface IForkChoice { * called by `predictProposerHead()` during `prepareNextSlot()`. */ shouldOverrideForkChoiceUpdate( - blockRoot: RootHex, + headBlock: ProtoBlock, secFromSlot: number, currentSlot: Slot ): ShouldOverrideForkChoiceUpdateResult; @@ -116,8 +124,8 @@ export interface IForkChoice { * Retrieve all nodes for the debug API. */ getAllNodes(): ProtoNode[]; - getFinalizedCheckpoint(): CheckpointWithHex; - getJustifiedCheckpoint(): CheckpointWithHex; + getFinalizedCheckpoint(): CheckpointWithPayload; + getJustifiedCheckpoint(): CheckpointWithPayload; /** * Add `block` to the fork choice DAG. * @@ -169,6 +177,38 @@ export interface IForkChoice { * https://github.com/ethereum/consensus-specs/blob/v1.2.0-rc.3/specs/phase0/fork-choice.md#on_attester_slashing */ onAttesterSlashing(slashing: AttesterSlashing): void; + /** + * Process PTC (Payload Timeliness Committee) messages from a block + * Updates the PTC votes for the attested beacon block + * + * ## Specification + * + * https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.0/specs/gloas/fork-choice.md#new-notify_ptc_messages + * + * @param blockRoot - The beacon block root being attested + * @param ptcIndices - Array of PTC committee indices that voted + * @param payloadPresent - Whether validators attest the payload is present + */ + notifyPtcMessages(blockRoot: RootHex, ptcIndices: number[], payloadPresent: boolean): void; + /** + * Notify fork choice that an execution payload has arrived (Gloas fork) + * Creates the FULL variant of a Gloas block when the payload becomes available + * + * ## Specification + * + * https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.1/specs/gloas/fork-choice.md#new-on_execution_payload + * + * @param blockRoot - The beacon block root for which the payload arrived + * @param executionPayloadBlockHash - The block hash of the execution payload + * @param executionPayloadNumber - The block number of the execution payload + * @param executionPayloadStateRoot - The execution payload state root ie. the root of post-state after processExecutionPayloadEnvelope() + */ + onExecutionPayload( + blockRoot: RootHex, + executionPayloadBlockHash: RootHex, + executionPayloadNumber: number, + executionPayloadStateRoot: RootHex + ): void; /** * Call `onTick` for all slots between `fcStore.getCurrentSlot()` and the provided `currentSlot`. */ @@ -192,8 +232,11 @@ export interface IForkChoice { /** * Returns a `ProtoBlock` if the block is known **and** a descendant of the finalized root. */ - getBlock(blockRoot: Root): ProtoBlock | null; - getBlockHex(blockRoot: RootHex): ProtoBlock | null; + getBlock(blockRoot: Root, payloadStatus: PayloadStatus): ProtoBlock | null; + getBlockHex(blockRoot: RootHex, payloadStatus: PayloadStatus): ProtoBlock | null; + getBlockDefaultStatus(blockRoot: Root): ProtoBlock | null; + getBlockHexDefaultStatus(blockRoot: RootHex): ProtoBlock | null; + getBlockHexAndBlockHash(blockRoot: RootHex, blockHash: RootHex): ProtoBlock | null; getFinalizedBlock(): ProtoBlock; getJustifiedBlock(): ProtoBlock; getFinalizedCheckpointSlot(): Slot; diff --git a/packages/fork-choice/src/forkChoice/store.ts b/packages/fork-choice/src/forkChoice/store.ts index fda01689f96c..8e817987d7e6 100644 --- a/packages/fork-choice/src/forkChoice/store.ts +++ b/packages/fork-choice/src/forkChoice/store.ts @@ -1,7 +1,8 @@ import {CachedBeaconStateAllForks, EffectiveBalanceIncrements} from "@lodestar/state-transition"; import {RootHex, Slot, ValidatorIndex, phase0} from "@lodestar/types"; import {toRootHex} from "@lodestar/utils"; -import {CheckpointHexWithBalance, CheckpointHexWithTotalBalance} from "./interface.js"; +import {PayloadStatus} from "../protoArray/interface.js"; +import {CheckpointWithPayloadAndBalance, CheckpointWithPayloadAndTotalBalance} from "./interface.js"; /** * Stores checkpoints in a hybrid format: @@ -10,6 +11,15 @@ import {CheckpointHexWithBalance, CheckpointHexWithTotalBalance} from "./interfa */ export type CheckpointWithHex = phase0.Checkpoint & {rootHex: RootHex}; +/** + * Checkpoint with payload status for Gloas fork choice. + * Used to track which variant (EMPTY or FULL) of the finalized/justified block to use. + * + * Pre-Gloas: payloadStatus is always FULL (payload embedded in block) + * Gloas: determined by state.execution_payload_availability + */ +export type CheckpointWithPayload = CheckpointWithHex & {payloadStatus: PayloadStatus}; + export type JustifiedBalances = EffectiveBalanceIncrements; /** @@ -19,7 +29,7 @@ export type JustifiedBalances = EffectiveBalanceIncrements; * @param blockState state that declares justified checkpoint `checkpoint` */ export type JustifiedBalancesGetter = ( - checkpoint: CheckpointWithHex, + checkpoint: CheckpointWithPayload, blockState: CachedBeaconStateAllForks ) => JustifiedBalances; @@ -37,11 +47,11 @@ export type JustifiedBalancesGetter = ( */ export interface IForkChoiceStore { currentSlot: Slot; - get justified(): CheckpointHexWithTotalBalance; - set justified(justified: CheckpointHexWithBalance); - unrealizedJustified: CheckpointHexWithBalance; - finalizedCheckpoint: CheckpointWithHex; - unrealizedFinalizedCheckpoint: CheckpointWithHex; + get justified(): CheckpointWithPayloadAndTotalBalance; + set justified(justified: CheckpointWithPayloadAndBalance); + unrealizedJustified: CheckpointWithPayloadAndBalance; + finalizedCheckpoint: CheckpointWithPayload; + unrealizedFinalizedCheckpoint: CheckpointWithPayload; justifiedBalancesGetter: JustifiedBalancesGetter; equivocatingIndices: Set; } @@ -50,10 +60,10 @@ export interface IForkChoiceStore { * IForkChoiceStore implementer which emits forkChoice events on updated justified and finalized checkpoints. */ export class ForkChoiceStore implements IForkChoiceStore { - private _justified: CheckpointHexWithTotalBalance; - unrealizedJustified: CheckpointHexWithBalance; - private _finalizedCheckpoint: CheckpointWithHex; - unrealizedFinalizedCheckpoint: CheckpointWithHex; + private _justified: CheckpointWithPayloadAndTotalBalance; + unrealizedJustified: CheckpointWithPayloadAndBalance; + private _finalizedCheckpoint: CheckpointWithPayload; + unrealizedFinalizedCheckpoint: CheckpointWithPayload; equivocatingIndices = new Set(); justifiedBalancesGetter: JustifiedBalancesGetter; currentSlot: Slot; @@ -64,37 +74,49 @@ export class ForkChoiceStore implements IForkChoiceStore { finalizedCheckpoint: phase0.Checkpoint, justifiedBalances: EffectiveBalanceIncrements, justifiedBalancesGetter: JustifiedBalancesGetter, + /** + * Payload status for justified checkpoint. + * Pre-Gloas: always FULL + * Gloas: determined by state.execution_payload_availability + */ + justifiedPayloadStatus: PayloadStatus, + /** + * Payload status for finalized checkpoint. + * Pre-Gloas: always FULL + * Gloas: determined by state.execution_payload_availability + */ + finalizedPayloadStatus: PayloadStatus, private readonly events?: { - onJustified: (cp: CheckpointWithHex) => void; - onFinalized: (cp: CheckpointWithHex) => void; + onJustified: (cp: CheckpointWithPayload) => void; + onFinalized: (cp: CheckpointWithPayload) => void; } ) { this.justifiedBalancesGetter = justifiedBalancesGetter; this.currentSlot = currentSlot; const justified = { - checkpoint: toCheckpointWithHex(justifiedCheckpoint), + checkpoint: toCheckpointWithPayload(justifiedCheckpoint, justifiedPayloadStatus), balances: justifiedBalances, totalBalance: computeTotalBalance(justifiedBalances), }; this._justified = justified; this.unrealizedJustified = justified; - this._finalizedCheckpoint = toCheckpointWithHex(finalizedCheckpoint); + this._finalizedCheckpoint = toCheckpointWithPayload(finalizedCheckpoint, finalizedPayloadStatus); this.unrealizedFinalizedCheckpoint = this._finalizedCheckpoint; } - get justified(): CheckpointHexWithTotalBalance { + get justified(): CheckpointWithPayloadAndTotalBalance { return this._justified; } - set justified(justified: CheckpointHexWithBalance) { + set justified(justified: CheckpointWithPayloadAndBalance) { this._justified = {...justified, totalBalance: computeTotalBalance(justified.balances)}; this.events?.onJustified(justified.checkpoint); } - get finalizedCheckpoint(): CheckpointWithHex { + get finalizedCheckpoint(): CheckpointWithPayload { return this._finalizedCheckpoint; } - set finalizedCheckpoint(checkpoint: CheckpointWithHex) { - const cp = toCheckpointWithHex(checkpoint); + set finalizedCheckpoint(checkpoint: CheckpointWithPayload) { + const cp = toCheckpointWithPayload(checkpoint, checkpoint.payloadStatus); this._finalizedCheckpoint = cp; this.events?.onFinalized(cp); } @@ -111,6 +133,16 @@ export function toCheckpointWithHex(checkpoint: phase0.Checkpoint): CheckpointWi }; } +export function toCheckpointWithPayload( + checkpoint: phase0.Checkpoint, + payloadStatus: PayloadStatus +): CheckpointWithPayload { + return { + ...toCheckpointWithHex(checkpoint), + payloadStatus, + }; +} + export function equalCheckpointWithHex(a: CheckpointWithHex, b: CheckpointWithHex): boolean { return a.epoch === b.epoch && a.rootHex === b.rootHex; } diff --git a/packages/fork-choice/src/index.ts b/packages/fork-choice/src/index.ts index e476660ee9ad..99246e5225f8 100644 --- a/packages/fork-choice/src/index.ts +++ b/packages/fork-choice/src/index.ts @@ -6,10 +6,17 @@ export { type InvalidBlock, InvalidBlockCode, } from "./forkChoice/errors.js"; -export {ForkChoice, type ForkChoiceOpts, UpdateHeadOpt} from "./forkChoice/forkChoice.js"; +export { + ForkChoice, + type ForkChoiceOpts, + UpdateHeadOpt, + getCheckpointPayloadStatus, +} from "./forkChoice/forkChoice.js"; export { type AncestorResult, AncestorStatus, + type CheckpointWithPayloadAndBalance, + type CheckpointWithPayloadAndTotalBalance, EpochDifference, type IForkChoice, NotReorgedReason, @@ -17,6 +24,7 @@ export { export * from "./forkChoice/safeBlocks.js"; export { type CheckpointWithHex, + type CheckpointWithPayload, ForkChoiceStore, type IForkChoiceStore, type JustifiedBalancesGetter, @@ -30,5 +38,5 @@ export type { ProtoBlock, ProtoNode, } from "./protoArray/interface.js"; -export {ExecutionStatus} from "./protoArray/interface.js"; +export {ExecutionStatus, PayloadStatus} from "./protoArray/interface.js"; export {ProtoArray} from "./protoArray/protoArray.js"; diff --git a/packages/fork-choice/src/protoArray/computeDeltas.ts b/packages/fork-choice/src/protoArray/computeDeltas.ts index 844d620f9998..44db719846b8 100644 --- a/packages/fork-choice/src/protoArray/computeDeltas.ts +++ b/packages/fork-choice/src/protoArray/computeDeltas.ts @@ -66,6 +66,7 @@ export function computeDeltas( for (let vIndex = 0; vIndex < voteNextIndices.length; vIndex++) { currentIndex = voteCurrentIndices[vIndex]; nextIndex = voteNextIndices[vIndex]; + // There is no need to create a score change if the validator has never voted or both of their // votes are for the zero hash (genesis block) if (currentIndex === NULL_VOTE_INDEX && nextIndex === NULL_VOTE_INDEX) { @@ -106,6 +107,9 @@ export function computeDeltas( continue; } + // Deduct old balance from current index, add new balance to next index + // currentIndex and nextIndex already point to the correct node variants + // Note: If a validator changes from EMPTY to FULL variant of the same block, indexChanged will be true if (currentIndex !== nextIndex || oldBalance !== newBalance) { // We ignore the vote if it is not known in `indices . // We assume that it is outside of our tree (ie: pre-finalization) and therefore not interesting @@ -116,6 +120,7 @@ export function computeDeltas( index: currentIndex, }); } + deltas[currentIndex] -= oldBalance; } @@ -128,6 +133,7 @@ export function computeDeltas( index: nextIndex, }); } + deltas[nextIndex] += newBalance; } voteCurrentIndices[vIndex] = nextIndex; diff --git a/packages/fork-choice/src/protoArray/errors.ts b/packages/fork-choice/src/protoArray/errors.ts index 650fd1c0b244..92b82242a4ac 100644 --- a/packages/fork-choice/src/protoArray/errors.ts +++ b/packages/fork-choice/src/protoArray/errors.ts @@ -12,6 +12,8 @@ export type LVHExecError = {lvhCode: LVHExecErrorCode; blockRoot: RootHex; execH export enum ProtoArrayErrorCode { FINALIZED_NODE_UNKNOWN = "PROTO_ARRAY_ERROR_FINALIZED_NODE_UNKNOWN", JUSTIFIED_NODE_UNKNOWN = "PROTO_ARRAY_ERROR_JUSTIFIED_NODE_UNKNOWN", + UNKNOWN_BLOCK = "PROTO_ARRAY_ERROR_UNKNOWN_BLOCK", + UNKNOWN_PARENT_BLOCK = "PROTO_ARRAY_ERROR_UNKNOWN_PARENT_BLOCK", INVALID_FINALIZED_ROOT_CHANGE = "PROTO_ARRAY_ERROR_INVALID_FINALIZED_ROOT_CHANGE", INVALID_NODE_INDEX = "PROTO_ARRAY_ERROR_INVALID_NODE_INDEX", INVALID_PARENT_INDEX = "PROTO_ARRAY_ERROR_INVALID_PARENT_INDEX", @@ -27,11 +29,14 @@ export enum ProtoArrayErrorCode { INVALID_BLOCK_EXECUTION_STATUS = "PROTO_ARRAY_INVALID_BLOCK_EXECUTION_STATUS", INVALID_JUSTIFIED_EXECUTION_STATUS = "PROTO_ARRAY_INVALID_JUSTIFIED_EXECUTION_STATUS", INVALID_LVH_EXECUTION_RESPONSE = "PROTO_ARRAY_INVALID_LVH_EXECUTION_RESPONSE", + PRE_GLOAS_BLOCK = "PROTO_ARRAY_ERROR_PRE_GLOAS_BLOCK", } export type ProtoArrayErrorType = | {code: ProtoArrayErrorCode.FINALIZED_NODE_UNKNOWN; root: RootHex} | {code: ProtoArrayErrorCode.JUSTIFIED_NODE_UNKNOWN; root: RootHex} + | {code: ProtoArrayErrorCode.UNKNOWN_BLOCK; root: RootHex} + | {code: ProtoArrayErrorCode.UNKNOWN_PARENT_BLOCK; parentRoot: RootHex; parentHash: RootHex | null} | {code: ProtoArrayErrorCode.INVALID_FINALIZED_ROOT_CHANGE} | {code: ProtoArrayErrorCode.INVALID_NODE_INDEX; index: number} | {code: ProtoArrayErrorCode.INVALID_PARENT_INDEX; index: number} @@ -54,6 +59,7 @@ export type ProtoArrayErrorType = } | {code: ProtoArrayErrorCode.INVALID_BLOCK_EXECUTION_STATUS; root: RootHex} | {code: ProtoArrayErrorCode.INVALID_JUSTIFIED_EXECUTION_STATUS; root: RootHex} - | ({code: ProtoArrayErrorCode.INVALID_LVH_EXECUTION_RESPONSE} & LVHExecError); + | ({code: ProtoArrayErrorCode.INVALID_LVH_EXECUTION_RESPONSE} & LVHExecError) + | {code: ProtoArrayErrorCode.PRE_GLOAS_BLOCK; root: RootHex}; export class ProtoArrayError extends LodestarError {} diff --git a/packages/fork-choice/src/protoArray/interface.ts b/packages/fork-choice/src/protoArray/interface.ts index 480ac7ffe04f..5c42e6f33c03 100644 --- a/packages/fork-choice/src/protoArray/interface.ts +++ b/packages/fork-choice/src/protoArray/interface.ts @@ -36,6 +36,23 @@ export enum ExecutionStatus { PayloadSeparated = "PayloadSeparated", } +/** + * Payload status for ePBS (Gloas fork) + * Spec: gloas/fork-choice.md#constants + */ +export enum PayloadStatus { + PENDING = 0, + EMPTY = 1, + FULL = 2, +} + +/** + * Check if a block is in the Gloas fork (ePBS enabled) + */ +export function isGloasBlock(block: ProtoBlock): boolean { + return block.parentBlockHash !== null; +} + export type LVHValidResponse = { executionStatus: ExecutionStatus.Valid; latestValidExecHash: RootHex; @@ -51,6 +68,11 @@ export type MaybeValidExecutionStatus = Exclude; @@ -102,14 +124,25 @@ export type ProtoBlock = BlockExtraMeta & { // Indicate whether block arrives in a timely manner ie. before the 4 second mark timeliness: boolean; - // GLOAS: The followings are from bids. Used for execution payload gossip validation - builderIndex?: number; - blockHashHex?: RootHex; + /** Payload status for this node (Gloas fork). Always FULL in pre-gloas */ + payloadStatus: PayloadStatus; + + // GLOAS: The followings are from bids. They are null in pre-gloas + // Used for execution payload gossip validation + builderIndex: number | null; + // Used for execution payload gossip validation. Not to be confused with executionPayloadBlockHash + blockHashFromBid: RootHex | null; + + // Used to determine if this block extends EMPTY or FULL parent variant + // Spec: gloas/fork-choice.md#new-get_parent_payload_status + parentBlockHash: RootHex | null; }; /** * A block root with additional metadata required to form a DAG * with vote weights and best blocks stored as metadata + * + * It is also used as ForkChoiceNode in fork choice spec */ export type ProtoNode = ProtoBlock & { parent?: number; diff --git a/packages/fork-choice/src/protoArray/protoArray.ts b/packages/fork-choice/src/protoArray/protoArray.ts index a006de102b2f..a8d13c3e5da1 100644 --- a/packages/fork-choice/src/protoArray/protoArray.ts +++ b/packages/fork-choice/src/protoArray/protoArray.ts @@ -1,16 +1,40 @@ -import {GENESIS_EPOCH} from "@lodestar/params"; +import {GENESIS_EPOCH, PTC_SIZE} from "@lodestar/params"; import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {Epoch, RootHex, Slot} from "@lodestar/types"; import {toRootHex} from "@lodestar/utils"; import {ForkChoiceError, ForkChoiceErrorCode} from "../forkChoice/errors.js"; import {LVHExecError, LVHExecErrorCode, ProtoArrayError, ProtoArrayErrorCode} from "./errors.js"; -import {ExecutionStatus, HEX_ZERO_HASH, LVHExecResponse, ProtoBlock, ProtoNode} from "./interface.js"; +import { + ExecutionStatus, + HEX_ZERO_HASH, + LVHExecResponse, + PayloadStatus, + ProtoBlock, + ProtoNode, + isGloasBlock, +} from "./interface.js"; + +/** + * Threshold for payload timeliness (>50% of PTC must vote) + * Spec: gloas/fork-choice.md (PAYLOAD_TIMELY_THRESHOLD = PTC_SIZE // 2) + */ +const PAYLOAD_TIMELY_THRESHOLD = Math.floor(PTC_SIZE / 2); export const DEFAULT_PRUNE_THRESHOLD = 0; type ProposerBoost = {root: RootHex; score: number}; const ZERO_HASH_HEX = toRootHex(Buffer.alloc(32, 0)); +/** Pre-Gloas: single element, FULL index (for backward compatibility) */ +type PreGloasVariantIndex = number; +/** + * Post-Gloas: array length is 2 or 3 + * - Length 2: [PENDING_INDEX, EMPTY_INDEX] when payload hasn't arrived yet + * - Length 3: [PENDING_INDEX, EMPTY_INDEX, FULL_INDEX] when payload has arrived + */ +type GloasVariantIndices = [number, number] | [number, number, number]; +type VariantIndices = PreGloasVariantIndex | GloasVariantIndices; + export class ProtoArray { // Do not attempt to prune the tree unless it has at least this many nodes. // Small prunes simply waste time @@ -20,11 +44,31 @@ export class ProtoArray { finalizedEpoch: Epoch; finalizedRoot: RootHex; nodes: ProtoNode[] = []; - indices = new Map(); + /** + * Maps block root to array of node indices for each payload status variant + * + * Array structure: [PENDING, EMPTY, FULL] where indices correspond to PayloadStatus enum values + * - number[0] = PENDING variant index (PayloadStatus.PENDING = 0) + * - number[1] = EMPTY variant index (PayloadStatus.EMPTY = 1) + * - number[2] = FULL variant index (PayloadStatus.FULL = 2) + * + * Note: undefined array elements indicate that variant doesn't exist for this block + */ + indices = new Map(); lvhError?: LVHExecError; private previousProposerBoost: ProposerBoost | null = null; + /** + * PTC (Payload Timeliness Committee) votes per block + * Maps block root to boolean array of size PTC_SIZE (from params: 512 mainnet, 2 minimal) + * Spec: gloas/fork-choice.md#modified-store (line 148) + * + * ptcVotes[blockRoot][i] = true if PTC member i voted payload_present=true + * Used by is_payload_timely() to determine if payload is timely + */ + private ptcVotes = new Map(); + constructor({ pruneThreshold, justifiedEpoch, @@ -59,11 +103,166 @@ export class ProtoArray { // We are using the blockROot as the targetRoot, since it always lies on an epoch boundary targetRoot: block.blockRoot, } as ProtoBlock, - currentSlot + currentSlot, + null ); return protoArray; } + /** + * Get node index for a block root and payload status + * + * @param root - The block root to look up + * @param payloadStatus - The specific payload status variant (PENDING/EMPTY/FULL) + * @returns The node index for the specified variant, or undefined if not found + * + * Behavior: + * - Pre-Gloas blocks: only FULL is valid, PENDING/EMPTY throw error + * - Gloas blocks: returns the specified variant index, or undefined if that variant doesn't exist + * + * Note: payloadStatus is required. Use getDefaultVariant() to get the canonical variant. + */ + getNodeIndexByRootAndStatus(root: RootHex, payloadStatus: PayloadStatus): number | undefined { + const variantOrArr = this.indices.get(root); + if (variantOrArr == null) { + return undefined; + } + + // Pre-Gloas: only FULL variant exists + if (!Array.isArray(variantOrArr)) { + // Return FULL variant if no status specified or FULL explicitly requested + if (payloadStatus === PayloadStatus.FULL) { + return variantOrArr; + } + // PENDING and EMPTY are invalid for pre-Gloas blocks + throw new ProtoArrayError({ + code: ProtoArrayErrorCode.INVALID_NODE_INDEX, + index: payloadStatus, + }); + } + + // Gloas: return the specified variant, or PENDING if not specified + return variantOrArr[payloadStatus]; + } + + /** + * Get the default/canonical payload status for a block root + * - Pre-Gloas blocks: Returns FULL (payload embedded in block) + * - Gloas blocks: Returns PENDING (canonical variant) + * + * @param blockRoot - The block root to check + * @returns PayloadStatus.FULL for pre-Gloas, PayloadStatus.PENDING for Gloas, undefined if block not found + */ + getDefaultVariant(blockRoot: RootHex): PayloadStatus | undefined { + const variantOrArr = this.indices.get(blockRoot); + if (variantOrArr == null) { + return undefined; + } + + // Pre-Gloas: only FULL variant exists + if (!Array.isArray(variantOrArr)) { + return PayloadStatus.FULL; + } + + // Gloas: multiple variants exist, PENDING is canonical + return PayloadStatus.PENDING; + } + + /** + * Determine which parent payload status a block extends + * Spec: gloas/fork-choice.md#new-get_parent_payload_status + * def get_parent_payload_status(store: Store, block: BeaconBlock) -> PayloadStatus: + * parent = store.blocks[block.parent_root] + * parent_block_hash = block.body.signed_execution_payload_bid.message.parent_block_hash + * message_block_hash = parent.body.signed_execution_payload_bid.message.block_hash + * return PAYLOAD_STATUS_FULL if parent_block_hash == message_block_hash else PAYLOAD_STATUS_EMPTY + * + * In lodestar forkchoice, we don't store the full bid, so we compares parent_block_hash in child's bid with executionPayloadBlockHash in parent: + * - If it matches EMPTY variant, return EMPTY + * - If it matches FULL variant, return FULL + * - If no match, throw UNKNOWN_PARENT_BLOCK error + * + * For pre-Gloas blocks: always returns FULL + */ + getParentPayloadStatus(block: ProtoBlock): PayloadStatus { + // Pre-Gloas blocks have payloads embedded, so parents are always FULL + const {parentBlockHash} = block; + if (parentBlockHash === null) { + return PayloadStatus.FULL; + } + + const parentBlock = this.getBlockHexAndBlockHash(block.parentRoot, parentBlockHash); + if (parentBlock == null) { + throw new ProtoArrayError({ + code: ProtoArrayErrorCode.UNKNOWN_PARENT_BLOCK, + parentRoot: block.parentRoot, + parentHash: parentBlockHash, + }); + } + + return parentBlock.payloadStatus; + } + + /** + * Return the parent `ProtoBlock` given its root and block hash. + */ + getParent(parentRoot: RootHex, parentBlockHash: RootHex | null): ProtoBlock | null { + // pre-gloas + if (parentBlockHash === null) { + const parentIndex = this.indices.get(parentRoot); + if (parentIndex === undefined) { + return null; + } + if (Array.isArray(parentIndex)) { + // Gloas block found when pre-gloas expected + throw new ProtoArrayError({ + code: ProtoArrayErrorCode.UNKNOWN_PARENT_BLOCK, + parentRoot, + parentHash: parentBlockHash, + }); + } + return this.nodes[parentIndex] ?? null; + } + + // post-gloas + return this.getBlockHexAndBlockHash(parentRoot, parentBlockHash); + } + + /** + * Returns an EMPTY or FULL `ProtoBlock` that has matching block root and block hash + */ + getBlockHexAndBlockHash(blockRoot: RootHex, blockHash: RootHex): ProtoBlock | null { + const variantIndices = this.indices.get(blockRoot); + if (variantIndices === undefined) { + return null; + } + + // Pre-Gloas + if (!Array.isArray(variantIndices)) { + const node = this.nodes[variantIndices]; + return node.executionPayloadBlockHash === blockHash ? node : null; + } + + // Post-Gloas, check empty and full variants + const fullNodeIndex = variantIndices[PayloadStatus.FULL]; + if (fullNodeIndex !== undefined) { + const fullNode = this.nodes[fullNodeIndex]; + if (fullNode && fullNode.executionPayloadBlockHash === blockHash) { + return fullNode; + } + } + + const emptyNode = this.nodes[variantIndices[PayloadStatus.EMPTY]]; + if (emptyNode && emptyNode.executionPayloadBlockHash === blockHash) { + return emptyNode; + } + + // PENDING is the same to EMPTY so not likely we can return it + // also it's only specific for fork-choice + + return null; + } + /** * Iterate backwards through the array, touching all nodes and their parents and potentially * the best-child of each parent. @@ -96,11 +295,11 @@ export class ProtoArray { finalizedRoot: RootHex; currentSlot: Slot; }): void { - if (deltas.length !== this.indices.size) { + if (deltas.length !== this.nodes.length) { throw new ProtoArrayError({ code: ProtoArrayErrorCode.INVALID_DELTA_LEN, deltas: deltas.length, - indices: this.indices.size, + indices: this.nodes.length, }); } @@ -183,7 +382,7 @@ export class ProtoArray { // If the node has a parent, try to update its best-child and best-descendant. const parentIndex = node.parent; if (parentIndex !== undefined) { - this.maybeUpdateBestChildAndDescendant(parentIndex, nodeIndex, currentSlot); + this.maybeUpdateBestChildAndDescendant(parentIndex, nodeIndex, currentSlot, proposerBoost?.root ?? null); } } // Update the previous proposer boost @@ -195,9 +394,9 @@ export class ProtoArray { * * It is only sane to supply an undefined parent for the genesis block */ - onBlock(block: ProtoBlock, currentSlot: Slot): void { + onBlock(block: ProtoBlock, currentSlot: Slot, proposerBoostRoot: RootHex | null): void { // If the block is already known, simply ignore it - if (this.indices.has(block.blockRoot)) { + if (this.hasBlock(block.blockRoot)) { return; } if (block.executionStatus === ExecutionStatus.Invalid) { @@ -207,28 +406,294 @@ export class ProtoArray { }); } - const node: ProtoNode = { - ...block, - parent: this.indices.get(block.parentRoot), + if (isGloasBlock(block)) { + // Gloas: Create PENDING + EMPTY nodes with correct parent relationships + // Parent of new PENDING node = parent block's EMPTY or FULL (inter-block edge) + // Parent of new EMPTY node = own PENDING node (intra-block edge) + + // For fork transition: if parent is pre-Gloas, point to parent's FULL + // Otherwise, determine which parent payload status this block extends + let parentIndex: number | undefined; + + // Check if parent exists by getting variants array + const parentVariants = this.indices.get(block.parentRoot); + if (parentVariants != null) { + const anyParentIndex = Array.isArray(parentVariants) ? parentVariants[0] : parentVariants; + const anyParentNode = this.nodes[anyParentIndex]; + + if (!isGloasBlock(anyParentNode)) { + // Fork transition: parent is pre-Gloas, so it only has FULL variant at variants[0] + parentIndex = anyParentIndex; + } else { + // Both blocks are Gloas: determine which parent payload status to extend + const parentPayloadStatus = this.getParentPayloadStatus(block); + parentIndex = this.getNodeIndexByRootAndStatus(block.parentRoot, parentPayloadStatus); + } + } + // else: parent doesn't exist, parentIndex remains undefined (orphan block) + + // Create PENDING node + const pendingNode: ProtoNode = { + ...block, + parent: parentIndex, // Points to parent's EMPTY/FULL or FULL (for transition) + payloadStatus: PayloadStatus.PENDING, + weight: 0, + bestChild: undefined, + bestDescendant: undefined, + }; + + const pendingIndex = this.nodes.length; + this.nodes.push(pendingNode); + + // Create EMPTY variant as a child of PENDING + const emptyNode: ProtoNode = { + ...block, + parent: pendingIndex, // Points to own PENDING + payloadStatus: PayloadStatus.EMPTY, + weight: 0, + bestChild: undefined, + bestDescendant: undefined, + }; + + const emptyIndex = this.nodes.length; + this.nodes.push(emptyNode); + + // Store both variants in the indices array + // [PENDING, EMPTY, undefined] - FULL will be added later if payload arrives + this.indices.set(block.blockRoot, [pendingIndex, emptyIndex]); + + // Update bestChild pointers + if (parentIndex !== undefined) { + this.maybeUpdateBestChildAndDescendant(parentIndex, pendingIndex, currentSlot, proposerBoostRoot); + + if (pendingNode.executionStatus === ExecutionStatus.Valid) { + this.propagateValidExecutionStatusByIndex(parentIndex); + } + } + + // Update bestChild for PENDING → EMPTY edge + this.maybeUpdateBestChildAndDescendant(pendingIndex, emptyIndex, currentSlot, proposerBoostRoot); + + // Initialize PTC votes for this block (all false initially) + // Spec: gloas/fork-choice.md#modified-on_block (line 645) + this.ptcVotes.set(block.blockRoot, new Array(PTC_SIZE).fill(false)); + } else { + // Pre-Gloas: Only create FULL node (payload embedded in block) + const node: ProtoNode = { + ...block, + parent: this.getNodeIndexByRootAndStatus(block.parentRoot, PayloadStatus.FULL), + payloadStatus: PayloadStatus.FULL, + weight: 0, + bestChild: undefined, + bestDescendant: undefined, + }; + + const nodeIndex = this.nodes.length; + this.nodes.push(node); + + // Pre-Gloas: store FULL index instead of array + this.indices.set(block.blockRoot, nodeIndex); + + // If this node is valid, lets propagate the valid status up the chain + // and throw error if we counter invalid, as this breaks consensus + if (node.parent !== undefined) { + this.maybeUpdateBestChildAndDescendant(node.parent, nodeIndex, currentSlot, proposerBoostRoot); + + if (node.executionStatus === ExecutionStatus.Valid) { + this.propagateValidExecutionStatusByIndex(node.parent); + } + } + } + } + + /** + * Called when an execution payload is received for a block (Gloas only) + * Creates a FULL variant node as a sibling to the existing EMPTY variant + * Both EMPTY and FULL have parent = own PENDING node + * + * Spec: gloas/fork-choice.md (on_execution_payload event) + */ + onExecutionPayload( + blockRoot: RootHex, + currentSlot: Slot, + executionPayloadBlockHash: RootHex, + executionPayloadNumber: number, + executionPayloadStateRoot: RootHex, + proposerBoostRoot: RootHex | null + ): void { + // First check if block exists + const variants = this.indices.get(blockRoot); + if (variants == null) { + // Equivalent to `assert envelope.beacon_block_root in store.block_states` + throw new ProtoArrayError({ + code: ProtoArrayErrorCode.UNKNOWN_BLOCK, + root: blockRoot, + }); + } + + if (!Array.isArray(variants)) { + // Pre-gloas block should not be calling this method + throw new ProtoArrayError({ + code: ProtoArrayErrorCode.PRE_GLOAS_BLOCK, + root: blockRoot, + }); + } + + // Check if FULL already exists for Gloas blocks + if (variants[PayloadStatus.FULL] !== undefined) { + return; + } + + // Get PENDING node for Gloas blocks + const pendingIndex = variants[PayloadStatus.PENDING]; + if (pendingIndex === undefined) { + throw new ProtoArrayError({ + code: ProtoArrayErrorCode.UNKNOWN_BLOCK, + root: blockRoot, + }); + } + + const pendingNode = this.nodes[pendingIndex]; + if (!pendingNode) { + throw new ProtoArrayError({ + code: ProtoArrayErrorCode.INVALID_NODE_INDEX, + index: pendingIndex, + }); + } + + // Create FULL variant as a child of PENDING (sibling to EMPTY) + const fullNode: ProtoNode = { + ...pendingNode, + parent: pendingIndex, // Points to own PENDING (same as EMPTY) + payloadStatus: PayloadStatus.FULL, weight: 0, bestChild: undefined, bestDescendant: undefined, + executionStatus: ExecutionStatus.Valid, + executionPayloadBlockHash, + executionPayloadNumber, + stateRoot: executionPayloadStateRoot, }; - const nodeIndex = this.nodes.length; + const fullIndex = this.nodes.length; + this.nodes.push(fullNode); + + // Add FULL variant to the indices array + variants[PayloadStatus.FULL] = fullIndex; - this.indices.set(node.blockRoot, nodeIndex); - this.nodes.push(node); + // Update bestChild for PENDING node (may now prefer FULL over EMPTY) + this.maybeUpdateBestChildAndDescendant(pendingIndex, fullIndex, currentSlot, proposerBoostRoot); + } - // If this node is valid, lets propagate the valid status up the chain - // and throw error if we counter invalid, as this breaks consensus - if (node.parent !== undefined) { - this.maybeUpdateBestChildAndDescendant(node.parent, nodeIndex, currentSlot); + /** + * Update PTC votes for multiple validators attesting to a block + * Spec: gloas/fork-choice.md#new-on_payload_attestation_message + * + * @param blockRoot - The beacon block root being attested + * @param ptcIndices - Array of PTC committee indices that voted (0..PTC_SIZE-1) + * @param payloadPresent - Whether the validators attest the payload is present + */ + notifyPtcMessages(blockRoot: RootHex, ptcIndices: number[], payloadPresent: boolean): void { + const votes = this.ptcVotes.get(blockRoot); + if (votes === undefined) { + // Block not found or not a Gloas block, ignore + return; + } - if (node.executionStatus === ExecutionStatus.Valid) { - this.propagateValidExecutionStatusByIndex(node.parent); + for (const ptcIndex of ptcIndices) { + if (ptcIndex < 0 || ptcIndex >= PTC_SIZE) { + throw new Error(`Invalid PTC index: ${ptcIndex}, must be 0..${PTC_SIZE - 1}`); } + + // Update the vote + votes[ptcIndex] = payloadPresent; + } + } + + /** + * Check if execution payload for a block is timely + * Spec: gloas/fork-choice.md#new-is_payload_timely + * + * Returns true if: + * 1. Block has PTC votes tracked + * 2. Payload is locally available (FULL variant exists in proto array) + * 3. More than PAYLOAD_TIMELY_THRESHOLD (>50% of PTC) members voted payload_present=true + * + * @param blockRoot - The beacon block root to check + */ + isPayloadTimely(blockRoot: RootHex): boolean { + const votes = this.ptcVotes.get(blockRoot); + if (votes === undefined) { + // Block not found or not a Gloas block + return false; + } + + // If payload is not locally available, it's not timely + // In our implementation, payload is locally available if proto array has FULL variant of the block + const fullNodeIndex = this.getNodeIndexByRootAndStatus(blockRoot, PayloadStatus.FULL); + if (fullNodeIndex === undefined) { + return false; + } + + // Count votes for payload_present=true + const yesVotes = votes.filter((v) => v).length; + return yesVotes > PAYLOAD_TIMELY_THRESHOLD; + } + + /** + * Check if parent node is FULL + * Spec: gloas/fork-choice.md#new-is_parent_node_full + * + * Returns true if the parent payload status (determined by block.parentBlockHash) is FULL + */ + isParentNodeFull(block: ProtoBlock): boolean { + return this.getParentPayloadStatus(block) === PayloadStatus.FULL; + } + + /** + * Determine if we should extend the payload (prefer FULL over EMPTY) + * Spec: gloas/fork-choice.md#new-should_extend_payload + * + * Returns true if: + * 1. Payload is timely, OR + * 2. No proposer boost root (empty/zero hash), OR + * 3. Proposer boost root's parent is not this block, OR + * 4. Proposer boost root extends FULL parent + * + * @param blockRoot - The block root to check + * @param proposerBoostRoot - Current proposer boost root (from ForkChoice) + */ + shouldExtendPayload(blockRoot: RootHex, proposerBoostRoot: RootHex | null): boolean { + // Condition 1: Payload is timely + if (this.isPayloadTimely(blockRoot)) { + return true; + } + + // Condition 2: No proposer boost root + if (proposerBoostRoot === null || proposerBoostRoot === HEX_ZERO_HASH) { + return true; + } + + // Get proposer boost block + // We don't care about variant here, just need proposer boost block info + const defaultStatus = this.getDefaultVariant(proposerBoostRoot); + const proposerBoostBlock = defaultStatus !== undefined ? this.getNode(proposerBoostRoot, defaultStatus) : undefined; + if (!proposerBoostBlock) { + // Proposer boost block not found, default to extending payload + return true; + } + + // Condition 3: Proposer boost root's parent is not this block + if (proposerBoostBlock.parentRoot !== blockRoot) { + return true; } + + // Condition 4: Proposer boost root extends FULL parent + if (this.isParentNodeFull(proposerBoostBlock)) { + return true; + } + + return false; } /** @@ -236,6 +701,7 @@ export class ProtoArray { * if invalidate till hash provided. If consensus fails, this will invalidate entire * forkChoice which will throw on any call to findHead */ + // TODO GLOAS: Review usage of this post-gloas validateLatestHash(execResponse: LVHExecResponse, currentSlot: Slot): void { // Look reverse because its highly likely node with latestValidExecHash is towards the // the leaves of the forkchoice @@ -279,7 +745,12 @@ export class ProtoArray { // if its in fcU. // const {invalidateFromParentBlockRoot, latestValidExecHash} = execResponse; - const invalidateFromParentIndex = this.indices.get(invalidateFromParentBlockRoot); + // TODO GLOAS: verify if getting default variant is correct here + const defaultStatus = this.getDefaultVariant(invalidateFromParentBlockRoot); + const invalidateFromParentIndex = + defaultStatus !== undefined + ? this.getNodeIndexByRootAndStatus(invalidateFromParentBlockRoot, defaultStatus) + : undefined; if (invalidateFromParentIndex === undefined) { throw Error(`Unable to find invalidateFromParentBlockRoot=${invalidateFromParentBlockRoot} in forkChoice`); } @@ -444,10 +915,46 @@ export class ProtoArray { return validNode; } + /** + * Get payload status tiebreaker for fork choice comparison + * Spec: gloas/fork-choice.md#new-get_payload_status_tiebreaker + * + * For PENDING nodes: always returns 0 + * For EMPTY/FULL variants from slot n-1: implements tiebreaker logic based on should_extend_payload + * For older blocks: returns node.payloadStatus + * + * Note: pre-gloas logic won't reach here. Pre-Gloas blocks have different roots, so they are always resolved by the weight and root tiebreaker before reaching here. + */ + private getPayloadStatusTiebreaker(node: ProtoNode, currentSlot: Slot, proposerBoostRoot: RootHex | null): number { + // PENDING nodes always return PENDING (no tiebreaker needed) + // PENDING=0, EMPTY=1, FULL=2 + if (node.payloadStatus === PayloadStatus.PENDING) { + return node.payloadStatus; + } + + // For Gloas: check if from previous slot + if (node.slot + 1 !== currentSlot) { + return node.payloadStatus; + } + + // For previous slot blocks in Gloas, decide between FULL and EMPTY + // based on should_extend_payload + if (node.payloadStatus === PayloadStatus.EMPTY) { + return PayloadStatus.EMPTY; + } + // FULL - check should_extend_payload + const shouldExtend = this.shouldExtendPayload(node.blockRoot, proposerBoostRoot); + return shouldExtend ? PayloadStatus.FULL : PayloadStatus.PENDING; + } + /** * Follows the best-descendant links to find the best-block (i.e., head-block). + * + * Returns the ProtoNode representing the head. + * For pre-Gloas forks, only FULL variants exist (payload embedded). + * For Gloas, may return PENDING/EMPTY/FULL variants. */ - findHead(justifiedRoot: RootHex, currentSlot: Slot): RootHex { + findHead(justifiedRoot: RootHex, currentSlot: Slot): ProtoNode { if (this.lvhError) { throw new ProtoArrayError({ code: ProtoArrayErrorCode.INVALID_LVH_EXECUTION_RESPONSE, @@ -455,7 +962,10 @@ export class ProtoArray { }); } - const justifiedIndex = this.indices.get(justifiedRoot); + // Get canonical node: FULL for pre-Gloas, PENDING for Gloas + const defaultStatus = this.getDefaultVariant(justifiedRoot); + const justifiedIndex = + defaultStatus !== undefined ? this.getNodeIndexByRootAndStatus(justifiedRoot, defaultStatus) : undefined; if (justifiedIndex === undefined) { throw new ProtoArrayError({ code: ProtoArrayErrorCode.JUSTIFIED_NODE_UNKNOWN, @@ -507,7 +1017,7 @@ export class ProtoArray { }); } - return bestNode.blockRoot; + return bestNode; } /** @@ -526,26 +1036,40 @@ export class ProtoArray { * - There is some internal error relating to invalid indices inside `this`. */ maybePrune(finalizedRoot: RootHex): ProtoBlock[] { - const finalizedIndex = this.indices.get(finalizedRoot); - if (finalizedIndex === undefined) { + const variants = this.indices.get(finalizedRoot); + if (variants == null) { throw new ProtoArrayError({ code: ProtoArrayErrorCode.FINALIZED_NODE_UNKNOWN, root: finalizedRoot, }); } + // Find the minimum index among all variants to ensure we don't prune too much + const finalizedIndex = Array.isArray(variants) + ? Math.min(...variants.filter((idx) => idx !== undefined)) + : variants; + if (finalizedIndex < this.pruneThreshold) { // Pruning at small numbers incurs more cost than benefit return []; } - // Remove the this.indices key/values for all the to-be-deleted nodes - for (let nodeIndex = 0; nodeIndex < finalizedIndex; nodeIndex++) { - const node = this.nodes[nodeIndex]; + // Collect all block roots that will be pruned + const prunedRoots = new Set(); + for (let i = 0; i < finalizedIndex; i++) { + const node = this.nodes[i]; if (node === undefined) { - throw new ProtoArrayError({code: ProtoArrayErrorCode.INVALID_NODE_INDEX, index: nodeIndex}); + throw new ProtoArrayError({code: ProtoArrayErrorCode.INVALID_NODE_INDEX, index: i}); } - this.indices.delete(node.blockRoot); + prunedRoots.add(node.blockRoot); + } + + // Remove indices for pruned blocks and PTC votes + for (const root of prunedRoots) { + this.indices.delete(root); + // Prune PTC votes for this block to prevent memory leak + // Spec: gloas/fork-choice.md (implicit - finalized blocks don't need PTC votes) + this.ptcVotes.delete(root); } // Store nodes prior to finalization @@ -553,15 +1077,35 @@ export class ProtoArray { // Drop all the nodes prior to finalization this.nodes = this.nodes.slice(finalizedIndex); - // Adjust the indices map - for (const [key, value] of this.indices.entries()) { - if (value < finalizedIndex) { - throw new ProtoArrayError({ - code: ProtoArrayErrorCode.INDEX_OVERFLOW, - value: "indices", - }); + // Adjust the indices map - subtract finalizedIndex from all node indices + for (const [root, variantIndices] of this.indices.entries()) { + // Pre-Gloas: single index + if (!Array.isArray(variantIndices)) { + if (variantIndices < finalizedIndex) { + throw new ProtoArrayError({ + code: ProtoArrayErrorCode.INDEX_OVERFLOW, + value: "indices", + }); + } + this.indices.set(root, variantIndices - finalizedIndex); + continue; } - this.indices.set(key, value - finalizedIndex); + + // Post-Gloas: array of variant indices + const adjustedVariants = variantIndices.map((variantIndex) => { + if (variantIndex === undefined) { + return undefined; + } + + if (variantIndex < finalizedIndex) { + throw new ProtoArrayError({ + code: ProtoArrayErrorCode.INDEX_OVERFLOW, + value: "indices", + }); + } + return variantIndex - finalizedIndex; + }); + this.indices.set(root, adjustedVariants as GloasVariantIndices); } // Iterate through all the existing nodes and adjust their indices to match the new layout of this.nodes @@ -610,7 +1154,13 @@ export class ProtoArray { * - The child is not the best child but becomes the best child. * - The child is not the best child and does not become the best child. */ - maybeUpdateBestChildAndDescendant(parentIndex: number, childIndex: number, currentSlot: Slot): void { + + maybeUpdateBestChildAndDescendant( + parentIndex: number, + childIndex: number, + currentSlot: Slot, + proposerBoostRoot: RootHex | null + ): void { const childNode = this.nodes[childIndex]; if (childNode === undefined) { throw new ProtoArrayError({ @@ -640,54 +1190,90 @@ export class ProtoArray { let newChildAndDescendant: ChildAndDescendant; const bestChildIndex = parentNode.bestChild; - if (bestChildIndex !== undefined) { - if (bestChildIndex === childIndex && !childLeadsToViableHead) { - // the child is already the best-child of the parent but its not viable for the head - // so remove it - newChildAndDescendant = changeToNull; - } else if (bestChildIndex === childIndex) { - // the child is the best-child already - // set it again to ensure that the best-descendent of the parent is updated - newChildAndDescendant = changeToChild; - } else { - const bestChildNode = this.nodes[bestChildIndex]; - if (bestChildNode === undefined) { - throw new ProtoArrayError({ - code: ProtoArrayErrorCode.INVALID_BEST_CHILD_INDEX, - index: bestChildIndex, - }); - } + // biome-ignore lint/suspicious/noConfusingLabels: labeled block used for early exit from complex decision tree + outer: { + if (bestChildIndex !== undefined) { + if (bestChildIndex === childIndex && !childLeadsToViableHead) { + // the child is already the best-child of the parent but its not viable for the head + // so remove it + newChildAndDescendant = changeToNull; + } else if (bestChildIndex === childIndex) { + // the child is the best-child already + // set it again to ensure that the best-descendent of the parent is updated + newChildAndDescendant = changeToChild; + } else { + const bestChildNode = this.nodes[bestChildIndex]; + if (bestChildNode === undefined) { + throw new ProtoArrayError({ + code: ProtoArrayErrorCode.INVALID_BEST_CHILD_INDEX, + index: bestChildIndex, + }); + } - const bestChildLeadsToViableHead = this.nodeLeadsToViableHead(bestChildNode, currentSlot); + const bestChildLeadsToViableHead = this.nodeLeadsToViableHead(bestChildNode, currentSlot); - if (childLeadsToViableHead && !bestChildLeadsToViableHead) { - // the child leads to a viable head, but the current best-child doesn't - newChildAndDescendant = changeToChild; - } else if (!childLeadsToViableHead && bestChildLeadsToViableHead) { - // the best child leads to a viable head but the child doesn't - newChildAndDescendant = noChange; - } else if (childNode.weight === bestChildNode.weight) { - // tie-breaker of equal weights by root - if (childNode.blockRoot >= bestChildNode.blockRoot) { + if (childLeadsToViableHead && !bestChildLeadsToViableHead) { + // the child leads to a viable head, but the current best-child doesn't newChildAndDescendant = changeToChild; - } else { + break outer; + } + if (!childLeadsToViableHead && bestChildLeadsToViableHead) { + // the best child leads to a viable head but the child doesn't newChildAndDescendant = noChange; + break outer; } - } else { - // choose the winner by weight - if (childNode.weight >= bestChildNode.weight) { + // Both nodes lead to viable heads (or both don't), need to pick winner + + // Pre-fulu we pick whichever has higher weight, tie-breaker by root + // Post-fulu we pick whichever has higher weight, then tie-breaker by root, then tie-breaker by `getPayloadStatusTiebreaker` + // Gloas: nodes from previous slot (n-1) with EMPTY/FULL variant have weight hardcoded to 0. + // https://github.com/ethereum/consensus-specs/blob/69a2582d5d62c914b24894bdb65f4bd5d4e49ae4/specs/gloas/fork-choice.md?plain=1#L442 + const childEffectiveWeight = + !isGloasBlock(childNode) || + childNode.payloadStatus === PayloadStatus.PENDING || + childNode.slot + 1 !== currentSlot + ? childNode.weight + : 0; + const bestChildEffectiveWeight = + !isGloasBlock(bestChildNode) || + bestChildNode.payloadStatus === PayloadStatus.PENDING || + bestChildNode.slot + 1 !== currentSlot + ? bestChildNode.weight + : 0; + + if (childEffectiveWeight !== bestChildEffectiveWeight) { + // Different effective weights, choose the winner by weight + newChildAndDescendant = childEffectiveWeight >= bestChildEffectiveWeight ? changeToChild : noChange; + break outer; + } + + if (childNode.blockRoot !== bestChildNode.blockRoot) { + // Different blocks, tie-breaker by root + newChildAndDescendant = childNode.blockRoot >= bestChildNode.blockRoot ? changeToChild : noChange; + break outer; + } + + // Same effective weight and same root — Gloas EMPTY vs FULL from n-1, tie-breaker by payload status + // Note: pre-Gloas, each child node of a block has a unique root, so this point should not be reached + const childTiebreaker = this.getPayloadStatusTiebreaker(childNode, currentSlot, proposerBoostRoot); + const bestChildTiebreaker = this.getPayloadStatusTiebreaker(bestChildNode, currentSlot, proposerBoostRoot); + + if (childTiebreaker > bestChildTiebreaker) { newChildAndDescendant = changeToChild; + } else if (childTiebreaker < bestChildTiebreaker) { + newChildAndDescendant = noChange; } else { + // Equal in all aspects, noChange newChildAndDescendant = noChange; } } + } else if (childLeadsToViableHead) { + // There is no current best-child and the child is viable. + newChildAndDescendant = changeToChild; + } else { + // There is no current best-child but the child is not viable. + newChildAndDescendant = noChange; } - } else if (childLeadsToViableHead) { - // There is no current best-child and the child is viable. - newChildAndDescendant = changeToChild; - } else { - // There is no current best-child but the child is not viable. - newChildAndDescendant = noChange; } parentNode.bestChild = newChildAndDescendant[0]; @@ -767,13 +1353,14 @@ export class ProtoArray { } const finalizedSlot = computeStartSlotAtEpoch(this.finalizedEpoch); - return this.finalizedEpoch === 0 || this.finalizedRoot === this.getAncestorOrNull(node.blockRoot, finalizedSlot); + const ancestorNode = this.getAncestorOrNull(node.blockRoot, finalizedSlot); + return this.finalizedEpoch === 0 || (ancestorNode !== null && this.finalizedRoot === ancestorNode.blockRoot); } /** * Same to getAncestor but it may return null instead of throwing error */ - getAncestorOrNull(blockRoot: RootHex, ancestorSlot: Slot): RootHex | null { + getAncestorOrNull(blockRoot: RootHex, ancestorSlot: Slot): ProtoNode | null { try { return this.getAncestor(blockRoot, ancestorSlot); } catch (_) { @@ -782,49 +1369,119 @@ export class ProtoArray { } /** - * Returns the block root of an ancestor of `blockRoot` at the given `slot`. + * Returns the node identifier of an ancestor of `blockRoot` at the given `slot`. * (Note: `slot` refers to the block that is *returned*, not the one that is supplied.) * * NOTE: May be expensive: potentially walks through the entire fork of head to finalized block * * ### Specification * - * Equivalent to: + * Modified for Gloas to return node identifier instead of just root: + * https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.1/specs/gloas/fork-choice.md#modified-get_ancestor * - * https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/fork-choice.md#get_ancestor + * Pre-Gloas: Returns (root, PAYLOAD_STATUS_FULL) + * Gloas: Returns (root, payloadStatus) based on actual node state */ - getAncestor(blockRoot: RootHex, ancestorSlot: Slot): RootHex { - const block = this.getBlock(blockRoot); - if (!block) { + getAncestor(blockRoot: RootHex, ancestorSlot: Slot): ProtoNode { + // Get any variant to check the block (use variants[0]) + const variantOrArr = this.indices.get(blockRoot); + if (variantOrArr == null) { throw new ForkChoiceError({ code: ForkChoiceErrorCode.MISSING_PROTO_ARRAY_BLOCK, root: blockRoot, }); } - if (block.slot > ancestorSlot) { - // Search for a slot that is lte the target slot. - // We check for lower slots to account for skip slots. - for (const node of this.iterateAncestorNodes(blockRoot)) { - if (node.slot <= ancestorSlot) { - return node.blockRoot; - } + const blockIndex = Array.isArray(variantOrArr) ? variantOrArr[0] : variantOrArr; + const block = this.nodes[blockIndex]; + + // If block is at or before queried slot, return PENDING variant (or FULL for pre-Gloas) + if (block.slot <= ancestorSlot) { + // For pre-Gloas: only FULL exists at variants[0] + // For Gloas: PENDING is at variants[0] + return block; + } + + // Walk backwards through beacon blocks to find ancestor + // Start with the parent of the current block + let currentBlock = block; + const parentVariants = this.indices.get(currentBlock.parentRoot); + if (parentVariants == null) { + throw new ForkChoiceError({ + code: ForkChoiceErrorCode.UNKNOWN_ANCESTOR, + descendantRoot: blockRoot, + ancestorSlot, + }); + } + + let parentIndex = Array.isArray(parentVariants) ? parentVariants[0] : parentVariants; + let parentBlock = this.nodes[parentIndex]; + + // Walk backwards while parent.slot > ancestorSlot + while (parentBlock.slot > ancestorSlot) { + currentBlock = parentBlock; + + const nextParentVariants = this.indices.get(currentBlock.parentRoot); + if (nextParentVariants == null) { + throw new ForkChoiceError({ + code: ForkChoiceErrorCode.UNKNOWN_ANCESTOR, + descendantRoot: blockRoot, + ancestorSlot, + }); } + + parentIndex = Array.isArray(nextParentVariants) ? nextParentVariants[0] : nextParentVariants; + parentBlock = this.nodes[parentIndex]; + } + + // Now parentBlock.slot <= ancestorSlot + // Return the parent with the correct payload status based on currentBlock + if (!isGloasBlock(currentBlock)) { + // Pre-Gloas: return FULL variant (only one that exists) + return parentBlock; + } + + // Gloas: determine which parent variant (EMPTY or FULL) based on parent_block_hash + const parentPayloadStatus = this.getParentPayloadStatus(currentBlock); + const parentVariantIndex = this.getNodeIndexByRootAndStatus(currentBlock.parentRoot, parentPayloadStatus); + + if (parentVariantIndex === undefined) { throw new ForkChoiceError({ code: ForkChoiceErrorCode.UNKNOWN_ANCESTOR, descendantRoot: blockRoot, ancestorSlot, }); } - // Root is older or equal than queried slot, thus a skip slot. Return most recent root prior to slot. - return blockRoot; + + return this.nodes[parentVariantIndex]; + } + + /** + * Get the parent node index for traversal + * For Gloas blocks: returns the correct EMPTY/FULL variant based on parent payload status + * For pre-Gloas blocks: returns the simple parent index + * Returns undefined if parent doesn't exist or can't be found + */ + private getParentNodeIndex(node: ProtoNode): number | undefined { + if (isGloasBlock(node)) { + // Use getParentPayloadStatus for Gloas blocks to get correct EMPTY/FULL variant + const parentPayloadStatus = this.getParentPayloadStatus(node); + return this.getNodeIndexByRootAndStatus(node.parentRoot, parentPayloadStatus); + } + // Simple parent traversal for pre-Gloas blocks (includes fork transition) + return node.parent; } /** * Iterate from a block root backwards over nodes + * For Gloas blocks: returns EMPTY/FULL variants (not PENDING) based on parent payload status + * For pre-Gloas blocks: returns FULL variants */ *iterateAncestorNodes(blockRoot: RootHex): IterableIterator { - const startIndex = this.indices.get(blockRoot); + // Get canonical node: FULL for pre-Gloas, PENDING for Gloas + const defaultStatus = this.getDefaultVariant(blockRoot); + const startIndex = + defaultStatus !== undefined ? this.getNodeIndexByRootAndStatus(blockRoot, defaultStatus) : undefined; if (startIndex === undefined) { return; } @@ -841,20 +1498,33 @@ export class ProtoArray { } /** - * Iterate from a block root backwards over nodes + * Iterate from a node backwards over ancestor nodes + * For Gloas blocks: returns EMPTY/FULL variants (not PENDING) based on parent payload status + * For pre-Gloas blocks: returns FULL variants + * Handles fork transition from Gloas to pre-Gloas blocks */ *iterateAncestorNodesFromNode(node: ProtoNode): IterableIterator { while (node.parent !== undefined) { - node = this.getNodeFromIndex(node.parent); + const parentIndex = this.getParentNodeIndex(node); + if (parentIndex === undefined) { + break; + } + + node = this.nodes[parentIndex]; yield node; } } /** * Get all nodes from a block root backwards + * For Gloas blocks: returns EMPTY/FULL variants (not PENDING) based on parent payload status + * For pre-Gloas blocks: returns FULL variants */ getAllAncestorNodes(blockRoot: RootHex): ProtoNode[] { - const startIndex = this.indices.get(blockRoot); + // Get canonical node: FULL for pre-Gloas, PENDING for Gloas + const defaultStatus = this.getDefaultVariant(blockRoot); + const startIndex = + defaultStatus !== undefined ? this.getNodeIndexByRootAndStatus(blockRoot, defaultStatus) : undefined; if (startIndex === undefined) { return []; } @@ -867,10 +1537,22 @@ export class ProtoArray { }); } - const nodes = [node]; + // Include starting node if node is pre-gloas + // Reason why we exclude post-gloas is because node is always default variant (PENDING) + // which we want to exclude. + const nodes: ProtoNode[] = []; + + if (!isGloasBlock(node)) { + nodes.push(node); + } while (node.parent !== undefined) { - node = this.getNodeFromIndex(node.parent); + const parentIndex = this.getParentNodeIndex(node); + if (parentIndex === undefined) { + break; + } + + node = this.nodes[parentIndex]; nodes.push(node); } @@ -881,9 +1563,17 @@ export class ProtoArray { * The opposite of iterateNodes. * iterateNodes is to find ancestor nodes of a blockRoot. * this is to find non-ancestor nodes of a blockRoot. + * + * For Gloas blocks: returns EMPTY/FULL variants (not PENDING) based on parent payload status + * For pre-Gloas blocks: returns FULL variants */ getAllNonAncestorNodes(blockRoot: RootHex): ProtoNode[] { - const startIndex = this.indices.get(blockRoot); + // Get canonical node: FULL for pre-Gloas, PENDING for Gloas + const defaultStatus = this.getDefaultVariant(blockRoot); + if (defaultStatus === undefined) { + return []; + } + const startIndex = this.getNodeIndexByRootAndStatus(blockRoot, defaultStatus); if (startIndex === undefined) { return []; } @@ -895,24 +1585,39 @@ export class ProtoArray { index: startIndex, }); } + + // For both Gloas and pre-Gloas blocks const result: ProtoNode[] = []; let nodeIndex = startIndex; while (node.parent !== undefined) { - const parentIndex = node.parent; - node = this.getNodeFromIndex(parentIndex); - // nodes between nodeIndex and parentIndex means non-ancestor nodes - result.push(...this.getNodesBetween(nodeIndex, parentIndex)); + const parentIndex = this.getParentNodeIndex(node); + if (parentIndex === undefined) { + break; + } + + node = this.nodes[parentIndex]; + // Collect non-ancestor nodes between current and parent + // Filter to exclude PENDING nodes (FULL variant pre-gloas, EMPTY or FULL variant post-gloas) + result.push( + ...this.getNodesBetween(nodeIndex, parentIndex).filter((n) => n.payloadStatus !== PayloadStatus.PENDING) + ); nodeIndex = parentIndex; } - result.push(...this.getNodesBetween(nodeIndex, 0)); + // Collect remaining nodes from nodeIndex to beginning + result.push(...this.getNodesBetween(nodeIndex, 0).filter((n) => n.payloadStatus !== PayloadStatus.PENDING)); return result; } /** * Returns both ancestor and non-ancestor nodes in a single traversal. + * For Gloas blocks: returns EMPTY/FULL variants (not PENDING) based on parent payload status + * For pre-Gloas blocks: returns FULL variants */ getAllAncestorAndNonAncestorNodes(blockRoot: RootHex): {ancestors: ProtoNode[]; nonAncestors: ProtoNode[]} { - const startIndex = this.indices.get(blockRoot); + // Get canonical node: FULL for pre-Gloas, PENDING for Gloas + const defaultStatus = this.getDefaultVariant(blockRoot); + const startIndex = + defaultStatus !== undefined ? this.getNodeIndexByRootAndStatus(blockRoot, defaultStatus) : undefined; if (startIndex === undefined) { return {ancestors: [], nonAncestors: []}; } @@ -928,39 +1633,78 @@ export class ProtoArray { const ancestors: ProtoNode[] = []; const nonAncestors: ProtoNode[] = []; + // Include starting node if it's not PENDING (i.e., pre-Gloas or EMPTY/FULL variant post-Gloas) + if (node.payloadStatus !== PayloadStatus.PENDING) { + ancestors.push(node); + } + let nodeIndex = startIndex; while (node.parent !== undefined) { - ancestors.push(node); + const parentIndex = this.getParentNodeIndex(node); + if (parentIndex === undefined) { + break; + } - const parentIndex = node.parent; - node = this.getNodeFromIndex(parentIndex); + node = this.nodes[parentIndex]; + ancestors.push(node); - // Nodes between nodeIndex and parentIndex are non-ancestor nodes - nonAncestors.push(...this.getNodesBetween(nodeIndex, parentIndex)); + // Collect non-ancestor nodes between current and parent + // Filter to exclude PENDING nodes (include all FULL/EMPTY for both pre-Gloas and Gloas) + nonAncestors.push( + ...this.getNodesBetween(nodeIndex, parentIndex).filter((n) => n.payloadStatus !== PayloadStatus.PENDING) + ); nodeIndex = parentIndex; } - ancestors.push(node); - nonAncestors.push(...this.getNodesBetween(nodeIndex, 0)); + // Collect remaining non-ancestor nodes from nodeIndex to beginning + nonAncestors.push(...this.getNodesBetween(nodeIndex, 0).filter((n) => n.payloadStatus !== PayloadStatus.PENDING)); return {ancestors, nonAncestors}; } + /** + * Check if a block exists in the proto array + * Uses default variant (PENDING for Gloas, FULL for pre-Gloas) + */ hasBlock(blockRoot: RootHex): boolean { - return this.indices.has(blockRoot); + const defaultVariant = this.getDefaultVariant(blockRoot); + if (defaultVariant === undefined) { + return false; + } + const index = this.getNodeIndexByRootAndStatus(blockRoot, defaultVariant); + return index !== undefined; } - getNode(blockRoot: RootHex): ProtoNode | undefined { - const blockIndex = this.indices.get(blockRoot); + /** + * Return ProtoNode for blockRoot with explicit payload status + * + * @param blockRoot - The block root to look up + * @param payloadStatus - The specific payload status variant (PENDING/EMPTY/FULL) + * @returns The ProtoNode for the specified variant, or undefined if not found + * + * Note: Callers must explicitly specify which variant they need. + * Use getDefaultVariant() to get the canonical variant for a block. + */ + getNode(blockRoot: RootHex, payloadStatus: PayloadStatus): ProtoNode | undefined { + const blockIndex = this.getNodeIndexByRootAndStatus(blockRoot, payloadStatus); if (blockIndex === undefined) { return undefined; } return this.getNodeByIndex(blockIndex); } - /** Return MUTABLE ProtoBlock for blockRoot (spreads properties) */ - getBlock(blockRoot: RootHex): ProtoBlock | undefined { - const node = this.getNode(blockRoot); + /** + * Return MUTABLE ProtoBlock for blockRoot with explicit payload status + * + * @param blockRoot - The block root to look up + * @param payloadStatus - The specific payload status variant (PENDING/EMPTY/FULL) + * @returns The ProtoBlock for the specified variant (spreads properties), or undefined if not found + * + * Note: Callers must explicitly specify which variant they need. + * Use getDefaultVariant() to get the canonical variant for a block. + */ + getBlock(blockRoot: RootHex, payloadStatus: PayloadStatus): ProtoBlock | undefined { + const node = this.getNode(blockRoot, payloadStatus); if (!node) { return undefined; } @@ -969,9 +1713,19 @@ export class ProtoArray { }; } - /** Return NON-MUTABLE ProtoBlock for blockRoot (does not spread properties) */ - getBlockReadonly(blockRoot: RootHex): ProtoBlock { - const node = this.getNode(blockRoot); + /** + * Return NON-MUTABLE ProtoBlock for blockRoot with explicit payload status + * + * @param blockRoot - The block root to look up + * @param payloadStatus - The specific payload status variant (PENDING/EMPTY/FULL) + * @returns The ProtoBlock for the specified variant (does not spread properties) + * @throws Error if block not found + * + * Note: Callers must explicitly specify which variant they need. + * Use getDefaultVariant() to get the canonical variant for a block. + */ + getBlockReadonly(blockRoot: RootHex, payloadStatus: PayloadStatus): ProtoBlock { + const node = this.getNode(blockRoot, payloadStatus); if (!node) { throw Error(`No block for root ${blockRoot}`); } @@ -984,7 +1738,10 @@ export class ProtoArray { * Still returns `true` if `ancestorRoot` === `descendantRoot` (and the roots are known) */ isDescendant(ancestorRoot: RootHex, descendantRoot: RootHex): boolean { - const ancestorNode = this.getNode(ancestorRoot); + // We use the default variant (PENDING for Gloas, FULL for pre-Gloas) + // We cannot use FULL/EMPTY variants for Gloas because they may not be canonical + const defaultStatus = this.getDefaultVariant(ancestorRoot); + const ancestorNode = defaultStatus !== undefined ? this.getNode(ancestorRoot, defaultStatus) : undefined; if (!ancestorNode) { return false; } diff --git a/packages/fork-choice/test/perf/forkChoice/updateHead.test.ts b/packages/fork-choice/test/perf/forkChoice/updateHead.test.ts index 9643d3f2e64b..60c18d7d73fc 100644 --- a/packages/fork-choice/test/perf/forkChoice/updateHead.test.ts +++ b/packages/fork-choice/test/perf/forkChoice/updateHead.test.ts @@ -1,6 +1,5 @@ import {bench, describe} from "@chainsafe/benchmark"; -import {computeEpochAtSlot} from "@lodestar/state-transition"; -import {ForkChoice, ProtoBlock} from "../../../src/index.js"; +import {ForkChoice, PayloadStatus, ProtoBlock} from "../../../src/index.js"; import {Opts, initializeForkChoice} from "./util.js"; describe("forkchoice updateHead", () => { @@ -30,7 +29,7 @@ describe("forkchoice updateHead", () => { const forkChoice = initializeForkChoice(opts); const vote1 = forkChoice.updateHead(); - const vote2 = forkChoice.getBlockHex(vote1.parentRoot); + const vote2 = forkChoice.getBlockHexDefaultStatus(vote1.parentRoot); if (!vote2) throw Error("no vote2"); if (vote1.blockRoot === vote2.blockRoot) throw Error("blockRoot vote1 == vote2"); if (vote1.slot === vote2.slot) throw Error("slot vote1 == vote2"); @@ -56,9 +55,8 @@ describe("forkchoice updateHead", () => { }); function everyoneVotes(vote: ProtoBlock, forkChoice: ForkChoice): void { - const nextEpoch = computeEpochAtSlot(vote.slot); const nextRoot = vote.blockRoot; for (let i = 0; i < forkChoice["balances"].length; i++) { - forkChoice["addLatestMessage"](i, nextEpoch, nextRoot); + forkChoice["addLatestMessage"](i, vote.slot, nextRoot, PayloadStatus.FULL); } } diff --git a/packages/fork-choice/test/perf/forkChoice/util.ts b/packages/fork-choice/test/perf/forkChoice/util.ts index 5b72dd88b9f5..9cc0cfe9a1fe 100644 --- a/packages/fork-choice/test/perf/forkChoice/util.ts +++ b/packages/fork-choice/test/perf/forkChoice/util.ts @@ -2,7 +2,14 @@ import {fromHexString} from "@chainsafe/ssz"; import {config} from "@lodestar/config/default"; import {DataAvailabilityStatus} from "@lodestar/state-transition"; import {computeTotalBalance} from "../../../src/forkChoice/store.js"; -import {ExecutionStatus, ForkChoice, IForkChoiceStore, ProtoArray, ProtoBlock} from "../../../src/index.js"; +import { + ExecutionStatus, + ForkChoice, + IForkChoiceStore, + PayloadStatus, + ProtoArray, + ProtoBlock, +} from "../../../src/index.js"; const genesisSlot = 0; const genesisEpoch = 0; @@ -40,16 +47,36 @@ export function initializeForkChoice(opts: Opts): ForkChoice { const fcStore: IForkChoiceStore = { currentSlot: genesisSlot, justified: { - checkpoint: {epoch: genesisEpoch, root: fromHexString(genesisRoot), rootHex: genesisRoot}, + checkpoint: { + epoch: genesisEpoch, + root: fromHexString(genesisRoot), + rootHex: genesisRoot, + payloadStatus: PayloadStatus.FULL, + }, balances, totalBalance: computeTotalBalance(balances), }, unrealizedJustified: { - checkpoint: {epoch: genesisEpoch, root: fromHexString(genesisRoot), rootHex: genesisRoot}, + checkpoint: { + epoch: genesisEpoch, + root: fromHexString(genesisRoot), + rootHex: genesisRoot, + payloadStatus: PayloadStatus.FULL, + }, balances, }, - finalizedCheckpoint: {epoch: genesisEpoch, root: fromHexString(genesisRoot), rootHex: genesisRoot}, - unrealizedFinalizedCheckpoint: {epoch: genesisEpoch, root: fromHexString(genesisRoot), rootHex: genesisRoot}, + finalizedCheckpoint: { + epoch: genesisEpoch, + root: fromHexString(genesisRoot), + rootHex: genesisRoot, + payloadStatus: PayloadStatus.FULL, + }, + unrealizedFinalizedCheckpoint: { + epoch: genesisEpoch, + root: fromHexString(genesisRoot), + rootHex: genesisRoot, + payloadStatus: PayloadStatus.FULL, + }, justifiedBalancesGetter: () => balances, equivocatingIndices: new Set(Array.from({length: opts.initialEquivocatedCount}, (_, i) => i)), }; @@ -80,9 +107,14 @@ export function initializeForkChoice(opts: Opts): ForkChoice { timeliness: false, dataAvailabilityStatus: DataAvailabilityStatus.PreData, + + parentBlockHash: null, + payloadStatus: PayloadStatus.FULL, + builderIndex: null, + blockHashFromBid: null, }; - protoArr.onBlock(block, block.slot); + protoArr.onBlock(block, block.slot, null); parentBlockRoot = blockRoot; } diff --git a/packages/fork-choice/test/unit/forkChoice/forkChoice.test.ts b/packages/fork-choice/test/unit/forkChoice/forkChoice.test.ts index 91f1fc8614e9..bb81100795f5 100644 --- a/packages/fork-choice/test/unit/forkChoice/forkChoice.test.ts +++ b/packages/fork-choice/test/unit/forkChoice/forkChoice.test.ts @@ -10,6 +10,7 @@ import { ExecutionStatus, ForkChoice, IForkChoiceStore, + PayloadStatus, ProtoArray, ProtoBlock, } from "../../../src/index.js"; @@ -41,6 +42,11 @@ describe("Forkchoice", () => { executionPayloadBlockHash: null, executionStatus: ExecutionStatus.PreMerge, dataAvailabilityStatus: DataAvailabilityStatus.PreData, + + // Pre-Gloas block fields (required to avoid being treated as Gloas) + parentBlockHash: null, + payloadStatus: PayloadStatus.FULL, + timeliness: false, } as Omit, genesisSlot ); @@ -49,16 +55,36 @@ describe("Forkchoice", () => { const fcStore: IForkChoiceStore = { currentSlot: genesisSlot + 1, justified: { - checkpoint: {epoch: genesisEpoch, root: fromHexString(finalizedRoot), rootHex: finalizedRoot}, + checkpoint: { + epoch: genesisEpoch, + root: fromHexString(finalizedRoot), + rootHex: finalizedRoot, + payloadStatus: PayloadStatus.FULL, + }, balances: new Uint16Array([32]), totalBalance: 32, }, unrealizedJustified: { - checkpoint: {epoch: genesisEpoch, root: fromHexString(finalizedRoot), rootHex: finalizedRoot}, + checkpoint: { + epoch: genesisEpoch, + root: fromHexString(finalizedRoot), + rootHex: finalizedRoot, + payloadStatus: PayloadStatus.FULL, + }, balances: new Uint16Array([32]), }, - finalizedCheckpoint: {epoch: genesisEpoch, root: fromHexString(finalizedRoot), rootHex: finalizedRoot}, - unrealizedFinalizedCheckpoint: {epoch: genesisEpoch, root: fromHexString(finalizedRoot), rootHex: finalizedRoot}, + finalizedCheckpoint: { + epoch: genesisEpoch, + root: fromHexString(finalizedRoot), + rootHex: finalizedRoot, + payloadStatus: PayloadStatus.FULL, + }, + unrealizedFinalizedCheckpoint: { + epoch: genesisEpoch, + root: fromHexString(finalizedRoot), + rootHex: finalizedRoot, + payloadStatus: PayloadStatus.FULL, + }, justifiedBalancesGetter: () => new Uint16Array([32]), equivocatingIndices: new Set(), }; @@ -104,6 +130,11 @@ describe("Forkchoice", () => { timeliness: false, dataAvailabilityStatus: DataAvailabilityStatus.PreData, + + parentBlockHash: null, + payloadStatus: PayloadStatus.FULL, + builderIndex: null, + blockHashFromBid: null, }; }; @@ -111,7 +142,7 @@ describe("Forkchoice", () => { for (let slot = genesisSlot + 1; slot <= tillSlot; slot++) { if (!skippedSlots.includes(slot)) { const block = getBlock(slot, skippedSlots); - protoArr.onBlock(block, block.slot); + protoArr.onBlock(block, block.slot, null); } } }; @@ -119,12 +150,19 @@ describe("Forkchoice", () => { it("getAllAncestorBlocks", () => { // Add block that is a finalized descendant. const block = getBlock(genesisSlot + 1); - protoArr.onBlock(block, block.slot); + protoArr.onBlock(block, block.slot, null); const forkchoice = new ForkChoice(config, fcStore, protoArr, validatorCount, null); const summaries = forkchoice.getAllAncestorBlocks(getBlockRoot(genesisSlot + 1)); // there are 2 blocks in protoArray but iterateAncestorBlocks should only return non-finalized blocks expect(summaries).toHaveLength(1); - expect(summaries[0]).toEqual({...block, bestChild: undefined, bestDescendant: undefined, parent: 0, weight: 0}); + expect(summaries[0]).toEqual({ + ...block, + bestChild: undefined, + bestDescendant: undefined, + parent: 0, + weight: 0, + payloadStatus: 2, // Pre-Gloas blocks always have PAYLOAD_STATUS_FULL + }); }); it("getAllAncestorAndNonAncestorBlocks equals getAllAncestorBlocks + getAllNonAncestorBlocks", () => { @@ -136,7 +174,7 @@ describe("Forkchoice", () => { ...getBlock(genesisSlot + 10), parentRoot: finalizedRoot, // Connect directly to genesis }; - protoArr.onBlock(forkBlock, forkBlock.slot); + protoArr.onBlock(forkBlock, forkBlock.slot, null); const forkchoice = new ForkChoice(config, fcStore, protoArr, validatorCount, null); @@ -193,7 +231,7 @@ describe("Forkchoice", () => { const forkchoice = new ForkChoice(config, fcStore, protoArr, validatorCount, null); const blockRoot = getBlockRoot(atSlot); - const block = forkchoice.getBlockHex(blockRoot); + const block = forkchoice.getBlockHexDefaultStatus(blockRoot); if (!block) throw Error(`No block for blockRoot ${blockRoot}`); const expectedDependentRoot = getBlockRoot(pivotSlot); diff --git a/packages/fork-choice/test/unit/forkChoice/getProposerHead.test.ts b/packages/fork-choice/test/unit/forkChoice/getProposerHead.test.ts index 0eb01f3580c3..0d556343e5b7 100644 --- a/packages/fork-choice/test/unit/forkChoice/getProposerHead.test.ts +++ b/packages/fork-choice/test/unit/forkChoice/getProposerHead.test.ts @@ -6,7 +6,14 @@ import {DataAvailabilityStatus} from "@lodestar/state-transition"; import {Slot} from "@lodestar/types"; import {toHex} from "@lodestar/utils"; import {NotReorgedReason} from "../../../src/forkChoice/interface.js"; -import {ExecutionStatus, ForkChoice, IForkChoiceStore, ProtoArray, ProtoBlock} from "../../../src/index.js"; +import { + ExecutionStatus, + ForkChoice, + IForkChoiceStore, + PayloadStatus, + ProtoArray, + ProtoBlock, +} from "../../../src/index.js"; import {getBlockRoot, getStateRoot} from "../../utils/index.js"; type ProtoBlockWithWeight = ProtoBlock & {weight: number}; // weight of the block itself @@ -42,6 +49,11 @@ describe("Forkchoice / GetProposerHead", () => { timeliness: false, dataAvailabilityStatus: DataAvailabilityStatus.PreData, + + parentBlockHash: null, + payloadStatus: PayloadStatus.FULL, + builderIndex: null, + blockHashFromBid: null, }; const baseHeadBlock: ProtoBlockWithWeight = { @@ -67,6 +79,11 @@ describe("Forkchoice / GetProposerHead", () => { weight: 29, dataAvailabilityStatus: DataAvailabilityStatus.PreData, + + parentBlockHash: null, + payloadStatus: PayloadStatus.FULL, + builderIndex: null, + blockHashFromBid: null, }; const baseParentHeadBlock: ProtoBlockWithWeight = { @@ -91,28 +108,45 @@ describe("Forkchoice / GetProposerHead", () => { timeliness: false, weight: 212, // 240 - 29 + 1 dataAvailabilityStatus: DataAvailabilityStatus.PreData, + + parentBlockHash: null, + payloadStatus: PayloadStatus.FULL, + builderIndex: null, + blockHashFromBid: null, }; const fcStore: IForkChoiceStore = { currentSlot: genesisSlot + 1, justified: { - checkpoint: {epoch: genesisEpoch, root: fromHexString(genesisBlock.blockRoot), rootHex: genesisBlock.blockRoot}, + checkpoint: { + epoch: genesisEpoch, + root: fromHexString(genesisBlock.blockRoot), + rootHex: genesisBlock.blockRoot, + payloadStatus: PayloadStatus.FULL, + }, balances: new Uint16Array(Array(32).fill(150)), totalBalance: 32 * 150, }, unrealizedJustified: { - checkpoint: {epoch: genesisEpoch, root: fromHexString(genesisBlock.blockRoot), rootHex: genesisBlock.blockRoot}, + checkpoint: { + epoch: genesisEpoch, + root: fromHexString(genesisBlock.blockRoot), + rootHex: genesisBlock.blockRoot, + payloadStatus: PayloadStatus.FULL, + }, balances: new Uint16Array(Array(32).fill(150)), }, finalizedCheckpoint: { epoch: genesisEpoch, root: fromHexString(genesisBlock.blockRoot), rootHex: genesisBlock.blockRoot, + payloadStatus: PayloadStatus.FULL, }, unrealizedFinalizedCheckpoint: { epoch: genesisEpoch, root: fromHexString(genesisBlock.blockRoot), rootHex: genesisBlock.blockRoot, + payloadStatus: PayloadStatus.FULL, }, justifiedBalancesGetter: () => new Uint16Array(Array(32).fill(150)), equivocatingIndices: new Set(), @@ -224,8 +258,8 @@ describe("Forkchoice / GetProposerHead", () => { expectedNotReorgedReason, } of testCases) { it(`${id}`, async () => { - protoArr.onBlock(parentBlock, parentBlock.slot); - protoArr.onBlock(headBlock, headBlock.slot); + protoArr.onBlock(parentBlock, parentBlock.slot, null); + protoArr.onBlock(headBlock, headBlock.slot, null); const currentSlot = proposalSlot ?? headBlock.slot + 1; const currentSecFromSlot = secFromSlot ?? 0; diff --git a/packages/fork-choice/test/unit/forkChoice/shouldOverrideForkChoiceUpdate.test.ts b/packages/fork-choice/test/unit/forkChoice/shouldOverrideForkChoiceUpdate.test.ts index 803c5fcbd5bb..a613962a0740 100644 --- a/packages/fork-choice/test/unit/forkChoice/shouldOverrideForkChoiceUpdate.test.ts +++ b/packages/fork-choice/test/unit/forkChoice/shouldOverrideForkChoiceUpdate.test.ts @@ -6,7 +6,14 @@ import {DataAvailabilityStatus} from "@lodestar/state-transition"; import {Slot} from "@lodestar/types"; import {toHex} from "@lodestar/utils"; import {NotReorgedReason} from "../../../src/forkChoice/interface.js"; -import {ExecutionStatus, ForkChoice, IForkChoiceStore, ProtoArray, ProtoBlock} from "../../../src/index.js"; +import { + ExecutionStatus, + ForkChoice, + IForkChoiceStore, + PayloadStatus, + ProtoArray, + ProtoBlock, +} from "../../../src/index.js"; import {getBlockRoot, getStateRoot} from "../../utils/index.js"; type ProtoBlockWithWeight = ProtoBlock & {weight: number}; // weight of the block itself @@ -42,6 +49,11 @@ describe("Forkchoice / shouldOverrideForkChoiceUpdate", () => { timeliness: false, dataAvailabilityStatus: DataAvailabilityStatus.PreData, + + parentBlockHash: null, + payloadStatus: PayloadStatus.FULL, + builderIndex: null, + blockHashFromBid: null, }; const baseHeadBlock: ProtoBlockWithWeight = { @@ -67,6 +79,11 @@ describe("Forkchoice / shouldOverrideForkChoiceUpdate", () => { weight: 29, dataAvailabilityStatus: DataAvailabilityStatus.PreData, + + parentBlockHash: null, + payloadStatus: PayloadStatus.FULL, + builderIndex: null, + blockHashFromBid: null, }; const baseParentHeadBlock: ProtoBlockWithWeight = { @@ -91,28 +108,45 @@ describe("Forkchoice / shouldOverrideForkChoiceUpdate", () => { timeliness: false, weight: 212, // 240 - 29 + 1 dataAvailabilityStatus: DataAvailabilityStatus.PreData, + + parentBlockHash: null, + payloadStatus: PayloadStatus.FULL, + builderIndex: null, + blockHashFromBid: null, }; const fcStore: IForkChoiceStore = { currentSlot: genesisSlot + 1, justified: { - checkpoint: {epoch: genesisEpoch, root: fromHexString(genesisBlock.blockRoot), rootHex: genesisBlock.blockRoot}, + checkpoint: { + epoch: genesisEpoch, + root: fromHexString(genesisBlock.blockRoot), + rootHex: genesisBlock.blockRoot, + payloadStatus: PayloadStatus.FULL, + }, balances: new Uint16Array(Array(32).fill(150)), totalBalance: 32 * 150, }, unrealizedJustified: { - checkpoint: {epoch: genesisEpoch, root: fromHexString(genesisBlock.blockRoot), rootHex: genesisBlock.blockRoot}, + checkpoint: { + epoch: genesisEpoch, + root: fromHexString(genesisBlock.blockRoot), + rootHex: genesisBlock.blockRoot, + payloadStatus: PayloadStatus.FULL, + }, balances: new Uint16Array(Array(32).fill(150)), }, finalizedCheckpoint: { epoch: genesisEpoch, root: fromHexString(genesisBlock.blockRoot), rootHex: genesisBlock.blockRoot, + payloadStatus: PayloadStatus.FULL, }, unrealizedFinalizedCheckpoint: { epoch: genesisEpoch, root: fromHexString(genesisBlock.blockRoot), rootHex: genesisBlock.blockRoot, + payloadStatus: PayloadStatus.FULL, }, justifiedBalancesGetter: () => new Uint16Array(Array(32).fill(150)), equivocatingIndices: new Set(), @@ -190,8 +224,8 @@ describe("Forkchoice / shouldOverrideForkChoiceUpdate", () => { expectedNotReorgedReason, } of testCases) { it(id, async () => { - protoArr.onBlock(parentBlock, parentBlock.slot); - protoArr.onBlock(headBlock, headBlock.slot); + protoArr.onBlock(parentBlock, parentBlock.slot, null); + protoArr.onBlock(headBlock, headBlock.slot, null); const secFromSlot = 0; const currentSlot = blockSeenSlot ?? headBlock.slot; @@ -200,7 +234,7 @@ describe("Forkchoice / shouldOverrideForkChoiceUpdate", () => { proposerBoostReorg: true, }); - const result = forkChoice.shouldOverrideForkChoiceUpdate(headBlock.blockRoot, secFromSlot, currentSlot); + const result = forkChoice.shouldOverrideForkChoiceUpdate(headBlock, secFromSlot, currentSlot); expect(result.shouldOverrideFcu).toBe(expectReorg); diff --git a/packages/fork-choice/test/unit/protoArray/executionStatusUpdates.test.ts b/packages/fork-choice/test/unit/protoArray/executionStatusUpdates.test.ts index 5e77b48a8a23..9e8b79261ad0 100644 --- a/packages/fork-choice/test/unit/protoArray/executionStatusUpdates.test.ts +++ b/packages/fork-choice/test/unit/protoArray/executionStatusUpdates.test.ts @@ -4,6 +4,7 @@ import { BlockExtraMeta, ExecutionStatus, MaybeValidExecutionStatus, + PayloadStatus, ProtoArray, ProtoBlock, } from "../../../src/index.js"; @@ -74,7 +75,12 @@ function setupForkChoice(): ProtoArray { finalizedEpoch: 0, finalizedRoot: "-", - ...{executionPayloadBlockHash: null, executionStatus: ExecutionStatus.PreMerge}, + executionPayloadBlockHash: null, + executionStatus: ExecutionStatus.PreMerge, + dataAvailabilityStatus: DataAvailabilityStatus.PreData, + + parentBlockHash: null, + payloadStatus: PayloadStatus.FULL, } as Omit, 0 ); @@ -115,8 +121,14 @@ function setupForkChoice(): ProtoArray { timeliness: false, ...executionData, + + parentBlockHash: null, + payloadStatus: PayloadStatus.FULL, + builderIndex: null, + blockHashFromBid: null, }, - block.slot + block.slot, + null ); } @@ -289,7 +301,10 @@ describe("executionStatus / invalidate all postmerge chain", () => { const fcHead = fc.findHead("0", 3); it("pre merge block should be the FC head", () => { - expect(fcHead).toBe("0"); + // findHead returns ProtoNode + // For pre-Gloas blocks, this should have blockRoot "0" and payloadStatus FULL (2) + expect(fcHead.blockRoot).toBe("0"); + expect(fcHead.payloadStatus).toBe(2); // PayloadStatus.FULL }); }); @@ -421,7 +436,8 @@ function collectProtoarrayValidationStatus(fcArray: ProtoArray): ValidationTestC const expectedForkChoice: ValidationTestCase[] = []; for (const fcRoot of fcRoots) { - const fcNode = fcArray.getNode(fcRoot); + const defaultStatus = fcArray.getDefaultVariant(fcRoot); + const fcNode = defaultStatus !== undefined ? fcArray.getNode(fcRoot, defaultStatus) : undefined; const bestChild = fcNode?.bestChild !== undefined ? fcArray["getNodeFromIndex"](fcNode.bestChild).blockRoot : undefined; const bestDescendant = diff --git a/packages/fork-choice/test/unit/protoArray/getCommonAncestor.test.ts b/packages/fork-choice/test/unit/protoArray/getCommonAncestor.test.ts index b6a3f50af7eb..c72f97ebf65a 100644 --- a/packages/fork-choice/test/unit/protoArray/getCommonAncestor.test.ts +++ b/packages/fork-choice/test/unit/protoArray/getCommonAncestor.test.ts @@ -1,6 +1,6 @@ import {describe, expect, it} from "vitest"; import {DataAvailabilityStatus} from "@lodestar/state-transition"; -import {ExecutionStatus, ProtoArray} from "../../../src/index.js"; +import {ExecutionStatus, PayloadStatus, ProtoArray} from "../../../src/index.js"; describe("getCommonAncestor", () => { const blocks: {slot: number; root: string; parent: string}[] = [ @@ -45,6 +45,11 @@ describe("getCommonAncestor", () => { ...{executionPayloadBlockHash: null, executionStatus: ExecutionStatus.PreMerge}, dataAvailabilityStatus: DataAvailabilityStatus.PreData, + + parentBlockHash: null, + payloadStatus: PayloadStatus.FULL, + builderIndex: null, + blockHashFromBid: null, }, 0 ); @@ -71,15 +76,25 @@ describe("getCommonAncestor", () => { ...{executionPayloadBlockHash: null, executionStatus: ExecutionStatus.PreMerge}, dataAvailabilityStatus: DataAvailabilityStatus.PreData, + + parentBlockHash: null, + payloadStatus: PayloadStatus.FULL, + builderIndex: null, + blockHashFromBid: null, }, - block.slot + block.slot, + null ); } for (const {nodeA, nodeB, ancestor} of testCases) { it(`${nodeA} & ${nodeB} -> ${ancestor}`, () => { + const defaultStatusA = fc.getDefaultVariant(nodeA); + const defaultStatusB = fc.getDefaultVariant(nodeB); + const nodeAResult = defaultStatusA !== undefined ? fc.getNode(nodeA, defaultStatusA) : undefined; + const nodeBResult = defaultStatusB !== undefined ? fc.getNode(nodeB, defaultStatusB) : undefined; // biome-ignore lint/style/noNonNullAssertion: We know the node can not be null here - const ancestorNode = fc.getCommonAncestor(fc.getNode(nodeA)!, fc.getNode(nodeB)!); + const ancestorNode = fc.getCommonAncestor(nodeAResult!, nodeBResult!); if (ancestor) { expect(ancestorNode?.blockRoot).toBe(ancestor); } else { diff --git a/packages/fork-choice/test/unit/protoArray/gloas.test.ts b/packages/fork-choice/test/unit/protoArray/gloas.test.ts new file mode 100644 index 000000000000..b1e5bb84d684 --- /dev/null +++ b/packages/fork-choice/test/unit/protoArray/gloas.test.ts @@ -0,0 +1,625 @@ +import {beforeEach, describe, expect, it} from "vitest"; +import {PTC_SIZE} from "@lodestar/params"; +import {DataAvailabilityStatus, computeStartSlotAtEpoch} from "@lodestar/state-transition"; +import {RootHex} from "@lodestar/types"; +import {ExecutionStatus, PayloadStatus, ProtoArray, ProtoBlock, ProtoNode} from "../../../src/index.js"; + +describe("Gloas Fork Choice", () => { + const genesisEpoch = 0; + const gloasForkEpoch = 5; + const gloasForkSlot = computeStartSlotAtEpoch(gloasForkEpoch); + + const stateRoot = "0x00"; + const genesisRoot = "0x01"; + + /** + * Helper to get a specific node variant (PENDING/EMPTY/FULL) from ProtoArray + * Replacement for removed getForkChoiceNode() method + */ + function getNodeByPayloadStatus( + protoArray: ProtoArray, + blockRoot: RootHex, + payloadStatus: PayloadStatus + ): ProtoNode | undefined { + const index = protoArray.getNodeIndexByRootAndStatus(blockRoot, payloadStatus); + if (index === undefined) return undefined; + return (protoArray as any).nodes[index]; + } + + function createTestBlock( + slot: number, + blockRoot: RootHex, + parentRoot: RootHex, + parentBlockHash?: RootHex + ): ProtoBlock { + return { + slot, + blockRoot, + parentRoot, + stateRoot, + targetRoot: genesisRoot, + justifiedEpoch: genesisEpoch, + justifiedRoot: genesisRoot, + finalizedEpoch: genesisEpoch, + finalizedRoot: genesisRoot, + unrealizedJustifiedEpoch: genesisEpoch, + unrealizedJustifiedRoot: genesisRoot, + unrealizedFinalizedEpoch: genesisEpoch, + unrealizedFinalizedRoot: genesisRoot, + timeliness: true, + executionPayloadBlockHash: blockRoot, // Use blockRoot as execution hash + executionPayloadNumber: slot, + executionStatus: ExecutionStatus.Valid, + dataAvailabilityStatus: DataAvailabilityStatus.Available, + parentBlockHash: parentBlockHash === undefined ? null : parentBlockHash, + payloadStatus: PayloadStatus.FULL, + builderIndex: null, + blockHashFromBid: null, + }; + } + + describe("ProtoArray indices lookup", () => { + it("indices map stores variants correctly for pre-Gloas blocks", () => { + const protoArray = ProtoArray.initialize(createTestBlock(0, genesisRoot, "0x00"), 0); + const variants = (protoArray as any).indices.get(genesisRoot); + expect(variants).toBeDefined(); + // Pre-Gloas: variants is the FULL index + expect(variants).toBe(0); + }); + + it("getNodeByPayloadStatus() retrieves correct variants", () => { + const protoArray = ProtoArray.initialize(createTestBlock(0, genesisRoot, "0x00"), 0); + const node = getNodeByPayloadStatus(protoArray, genesisRoot, PayloadStatus.FULL); + expect(node).toBeDefined(); + expect(node?.blockRoot).toBe(genesisRoot); + expect(node?.payloadStatus).toBe(PayloadStatus.FULL); + }); + + it("indices map stores multiple variants for Gloas blocks", () => { + const protoArray = ProtoArray.initialize(createTestBlock(0, genesisRoot, "0x00"), 0); + + // Add a Gloas block + const gloasBlock = createTestBlock(gloasForkSlot, "0x02", genesisRoot, genesisRoot); + protoArray.onBlock(gloasBlock, gloasForkSlot, null); + + const variants = (protoArray as any).indices.get("0x02"); + expect(variants).toBeDefined(); + // Gloas: variants[PENDING] and variants[EMPTY] should be defined + expect(variants[PayloadStatus.PENDING]).toBeDefined(); + expect(variants[PayloadStatus.EMPTY]).toBeDefined(); + expect(variants[PayloadStatus.FULL]).toBeUndefined(); + }); + }); + + describe("Pre-Gloas (Fulu) behavior", () => { + let protoArray: ProtoArray; + + beforeEach(() => { + // Test pre-Gloas behavior by creating blocks with parentBlockHash: null + protoArray = new ProtoArray({ + pruneThreshold: 0, + justifiedEpoch: genesisEpoch, + justifiedRoot: genesisRoot, + finalizedEpoch: genesisEpoch, + finalizedRoot: genesisRoot, + }); + }); + + it("creates only FULL nodes for pre-Gloas blocks", () => { + const block = createTestBlock(1, "0x02", genesisRoot); + protoArray.onBlock(block, 1, null); + + // Should only have FULL variant + const fullNode = getNodeByPayloadStatus(protoArray, "0x02", PayloadStatus.FULL); + expect(fullNode).toBeDefined(); + expect(fullNode?.payloadStatus).toBe(PayloadStatus.FULL); + + // Should not have PENDING or EMPTY variants + expect(() => getNodeByPayloadStatus(protoArray, "0x02", PayloadStatus.PENDING)).toThrow(); + expect(() => getNodeByPayloadStatus(protoArray, "0x02", PayloadStatus.EMPTY)).toThrow(); + }); + + it("getNode() finds pre-Gloas blocks by root (FULL)", () => { + const block = createTestBlock(1, "0x02", genesisRoot); + protoArray.onBlock(block, 1, null); + + const defaultStatus = protoArray.getDefaultVariant("0x02"); + expect(defaultStatus).toBe(PayloadStatus.FULL); + const node = defaultStatus !== undefined ? protoArray.getNode("0x02", defaultStatus) : undefined; + expect(node).toBeDefined(); + expect(node?.payloadStatus).toBe(PayloadStatus.FULL); + }); + + it("hasBlock() returns true for pre-Gloas blocks", () => { + const block = createTestBlock(1, "0x02", genesisRoot); + protoArray.onBlock(block, 1, null); + + expect(protoArray.hasBlock("0x02")).toBe(true); + expect(protoArray.hasBlock("0x99")).toBe(false); + }); + }); + + describe("Gloas fork activation", () => { + let protoArray: ProtoArray; + + beforeEach(() => { + protoArray = new ProtoArray({ + pruneThreshold: 0, + justifiedEpoch: genesisEpoch, + justifiedRoot: genesisRoot, + finalizedEpoch: genesisEpoch, + finalizedRoot: genesisRoot, + }); + }); + + it("creates PENDING + EMPTY nodes for Gloas blocks", () => { + const block = createTestBlock(gloasForkSlot, "0x02", genesisRoot, genesisRoot); + protoArray.onBlock(block, gloasForkSlot, null); + + // Should have PENDING variant + const pendingNode = getNodeByPayloadStatus(protoArray, "0x02", PayloadStatus.PENDING); + expect(pendingNode).toBeDefined(); + expect(pendingNode?.payloadStatus).toBe(PayloadStatus.PENDING); + + // Should have EMPTY variant + const emptyNode = getNodeByPayloadStatus(protoArray, "0x02", PayloadStatus.EMPTY); + expect(emptyNode).toBeDefined(); + expect(emptyNode?.payloadStatus).toBe(PayloadStatus.EMPTY); + + // Should not have FULL variant yet + const fullNode = getNodeByPayloadStatus(protoArray, "0x02", PayloadStatus.FULL); + expect(fullNode).toBeUndefined(); + }); + + it("EMPTY node has PENDING as parent", () => { + const block = createTestBlock(gloasForkSlot, "0x02", genesisRoot, genesisRoot); + protoArray.onBlock(block, gloasForkSlot, null); + + const emptyNode = getNodeByPayloadStatus(protoArray, "0x02", PayloadStatus.EMPTY); + const pendingIndex = protoArray.getNodeIndexByRootAndStatus("0x02", PayloadStatus.PENDING); + + expect(emptyNode?.parent).toBe(pendingIndex); + }); + + it("initializes PTC votes for Gloas blocks", () => { + const block = createTestBlock(gloasForkSlot, "0x02", genesisRoot, genesisRoot); + protoArray.onBlock(block, gloasForkSlot, null); + + // All PTC votes should be false initially + const isTimely = protoArray.isPayloadTimely("0x02"); + expect(isTimely).toBe(false); + }); + + it("does not create PENDING/EMPTY for pre-fork blocks", () => { + const block = createTestBlock(gloasForkSlot - 1, "0x02", genesisRoot); + protoArray.onBlock(block, gloasForkSlot - 1, null); + + // Should only have FULL (pre-Gloas behavior) + const fullNode = getNodeByPayloadStatus(protoArray, "0x02", PayloadStatus.FULL); + expect(fullNode).toBeDefined(); + + expect(() => getNodeByPayloadStatus(protoArray, "0x02", PayloadStatus.PENDING)).toThrow(); + }); + }); + + describe("Fork transition (Fulu → Gloas)", () => { + let protoArray: ProtoArray; + + beforeEach(() => { + protoArray = new ProtoArray({ + pruneThreshold: 0, + justifiedEpoch: genesisEpoch, + justifiedRoot: genesisRoot, + finalizedEpoch: genesisEpoch, + finalizedRoot: genesisRoot, + }); + }); + + it("first Gloas block points to FULL parent (Fulu block)", () => { + // Add pre-Gloas block + const fuluBlock = createTestBlock(gloasForkSlot - 1, "0x02", genesisRoot); + protoArray.onBlock(fuluBlock, gloasForkSlot - 1, null); + + // Add first Gloas block + const gloasBlock = createTestBlock(gloasForkSlot, "0x03", "0x02", "0x02"); + protoArray.onBlock(gloasBlock, gloasForkSlot, null); + + const gloasPendingNode = getNodeByPayloadStatus(protoArray, "0x03", PayloadStatus.PENDING); + const fuluFullIndex = protoArray.getNodeIndexByRootAndStatus("0x02", PayloadStatus.FULL); + + // First Gloas block's PENDING should point to parent's FULL + expect(gloasPendingNode?.parent).toBe(fuluFullIndex); + }); + + it("getNode() finds blocks across fork transition", () => { + // Add pre-Gloas block + const fuluBlock = createTestBlock(gloasForkSlot - 1, "0x02", genesisRoot); + protoArray.onBlock(fuluBlock, gloasForkSlot - 1, null); + + // Add Gloas block + const gloasBlock = createTestBlock(gloasForkSlot, "0x03", "0x02", "0x02"); + protoArray.onBlock(gloasBlock, gloasForkSlot, null); + + // Should find both blocks with correct default variants + const fuluDefaultStatus = protoArray.getDefaultVariant("0x02"); + expect(fuluDefaultStatus).toBe(PayloadStatus.FULL); + const fuluNode = fuluDefaultStatus !== undefined ? protoArray.getNode("0x02", fuluDefaultStatus) : undefined; + expect(fuluNode?.payloadStatus).toBe(PayloadStatus.FULL); + + const gloasDefaultStatus = protoArray.getDefaultVariant("0x03"); + expect(gloasDefaultStatus).toBe(PayloadStatus.PENDING); + const gloasNode = gloasDefaultStatus !== undefined ? protoArray.getNode("0x03", gloasDefaultStatus) : undefined; + expect(gloasNode?.payloadStatus).toBe(PayloadStatus.PENDING); + }); + }); + + describe("onExecutionPayload()", () => { + let protoArray: ProtoArray; + + beforeEach(() => { + protoArray = new ProtoArray({ + pruneThreshold: 0, + justifiedEpoch: genesisEpoch, + justifiedRoot: genesisRoot, + finalizedEpoch: genesisEpoch, + finalizedRoot: genesisRoot, + }); + }); + + it("creates FULL variant when payload arrives", () => { + const block = createTestBlock(gloasForkSlot, "0x02", genesisRoot, genesisRoot); + protoArray.onBlock(block, gloasForkSlot, null); + + // FULL should not exist yet + expect(getNodeByPayloadStatus(protoArray, "0x02", PayloadStatus.FULL)).toBeUndefined(); + + // Call onExecutionPayload + protoArray.onExecutionPayload("0x02", gloasForkSlot, "0x02", gloasForkSlot, stateRoot, null); + + // FULL should now exist + const fullNode = getNodeByPayloadStatus(protoArray, "0x02", PayloadStatus.FULL); + expect(fullNode).toBeDefined(); + expect(fullNode?.payloadStatus).toBe(PayloadStatus.FULL); + }); + + it("FULL node has PENDING as parent", () => { + const block = createTestBlock(gloasForkSlot, "0x02", genesisRoot, genesisRoot); + protoArray.onBlock(block, gloasForkSlot, null); + + protoArray.onExecutionPayload("0x02", gloasForkSlot, "0x02", gloasForkSlot, stateRoot, null); + + const fullNode = getNodeByPayloadStatus(protoArray, "0x02", PayloadStatus.FULL); + const pendingIndex = protoArray.getNodeIndexByRootAndStatus("0x02", PayloadStatus.PENDING); + + expect(fullNode?.parent).toBe(pendingIndex); + }); + + it("is idempotent (calling twice does not create duplicate)", () => { + const block = createTestBlock(gloasForkSlot, "0x02", genesisRoot, genesisRoot); + protoArray.onBlock(block, gloasForkSlot, null); + + protoArray.onExecutionPayload("0x02", gloasForkSlot, "0x02", gloasForkSlot, stateRoot, null); + protoArray.onExecutionPayload("0x02", gloasForkSlot, "0x02", gloasForkSlot, stateRoot, null); + + // Should still only have one FULL node + const fullNode = getNodeByPayloadStatus(protoArray, "0x02", PayloadStatus.FULL); + expect(fullNode).toBeDefined(); + }); + + it("throws for pre-Gloas blocks", () => { + const block = createTestBlock(gloasForkSlot - 1, "0x02", genesisRoot); + protoArray.onBlock(block, gloasForkSlot - 1, null); + + // Pre-Gloas block already has FULL + expect(getNodeByPayloadStatus(protoArray, "0x02", PayloadStatus.FULL)).toBeDefined(); + + // Calling onExecutionPayload should throw for pre-Gloas blocks + expect(() => + protoArray.onExecutionPayload("0x02", gloasForkSlot - 1, "0x02", gloasForkSlot - 1, stateRoot, null) + ).toThrow(); + }); + + it("throws for unknown block", () => { + expect(() => + protoArray.onExecutionPayload("0x99", gloasForkSlot, "0x99", gloasForkSlot, stateRoot, null) + ).toThrow(); + }); + }); + + describe("PTC (Payload Timeliness Committee)", () => { + let protoArray: ProtoArray; + + beforeEach(() => { + protoArray = new ProtoArray({ + pruneThreshold: 0, + justifiedEpoch: genesisEpoch, + justifiedRoot: genesisRoot, + finalizedEpoch: genesisEpoch, + finalizedRoot: genesisRoot, + }); + }); + + it("notifyPtcMessages() updates votes for multiple validators", () => { + const block = createTestBlock(gloasForkSlot, "0x02", genesisRoot, genesisRoot); + protoArray.onBlock(block, gloasForkSlot, null); + + // Initially not timely (no votes) + expect(protoArray.isPayloadTimely("0x02")).toBe(false); + + // Vote yes from validators at indices 0, 1, 2 + protoArray.notifyPtcMessages("0x02", [0, 1, 2], true); + + // Still not timely (need >50% of PTC_SIZE) + expect(protoArray.isPayloadTimely("0x02")).toBe(false); + }); + + it("notifyPtcMessages() validates ptcIndex range", () => { + const block = createTestBlock(gloasForkSlot, "0x02", genesisRoot, genesisRoot); + protoArray.onBlock(block, gloasForkSlot, null); + + expect(() => protoArray.notifyPtcMessages("0x02", [-1], true)).toThrow(/Invalid PTC index/); + expect(() => protoArray.notifyPtcMessages("0x02", [PTC_SIZE], true)).toThrow(/Invalid PTC index/); + expect(() => protoArray.notifyPtcMessages("0x02", [PTC_SIZE + 1], true)).toThrow(/Invalid PTC index/); + expect(() => protoArray.notifyPtcMessages("0x02", [0, 1, PTC_SIZE], true)).toThrow(/Invalid PTC index/); + }); + + it("notifyPtcMessages() handles unknown block gracefully", () => { + // Should not throw for unknown block + expect(() => protoArray.notifyPtcMessages("0x99", [0], true)).not.toThrow(); + }); + + it("isPayloadTimely() returns false when payload not locally available", () => { + const block = createTestBlock(gloasForkSlot, "0x02", genesisRoot, genesisRoot); + protoArray.onBlock(block, gloasForkSlot, null); + + // Vote yes from majority of PTC + const threshold = Math.floor(PTC_SIZE / 2) + 1; + const indices = Array.from({length: threshold}, (_, i) => i); + protoArray.notifyPtcMessages("0x02", indices, true); + + // Without execution payload (no FULL variant), should return false + expect(protoArray.isPayloadTimely("0x02")).toBe(false); + }); + + it("isPayloadTimely() returns true when threshold met and payload available", () => { + const block = createTestBlock(gloasForkSlot, "0x02", genesisRoot, genesisRoot); + protoArray.onBlock(block, gloasForkSlot, null); + + // Make execution payload available by creating FULL variant + protoArray.onExecutionPayload("0x02", gloasForkSlot, "0x02", gloasForkSlot, stateRoot, null); + + // Vote yes from majority of PTC (>50%) + const threshold = Math.floor(PTC_SIZE / 2) + 1; + const indices = Array.from({length: threshold}, (_, i) => i); + protoArray.notifyPtcMessages("0x02", indices, true); + + // Should now be timely + expect(protoArray.isPayloadTimely("0x02")).toBe(true); + }); + + it("isPayloadTimely() returns false when threshold not met", () => { + const block = createTestBlock(gloasForkSlot, "0x02", genesisRoot, genesisRoot); + protoArray.onBlock(block, gloasForkSlot, null); + + // Make execution payload available by creating FULL variant + protoArray.onExecutionPayload("0x02", gloasForkSlot, "0x02", gloasForkSlot, stateRoot, null); + + // Vote yes from exactly 50% (not >50%) + const threshold = Math.floor(PTC_SIZE / 2); + const indices = Array.from({length: threshold}, (_, i) => i); + protoArray.notifyPtcMessages("0x02", indices, true); + + // Should not be timely (need >50%, not >=50%) + expect(protoArray.isPayloadTimely("0x02")).toBe(false); + }); + + it("isPayloadTimely() counts only 'true' votes", () => { + const block = createTestBlock(gloasForkSlot, "0x02", genesisRoot, genesisRoot); + protoArray.onBlock(block, gloasForkSlot, null); + + // Make execution payload available by creating FULL variant + protoArray.onExecutionPayload("0x02", gloasForkSlot, "0x02", gloasForkSlot, stateRoot, null); + + // Vote mixed yes/no + const threshold = Math.floor(PTC_SIZE / 2) + 1; + // Vote yes from indices 0..threshold-1 + const yesIndices = Array.from({length: threshold}, (_, i) => i); + protoArray.notifyPtcMessages("0x02", yesIndices, true); + // Vote no from indices threshold..PTC_SIZE-1 + const noIndices = Array.from({length: PTC_SIZE - threshold}, (_, i) => i + threshold); + protoArray.notifyPtcMessages("0x02", noIndices, false); + + // Should be timely (threshold met) + expect(protoArray.isPayloadTimely("0x02")).toBe(true); + + // Change some yes votes to no + protoArray.notifyPtcMessages("0x02", [0, 1], false); + + // Should no longer be timely + expect(protoArray.isPayloadTimely("0x02")).toBe(false); + }); + + it("isPayloadTimely() returns false for unknown block", () => { + expect(protoArray.isPayloadTimely("0x99")).toBe(false); + }); + + it("does not initialize PTC votes for pre-Gloas blocks", () => { + const block = createTestBlock(gloasForkSlot - 1, "0x02", genesisRoot); + protoArray.onBlock(block, gloasForkSlot - 1, null); + + // Pre-Gloas blocks should not have PTC tracking + expect(protoArray.isPayloadTimely("0x02")).toBe(false); + + // notifyPtcMessages should be no-op + expect(() => protoArray.notifyPtcMessages("0x02", [0], true)).not.toThrow(); + }); + }); + + describe("Parent relationships", () => { + let protoArray: ProtoArray; + + beforeEach(() => { + protoArray = new ProtoArray({ + pruneThreshold: 0, + justifiedEpoch: genesisEpoch, + justifiedRoot: genesisRoot, + finalizedEpoch: genesisEpoch, + finalizedRoot: genesisRoot, + }); + }); + + it("intra-block: EMPTY/FULL variants have PENDING as parent", () => { + const block = createTestBlock(gloasForkSlot, "0x02", genesisRoot, genesisRoot); + protoArray.onBlock(block, gloasForkSlot, null); + protoArray.onExecutionPayload("0x02", gloasForkSlot, "0x02", gloasForkSlot, stateRoot, null); + + const pendingIndex = protoArray.getNodeIndexByRootAndStatus("0x02", PayloadStatus.PENDING); + const emptyNode = getNodeByPayloadStatus(protoArray, "0x02", PayloadStatus.EMPTY); + const fullNode = getNodeByPayloadStatus(protoArray, "0x02", PayloadStatus.FULL); + + expect(emptyNode?.parent).toBe(pendingIndex); + expect(fullNode?.parent).toBe(pendingIndex); + }); + + it("inter-block: new PENDING extends parent's EMPTY or FULL", () => { + // Block A + const blockA = createTestBlock(gloasForkSlot, "0x02Root", genesisRoot, genesisRoot); + protoArray.onBlock(blockA, gloasForkSlot, null); + protoArray.onExecutionPayload("0x02Root", gloasForkSlot, "0x02Hash", gloasForkSlot, stateRoot, null); + + // Block B extends A's FULL (parentBlockHash matches) + const blockB = createTestBlock(gloasForkSlot + 1, "0x03Root", "0x02Root", "0x02Hash"); + protoArray.onBlock(blockB, gloasForkSlot + 1, null); + + const blockAPending = protoArray.getNodeIndexByRootAndStatus("0x02Root", PayloadStatus.PENDING); + const blockAFull = protoArray.getNodeIndexByRootAndStatus("0x02Root", PayloadStatus.FULL); + const blockBPending = getNodeByPayloadStatus(protoArray, "0x03Root", PayloadStatus.PENDING); + + // Block B's PENDING should NOT point to A's PENDING + expect(blockBPending?.parent).not.toBe(blockAPending); + // Block B's PENDING should point to A's FULL (because parentBlockHash matches) + expect(blockBPending?.parent).toBe(blockAFull); + }); + }); + + describe("Explicit EMPTY vs FULL tiebreaker for recent slots", () => { + let protoArray: ProtoArray; + + beforeEach(() => { + // Initialize with genesis block to avoid INVALID_PARENT_DELTA errors + protoArray = ProtoArray.initialize(createTestBlock(0, genesisRoot, "0x00"), 0); + }); + + it("EMPTY vs FULL comparison uses explicit tiebreaker for slot n-1 blocks", () => { + const blockSlot = gloasForkSlot + 10; + const block = createTestBlock(blockSlot, "0x02", genesisRoot, genesisRoot); + protoArray.onBlock(block, blockSlot, null); + protoArray.onExecutionPayload("0x02", blockSlot, "0x02", blockSlot, stateRoot, null); + + const emptyIndex = protoArray.getNodeIndexByRootAndStatus("0x02", PayloadStatus.EMPTY); + if (emptyIndex === undefined) throw new Error("Expected emptyIndex to exist"); + const fullIndex = protoArray.getNodeIndexByRootAndStatus("0x02", PayloadStatus.FULL); + if (fullIndex === undefined) throw new Error("Expected fullIndex to exist"); + + // Give EMPTY more weight than FULL + const deltas = new Array(protoArray.length()).fill(0); + deltas[emptyIndex] = 200; + deltas[fullIndex] = 100; + + // Apply at currentSlot = blockSlot + 1 (makes block from slot n-1) + protoArray.applyScoreChanges({ + deltas, + proposerBoost: null, + justifiedEpoch: genesisEpoch, + justifiedRoot: genesisRoot, + finalizedEpoch: genesisEpoch, + finalizedRoot: genesisRoot, + currentSlot: blockSlot + 1, + }); + + const emptyNode = getNodeByPayloadStatus(protoArray, "0x02", PayloadStatus.EMPTY); + const fullNode = getNodeByPayloadStatus(protoArray, "0x02", PayloadStatus.FULL); + + // Both nodes should have accumulated their weights + expect(emptyNode?.weight).toBe(200); + expect(fullNode?.weight).toBe(100); + + // But when comparing for bestChild, the tiebreaker should be used + // (this is implicitly tested by the comparison logic, weights are ignored) + }); + + it("different blocks at slot n-1 still use weight comparison", () => { + const blockSlot = gloasForkSlot + 10; + + const blockA = createTestBlock(blockSlot, "0x02", genesisRoot, genesisRoot); + const blockB = createTestBlock(blockSlot, "0x03", genesisRoot, genesisRoot); + + protoArray.onBlock(blockA, blockSlot, null); + protoArray.onBlock(blockB, blockSlot, null); + + const emptyAIndex = protoArray.getNodeIndexByRootAndStatus("0x02", PayloadStatus.EMPTY); + if (emptyAIndex === undefined) throw new Error("Expected emptyAIndex to exist"); + const emptyBIndex = protoArray.getNodeIndexByRootAndStatus("0x03", PayloadStatus.EMPTY); + if (emptyBIndex === undefined) throw new Error("Expected emptyBIndex to exist"); + + // Give A more votes than B + // Note: Use nodes.length (not protoArray.length()) since Gloas blocks have multiple nodes per root + const deltas = new Array(protoArray.nodes.length).fill(0); + deltas[emptyAIndex] = 200; + deltas[emptyBIndex] = 100; + + protoArray.applyScoreChanges({ + deltas, + proposerBoost: null, + justifiedEpoch: genesisEpoch, + justifiedRoot: genesisRoot, + finalizedEpoch: genesisEpoch, + finalizedRoot: genesisRoot, + currentSlot: blockSlot + 1, + }); + + const emptyANode = getNodeByPayloadStatus(protoArray, "0x02", PayloadStatus.EMPTY); + const emptyBNode = getNodeByPayloadStatus(protoArray, "0x03", PayloadStatus.EMPTY); + + // Different blocks should use weight comparison, not tiebreaker + expect(emptyANode?.weight).toBe(200); + expect(emptyBNode?.weight).toBe(100); + // Block A should be preferred due to higher weight + }); + + it("EMPTY vs FULL from older slots (n-2) uses weight comparison", () => { + const blockSlot = gloasForkSlot + 10; + const block = createTestBlock(blockSlot, "0x02", genesisRoot, genesisRoot); + protoArray.onBlock(block, blockSlot, null); + protoArray.onExecutionPayload("0x02", blockSlot, "0x02", blockSlot, stateRoot, null); + + const emptyIndex = protoArray.getNodeIndexByRootAndStatus("0x02", PayloadStatus.EMPTY); + if (emptyIndex === undefined) throw new Error("Expected emptyIndex to exist"); + const fullIndex = protoArray.getNodeIndexByRootAndStatus("0x02", PayloadStatus.FULL); + if (fullIndex === undefined) throw new Error("Expected fullIndex to exist"); + + const deltas = new Array(protoArray.length()).fill(0); + deltas[emptyIndex] = 100; + deltas[fullIndex] = 200; + + // currentSlot = blockSlot + 2, so block is from slot n-2 (not n-1) + protoArray.applyScoreChanges({ + deltas, + proposerBoost: null, + justifiedEpoch: genesisEpoch, + justifiedRoot: genesisRoot, + finalizedEpoch: genesisEpoch, + finalizedRoot: genesisRoot, + currentSlot: blockSlot + 2, + }); + + const emptyNode = getNodeByPayloadStatus(protoArray, "0x02", PayloadStatus.EMPTY); + const fullNode = getNodeByPayloadStatus(protoArray, "0x02", PayloadStatus.FULL); + + // Older blocks use weight comparison, not tiebreaker + expect(emptyNode?.weight).toBe(100); + expect(fullNode?.weight).toBe(200); + // FULL should be preferred due to higher weight + }); + }); +}); diff --git a/packages/fork-choice/test/unit/protoArray/protoArray.test.ts b/packages/fork-choice/test/unit/protoArray/protoArray.test.ts index 6be99553ba3f..bbfa95e59aa4 100644 --- a/packages/fork-choice/test/unit/protoArray/protoArray.test.ts +++ b/packages/fork-choice/test/unit/protoArray/protoArray.test.ts @@ -1,7 +1,7 @@ import {describe, expect, it} from "vitest"; import {DataAvailabilityStatus} from "@lodestar/state-transition"; import {RootHex} from "@lodestar/types"; -import {ExecutionStatus, ProtoArray} from "../../../src/index.js"; +import {ExecutionStatus, PayloadStatus, ProtoArray} from "../../../src/index.js"; describe("ProtoArray", () => { it("finalized descendant", () => { @@ -34,6 +34,11 @@ describe("ProtoArray", () => { ...{executionPayloadBlockHash: null, executionStatus: ExecutionStatus.PreMerge}, dataAvailabilityStatus: DataAvailabilityStatus.PreData, + + parentBlockHash: null, + payloadStatus: PayloadStatus.FULL, + builderIndex: null, + blockHashFromBid: null, }, genesisSlot ); @@ -60,8 +65,14 @@ describe("ProtoArray", () => { ...{executionPayloadBlockHash: null, executionStatus: ExecutionStatus.PreMerge}, dataAvailabilityStatus: DataAvailabilityStatus.PreData, + + parentBlockHash: null, + payloadStatus: PayloadStatus.FULL, + builderIndex: null, + blockHashFromBid: null, }, - genesisSlot + 1 + genesisSlot + 1, + null ); // Add block that is *not* a finalized descendant. @@ -86,8 +97,14 @@ describe("ProtoArray", () => { ...{executionPayloadBlockHash: null, executionStatus: ExecutionStatus.PreMerge}, dataAvailabilityStatus: DataAvailabilityStatus.PreData, + + parentBlockHash: null, + payloadStatus: PayloadStatus.FULL, + builderIndex: null, + blockHashFromBid: null, }, - genesisSlot + 1 + genesisSlot + 1, + null ); // ancestorRoot, descendantRoot, isDescendant From 45864cc073905b973908d93cd47f6a6a60676d84 Mon Sep 17 00:00:00 2001 From: Lodekeeper <258435968+lodekeeper@users.noreply.github.com> Date: Wed, 4 Mar 2026 21:06:43 +0000 Subject: [PATCH 18/63] chore: bump libp2p deps to latest patch releases (#8984) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Bump libp2p and related @libp2p/* dependencies to the latest patch releases that include the recent upstream fixes. ### Updated - libp2p: 3.1.4 -> 3.1.5 - @libp2p/identify: ^4.0.11 -> ^4.0.12 - @libp2p/utils: ^7.0.11 -> ^7.0.12 - @libp2p/prometheus-metrics: ^5.0.11 -> ^5.0.13 - plus aligned patch bumps for bootstrap/mdns/mplex/tcp/gossipsub/interface-internal Also adds `libp2p` to `minimumReleaseAgeExclude` in `pnpm-workspace.yaml` so fresh libp2p patch releases can be pulled immediately (matching existing `@libp2p/*` exclude). ## Validation - pnpm build - pnpm lint (warning-only: pre-existing unused biome suppression in packages/light-client/test/unit/webEsmBundle.browser.test.ts) --- This PR was authored with AI assistance (Lodekeeper 🌟). Co-authored-by: lodekeeper --- package.json | 2 +- packages/beacon-node/package.json | 20 ++-- packages/reqresp/package.json | 4 +- pnpm-lock.yaml | 168 +++++++++++++++--------------- pnpm-workspace.yaml | 1 + 5 files changed, 98 insertions(+), 97 deletions(-) diff --git a/package.json b/package.json index a46a30be7eb3..462aceaf183c 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,7 @@ "https-browserify": "^1.0.0", "inquirer": "^9.1.5", "jsdom": "^23.0.1", - "libp2p": "3.1.4", + "libp2p": "3.1.5", "node-gyp": "^9.4.0", "npm-run-all": "^4.1.5", "path-browserify": "^1.0.1", diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index 6264540b3627..3d6340aae6b7 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -124,16 +124,16 @@ "@fastify/cors": "^10.0.1", "@fastify/swagger": "^9.0.0", "@fastify/swagger-ui": "^5.0.1", - "@libp2p/bootstrap": "^12.0.12", + "@libp2p/bootstrap": "^12.0.13", "@libp2p/crypto": "^5.1.13", - "@libp2p/gossipsub": "^15.0.13", - "@libp2p/identify": "^4.0.11", + "@libp2p/gossipsub": "^15.0.14", + "@libp2p/identify": "^4.0.12", "@libp2p/interface": "^3.1.0", - "@libp2p/mdns": "^12.0.12", - "@libp2p/mplex": "^12.0.12", + "@libp2p/mdns": "^12.0.13", + "@libp2p/mplex": "^12.0.13", "@libp2p/peer-id": "^6.0.4", - "@libp2p/prometheus-metrics": "^5.0.11", - "@libp2p/tcp": "^11.0.11", + "@libp2p/prometheus-metrics": "^5.0.13", + "@libp2p/tcp": "^11.0.12", "@lodestar/api": "workspace:^", "@lodestar/config": "workspace:^", "@lodestar/db": "workspace:^", @@ -154,7 +154,7 @@ "fastify": "^5.7.4", "interface-datastore": "^9.0.2", "jwt-simple": "0.5.6", - "libp2p": "3.1.4", + "libp2p": "3.1.5", "multiformats": "^13.4.2", "prom-client": "^15.1.0", "qs": "^6.11.1", @@ -165,9 +165,9 @@ }, "devDependencies": { "@chainsafe/swap-or-not-shuffle": "^1.2.1", - "@libp2p/interface-internal": "^3.0.11", + "@libp2p/interface-internal": "^3.0.12", "@libp2p/logger": "^6.2.2", - "@libp2p/utils": "^7.0.11", + "@libp2p/utils": "^7.0.12", "@lodestar/spec-test-util": "workspace:^", "@types/js-yaml": "^4.0.5", "@types/qs": "^6.9.7", diff --git a/packages/reqresp/package.json b/packages/reqresp/package.json index 8e1dbb5a14d6..a6dd03b75597 100644 --- a/packages/reqresp/package.json +++ b/packages/reqresp/package.json @@ -46,7 +46,7 @@ "dependencies": { "@chainsafe/fast-crc32c": "^4.2.0", "@libp2p/interface": "^3.1.0", - "@libp2p/utils": "^7.0.11", + "@libp2p/utils": "^7.0.12", "@lodestar/config": "workspace:^", "@lodestar/params": "workspace:^", "@lodestar/utils": "workspace:^", @@ -62,7 +62,7 @@ "@libp2p/peer-id": "^6.0.4", "@lodestar/logger": "workspace:^", "@lodestar/types": "workspace:^", - "libp2p": "3.1.4" + "libp2p": "3.1.5" }, "peerDependencies": { "libp2p": "^3.1.4" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 45128f8438f2..a7857b965c8a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -105,8 +105,8 @@ importers: specifier: ^23.0.1 version: 23.0.1 libp2p: - specifier: 3.1.4 - version: 3.1.4 + specifier: 3.1.5 + version: 3.1.5 node-gyp: specifier: ^9.4.0 version: 9.4.1 @@ -262,35 +262,35 @@ importers: specifier: ^5.0.1 version: 5.0.1 '@libp2p/bootstrap': - specifier: ^12.0.12 - version: 12.0.12 + specifier: ^12.0.13 + version: 12.0.13 '@libp2p/crypto': specifier: ^5.1.13 version: 5.1.13 '@libp2p/gossipsub': - specifier: ^15.0.13 - version: 15.0.13 + specifier: ^15.0.14 + version: 15.0.14 '@libp2p/identify': - specifier: ^4.0.11 - version: 4.0.11 + specifier: ^4.0.12 + version: 4.0.12 '@libp2p/interface': specifier: ^3.1.0 version: 3.1.0 '@libp2p/mdns': - specifier: ^12.0.12 - version: 12.0.12 + specifier: ^12.0.13 + version: 12.0.13 '@libp2p/mplex': - specifier: ^12.0.12 - version: 12.0.12 + specifier: ^12.0.13 + version: 12.0.13 '@libp2p/peer-id': specifier: ^6.0.4 version: 6.0.4 '@libp2p/prometheus-metrics': - specifier: ^5.0.11 - version: 5.0.11 + specifier: ^5.0.13 + version: 5.0.13 '@libp2p/tcp': - specifier: ^11.0.11 - version: 11.0.11 + specifier: ^11.0.12 + version: 11.0.12 '@lodestar/api': specifier: workspace:^ version: link:../api @@ -352,8 +352,8 @@ importers: specifier: 0.5.6 version: 0.5.6 libp2p: - specifier: 3.1.4 - version: 3.1.4 + specifier: 3.1.5 + version: 3.1.5 multiformats: specifier: ^13.4.2 version: 13.4.2 @@ -380,14 +380,14 @@ importers: specifier: ^1.2.1 version: 1.2.1 '@libp2p/interface-internal': - specifier: ^3.0.11 - version: 3.0.11 + specifier: ^3.0.12 + version: 3.0.12 '@libp2p/logger': specifier: ^6.2.2 version: 6.2.2 '@libp2p/utils': - specifier: ^7.0.11 - version: 7.0.11 + specifier: ^7.0.12 + version: 7.0.12 '@lodestar/spec-test-util': specifier: workspace:^ version: link:../spec-test-util @@ -890,8 +890,8 @@ importers: specifier: ^3.1.0 version: 3.1.0 '@libp2p/utils': - specifier: ^7.0.11 - version: 7.0.11 + specifier: ^7.0.12 + version: 7.0.12 '@lodestar/config': specifier: workspace:^ version: link:../config @@ -933,8 +933,8 @@ importers: specifier: workspace:^ version: link:../types libp2p: - specifier: 3.1.4 - version: 3.1.4 + specifier: 3.1.5 + version: 3.1.5 packages/spec-test-util: dependencies: @@ -2194,21 +2194,21 @@ packages: resolution: {integrity: sha512-hRFGB47RWHNKZkh9t+ka34kNlv5DQwrIDBvjI+mmzZqZK7UAtPRsJoE+MlQHVlQApDFwUPiSTdxY9UXzJrYFdQ==} engines: {node: ^20.17.0 || >=22.9.0} - '@libp2p/bootstrap@12.0.12': - resolution: {integrity: sha512-OFVBDm2UO5J/0olQKypj9VjhQOs4SCdVXqCEdItOrZeahsgB6XeVS+eHlbAle87jzh6r6gO/yVMGCEeJeRiMHQ==} + '@libp2p/bootstrap@12.0.13': + resolution: {integrity: sha512-txdW/5IK/2gm76NpnRYcTpZgaT7FIH/t59171ENc1t2lnvwXeaP6l/6bTXX23TSDCINuZUHOv0Hm8f+CpGNB8Q==} '@libp2p/crypto@5.1.13': resolution: {integrity: sha512-8NN9cQP3jDn+p9+QE9ByiEoZ2lemDFf/unTgiKmS3JF93ph240EUVdbCyyEgOMfykzb0okTM4gzvwfx9osJebQ==} - '@libp2p/gossipsub@15.0.13': - resolution: {integrity: sha512-cqDaJ8wdJ0n6sbhk1xe5dZIFfAvB5laaorXPIw7jWhkG+WA9h02INgbUPz+YkMhBJdsNwa5gKRMjVU3sdaaAPA==} + '@libp2p/gossipsub@15.0.14': + resolution: {integrity: sha512-F+Rjyz6sPvKvZr588D1l7rRZPefPTOxvVYtH3/WXUyu4h6EA+trwMax/WOrbrAYopfwJd++/1RXHMEuZpGzirg==} engines: {npm: '>=8.7.0'} - '@libp2p/identify@4.0.11': - resolution: {integrity: sha512-DfQcNVkRROtSChTxoTlqBWKO9+ph4G8vv4Z7CtiN2wFALVudssIUcPwCeTdj10fcjL0NN/rckf09c9HJHt3/rw==} + '@libp2p/identify@4.0.12': + resolution: {integrity: sha512-jolOVcJ9zj4306btDuDQw5mZId4tE20UndS4w5+v+KhXcNZWsntZ6p/F0l3AgOjPRxHUw1d0VabQQEKjHDbOjw==} - '@libp2p/interface-internal@3.0.11': - resolution: {integrity: sha512-tByDMIf3LGOEgmSxxe/zamKxYIWHD/2pLWntOdtcZ/fHB7Wnhk2HvOlPgD0nPVqIDWHwFfmS6uajGRWOtS150w==} + '@libp2p/interface-internal@3.0.12': + resolution: {integrity: sha512-WfUwypkIEwy0YwS05a1E8o7VicbUa259rSjzSpbGFFw8dlVqGCaKKU+6hj4mSbkXhRWq56xXkvt9MjzdEZ/fcA==} '@libp2p/interface@3.1.0': resolution: {integrity: sha512-RE7/XyvC47fQBe1cHxhMvepYKa5bFCUyFrrpj8PuM0E7JtzxU7F+Du5j4VXbg2yLDcToe0+j8mB7jvwE2AThYw==} @@ -2216,17 +2216,17 @@ packages: '@libp2p/logger@6.2.2': resolution: {integrity: sha512-XtanXDT+TuMuZoCK760HGV1AmJsZbwAw5AiRUxWDbsZPwAroYq64nb41AHRu9Gyc0TK9YD+p72+5+FIxbw0hzw==} - '@libp2p/mdns@12.0.12': - resolution: {integrity: sha512-Tu4FaLrguteEaD+MUwi9fFXxfjVm306czSEmZWxnoGjPc3+iVG0B4c4VVNunWHedhMktJALNKHw6FoM9PfQ/uQ==} + '@libp2p/mdns@12.0.13': + resolution: {integrity: sha512-XDQbX8PBvGWHeL+pvGGTGB2NYY2ib1lTZg7e4CV3pqnK9TxizIVQNtp7sdRx9YpxvvJdxuBtHezUZT714VtLJw==} - '@libp2p/mplex@12.0.12': - resolution: {integrity: sha512-xGGOR4tS1NBeLVzvNXVewEVIYz5y2DkuOXgUUrtufSRZqBS5Pyy+mgchn6qR2X73Tmx5AzD783T7RlhSsAhnVg==} + '@libp2p/mplex@12.0.13': + resolution: {integrity: sha512-fDscM89OmkQngE/CUMDvdQDAP3GLp7lqC/95OXvl/fmNxQBDd1mV3UoHsmmrbcVI0auinC/keIqBGZG9NJGDqQ==} - '@libp2p/multistream-select@7.0.11': - resolution: {integrity: sha512-zoIeiA/jl+/pwLAU4hDo1l4YVxr/ZMh1IOYpnCz6VwRBh9+2uuUFjbLRanQdP4hGQXdg0L6BsPChxgcag5MJrg==} + '@libp2p/multistream-select@7.0.12': + resolution: {integrity: sha512-fYvGllfzb3J5CJqM8E/uKmcsPqQc0mNR5zdYDFhYpqz7YjPrx2r9tYXisDrH8J2nW71a68RMPfRavSwi7Bs1DQ==} - '@libp2p/peer-collections@7.0.11': - resolution: {integrity: sha512-LJ+3lew9FcwFpL0zA4ZYhUHcFR+mekr8mHPHo/6YM9lY4feIBekZPidzkpKHNY8nn2SCXJb2e47ruqjy+whijg==} + '@libp2p/peer-collections@7.0.12': + resolution: {integrity: sha512-1QtoSWo/vJv2pXDjUW1/QYMtYS4WzmJT9LPjBqoosKC6fJUiMoumcF9uh0bZKKDH74Biy5ZEKx6kaW18Azwsnw==} '@libp2p/peer-id@6.0.4': resolution: {integrity: sha512-Z3xK0lwwKn4bPg3ozEpPr1HxsRi2CxZdghOL+MXoFah/8uhJJHxHFA8A/jxtKn4BB8xkk6F8R5vKNIS05yaCYw==} @@ -2234,17 +2234,17 @@ packages: '@libp2p/peer-record@9.0.5': resolution: {integrity: sha512-disk23OO00yD52O4VmItbDkjJZ/YZJsKbMsqNgVhr+D3PcM+KRpu9VVbiCnN5Tzn9XvFEHhrMJY7BPE+rvT5MQ==} - '@libp2p/peer-store@12.0.11': - resolution: {integrity: sha512-vOoUPv1UqFWFyRrheh31AQgUxooNPwdJz5riklXsv6d/BpCwIQjGLsTi9WCQ/S7QdPVG71hfBDWtfUg8dO8hLA==} + '@libp2p/peer-store@12.0.12': + resolution: {integrity: sha512-Ra4yqaE4RSSmYSbfWLw+Gfi6Zkb/Khf4r5tAZiafO5UyX8mBnaDqTnL1iInUwTrHVJX/neB/lfLYLD9m6Yb89g==} - '@libp2p/prometheus-metrics@5.0.11': - resolution: {integrity: sha512-sVB7GrUZ5UlfSaXR1QsivBKU+REUHNUrBrPAo/N1Lv9UdKnru4p49HEgIzA80IMW10RnM/xJqwR26tTznNyusw==} + '@libp2p/prometheus-metrics@5.0.13': + resolution: {integrity: sha512-VovjUEMzolaKtxGUn+AY7YCNjMgh7gX2rjjoUyWBBZ/tv+KJjX0kJZ5tHVpwqrOsXsuIzRl9oRiaPLQ+m0ZtRA==} - '@libp2p/tcp@11.0.11': - resolution: {integrity: sha512-6Tg8+0isJCZ/ci6+IjDuRQ6NVovT0xxqFoICtnU3u1Tu6mSZvEbebUZjGNrq7kAN9EcXYiEBwI/fRhXQtmFQGg==} + '@libp2p/tcp@11.0.12': + resolution: {integrity: sha512-oz77ksfE/eQbpz2Vkx8Kr6rklOLkhDq0NC0Plc+ScIqTyYbnWJ6L3tMd/+3vaEArF+uKeeeBA7VfGgI1+TOzaQ==} - '@libp2p/utils@7.0.11': - resolution: {integrity: sha512-M7jAWD9V0Hb7FYsPgzbmNxNwp3P8U9hmdllVhyz26rhVAZxoDdhSf0G+9Hey7DfZvMNxyMER/h/Sojqk0rYhPg==} + '@libp2p/utils@7.0.12': + resolution: {integrity: sha512-+qHXhPubw1KJtuzNH2IR02frGJxebnD/wrDIP+0GEwfUS3h76dN0TzPWLjSghzDQaUybbzlaCHTsN6c3OVcj4Q==} '@lukeed/ms@2.0.2': resolution: {integrity: sha512-9I2Zn6+NJLfaGoz9jN3lpwDgAYvfGeNYdbAIjJOqzs4Tpc+VU3Jqq4IofSUBKajiDS8k9fZIg18/z13mpk1bsA==} @@ -5137,8 +5137,8 @@ packages: resolution: {integrity: sha512-NVPTth/71cfbdYHqypcO9Lt5WFGTzFEcx81lWd7GDJIgZ95ERdYHGUfCtFejHCyqodKsQkNEx2JCkMpreDty/A==} engines: {node: ^20.17.0 || >=22.9.0} - libp2p@3.1.4: - resolution: {integrity: sha512-GiF4ZGFuldDHHEE12R8R0Pvv8mRBmiksgSgMds+zXRmcu08X1fpXfVV/YknuWrLteOIxDdgY2E3Ur6d6XmKWJA==} + libp2p@3.1.5: + resolution: {integrity: sha512-C+ZL03De0WFr9e09A27hvoCmlvWciZ9FyXVonuXUgPEoob5XyTxjJedfmQ2gXxkh2Pqz1W2vZSUqi9f0BYIT+A==} light-my-request@6.0.0: resolution: {integrity: sha512-kFkFXrmKCL0EEeOmJybMH5amWFd+AFvlvMlvFTRxCUwbhfapZqDmeLMPoWihntnYY6JpoQDE9k+vOzObF1fDqg==} @@ -7752,7 +7752,7 @@ snapshots: '@libp2p/crypto': 5.1.13 '@libp2p/interface': 3.1.0 '@libp2p/peer-id': 6.0.4 - '@libp2p/utils': 7.0.11 + '@libp2p/utils': 7.0.12 '@noble/ciphers': 2.1.1 '@noble/curves': 2.0.1 '@noble/hashes': 2.0.1 @@ -8862,10 +8862,10 @@ snapshots: - conventional-commits-filter - supports-color - '@libp2p/bootstrap@12.0.12': + '@libp2p/bootstrap@12.0.13': dependencies: '@libp2p/interface': 3.1.0 - '@libp2p/interface-internal': 3.0.11 + '@libp2p/interface-internal': 3.0.12 '@libp2p/peer-id': 6.0.4 '@multiformats/multiaddr': 13.0.1 '@multiformats/multiaddr-matcher': 3.0.1 @@ -8881,13 +8881,13 @@ snapshots: uint8arraylist: 2.4.8 uint8arrays: 5.1.0 - '@libp2p/gossipsub@15.0.13': + '@libp2p/gossipsub@15.0.14': dependencies: '@libp2p/crypto': 5.1.13 '@libp2p/interface': 3.1.0 - '@libp2p/interface-internal': 3.0.11 + '@libp2p/interface-internal': 3.0.12 '@libp2p/peer-id': 6.0.4 - '@libp2p/utils': 7.0.11 + '@libp2p/utils': 7.0.12 '@multiformats/multiaddr': 13.0.1 denque: 2.1.0 it-length-prefixed: 10.0.1 @@ -8898,14 +8898,14 @@ snapshots: uint8arraylist: 2.4.8 uint8arrays: 5.1.0 - '@libp2p/identify@4.0.11': + '@libp2p/identify@4.0.12': dependencies: '@libp2p/crypto': 5.1.13 '@libp2p/interface': 3.1.0 - '@libp2p/interface-internal': 3.0.11 + '@libp2p/interface-internal': 3.0.12 '@libp2p/peer-id': 6.0.4 '@libp2p/peer-record': 9.0.5 - '@libp2p/utils': 7.0.11 + '@libp2p/utils': 7.0.12 '@multiformats/multiaddr': 13.0.1 '@multiformats/multiaddr-matcher': 3.0.1 it-drain: 3.0.10 @@ -8915,10 +8915,10 @@ snapshots: uint8arraylist: 2.4.8 uint8arrays: 5.1.0 - '@libp2p/interface-internal@3.0.11': + '@libp2p/interface-internal@3.0.12': dependencies: '@libp2p/interface': 3.1.0 - '@libp2p/peer-collections': 7.0.11 + '@libp2p/peer-collections': 7.0.12 '@multiformats/multiaddr': 13.0.1 progress-events: 1.0.1 @@ -8939,40 +8939,40 @@ snapshots: multiformats: 13.4.2 weald: 1.1.1 - '@libp2p/mdns@12.0.12': + '@libp2p/mdns@12.0.13': dependencies: '@libp2p/interface': 3.1.0 - '@libp2p/interface-internal': 3.0.11 + '@libp2p/interface-internal': 3.0.12 '@libp2p/peer-id': 6.0.4 - '@libp2p/utils': 7.0.11 + '@libp2p/utils': 7.0.12 '@multiformats/multiaddr': 13.0.1 '@types/multicast-dns': 7.2.4 dns-packet: 5.6.1 main-event: 1.0.1 multicast-dns: 7.2.5 - '@libp2p/mplex@12.0.12': + '@libp2p/mplex@12.0.13': dependencies: '@libp2p/interface': 3.1.0 - '@libp2p/utils': 7.0.11 + '@libp2p/utils': 7.0.12 it-pushable: 3.2.3 uint8-varint: 2.0.4 uint8arraylist: 2.4.8 uint8arrays: 5.1.0 - '@libp2p/multistream-select@7.0.11': + '@libp2p/multistream-select@7.0.12': dependencies: '@libp2p/interface': 3.1.0 - '@libp2p/utils': 7.0.11 + '@libp2p/utils': 7.0.12 it-length-prefixed: 10.0.1 uint8arraylist: 2.4.8 uint8arrays: 5.1.0 - '@libp2p/peer-collections@7.0.11': + '@libp2p/peer-collections@7.0.12': dependencies: '@libp2p/interface': 3.1.0 '@libp2p/peer-id': 6.0.4 - '@libp2p/utils': 7.0.11 + '@libp2p/utils': 7.0.12 multiformats: 13.4.2 '@libp2p/peer-id@6.0.4': @@ -8994,11 +8994,11 @@ snapshots: uint8arraylist: 2.4.8 uint8arrays: 5.1.0 - '@libp2p/peer-store@12.0.11': + '@libp2p/peer-store@12.0.12': dependencies: '@libp2p/crypto': 5.1.13 '@libp2p/interface': 3.1.0 - '@libp2p/peer-collections': 7.0.11 + '@libp2p/peer-collections': 7.0.12 '@libp2p/peer-id': 6.0.4 '@libp2p/peer-record': 9.0.5 '@multiformats/multiaddr': 13.0.1 @@ -9011,15 +9011,15 @@ snapshots: uint8arraylist: 2.4.8 uint8arrays: 5.1.0 - '@libp2p/prometheus-metrics@5.0.11': + '@libp2p/prometheus-metrics@5.0.13': dependencies: '@libp2p/interface': 3.1.0 prom-client: 15.1.3 - '@libp2p/tcp@11.0.11': + '@libp2p/tcp@11.0.12': dependencies: '@libp2p/interface': 3.1.0 - '@libp2p/utils': 7.0.11 + '@libp2p/utils': 7.0.12 '@multiformats/multiaddr': 13.0.1 '@multiformats/multiaddr-matcher': 3.0.1 '@types/sinon': 20.0.0 @@ -9028,7 +9028,7 @@ snapshots: progress-events: 1.0.1 uint8arraylist: 2.4.8 - '@libp2p/utils@7.0.11': + '@libp2p/utils@7.0.12': dependencies: '@chainsafe/is-ip': 2.1.0 '@chainsafe/netmask': 2.0.0 @@ -12305,19 +12305,19 @@ snapshots: transitivePeerDependencies: - supports-color - libp2p@3.1.4: + libp2p@3.1.5: dependencies: '@chainsafe/is-ip': 2.1.0 '@chainsafe/netmask': 2.0.0 '@libp2p/crypto': 5.1.13 '@libp2p/interface': 3.1.0 - '@libp2p/interface-internal': 3.0.11 + '@libp2p/interface-internal': 3.0.12 '@libp2p/logger': 6.2.2 - '@libp2p/multistream-select': 7.0.11 - '@libp2p/peer-collections': 7.0.11 + '@libp2p/multistream-select': 7.0.12 + '@libp2p/peer-collections': 7.0.12 '@libp2p/peer-id': 6.0.4 - '@libp2p/peer-store': 12.0.11 - '@libp2p/utils': 7.0.11 + '@libp2p/peer-store': 12.0.12 + '@libp2p/utils': 7.0.12 '@multiformats/dns': 1.0.13 '@multiformats/multiaddr': 13.0.1 '@multiformats/multiaddr-matcher': 3.0.1 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 066840a788f0..1bf49212904a 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -7,6 +7,7 @@ minimumReleaseAge: 2880 minimumReleaseAgeExclude: - "@libp2p/*" + - "libp2p" nodeLinker: isolated From 477bc362cedd630fb3bdf4e40d6a825b0d9ffb00 Mon Sep 17 00:00:00 2001 From: Lodekeeper <258435968+lodekeeper@users.noreply.github.com> Date: Wed, 4 Mar 2026 21:44:02 +0000 Subject: [PATCH 19/63] chore(ci): pin ethspecify to 0.3.7 in check-specrefs (#8986) ## Summary\nPin `ethspecify` to `0.3.7` in the `check-specrefs` workflow to avoid the current regression in `0.3.8` (false `MISSING FILE` failures).\n\n## Context\nPR #8984 merged before the pin commit landed, so this is a standalone follow-up PR carrying only the CI pin.\n\n## Validation\n- Reproduced failure with `ethspecify==0.3.8`\n- Confirmed pass with `ethspecify==0.3.7` (`All specification references (935) are valid`) --------- Co-authored-by: lodekeeper --- .github/workflows/check-specrefs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-specrefs.yml b/.github/workflows/check-specrefs.yml index b7dc415253e8..229b832b4f97 100644 --- a/.github/workflows/check-specrefs.yml +++ b/.github/workflows/check-specrefs.yml @@ -32,7 +32,7 @@ jobs: fi - name: Install ethspecify - run: python3 -mpip install ethspecify + run: python3 -mpip install ethspecify==0.3.9 - name: Update spec references run: ethspecify process --path=specrefs From 20f622cc52f9f322d4714265d25dfc0690105945 Mon Sep 17 00:00:00 2001 From: Lodekeeper <258435968+lodekeeper@users.noreply.github.com> Date: Thu, 5 Mar 2026 10:43:36 +0000 Subject: [PATCH 20/63] chore: bump fastify to v5.8.1 (#8989) ## Summary - bump `fastify` from `^5.7.4` to `^5.8.1` in `@lodestar/api`, `@lodestar/beacon-node`, `@lodestar/cli`, and `@lodestar/light-client` - refresh the lockfile to pin `fastify@5.8.1` and compatible transitive updates ## Validation - `pnpm lint` (biome check passes with one pre-existing warning in `packages/light-client/test/unit/webEsmBundle.browser.test.ts`) AI assistance: Prepared with OpenClaw (Lodekeeper). Co-authored-by: lodekeeper --- packages/api/package.json | 2 +- packages/beacon-node/package.json | 2 +- packages/cli/package.json | 2 +- packages/light-client/package.json | 2 +- pnpm-lock.yaml | 248 ++++++++++++++++------------- 5 files changed, 144 insertions(+), 112 deletions(-) diff --git a/packages/api/package.json b/packages/api/package.json index 2b6b616151ac..d8ed1fb2cebc 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -87,7 +87,7 @@ "@types/eventsource": "^1.1.11", "@types/qs": "^6.9.7", "ajv": "^8.12.0", - "fastify": "^5.7.4" + "fastify": "^5.8.1" }, "keywords": [ "ethereum", diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index 3d6340aae6b7..615889647afa 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -151,7 +151,7 @@ "datastore-fs": "^11.0.2", "datastore-level": "^12.0.2", "deepmerge": "^4.3.1", - "fastify": "^5.7.4", + "fastify": "^5.8.1", "interface-datastore": "^9.0.2", "jwt-simple": "0.5.6", "libp2p": "3.1.5", diff --git a/packages/cli/package.json b/packages/cli/package.json index ef7608b55bc8..a879acf1f913 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -104,7 +104,7 @@ "@types/tmp": "^0.2.3", "@types/yargs": "^17.0.24", "ethereum-cryptography": "^2.2.1", - "fastify": "^5.7.4", + "fastify": "^5.8.1", "tmp": "^0.2.4", "web3": "^4.0.3", "web3-eth-accounts": "^4.0.3" diff --git a/packages/light-client/package.json b/packages/light-client/package.json index af18c1d13b87..aa88860bac9f 100644 --- a/packages/light-client/package.json +++ b/packages/light-client/package.json @@ -79,7 +79,7 @@ "@fastify/cors": "^10.0.1", "@lodestar/state-transition": "workspace:^", "@types/qs": "^6.9.7", - "fastify": "^5.7.4", + "fastify": "^5.8.1", "qs": "^6.11.1", "uint8arrays": "^5.0.1" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a7857b965c8a..273409581ac8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -208,8 +208,8 @@ importers: specifier: ^8.12.0 version: 8.18.0 fastify: - specifier: ^5.7.4 - version: 5.7.4 + specifier: ^5.8.1 + version: 5.8.1 packages/beacon-node: dependencies: @@ -343,8 +343,8 @@ importers: specifier: ^4.3.1 version: 4.3.1 fastify: - specifier: ^5.7.4 - version: 5.7.4 + specifier: ^5.8.1 + version: 5.8.1 interface-datastore: specifier: ^9.0.2 version: 9.0.2 @@ -558,8 +558,8 @@ importers: specifier: ^2.2.1 version: 2.2.1 fastify: - specifier: ^5.7.4 - version: 5.7.4 + specifier: ^5.8.1 + version: 5.8.1 tmp: specifier: ^0.2.4 version: 0.2.4 @@ -735,8 +735,8 @@ importers: specifier: ^6.9.7 version: 6.9.7 fastify: - specifier: ^5.7.4 - version: 5.7.4 + specifier: ^5.8.1 + version: 5.8.1 qs: specifier: ^6.11.1 version: 6.14.1 @@ -1973,17 +1973,20 @@ packages: '@fastify/error@4.0.0': resolution: {integrity: sha512-OO/SA8As24JtT1usTUTKgGH7uLvhfwZPwlptRi2Dp5P4KKmJI3gvsZ8MIHnNwDs4sLf/aai5LzTyl66xr7qMxA==} - '@fastify/fast-json-stringify-compiler@5.0.1': - resolution: {integrity: sha512-f2d3JExJgFE3UbdFcpPwqNUEoHWmt8pAKf8f+9YuLESdefA0WgqxeT6DrGL4Yrf/9ihXNSKOqpjEmurV405meA==} + '@fastify/error@4.2.0': + resolution: {integrity: sha512-RSo3sVDXfHskiBZKBPRgnQTtIqpi/7zhJOEmAxCiBcM7d0uwdGdxLlsCaLzGs8v8NnxIRlfG0N51p5yFaOentQ==} - '@fastify/forwarded@3.0.0': - resolution: {integrity: sha512-kJExsp4JCms7ipzg7SJ3y8DwmePaELHxKYtg+tZow+k0znUTf3cb+npgyqm8+ATZOdmfgfydIebPDWM172wfyA==} + '@fastify/fast-json-stringify-compiler@5.0.3': + resolution: {integrity: sha512-uik7yYHkLr6fxd8hJSZ8c+xF4WafPK+XzneQDPU+D10r5X19GW8lJcom2YijX2+qtFF1ENJlHXKFM9ouXNJYgQ==} - '@fastify/merge-json-schemas@0.1.1': - resolution: {integrity: sha512-fERDVz7topgNjtXsJTTW1JKLy0rhuLRcquYqNR9rF7OcVpCa2OVW49ZPDIhaRRCaUuvVxI+N416xUoF76HNSXA==} + '@fastify/forwarded@3.0.1': + resolution: {integrity: sha512-JqDochHFqXs3C3Ml3gOY58zM7OqO9ENqPo0UqAjAjH8L01fRZqwX9iLeX34//kiJubF7r2ZQHtBRU36vONbLlw==} - '@fastify/proxy-addr@5.0.0': - resolution: {integrity: sha512-37qVVA1qZ5sgH7KpHkkC4z9SK6StIsIcOmpjvMPXNb3vx2GQxhZocogVYbr2PbbeLCQxYIPDok307xEvRZOzGA==} + '@fastify/merge-json-schemas@0.2.1': + resolution: {integrity: sha512-OA3KGBCy6KtIvLf8DINC5880o5iBlDX4SxzLQS8HorJAbqluzLRn80UXU0bxZn7UOFhFgpRJDasfwn9nG4FG4A==} + + '@fastify/proxy-addr@5.1.0': + resolution: {integrity: sha512-INS+6gh91cLUjB+PVHfu1UqcB76Sqtpyp7bnL+FYojhjygvOPA9ctiD/JDKsyD9Xgu4hUhCSJBPig/w7duNajw==} '@fastify/send@3.1.1': resolution: {integrity: sha512-LdiV2mle/2tH8vh6GwGl0ubfUAgvY+9yF9oGI1iiwVyNUVOQamvw5n+OFu6iCNNoyuCY80FFURBn4TZCbTe8LA==} @@ -3456,8 +3459,8 @@ packages: resolution: {integrity: sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==} engines: {node: '>= 0.4'} - avvio@9.0.0: - resolution: {integrity: sha512-UbYrOXgE/I+knFG+3kJr9AgC7uNo8DG+FGGODpH9Bj1O1kL/QDjBXnTem9leD3VdQKtaHjV3O85DQ7hHh4IIHw==} + avvio@9.2.0: + resolution: {integrity: sha512-2t/sy01ArdHHE0vRH5Hsay+RtCZt3dLPji7W7/MMOCEgze5b7SNDC4j5H6FnVgPkI1MTNFGzHdHrVXDDl7QSSQ==} aws-sdk@2.934.0: resolution: {integrity: sha512-k7p08ewrKcbs0ikCLFi9OI98Iv9dMND5244xPxUIjK5BLtuT/9Gr6eSCHfN70eCQyM5Y2xG1VJP6zhpkihu9Ew==} @@ -3830,9 +3833,9 @@ packages: engines: {node: '>=18'} hasBin: true - cookie@0.6.0: - resolution: {integrity: sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==} - engines: {node: '>= 0.6'} + cookie@1.1.1: + resolution: {integrity: sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==} + engines: {node: '>=18'} cookiejar@2.1.4: resolution: {integrity: sha512-LDx6oHrK+PhzLKJU9j5S7/Y3jM/mUHvD/DeI1WQmJn652iPC5Y4TBzC9l+5OMOXlyTTA+SmVUPm0HQUwpD5Jqw==} @@ -3998,6 +4001,10 @@ packages: deprecation@2.3.1: resolution: {integrity: sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ==} + dequal@2.0.3: + resolution: {integrity: sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==} + engines: {node: '>=6'} + des.js@1.0.1: resolution: {integrity: sha512-Q0I4pfFrv2VPd34/vfLrFOoRmlYj3OV50i7fskps1jZWK1kApMWWT9G6RRUeYedLcBDIhnSDaUvJMb3AhUlaEA==} @@ -4238,11 +4245,11 @@ packages: resolution: {integrity: sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==} engines: {node: '>=8.6.0'} - fast-json-stringify@6.0.0: - resolution: {integrity: sha512-FGMKZwniMTgZh7zQp9b6XnBVxUmKVahQLQeRQHqwYmPDqDhcEKZ3BaQsxelFFI5PY7nN71OEeiL47/zUWcYe1A==} + fast-json-stringify@6.3.0: + resolution: {integrity: sha512-oRCntNDY/329HJPlmdNLIdogNtt6Vyjb1WuT01Soss3slIdyUp8kAcDU3saQTOquEK8KFVfwIIF7FebxUAu+yA==} - fast-querystring@1.1.1: - resolution: {integrity: sha512-qR2r+e3HvhEFmpdHMv//U8FnFlnYjaC6QKDuaXALDkw2kvHO8WDjxH+f/rHGR4Me4pnk8p9JAkRNTjYHAKRn2Q==} + fast-querystring@1.1.2: + resolution: {integrity: sha512-g6KuKWmFXc0fID8WWH0jit4g0AGBoJhCkJMb1RmbsSEUNvQ+ZC8D6CUZ+GtF8nMzSPXnhiePyyqqipzNNEnHjg==} fast-safe-stringify@2.1.1: resolution: {integrity: sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==} @@ -4253,9 +4260,6 @@ packages: fast-string-width@3.0.2: resolution: {integrity: sha512-gX8LrtNEI5hq8DVUfRQMbr5lpaS4nMIWV+7XEbXk2b8kiQIizgnlr12B4dA3ZEx3308ze0O4Q1R+cHts8kyUJg==} - fast-uri@2.4.0: - resolution: {integrity: sha512-ypuAmmMKInk5q7XcepxlnUWDLWv4GFtaJqAzWKqn62IpQ3pejtr5dTVbt3vwqVaMKmkNR55sTT+CqUKIaT21BA==} - fast-uri@3.1.0: resolution: {integrity: sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==} @@ -4266,12 +4270,15 @@ packages: fastify-plugin@5.0.1: resolution: {integrity: sha512-HCxs+YnRaWzCl+cWRYFnHmeRFyR5GVnJTAaCJQiYzQSDwK9MgJdyAsuL3nh0EWRCYMgQ5MeziymvmAhUHYHDUQ==} - fastify@5.7.4: - resolution: {integrity: sha512-e6l5NsRdaEP8rdD8VR0ErJASeyaRbzXYpmkrpr2SuvuMq6Si3lvsaVy5C+7gLanEkvjpMDzBXWE5HPeb/hgTxA==} + fastify@5.8.1: + resolution: {integrity: sha512-y0kicFvvn7CYWoPOVLOcvn4YyKQz03DIY7UxmyOy21/J8eXm09R+tmb+tVDBW5h+pja30cHI5dqUcSlvY86V2A==} fastq@1.17.1: resolution: {integrity: sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w==} + fastq@1.20.1: + resolution: {integrity: sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==} + fd-package-json@2.0.0: resolution: {integrity: sha512-jKmm9YtsNXN789RS/0mSzOC1NUq9mkVd65vbSSVsKdjGvYXBuE4oWe2QOEoFeRmJg+lPuZxpmrfFclNhoRMneQ==} @@ -4305,9 +4312,9 @@ packages: resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} engines: {node: '>=8'} - find-my-way@9.0.1: - resolution: {integrity: sha512-/5NN/R0pFWuff16TMajeKt2JyiW+/OE8nOO8vo1DwZTxLaIURb7lcBYPIgRPh61yCNh9l8voeKwcrkUzmB00vw==} - engines: {node: '>=14'} + find-my-way@9.5.0: + resolution: {integrity: sha512-VW2RfnmscZO5KgBY5XVyKREMW5nMZcxDy+buTOsL+zIPnBlbKm+00sgzoQzq1EVh4aALZLfKdwv6atBGcjvjrQ==} + engines: {node: '>=20'} find-up@4.1.0: resolution: {integrity: sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==} @@ -4744,8 +4751,8 @@ packages: ip@2.0.1: resolution: {integrity: sha512-lJUL9imLTNi1ZfXT+DU6rBBdbiKGBuay9B6xGSPVjUeQwaH1RIGqef8RZkUtHioLmSNpPR5M4HVKJGm1j8FWVQ==} - ipaddr.js@2.2.0: - resolution: {integrity: sha512-Ag3wB2o37wslZS19hZqorUnrnzSkpOVy+IiiDEiTqNubEYpYuHWIf6K4psgN2ZWKExS4xhVCrRVfb/wfW8fWJA==} + ipaddr.js@2.3.0: + resolution: {integrity: sha512-Zv/pA+ciVFbCSBBjGfaKUya/CcGmUHzTydLMaTwrUUEM2DIEO3iZvueGxmacvmN50fGpGVKeTXpb2LcYQxeVdg==} engines: {node: '>= 10'} is-arguments@1.1.0: @@ -5055,8 +5062,8 @@ packages: resolution: {integrity: sha512-ZF1nxZ28VhQouRWhUcVlUIN3qwSgPuswK05s/HIaoetAoE/9tngVmCHjSxmSQPav1nd+lPtTL0YZ/2AFdR/iYQ==} engines: {node: ^20.17.0 || >=22.9.0} - json-schema-ref-resolver@1.0.1: - resolution: {integrity: sha512-EJAj1pgHc1hxF6vo2Z3s69fMjO1INq6eGHXZ8Z6wCQeldCuwxGK9Sxf4/cScGn3FZubCVUehfWtcDM/PLteCQw==} + json-schema-ref-resolver@3.0.0: + resolution: {integrity: sha512-hOrZIVL5jyYFjzk7+y7n5JDzGlU8rfWDuYyHwGa2WA8/pcmMHezp2xsVwxrebD/Q9t8Nc5DboieySDpCp4WG4A==} json-schema-resolver@2.0.0: resolution: {integrity: sha512-pJ4XLQP4Q9HTxl6RVDLJ8Cyh1uitSs0CzDBAz1uoJ4sRD/Bk7cFSXL1FUXDW3zJ7YnfliJx6eu8Jn283bpZ4Yg==} @@ -5140,8 +5147,8 @@ packages: libp2p@3.1.5: resolution: {integrity: sha512-C+ZL03De0WFr9e09A27hvoCmlvWciZ9FyXVonuXUgPEoob5XyTxjJedfmQ2gXxkh2Pqz1W2vZSUqi9f0BYIT+A==} - light-my-request@6.0.0: - resolution: {integrity: sha512-kFkFXrmKCL0EEeOmJybMH5amWFd+AFvlvMlvFTRxCUwbhfapZqDmeLMPoWihntnYY6JpoQDE9k+vOzObF1fDqg==} + light-my-request@6.6.0: + resolution: {integrity: sha512-CHYbu8RtboSIoVsHZ6Ye4cj4Aw/yg2oAFimlF7mNvfDV192LR7nDiKtSIfCuLT7KokPSTn/9kfVLm5OGN0A28A==} lilconfig@3.1.3: resolution: {integrity: sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==} @@ -5642,8 +5649,9 @@ packages: observable-fns@0.6.1: resolution: {integrity: sha512-9gRK4+sRWzeN6AOewNBTLXir7Zl/i3GB6Yl26gK4flxz8BXVpD3kt8amREmWNb0mxYOGDotvE5a4N+PtGGKdkg==} - on-exit-leak-free@2.1.0: - resolution: {integrity: sha512-VuCaZZAjReZ3vUwgOB8LxAosIurDiAW0s13rI1YwmaP++jvcxP77AWoQvenZebpCA2m8WC1/EosPYPMjnRAp/w==} + on-exit-leak-free@2.1.2: + resolution: {integrity: sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==} + engines: {node: '>=14.0.0'} once@1.4.0: resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} @@ -5883,8 +5891,8 @@ packages: pino-std-serializers@7.1.0: resolution: {integrity: sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw==} - pino@10.3.0: - resolution: {integrity: sha512-0GNPNzHXBKw6U/InGe79A3Crzyk9bcSyObF9/Gfo9DLEf5qj5RF50RSjsu0W1rZ6ZqRGdzDFCRBQvi9/rSGPtA==} + pino@10.3.1: + resolution: {integrity: sha512-r34yH/GlQpKZbU1BvFFqOjhISRo1MNx1tWYsYvmj6KIRHSPMT2+yHOEb1SG6NMvRoHRF0a07kCOox/9yakl1vg==} hasBin: true pixelmatch@7.1.0: @@ -5960,8 +5968,8 @@ packages: process-nextick-args@2.0.1: resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==} - process-warning@4.0.0: - resolution: {integrity: sha512-/MyYDxttz7DfGMMHiysAsFE4qF+pQYAA8ziO/3NcRVrQ5fSk+Mns4QZA/oRPFzvcqNoVJXQNWNAsdwBXLUkQKw==} + process-warning@4.0.1: + resolution: {integrity: sha512-3c2LzQ3rY9d0hc1emcsHhfT9Jwz0cChib/QN89oME2R451w5fy3f0afAhERFZAwrbDU43wk12d0ORBpDVME50Q==} process-warning@5.0.0: resolution: {integrity: sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA==} @@ -6059,8 +6067,8 @@ packages: queue-microtask@1.2.3: resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} - quick-format-unescaped@4.0.3: - resolution: {integrity: sha512-MaL/oqh02mhEo5m5J2rwsVL23Iw2PEaGVHgT2vFt8AAsr0lfvQA5dpXo9TPu0rz7tSBdUPgkbam0j/fj5ZM8yg==} + quick-format-unescaped@4.0.4: + resolution: {integrity: sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==} quick-lru@5.1.1: resolution: {integrity: sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==} @@ -6161,6 +6169,10 @@ packages: resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==} engines: {iojs: '>=1.0.0', node: '>=0.10.0'} + reusify@1.1.0: + resolution: {integrity: sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==} + engines: {iojs: '>=1.0.0', node: '>=0.10.0'} + rewiremock@3.14.5: resolution: {integrity: sha512-MdPutvaUd+kKVz/lcEz6N6337s4PxRUR5vhphIp2/TJRgfXIckomIkCsIAbwB53MjiSLwi7KBMdQ9lPWE5WpYA==} @@ -6231,13 +6243,17 @@ packages: safe-regex-test@1.0.0: resolution: {integrity: sha512-JBUUzyOgEwXQY1NuPtvcj/qcBDbDmEvWufhlnXZIm75DEHp+afM1r1ujJpJsV/gSM4t59tpDyPi1sd6ZaPFfsA==} - safe-regex2@4.0.0: - resolution: {integrity: sha512-Hvjfv25jPDVr3U+4LDzBuZPPOymELG3PYcSk5hcevooo1yxxamQL/bHs/GrEPGmMoMEwRrHVGiCA1pXi97B8Ew==} + safe-regex2@5.0.0: + resolution: {integrity: sha512-YwJwe5a51WlK7KbOJREPdjNrpViQBI3p4T50lfwPuDhZnE3XGVTlGvi+aolc5+RvxDD6bnUmjVsU9n1eboLUYw==} safe-stable-stringify@2.4.2: resolution: {integrity: sha512-gMxvPJYhP0O9n2pvcfYfIuYgbledAOJFcqRThtPRmjscaipiwcwPPKLytpVzMkG2HAN87Qmo2d4PtGiri1dSLA==} engines: {node: '>=10'} + safe-stable-stringify@2.5.0: + resolution: {integrity: sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==} + engines: {node: '>=10'} + safer-buffer@2.1.2: resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} @@ -6254,8 +6270,8 @@ packages: scrypt-js@3.0.1: resolution: {integrity: sha512-cdwTTnqPu0Hyvf5in5asVdZocVDTNRmR7XEcJuIzMjJeSHybHl7vpB66AzwTaIg6CLSbtjcxc8fqcySfnTkccA==} - secure-json-parse@4.0.0: - resolution: {integrity: sha512-dxtLJO6sc35jWidmLxo7ij+Eg48PM/kleBsxpC8QJE0qJICe+KawkDQmvCMZUr9u7WKVHgMW6vy3fQ7zMiFZMA==} + secure-json-parse@4.1.0: + resolution: {integrity: sha512-l4KnYfEyqYJxDwlNVyRfO2E4NTHfMKAWdUuA8J0yve2Dz/E/PdBepY03RvyJpssIpRFwJoCD55wA+mEDs6ByWA==} semver-compare@1.0.0: resolution: {integrity: sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==} @@ -6278,6 +6294,11 @@ packages: engines: {node: '>=10'} hasBin: true + semver@7.7.4: + resolution: {integrity: sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==} + engines: {node: '>=10'} + hasBin: true + serialize-error@7.0.1: resolution: {integrity: sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==} engines: {node: '>=10'} @@ -6285,8 +6306,8 @@ packages: set-blocking@2.0.0: resolution: {integrity: sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==} - set-cookie-parser@2.7.0: - resolution: {integrity: sha512-lXLOiqpkUumhRdFF3k1osNXCy9akgx/dyPZ5p8qAg9seJzXr5ZrlqZuWIMuY6ejOsVLE6flJ5/h3lsn57fQ/PQ==} + set-cookie-parser@2.7.2: + resolution: {integrity: sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw==} set-function-length@1.2.0: resolution: {integrity: sha512-4DBHDoyHlM1IRPGYcoxexgh67y4ueR53FKV1yyxwFMY7aCqcN/38M1+SwZ/qJQ8iLv7+ck385ot4CcisOAPT9w==} @@ -6393,8 +6414,8 @@ packages: resolution: {integrity: sha512-HLpt+uLy/pxB+bum/9DzAgiKS8CX1EvbWxI4zlmgGCExImLdiad2iCwXT5Z4c9c3Eq8rP2318mPW2c+QbtjK8A==} engines: {node: '>= 10.0.0', npm: '>= 3.0.0'} - sonic-boom@4.2.0: - resolution: {integrity: sha512-INb7TM37/mAcsGmc9hyyI6+QR3rR1zVRu36B0NeGXKnOOLiZOfER5SA+N7X7k3yUYRzLWafduTDvJAfDswwEww==} + sonic-boom@4.2.1: + resolution: {integrity: sha512-w6AxtubXa2wTXAUsZMMWERrsIRAdrK0Sc+FUytWvYAhBJLyuI4llrMIC1DtlNSdI99EI86KZum2MMq3EAZlF9Q==} sort-keys@5.1.0: resolution: {integrity: sha512-aSbHV0DaBcr7u0PVHXzM6NbZNAtrr9sF6+Qfs9UUVG7Ll3jQ6hHi8F/xqIIcn2rvIVbr0v/2zyjSdwSV47AgLQ==} @@ -8433,20 +8454,22 @@ snapshots: '@fastify/error@4.0.0': {} - '@fastify/fast-json-stringify-compiler@5.0.1': + '@fastify/error@4.2.0': {} + + '@fastify/fast-json-stringify-compiler@5.0.3': dependencies: - fast-json-stringify: 6.0.0 + fast-json-stringify: 6.3.0 - '@fastify/forwarded@3.0.0': {} + '@fastify/forwarded@3.0.1': {} - '@fastify/merge-json-schemas@0.1.1': + '@fastify/merge-json-schemas@0.2.1': dependencies: - fast-deep-equal: 3.1.3 + dequal: 2.0.3 - '@fastify/proxy-addr@5.0.0': + '@fastify/proxy-addr@5.1.0': dependencies: - '@fastify/forwarded': 3.0.0 - ipaddr.js: 2.2.0 + '@fastify/forwarded': 3.0.1 + ipaddr.js: 2.3.0 '@fastify/send@3.1.1': dependencies: @@ -10333,10 +10356,10 @@ snapshots: available-typed-arrays@1.0.5: {} - avvio@9.0.0: + avvio@9.2.0: dependencies: - '@fastify/error': 4.0.0 - fastq: 1.17.1 + '@fastify/error': 4.2.0 + fastq: 1.20.1 aws-sdk@2.934.0: dependencies: @@ -10777,7 +10800,7 @@ snapshots: conventional-commits-parser: 6.2.1 meow: 13.2.0 - cookie@0.6.0: {} + cookie@1.1.1: {} cookiejar@2.1.4: {} @@ -10961,6 +10984,8 @@ snapshots: deprecation@2.3.1: {} + dequal@2.0.3: {} + des.js@1.0.1: dependencies: inherits: 2.0.4 @@ -11310,17 +11335,16 @@ snapshots: merge2: 1.4.1 micromatch: 4.0.8 - fast-json-stringify@6.0.0: + fast-json-stringify@6.3.0: dependencies: - '@fastify/merge-json-schemas': 0.1.1 + '@fastify/merge-json-schemas': 0.2.1 ajv: 8.18.0 ajv-formats: 3.0.1(ajv@8.18.0) - fast-deep-equal: 3.1.3 - fast-uri: 2.4.0 - json-schema-ref-resolver: 1.0.1 + fast-uri: 3.1.0 + json-schema-ref-resolver: 3.0.0 rfdc: 1.4.1 - fast-querystring@1.1.1: + fast-querystring@1.1.2: dependencies: fast-decode-uri-component: 1.0.1 @@ -11332,8 +11356,6 @@ snapshots: dependencies: fast-string-truncated-width: 3.0.3 - fast-uri@2.4.0: {} - fast-uri@3.1.0: {} fast-xml-parser@4.5.1: @@ -11342,28 +11364,32 @@ snapshots: fastify-plugin@5.0.1: {} - fastify@5.7.4: + fastify@5.8.1: dependencies: '@fastify/ajv-compiler': 4.0.5 - '@fastify/error': 4.0.0 - '@fastify/fast-json-stringify-compiler': 5.0.1 - '@fastify/proxy-addr': 5.0.0 + '@fastify/error': 4.2.0 + '@fastify/fast-json-stringify-compiler': 5.0.3 + '@fastify/proxy-addr': 5.1.0 abstract-logging: 2.0.1 - avvio: 9.0.0 - fast-json-stringify: 6.0.0 - find-my-way: 9.0.1 - light-my-request: 6.0.0 - pino: 10.3.0 + avvio: 9.2.0 + fast-json-stringify: 6.3.0 + find-my-way: 9.5.0 + light-my-request: 6.6.0 + pino: 10.3.1 process-warning: 5.0.0 rfdc: 1.4.1 - secure-json-parse: 4.0.0 - semver: 7.7.3 + secure-json-parse: 4.1.0 + semver: 7.7.4 toad-cache: 3.7.0 fastq@1.17.1: dependencies: reusify: 1.0.4 + fastq@1.20.1: + dependencies: + reusify: 1.1.0 + fd-package-json@2.0.0: dependencies: walk-up-path: 4.0.0 @@ -11395,11 +11421,11 @@ snapshots: dependencies: to-regex-range: 5.0.1 - find-my-way@9.0.1: + find-my-way@9.5.0: dependencies: fast-deep-equal: 3.1.3 - fast-querystring: 1.1.1 - safe-regex2: 4.0.0 + fast-querystring: 1.1.2 + safe-regex2: 5.0.0 find-up@4.1.0: dependencies: @@ -11908,7 +11934,7 @@ snapshots: ip@2.0.1: {} - ipaddr.js@2.2.0: {} + ipaddr.js@2.3.0: {} is-arguments@1.1.0: dependencies: @@ -12215,9 +12241,9 @@ snapshots: json-parse-even-better-errors@5.0.0: {} - json-schema-ref-resolver@1.0.1: + json-schema-ref-resolver@3.0.0: dependencies: - fast-deep-equal: 3.1.3 + dequal: 2.0.3 json-schema-resolver@2.0.0: dependencies: @@ -12335,11 +12361,11 @@ snapshots: race-signal: 2.0.0 uint8arrays: 5.1.0 - light-my-request@6.0.0: + light-my-request@6.6.0: dependencies: - cookie: 0.6.0 - process-warning: 4.0.0 - set-cookie-parser: 2.7.0 + cookie: 1.1.1 + process-warning: 4.0.1 + set-cookie-parser: 2.7.2 lilconfig@3.1.3: {} @@ -12918,7 +12944,7 @@ snapshots: observable-fns@0.6.1: {} - on-exit-leak-free@2.1.0: {} + on-exit-leak-free@2.1.2: {} once@1.4.0: dependencies: @@ -13157,18 +13183,18 @@ snapshots: pino-std-serializers@7.1.0: {} - pino@10.3.0: + pino@10.3.1: dependencies: '@pinojs/redact': 0.4.0 atomic-sleep: 1.0.0 - on-exit-leak-free: 2.1.0 + on-exit-leak-free: 2.1.2 pino-abstract-transport: 3.0.0 pino-std-serializers: 7.1.0 process-warning: 5.0.0 - quick-format-unescaped: 4.0.3 + quick-format-unescaped: 4.0.4 real-require: 0.2.0 - safe-stable-stringify: 2.4.2 - sonic-boom: 4.2.0 + safe-stable-stringify: 2.5.0 + sonic-boom: 4.2.1 thread-stream: 4.0.0 pixelmatch@7.1.0: @@ -13242,7 +13268,7 @@ snapshots: process-nextick-args@2.0.1: {} - process-warning@4.0.0: {} + process-warning@4.0.1: {} process-warning@5.0.0: {} @@ -13324,7 +13350,7 @@ snapshots: queue-microtask@1.2.3: {} - quick-format-unescaped@4.0.3: {} + quick-format-unescaped@4.0.4: {} quick-lru@5.1.1: {} @@ -13424,6 +13450,8 @@ snapshots: reusify@1.0.4: {} + reusify@1.1.0: {} + rewiremock@3.14.5: dependencies: babel-runtime: 6.26.0 @@ -13532,12 +13560,14 @@ snapshots: get-intrinsic: 1.2.2 is-regex: 1.1.4 - safe-regex2@4.0.0: + safe-regex2@5.0.0: dependencies: ret: 0.5.0 safe-stable-stringify@2.4.2: {} + safe-stable-stringify@2.5.0: {} + safer-buffer@2.1.2: {} sax@1.2.1: {} @@ -13550,7 +13580,7 @@ snapshots: scrypt-js@3.0.1: {} - secure-json-parse@4.0.0: {} + secure-json-parse@4.1.0: {} semver-compare@1.0.0: optional: true @@ -13565,6 +13595,8 @@ snapshots: semver@7.7.3: {} + semver@7.7.4: {} + serialize-error@7.0.1: dependencies: type-fest: 0.13.1 @@ -13572,7 +13604,7 @@ snapshots: set-blocking@2.0.0: {} - set-cookie-parser@2.7.0: {} + set-cookie-parser@2.7.2: {} set-function-length@1.2.0: dependencies: @@ -13714,7 +13746,7 @@ snapshots: ip-address: 10.1.0 smart-buffer: 4.2.0 - sonic-boom@4.2.0: + sonic-boom@4.2.1: dependencies: atomic-sleep: 1.0.0 From f36ec314974a32ad300cf36227134f1a1a1aacc1 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Fri, 6 Mar 2026 09:10:57 +0000 Subject: [PATCH 21/63] feat: set attestation data index based on payload status (#8982) After Gloas, attestation.data.index signals payload status in fork-choice: - 0 = EMPTY / not present, 1 = FULL / present - same-slot attestations must always use index = 0 --------- Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- .../src/api/impl/validator/index.ts | 34 +++++++++++++------ .../fork-choice/src/forkChoice/forkChoice.ts | 15 ++++++++ .../fork-choice/src/forkChoice/interface.ts | 1 + .../validator/src/services/attestation.ts | 11 ++++-- .../validator/src/services/validatorStore.ts | 12 +++++-- .../test/unit/services/attestation.test.ts | 32 +++++++++++++---- 6 files changed, 84 insertions(+), 21 deletions(-) diff --git a/packages/beacon-node/src/api/impl/validator/index.ts b/packages/beacon-node/src/api/impl/validator/index.ts index 3312ebc7cae3..2a1e9b1d9fc9 100644 --- a/packages/beacon-node/src/api/impl/validator/index.ts +++ b/packages/beacon-node/src/api/impl/validator/index.ts @@ -1,6 +1,6 @@ import {routes} from "@lodestar/api"; import {ApplicationMethods} from "@lodestar/api/server"; -import {ExecutionStatus, ProtoBlock} from "@lodestar/fork-choice"; +import {ExecutionStatus, PayloadStatus, ProtoBlock} from "@lodestar/fork-choice"; import { BUILDER_INDEX_SELF_BUILD, ForkName, @@ -989,8 +989,30 @@ export function getValidatorApi( const headBlockRoot = fromHex(headBlockRootHex); const fork = config.getForkName(slot); + const beaconBlockRoot = + slot >= headSlot + ? // When attesting to the head slot or later, always use the head of the chain. + headBlockRoot + : // Permit attesting to slots *prior* to the current head. This is desirable when + // the VC and BN are out-of-sync due to time issues or overloading. + getBlockRootAtSlot(headState, slot); + let index: CommitteeIndex; - if (isForkPostElectra(fork)) { + if (isForkPostGloas(fork)) { + const canonicalBlock = chain.forkChoice.getCanonicalBlockByRoot(beaconBlockRoot); + if (!canonicalBlock) { + // This should never happen + throw Error(`Block not found in fork choice for slot=${slot}, root=${toRootHex(beaconBlockRoot)}`); + } + // After Gloas, attestation.data.index signals payload status in fork-choice: + // - 0 = EMPTY / not present, 1 = FULL / present + // - same-slot attestations must always use index = 0 + if (canonicalBlock.slot !== slot) { + index = canonicalBlock.payloadStatus === PayloadStatus.FULL ? 1 : 0; + } else { + index = 0; + } + } else if (isForkPostElectra(fork)) { index = 0; } else { if (committeeIndex === undefined) { @@ -999,14 +1021,6 @@ export function getValidatorApi( index = committeeIndex; } - const beaconBlockRoot = - slot >= headSlot - ? // When attesting to the head slot or later, always use the head of the chain. - headBlockRoot - : // Permit attesting to slots *prior* to the current head. This is desirable when - // the VC and BN are out-of-sync due to time issues or overloading. - getBlockRootAtSlot(headState, slot); - const targetSlot = computeStartSlotAtEpoch(attEpoch); const targetRoot = targetSlot >= headSlot diff --git a/packages/fork-choice/src/forkChoice/forkChoice.ts b/packages/fork-choice/src/forkChoice/forkChoice.ts index d280abbeead2..4f8bab2ca01b 100644 --- a/packages/fork-choice/src/forkChoice/forkChoice.ts +++ b/packages/fork-choice/src/forkChoice/forkChoice.ts @@ -1211,6 +1211,21 @@ export class ForkChoice implements IForkChoice { }; } + getCanonicalBlockByRoot(blockRoot: Root): ProtoBlock | null { + const blockRootHex = toRootHex(blockRoot); + if (blockRootHex === this.head.blockRoot) { + return this.head; + } + + for (const block of this.protoArray.iterateAncestorNodes(this.head.blockRoot)) { + if (block.blockRoot === blockRootHex) { + return block; + } + } + + return null; + } + getCanonicalBlockAtSlot(slot: Slot): ProtoBlock | null { if (slot > this.head.slot) { return null; diff --git a/packages/fork-choice/src/forkChoice/interface.ts b/packages/fork-choice/src/forkChoice/interface.ts index e104f7a31709..66bb5f8de753 100644 --- a/packages/fork-choice/src/forkChoice/interface.ts +++ b/packages/fork-choice/src/forkChoice/interface.ts @@ -265,6 +265,7 @@ export interface IForkChoice { * Returns both ancestor and non-ancestor blocks in a single traversal. */ getAllAncestorAndNonAncestorBlocks(blockRoot: RootHex): {ancestors: ProtoBlock[]; nonAncestors: ProtoBlock[]}; + getCanonicalBlockByRoot(blockRoot: Root): ProtoBlock | null; getCanonicalBlockAtSlot(slot: Slot): ProtoBlock | null; getCanonicalBlockClosestLteSlot(slot: Slot): ProtoBlock | null; /** diff --git a/packages/validator/src/services/attestation.ts b/packages/validator/src/services/attestation.ts index 2dbb2b847ef6..1f2b5c2338e4 100644 --- a/packages/validator/src/services/attestation.ts +++ b/packages/validator/src/services/attestation.ts @@ -1,6 +1,6 @@ import {ApiClient} from "@lodestar/api"; import {ChainForkConfig} from "@lodestar/config"; -import {ForkName, isForkPostElectra} from "@lodestar/params"; +import {ForkName, isForkPostElectra, isForkPostGloas} from "@lodestar/params"; import {computeEpochAtSlot} from "@lodestar/state-transition"; import {SignedAggregateAndProof, SingleAttestation, Slot, phase0, ssz} from "@lodestar/types"; import {prettyBytes, sleep, toRootHex} from "@lodestar/utils"; @@ -108,7 +108,8 @@ export class AttestationService { Array.from(dutiesByCommitteeIndex.entries()).map(([index, dutiesSameCommittee]) => { const attestationData: phase0.AttestationData = { ...attestationNoCommittee, - index: isForkPostElectra(fork) ? 0 : index, + // After Gloas, preserve index returned by beacon node (payload status signal) + index: isForkPostGloas(fork) ? attestationNoCommittee.index : isForkPostElectra(fork) ? 0 : index, }; return this.produceAndPublishAggregates(fork, attestationData, index, dutiesSameCommittee); }) @@ -145,7 +146,11 @@ export class AttestationService { await Promise.all( duties.map(async ({duty}) => { - const index = isForkPostElectra(fork) ? 0 : duty.committeeIndex; + const index = isForkPostGloas(fork) + ? attestationNoCommittee.index + : isForkPostElectra(fork) + ? 0 + : duty.committeeIndex; const attestationData: phase0.AttestationData = {...attestationNoCommittee, index}; const logCtxValidator = {slot, index, head: headRootHex, validatorIndex: duty.validatorIndex}; diff --git a/packages/validator/src/services/validatorStore.ts b/packages/validator/src/services/validatorStore.ts index 1c461af9d36d..4716d3498756 100644 --- a/packages/validator/src/services/validatorStore.ts +++ b/packages/validator/src/services/validatorStore.ts @@ -831,13 +831,21 @@ export class ValidatorStore { throw Error(`Inconsistent duties during signing: duty.slot ${duty.slot} != att.slot ${data.slot}`); } - const isPostElectra = this.config.getForkSeq(data.slot) >= ForkSeq.electra; + const forkSeq = this.config.getForkSeq(data.slot); + const isPostElectra = forkSeq >= ForkSeq.electra; + const isPostGloas = forkSeq >= ForkSeq.gloas; + if (!isPostElectra && duty.committeeIndex !== data.index) { throw Error( `Inconsistent duties during signing: duty.committeeIndex ${duty.committeeIndex} != att.committeeIndex ${data.index}` ); } - if (isPostElectra && data.index !== 0) { + if (isPostGloas) { + // After Gloas, data.index signals payload status: 0 (EMPTY) or 1 (FULL) + if (data.index !== 0 && data.index !== 1) { + throw Error(`Invalid payload status index post-gloas during signing: data.index=${data.index}`); + } + } else if (isPostElectra && data.index !== 0) { throw Error(`Non-zero committee index post-electra during signing: att.committeeIndex ${data.index}`); } } diff --git a/packages/validator/test/unit/services/attestation.test.ts b/packages/validator/test/unit/services/attestation.test.ts index f86e93b1f2a1..23431f37bc18 100644 --- a/packages/validator/test/unit/services/attestation.test.ts +++ b/packages/validator/test/unit/services/attestation.test.ts @@ -3,7 +3,8 @@ import {SecretKey} from "@chainsafe/blst"; import {toHexString} from "@chainsafe/ssz"; import {ChainConfig, createChainForkConfig} from "@lodestar/config"; import {config as defaultConfig} from "@lodestar/config/default"; -import {ForkName} from "@lodestar/params"; +import {ForkSeq} from "@lodestar/params"; +import {computeEpochAtSlot} from "@lodestar/state-transition"; import {ssz} from "@lodestar/types"; import {AttestationService, AttestationServiceOpts} from "../../../src/services/attestation.js"; import {AttDutyAndProof} from "../../../src/services/attestationDuties.js"; @@ -58,18 +59,22 @@ describe("AttestationService", () => { }); const electraConfig: Partial = {ELECTRA_FORK_EPOCH: 0}; + const gloasConfig: Partial = {GLOAS_FORK_EPOCH: 0}; const testContexts: [string, AttestationServiceOpts, Partial][] = [ ["With default configuration", {}, {}], ["With default configuration post-electra", {}, electraConfig], + ["With default configuration post-gloas", {}, gloasConfig], ]; for (const [title, opts, chainConfig] of testContexts) { describe(title, () => { it("Should produce, sign, and publish an attestation + aggregate", async () => { + const slot = 0; const clock = new ClockMock(); const config = createChainForkConfig({...defaultConfig, ...chainConfig}); - const isPostElectra = chainConfig.ELECTRA_FORK_EPOCH === 0; + const isPostElectra = config.getForkSeq(slot) >= ForkSeq.electra; + const isPostGloas = config.getForkSeq(slot) >= ForkSeq.gloas; const attestationService = new AttestationService( loggerVc, api, @@ -86,6 +91,9 @@ describe("AttestationService", () => { const singleAttestation = isPostElectra ? ssz.electra.SingleAttestation.defaultValue() : ssz.phase0.Attestation.defaultValue(); + if (isPostGloas) { + singleAttestation.data.index = 1; + } const aggregatedAttestation = isPostElectra ? ssz.electra.Attestation.defaultValue() : ssz.phase0.Attestation.defaultValue(); @@ -95,8 +103,8 @@ describe("AttestationService", () => { const duties: AttDutyAndProof[] = [ { duty: { - slot: 0, - committeeIndex: singleAttestation.data.index, + slot, + committeeIndex: 6, committeeLength: 120, committeesAtSlot: 120, validatorCommitteeIndex: 1, @@ -121,7 +129,7 @@ describe("AttestationService", () => { // Mock beacon's attestation and aggregates endpoints api.validator.produceAttestationData.mockResolvedValue(mockApiResponse({data: singleAttestation.data})); api.validator.getAggregatedAttestationV2.mockResolvedValue( - mockApiResponse({data: aggregatedAttestation, meta: {version: ForkName.electra}}) + mockApiResponse({data: aggregatedAttestation, meta: {version: config.getForkName(slot)}}) ); api.beacon.submitPoolAttestationsV2.mockResolvedValue(mockApiResponse({})); api.validator.publishAggregateAndProofsV2.mockResolvedValue(mockApiResponse({})); @@ -131,7 +139,7 @@ describe("AttestationService", () => { validatorStore.signAggregateAndProof.mockResolvedValue(aggregateAndProof); // Trigger clock onSlot for slot 0 - await clock.tickSlotFns(0, controller.signal); + await clock.tickSlotFns(slot, controller.signal); // Must submit the attestation received through produceAttestationData() expect(api.beacon.submitPoolAttestationsV2).toHaveBeenCalledOnce(); @@ -142,6 +150,18 @@ describe("AttestationService", () => { expect(api.validator.publishAggregateAndProofsV2).toHaveBeenCalledWith({ signedAggregateAndProofs: [aggregateAndProof], }); + + const expectedIndex = isPostGloas + ? singleAttestation.data.index + : isPostElectra + ? 0 + : duties[0].duty.committeeIndex; + + expect(validatorStore.signAttestation).toHaveBeenCalledWith( + duties[0].duty, + expect.objectContaining({index: expectedIndex}), + computeEpochAtSlot(slot) + ); }); }); } From 66fe43ae9ed234f73c1ffb6cb409ab3f2e593087 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Fri, 6 Mar 2026 18:07:29 +0000 Subject: [PATCH 22/63] fix: cache previous epoch payload timeliness committees (#8991) We need the PTC cached for previous epoch in case of epoch boundary lookups, eg. simple case is slot N block validating slot N-1 attestation ~~but also if there are deeper reorgs / missed slots.~~ (this isn't required as ptc messages are only valid for one slot). See https://github.com/ChainSafe/lodestar/pull/8983 for more details. --- .../state-transition/src/cache/epochCache.ts | 32 ++++++++++++++++--- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index 24207423a773..d87c998d1e8a 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -226,7 +226,9 @@ export class EpochCache { /** TODO: Indexed SyncCommitteeCache */ nextSyncCommitteeIndexed: SyncCommitteeCache; - // TODO GLOAS: See if we need to cached PTC for prev/next epoch + // TODO GLOAS: See if we need to cache PTC for next epoch + // PTC for previous epoch, required for slot N block validating slot N-1 attestations + previousPayloadTimelinessCommittees: Uint32Array[]; // PTC for current epoch, computed eagerly at epoch transition payloadTimelinessCommittees: Uint32Array[]; @@ -266,6 +268,7 @@ export class EpochCache { previousTargetUnslashedBalanceIncrements: number; currentSyncCommitteeIndexed: SyncCommitteeCache; nextSyncCommitteeIndexed: SyncCommitteeCache; + previousPayloadTimelinessCommittees: Uint32Array[]; payloadTimelinessCommittees: Uint32Array[]; epoch: Epoch; syncPeriod: SyncPeriod; @@ -296,6 +299,7 @@ export class EpochCache { this.previousTargetUnslashedBalanceIncrements = data.previousTargetUnslashedBalanceIncrements; this.currentSyncCommitteeIndexed = data.currentSyncCommitteeIndexed; this.nextSyncCommitteeIndexed = data.nextSyncCommitteeIndexed; + this.previousPayloadTimelinessCommittees = data.previousPayloadTimelinessCommittees; this.payloadTimelinessCommittees = data.payloadTimelinessCommittees; this.epoch = data.epoch; this.syncPeriod = data.syncPeriod; @@ -447,7 +451,8 @@ export class EpochCache { nextSyncCommitteeIndexed = new SyncCommitteeCacheEmpty(); } - // Compute PTC eagerly for all slots in the epoch + // Compute PTC for all slots in the prev/current epoch + let previousPayloadTimelinessCommittees: Uint32Array[] = []; let payloadTimelinessCommittees: Uint32Array[] = []; if (currentEpoch >= config.GLOAS_FORK_EPOCH) { payloadTimelinessCommittees = computePayloadTimelinessCommitteesForEpoch( @@ -456,6 +461,15 @@ export class EpochCache { currentShuffling.committees, effectiveBalanceIncrements ); + + if (!isGenesis && previousEpoch >= config.GLOAS_FORK_EPOCH) { + previousPayloadTimelinessCommittees = computePayloadTimelinessCommitteesForEpoch( + state, + previousEpoch, + previousShuffling.committees, + effectiveBalanceIncrements + ); + } } // Precompute churnLimit for efficient initiateValidatorExit() during block proposing MUST be recompute everytime the @@ -530,6 +544,7 @@ export class EpochCache { currentTargetUnslashedBalanceIncrements, currentSyncCommitteeIndexed, nextSyncCommitteeIndexed, + previousPayloadTimelinessCommittees, payloadTimelinessCommittees, epoch: currentEpoch, syncPeriod: computeSyncPeriodAtEpoch(currentEpoch), @@ -575,6 +590,7 @@ export class EpochCache { currentTargetUnslashedBalanceIncrements: this.currentTargetUnslashedBalanceIncrements, currentSyncCommitteeIndexed: this.currentSyncCommitteeIndexed, nextSyncCommitteeIndexed: this.nextSyncCommitteeIndexed, + previousPayloadTimelinessCommittees: this.previousPayloadTimelinessCommittees, payloadTimelinessCommittees: this.payloadTimelinessCommittees, epoch: this.epoch, syncPeriod: this.syncPeriod, @@ -686,6 +702,8 @@ export class EpochCache { this.proposersPrevEpoch = this.proposers; if (upcomingEpoch >= this.config.GLOAS_FORK_EPOCH) { + // Shift and compute current epoch PTC eagerly for all slots + this.previousPayloadTimelinessCommittees = this.payloadTimelinessCommittees; this.payloadTimelinessCommittees = computePayloadTimelinessCommitteesForEpoch( state, upcomingEpoch, @@ -1016,11 +1034,15 @@ export class EpochCache { throw new Error("Payload Timeliness Committee is not available before gloas fork"); } - if (epoch !== this.epoch) { - throw new Error(`Payload Timeliness Committee is not available for slot=${slot}`); + if (epoch === this.epoch) { + return this.payloadTimelinessCommittees[slot % SLOTS_PER_EPOCH]; + } + + if (epoch === this.epoch - 1 && this.previousPayloadTimelinessCommittees.length > 0) { + return this.previousPayloadTimelinessCommittees[slot % SLOTS_PER_EPOCH]; } - return this.payloadTimelinessCommittees[slot % SLOTS_PER_EPOCH]; + throw new Error(`Payload Timeliness Committee is not available for slot=${slot}`); } getIndexedPayloadAttestation( From dce096d70c77e320a1a23923e235cfb47d67c675 Mon Sep 17 00:00:00 2001 From: twoeths <10568965+twoeths@users.noreply.github.com> Date: Mon, 9 Mar 2026 18:30:54 +0700 Subject: [PATCH 23/63] refactor: separate writeBlockInputToDb into parallel block and column writes (#8974) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Motivation** - in preparation for [gloas](https://github.com/ChainSafe/lodestar/pull/8962/changes#diff-31556a76147b927f6429470ca658ba8ea013b8a869566247b9410f791988454fR32), columns and block could/should be written separately - the nice thing of having a `writeDataColumnsToDb()` function is gloas has to come up with the same interface, and reuse the code, especially `waitForComputedAllData()` to keep the same functionality we introduced in [allow block import after 64 columns](https://github.com/ChainSafe/lodestar/pull/8818) **Description** - separate`writeBlockInputToDb()` into `writeBlockAndBlobsToDb()` and `writeDataColumnsToDb()` - new `IDataColumnsInput` interface, gloas [PayloadEnvelopeInput](https://github.com/ChainSafe/lodestar/pull/8962/changes#diff-39f10cf452f670b2fe7018b493ee4c550bb6bf19d0eac2fed0a238ed7511cf36R54) is expected to implement this cc @ensi321 - only prune `this.serializedCache.clear();` for pre-gloas, need to call that after we write `PayloadEnvelopeInput` in gloas - simplify `writeDataColumnsToDb()` to accept a single `IDataColumnsInput` instead of an array because that's what the consumer need **AI Assistance Disclosure** 🤖 Generated with [Claude Code](https://claude.ai/code) --------- Co-authored-by: Tuyen Nguyen Co-authored-by: Claude Sonnet 4.6 Co-authored-by: NC <17676176+ensi321@users.noreply.github.com> --- .../src/chain/blocks/blockInput/types.ts | 12 + .../src/chain/blocks/importBlock.ts | 2 +- .../src/chain/blocks/writeBlockInputToDb.ts | 220 ++++++++++-------- packages/beacon-node/src/chain/chain.ts | 6 +- 4 files changed, 135 insertions(+), 105 deletions(-) diff --git a/packages/beacon-node/src/chain/blocks/blockInput/types.ts b/packages/beacon-node/src/chain/blocks/blockInput/types.ts index 587ad83c70e8..d6acd478fc1a 100644 --- a/packages/beacon-node/src/chain/blocks/blockInput/types.ts +++ b/packages/beacon-node/src/chain/blocks/blockInput/types.ts @@ -100,6 +100,18 @@ export type MissingColumnMeta = { versionedHashes: VersionedHashes; }; +/** + * Minimal interface required to write data columns to the DB. + * Used by `writeDataColumnsToDb` and designed to be reusable across forks (e.g. Fulu, Gloas). + */ +export interface IDataColumnsInput { + readonly slot: Slot; + readonly blockRootHex: string; + getCustodyColumns(): fulu.DataColumnSidecars; + hasComputedAllData(): boolean; + waitForComputedAllData(timeout: number, signal?: AbortSignal): Promise; +} + /** * This is used to validate that BlockInput implementations follow some minimal subset of operations * and that adding a new implementation won't break consumers that rely on this subset. diff --git a/packages/beacon-node/src/chain/blocks/importBlock.ts b/packages/beacon-node/src/chain/blocks/importBlock.ts index aaef371a480c..7e61ce6671d9 100644 --- a/packages/beacon-node/src/chain/blocks/importBlock.ts +++ b/packages/beacon-node/src/chain/blocks/importBlock.ts @@ -95,7 +95,7 @@ export async function importBlock( // Without this, a supernode syncing from behind can accumulate many blocks worth of column // data in memory (up to 128 columns per block) causing OOM before persistence catches up. await this.unfinalizedBlockWrites.waitForSpace(); - this.unfinalizedBlockWrites.push([blockInput]).catch((e) => { + this.unfinalizedBlockWrites.push(blockInput).catch((e) => { if (!isQueueErrorAborted(e)) { this.logger.error("Error pushing block to unfinalized write queue", {slot: blockSlot}, e as Error); } diff --git a/packages/beacon-node/src/chain/blocks/writeBlockInputToDb.ts b/packages/beacon-node/src/chain/blocks/writeBlockInputToDb.ts index bcc30c1e40c5..4a8ec0f7dbe8 100644 --- a/packages/beacon-node/src/chain/blocks/writeBlockInputToDb.ts +++ b/packages/beacon-node/src/chain/blocks/writeBlockInputToDb.ts @@ -1,7 +1,15 @@ -import {fulu} from "@lodestar/types"; -import {prettyPrintIndices, toRootHex} from "@lodestar/utils"; +import {ForkPostDeneb, isForkPostDeneb} from "@lodestar/params"; +import {SignedBeaconBlock} from "@lodestar/types"; +import {fromHex, toRootHex} from "@lodestar/utils"; +import {getBlobKzgCommitments} from "../../util/dataColumns.js"; import {BeaconChain} from "../chain.js"; -import {IBlockInput, isBlockInputBlobs, isBlockInputColumns} from "./blockInput/index.js"; +import { + IBlockInput, + IDataColumnsInput, + isBlockInputBlobs, + isBlockInputColumns, + isBlockInputNoData, +} from "./blockInput/index.js"; import {BLOB_AVAILABILITY_TIMEOUT} from "./verifyBlocksDataAvailability.js"; /** @@ -10,129 +18,139 @@ import {BLOB_AVAILABILITY_TIMEOUT} from "./verifyBlocksDataAvailability.js"; * * This operation may be performed before, during or after importing to the fork-choice. As long as errors * are handled properly for eventual consistency. + * + * Block+blobs (pre-fulu) and data columns (fulu+) are written in parallel. */ -export async function writeBlockInputToDb(this: BeaconChain, blocksInputs: IBlockInput[]): Promise { - const fnPromises: Promise[] = []; - // track slots for logging - const slots: number[] = []; - - for (const blockInput of blocksInputs) { - const block = blockInput.getBlock(); - const slot = block.message.slot; - slots.push(slot); - const blockRoot = this.config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message); - const blockRootHex = toRootHex(blockRoot); - const blockBytes = this.serializedCache.get(block); - if (blockBytes) { - // skip serializing data if we already have it - this.metrics?.importBlock.persistBlockWithSerializedDataCount.inc(); - fnPromises.push(this.db.block.putBinary(this.db.block.getId(block), blockBytes)); - } else { - this.metrics?.importBlock.persistBlockNoSerializedDataCount.inc(); - fnPromises.push(this.db.block.add(block)); - } +export async function writeBlockInputToDb(this: BeaconChain, blockInput: IBlockInput): Promise { + const promises: Promise[] = [writeBlockAndBlobsToDb.call(this, blockInput)]; - this.logger.debug("Persist block to hot DB", { - slot: block.message.slot, - root: blockRootHex, - inputType: blockInput.type, - }); + if (isBlockInputColumns(blockInput)) { + promises.push(writeDataColumnsToDb.call(this, blockInput)); + } - if (!blockInput.hasAllData()) { - await blockInput.waitForAllData(BLOB_AVAILABILITY_TIMEOUT); - } + await Promise.all(promises); + this.logger.debug("Persisted blockInput to db", {slot: blockInput.slot, root: blockInput.blockRootHex}); +} - // NOTE: Old data is pruned on archive - if (isBlockInputColumns(blockInput)) { - if (!blockInput.hasComputedAllData()) { - // Supernodes may only have a subset of the data columns by the time the block begins to be imported - // because full data availability can be assumed after NUMBER_OF_COLUMNS / 2 columns are available. - // Here, however, all data columns must be fully available/reconstructed before persisting to the DB. - await blockInput.waitForComputedAllData(BLOB_AVAILABILITY_TIMEOUT).catch(() => { - this.logger.debug("Failed to wait for computed all data", {slot, blockRoot: blockRootHex}); - }); - } +async function writeBlockAndBlobsToDb(this: BeaconChain, blockInput: IBlockInput): Promise { + const block = blockInput.getBlock(); + const slot = block.message.slot; + const blockRoot = this.config.getForkTypes(slot).BeaconBlock.hashTreeRoot(block.message); + const blockRootHex = toRootHex(blockRoot); + const numBlobs = isForkPostDeneb(blockInput.forkName) + ? getBlobKzgCommitments(blockInput.forkName, block as SignedBeaconBlock).length + : undefined; + const fnPromises: Promise[] = []; - const {custodyColumns} = this.custodyConfig; - const blobsLen = (block.message as fulu.BeaconBlock).body.blobKzgCommitments.length; - let dataColumnsLen: number; - if (blobsLen === 0) { - dataColumnsLen = 0; - } else { - dataColumnsLen = custodyColumns.length; - } + const blockBytes = this.serializedCache.get(block); + if (blockBytes) { + // skip serializing data if we already have it + this.metrics?.importBlock.persistBlockWithSerializedDataCount.inc(); + fnPromises.push(this.db.block.putBinary(this.db.block.getId(block), blockBytes)); + } else { + this.metrics?.importBlock.persistBlockNoSerializedDataCount.inc(); + fnPromises.push(this.db.block.add(block)); + } - const dataColumnSidecars = blockInput.getCustodyColumns(); - if (dataColumnSidecars.length !== dataColumnsLen) { - this.logger.debug( - `Invalid dataColumnSidecars=${dataColumnSidecars.length} for custody expected custodyColumnsLen=${dataColumnsLen}` - ); - } + this.logger.debug("Persist block to hot DB", {slot, root: blockRootHex, inputType: blockInput.type, numBlobs}); - const binaryPuts = []; - const nonbinaryPuts = []; - for (const dataColumnSidecar of dataColumnSidecars) { - // skip reserializing column if we already have it - const serialized = this.serializedCache.get(dataColumnSidecar); - if (serialized) { - binaryPuts.push({key: dataColumnSidecar.index, value: serialized}); - } else { - nonbinaryPuts.push(dataColumnSidecar); + if (isBlockInputBlobs(blockInput)) { + fnPromises.push( + (async () => { + if (!blockInput.hasAllData()) { + await blockInput.waitForAllData(BLOB_AVAILABILITY_TIMEOUT); } - } - fnPromises.push(this.db.dataColumnSidecar.putManyBinary(blockRoot, binaryPuts)); - fnPromises.push(this.db.dataColumnSidecar.putMany(blockRoot, nonbinaryPuts)); - this.logger.debug("Persisted dataColumnSidecars to hot DB", { - slot: block.message.slot, - root: blockRootHex, - dataColumnSidecars: dataColumnSidecars.length, - numBlobs: blobsLen, - custodyColumns: custodyColumns.length, - }); - } else if (isBlockInputBlobs(blockInput)) { - const blobSidecars = blockInput.getBlobs(); - fnPromises.push(this.db.blobSidecars.add({blockRoot, slot: block.message.slot, blobSidecars})); - this.logger.debug("Persisted blobSidecars to hot DB", { - blobsLen: blobSidecars.length, - slot: block.message.slot, - root: blockRootHex, - }); - } + const blobSidecars = blockInput.getBlobs(); + await this.db.blobSidecars.add({blockRoot, slot, blobSidecars}); + this.logger.debug("Persisted blobSidecars to hot DB", { + slot, + root: blockRootHex, + numBlobs: blobSidecars.length, + }); + })() + ); + } - await Promise.all(fnPromises); - this.logger.debug("Persisted blocksInput to db", { - blocksInput: blocksInputs.length, - slots: prettyPrintIndices(slots), + await Promise.all(fnPromises); +} + +/** + * Persists data columns to DB for a given block. Accepts a narrow sub-interface of IBlockInput + * so it can be reused across forks (e.g. Fulu, Gloas). + * + * NOTE: Old data is pruned on archive. + */ +export async function writeDataColumnsToDb(this: BeaconChain, blockInput: IDataColumnsInput): Promise { + const {slot, blockRootHex} = blockInput; + const blockRoot = fromHex(blockRootHex); + + if (!blockInput.hasComputedAllData()) { + // Supernodes may only have a subset of the data columns by the time the block begins to be imported + // because full data availability can be assumed after NUMBER_OF_COLUMNS / 2 columns are available. + // Here, however, all data columns must be fully available/reconstructed before persisting to the DB. + await blockInput.waitForComputedAllData(BLOB_AVAILABILITY_TIMEOUT).catch(() => { + this.logger.debug("Failed to wait for computed all data", {slot, blockRoot: blockRootHex}); }); } + + const {custodyColumns} = this.custodyConfig; + const dataColumnSidecars = blockInput.getCustodyColumns(); + + const binaryPuts: {key: number; value: Uint8Array}[] = []; + const nonbinaryPuts = []; + for (const dataColumnSidecar of dataColumnSidecars) { + // skip reserializing column if we already have it + const serialized = this.serializedCache.get(dataColumnSidecar); + if (serialized) { + binaryPuts.push({key: dataColumnSidecar.index, value: serialized}); + } else { + nonbinaryPuts.push(dataColumnSidecar); + } + } + + await Promise.all([ + this.db.dataColumnSidecar.putManyBinary(blockRoot, binaryPuts), + this.db.dataColumnSidecar.putMany(blockRoot, nonbinaryPuts), + ]); + + this.logger.debug("Persisted dataColumnSidecars to hot DB", { + slot, + root: blockRootHex, + dataColumnSidecars: dataColumnSidecars.length, + custodyColumns: custodyColumns.length, + numBlobs: dataColumnSidecars[0]?.column.length, + }); } -export async function persistBlockInputs(this: BeaconChain, blockInputs: IBlockInput[]): Promise { +export async function persistBlockInput(this: BeaconChain, blockInput: IBlockInput): Promise { await writeBlockInputToDb - .call(this, blockInputs) + .call(this, blockInput) .catch((e) => { this.logger.debug( "Error persisting block input in hot db", { - count: blockInputs.length, - slot: blockInputs[0].slot, - root: blockInputs[0].blockRootHex, + slot: blockInput.slot, + root: blockInput.blockRootHex, }, e ); }) .finally(() => { - for (const blockInput of blockInputs) { - this.seenBlockInputCache.prune(blockInput.blockRootHex); - } + this.seenBlockInputCache.prune(blockInput.blockRootHex); // Without forcefully clearing this cache, we would rely on WeakMap to evict memory which is not reliable. // Clear here (after the DB write) so that writeBlockInputToDb can still use the cached serialized bytes. - this.serializedCache.clear(); - if (blockInputs.length === 1) { - this.logger.debug("Pruned block input", { - slot: blockInputs[0].slot, - root: blockInputs[0].blockRootHex, - }); + // + // For Gloas (BlockInputNoData), the execution payload and columns arrive separately after the beacon block. + // Do NOT clear the cache here — it must remain available for writeDataColumnsToDb when the payload arrives. + // The cache is cleared in the Gloas payload persistence path instead. + if (!isBlockInputNoData(blockInput)) { + // TODO: enhance this SerializedCache for Gloas because payload may not come + // see https://github.com/ChainSafe/lodestar/pull/8974#discussion_r2885598229 + this.serializedCache.clear(); } + this.logger.debug("Pruned block input", { + slot: blockInput.slot, + root: blockInput.blockRootHex, + }); }); } diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 5a063713008f..e485e5cd7741 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -77,7 +77,7 @@ import {CheckpointBalancesCache} from "./balancesCache.js"; import {BeaconProposerCache} from "./beaconProposerCache.js"; import {IBlockInput, isBlockInputBlobs, isBlockInputColumns} from "./blocks/blockInput/index.js"; import {BlockProcessor, ImportBlockOpts} from "./blocks/index.js"; -import {persistBlockInputs} from "./blocks/writeBlockInputToDb.ts"; +import {persistBlockInput} from "./blocks/writeBlockInputToDb.ts"; import {BlsMultiThreadWorkerPool, BlsSingleThreadVerifier, IBlsVerifier} from "./bls/index.js"; import {ColumnReconstructionTracker} from "./ColumnReconstructionTracker.js"; import {ChainEvent, ChainEventEmitter} from "./emitter.js"; @@ -164,7 +164,7 @@ export class BeaconChain implements IBeaconChain { readonly lightClientServer?: LightClientServer; readonly reprocessController: ReprocessController; readonly archiveStore: ArchiveStore; - readonly unfinalizedBlockWrites: JobItemQueue<[IBlockInput[]], void>; + readonly unfinalizedBlockWrites: JobItemQueue<[IBlockInput], void>; // Ops pool readonly attestationPool: AttestationPool; @@ -429,7 +429,7 @@ export class BeaconChain implements IBeaconChain { ); this.unfinalizedBlockWrites = new JobItemQueue( - persistBlockInputs.bind(this), + persistBlockInput.bind(this), { maxLength: DEFAULT_MAX_PENDING_UNFINALIZED_BLOCK_WRITES, signal, From f9a3a81538bd605c20709a95cc999aa59f08aeb8 Mon Sep 17 00:00:00 2001 From: twoeths <10568965+twoeths@users.noreply.github.com> Date: Mon, 9 Mar 2026 18:35:55 +0700 Subject: [PATCH 24/63] fix(fork-choice): make payloadStatus explicit in ancestor traversal APIs (#8996) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation The Gloas fork introduced block variants (FULL/EMPTY/PENDING) that share the same `blockRoot`. Previously, traversal methods in `ProtoArray`/`ForkChoice` used `getDefaultVariant()` internally to pick a canonical variant (PENDING for Gloas, FULL for pre-Gloas). This made callers unaware of which variant was being traversed and caused ambiguity when the caller already held a specific variant. ## Changes **`packages/fork-choice`** - `iterateAncestorBlocks`, `getAllAncestorBlocks`, `getAllNonAncestorBlocks`, `getAllAncestorAndNonAncestorBlocks`, `forwardIterateDescendants`, `isDescendant` — now require an explicit `payloadStatus` parameter instead of calling `getDefaultVariant()` internally - `getAllAncestorNodes` / `getAllAncestorAndNonAncestorNodes` — PENDING variants are now explicitly filtered out from returned results (previously only pre-Gloas FULL nodes were excluded via `isGloasBlock()` check) - `isDescendant` — self-equality now also requires matching `payloadStatus`, not just matching `blockRoot` - Removed TODO comment on `forwardIterateDescendants` about payload ambiguity — resolved by explicit parameter **`packages/beacon-node`** - `CheckpointWithHex` replaced with `CheckpointWithPayload` (`CheckpointWithHex & {payloadStatus}`) in `ArchiveStore`, `archiveBlocks`, `regen`, and `chain.ts` where payload status is semantically relevant - All call sites updated to pass explicit `payloadStatus` from the `ProtoBlock` they already hold ## Test plan - [ ] `pnpm test:unit` in `packages/fork-choice` and `packages/beacon-node` - [ ] `pnpm check-types` > Note: AI-assisted (Claude Sonnet 4.6) 🤖 Generated with [Claude Code](https://claude.ai/code) --------- Co-authored-by: Tuyen Nguyen Co-authored-by: Claude Sonnet 4.6 --- .../src/chain/archiveStore/archiveStore.ts | 10 +-- .../chain/archiveStore/utils/archiveBlocks.ts | 9 ++- packages/beacon-node/src/chain/chain.ts | 22 +++++-- packages/beacon-node/src/chain/emitter.ts | 6 +- packages/beacon-node/src/chain/regen/regen.ts | 2 +- .../reqresp/handlers/beaconBlocksByRange.ts | 5 +- .../reqresp/handlers/blobSidecarsByRange.ts | 5 +- .../handlers/dataColumnSidecarsByRange.ts | 5 +- .../chain/archiveStore/blockArchiver.test.ts | 10 ++- .../unit/chain/forkChoice/forkChoice.test.ts | 15 +++-- .../fork-choice/src/forkChoice/forkChoice.ts | 66 +++++++++---------- .../fork-choice/src/forkChoice/interface.ts | 28 +++++--- packages/fork-choice/src/forkChoice/store.ts | 22 +++---- packages/fork-choice/src/index.ts | 2 +- .../fork-choice/src/protoArray/protoArray.ts | 59 +++++++---------- .../test/unit/forkChoice/forkChoice.test.ts | 14 ++-- .../test/unit/protoArray/protoArray.test.ts | 2 +- 17 files changed, 150 insertions(+), 132 deletions(-) diff --git a/packages/beacon-node/src/chain/archiveStore/archiveStore.ts b/packages/beacon-node/src/chain/archiveStore/archiveStore.ts index 50ca1ab33fa3..812f01034d53 100644 --- a/packages/beacon-node/src/chain/archiveStore/archiveStore.ts +++ b/packages/beacon-node/src/chain/archiveStore/archiveStore.ts @@ -1,4 +1,4 @@ -import {CheckpointWithHex} from "@lodestar/fork-choice"; +import {CheckpointWithPayloadStatus} from "@lodestar/fork-choice"; import {LoggerNode} from "@lodestar/logger/node"; import {ForkSeq} from "@lodestar/params"; import {Checkpoint} from "@lodestar/types/phase0"; @@ -44,7 +44,7 @@ export enum ArchiveStoreTask { */ export class ArchiveStore { private archiveMode: ArchiveMode; - private jobQueue: JobItemQueue<[CheckpointWithHex], void>; + private jobQueue: JobItemQueue<[CheckpointWithPayloadStatus], void>; private archiveDataEpochs?: number; private readonly statesArchiverStrategy: StateArchiveStrategy; @@ -67,7 +67,7 @@ export class ArchiveStore { this.archiveMode = opts.archiveMode; this.archiveDataEpochs = opts.archiveDataEpochs; - this.jobQueue = new JobItemQueue<[CheckpointWithHex], void>(this.processFinalizedCheckpoint, { + this.jobQueue = new JobItemQueue<[CheckpointWithPayloadStatus], void>(this.processFinalizedCheckpoint, { maxLength: PROCESS_FINALIZED_CHECKPOINT_QUEUE_LENGTH, signal, }); @@ -168,7 +168,7 @@ export class ArchiveStore { //------------------------------------------------------------------------- // Event handlers //------------------------------------------------------------------------- - private onFinalizedCheckpoint = (finalized: CheckpointWithHex): void => { + private onFinalizedCheckpoint = (finalized: CheckpointWithPayloadStatus): void => { this.jobQueue.push(finalized).catch((e) => { if (!isQueueErrorAborted(e)) { this.logger.error("Error queuing finalized checkpoint", {epoch: finalized.epoch}, e as Error); @@ -189,7 +189,7 @@ export class ArchiveStore { }); }; - private processFinalizedCheckpoint = async (finalized: CheckpointWithHex): Promise => { + private processFinalizedCheckpoint = async (finalized: CheckpointWithPayloadStatus): Promise => { try { const finalizedEpoch = finalized.epoch; const finalizedFork = this.chain.config.getForkSeqAtEpoch(finalizedEpoch); diff --git a/packages/beacon-node/src/chain/archiveStore/utils/archiveBlocks.ts b/packages/beacon-node/src/chain/archiveStore/utils/archiveBlocks.ts index 75cd1df77d1a..f8f62a244aff 100644 --- a/packages/beacon-node/src/chain/archiveStore/utils/archiveBlocks.ts +++ b/packages/beacon-node/src/chain/archiveStore/utils/archiveBlocks.ts @@ -1,10 +1,10 @@ import path from "node:path"; import {ChainForkConfig} from "@lodestar/config"; import {KeyValue} from "@lodestar/db"; -import {IForkChoice} from "@lodestar/fork-choice"; +import {CheckpointWithPayloadStatus, IForkChoice} from "@lodestar/fork-choice"; import {ForkSeq, SLOTS_PER_EPOCH} from "@lodestar/params"; import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition"; -import {Epoch, RootHex, Slot} from "@lodestar/types"; +import {Epoch, Slot} from "@lodestar/types"; import {Logger, fromAsync, fromHex, prettyPrintIndices, toRootHex} from "@lodestar/utils"; import {IBeaconDb} from "../../../db/index.js"; import {BlockArchiveBatchPutBinaryItem} from "../../../db/repositories/index.js"; @@ -19,7 +19,6 @@ const BLOCK_BATCH_SIZE = 256; const BLOB_SIDECAR_BATCH_SIZE = 32; type BlockRootSlot = {slot: Slot; root: Uint8Array}; -type CheckpointHex = {epoch: Epoch; rootHex: RootHex}; /** * Persist orphaned block to disk @@ -53,7 +52,7 @@ export async function archiveBlocks( forkChoice: IForkChoice, lightclientServer: LightClientServer | undefined, logger: Logger, - finalizedCheckpoint: CheckpointHex, + finalizedCheckpoint: CheckpointWithPayloadStatus, currentEpoch: Epoch, archiveDataEpochs?: number, persistOrphanedBlocks?: boolean, @@ -62,7 +61,7 @@ export async function archiveBlocks( // Use fork choice to determine the blocks to archive and delete // getAllAncestorBlocks response includes the finalized block, so it's also moved to the cold db const {ancestors: finalizedCanonicalBlocks, nonAncestors: finalizedNonCanonicalBlocks} = - forkChoice.getAllAncestorAndNonAncestorBlocks(finalizedCheckpoint.rootHex); + forkChoice.getAllAncestorAndNonAncestorBlocks(finalizedCheckpoint.rootHex, finalizedCheckpoint.payloadStatus); // NOTE: The finalized block will be exactly the first block of `epoch` or previous const finalizedPostDeneb = finalizedCheckpoint.epoch >= config.DENEB_FORK_EPOCH; diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index e485e5cd7741..4aefd4b9aef3 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -2,7 +2,13 @@ import path from "node:path"; import {PrivateKey} from "@libp2p/interface"; import {CompositeTypeAny, TreeView, Type} from "@chainsafe/ssz"; import {BeaconConfig} from "@lodestar/config"; -import {CheckpointWithHex, IForkChoice, ProtoBlock, UpdateHeadOpt} from "@lodestar/fork-choice"; +import { + CheckpointWithHex, + CheckpointWithPayloadStatus, + IForkChoice, + ProtoBlock, + UpdateHeadOpt, +} from "@lodestar/fork-choice"; import {LoggerNode} from "@lodestar/logger/node"; import { BUILDER_INDEX_SELF_BUILD, @@ -1186,7 +1192,7 @@ export class BeaconChain implements IBeaconChain { * @param blockState state that declares justified checkpoint `checkpoint` */ private justifiedBalancesGetter( - checkpoint: CheckpointWithHex, + checkpoint: CheckpointWithPayloadStatus, blockState: CachedBeaconStateAllForks ): EffectiveBalanceIncrements { this.metrics?.balancesCache.requests.inc(); @@ -1225,7 +1231,7 @@ export class BeaconChain implements IBeaconChain { * @param blockState state that declares justified checkpoint `checkpoint` */ private closestJustifiedBalancesStateToCheckpoint( - checkpoint: CheckpointWithHex, + checkpoint: CheckpointWithPayloadStatus, blockState: CachedBeaconStateAllForks ): {state: CachedBeaconStateAllForks; stateId: string; shouldWarn: boolean} { const state = this.regen.getCheckpointStateSync(checkpoint); @@ -1239,7 +1245,10 @@ export class BeaconChain implements IBeaconChain { } // Find a state in the same branch of checkpoint at same epoch. Balances should exactly the same - for (const descendantBlock of this.forkChoice.forwardIterateDescendants(checkpoint.rootHex)) { + for (const descendantBlock of this.forkChoice.forwardIterateDescendants( + checkpoint.rootHex, + checkpoint.payloadStatus + )) { if (computeEpochAtSlot(descendantBlock.slot) === checkpoint.epoch) { const descendantBlockState = this.regen.getStateSync(descendantBlock.stateRoot); if (descendantBlockState) { @@ -1255,7 +1264,10 @@ export class BeaconChain implements IBeaconChain { // Find a state in the same branch of checkpoint at a latter epoch. Balances are not the same, but should be close // Note: must call .forwardIterateDescendants() again since nodes are not sorted - for (const descendantBlock of this.forkChoice.forwardIterateDescendants(checkpoint.rootHex)) { + for (const descendantBlock of this.forkChoice.forwardIterateDescendants( + checkpoint.rootHex, + checkpoint.payloadStatus + )) { if (computeEpochAtSlot(descendantBlock.slot) > checkpoint.epoch) { const descendantBlockState = this.regen.getStateSync(descendantBlock.stateRoot); if (descendantBlockState) { diff --git a/packages/beacon-node/src/chain/emitter.ts b/packages/beacon-node/src/chain/emitter.ts index 95a12ead032b..17818b122021 100644 --- a/packages/beacon-node/src/chain/emitter.ts +++ b/packages/beacon-node/src/chain/emitter.ts @@ -1,7 +1,7 @@ import {EventEmitter} from "node:events"; import {StrictEventEmitter} from "strict-event-emitter-types"; import {routes} from "@lodestar/api"; -import {CheckpointWithPayload} from "@lodestar/fork-choice"; +import {CheckpointWithPayloadStatus} from "@lodestar/fork-choice"; import {CachedBeaconStateAllForks} from "@lodestar/state-transition"; import {DataColumnSidecars, RootHex, deneb, phase0} from "@lodestar/types"; import {PeerIdStr} from "../util/peerId.js"; @@ -83,8 +83,8 @@ export type ChainEventData = { export type IChainEvents = ApiEvents & { [ChainEvent.checkpoint]: (checkpoint: phase0.Checkpoint, state: CachedBeaconStateAllForks) => void; - [ChainEvent.forkChoiceJustified]: (checkpoint: CheckpointWithPayload) => void; - [ChainEvent.forkChoiceFinalized]: (checkpoint: CheckpointWithPayload) => void; + [ChainEvent.forkChoiceJustified]: (checkpoint: CheckpointWithPayloadStatus) => void; + [ChainEvent.forkChoiceFinalized]: (checkpoint: CheckpointWithPayloadStatus) => void; [ChainEvent.updateTargetCustodyGroupCount]: (targetGroupCount: number) => void; diff --git a/packages/beacon-node/src/chain/regen/regen.ts b/packages/beacon-node/src/chain/regen/regen.ts index 6ad654280293..08e580f1295c 100644 --- a/packages/beacon-node/src/chain/regen/regen.ts +++ b/packages/beacon-node/src/chain/regen/regen.ts @@ -158,7 +158,7 @@ export class StateRegenerator implements IStateRegeneratorInternal { const getSeedStateTimer = this.modules.metrics?.regenGetState.getSeedState.startTimer({caller}); // iterateAncestorBlocks only returns ancestor blocks, not the block itself - for (const b of this.modules.forkChoice.iterateAncestorBlocks(block.blockRoot)) { + for (const b of this.modules.forkChoice.iterateAncestorBlocks(block.blockRoot, block.payloadStatus)) { state = this.modules.blockStateCache.get(b.stateRoot); if (state) { break; diff --git a/packages/beacon-node/src/network/reqresp/handlers/beaconBlocksByRange.ts b/packages/beacon-node/src/network/reqresp/handlers/beaconBlocksByRange.ts index b4eda0db765e..e659266c23a3 100644 --- a/packages/beacon-node/src/network/reqresp/handlers/beaconBlocksByRange.ts +++ b/packages/beacon-node/src/network/reqresp/handlers/beaconBlocksByRange.ts @@ -47,9 +47,10 @@ export async function* onBeaconBlocksByRange( // Non-finalized range of blocks if (endSlot > finalizedSlot) { - const headRoot = chain.forkChoice.getHeadRoot(); + const headBlock = chain.forkChoice.getHead(); + const headRoot = headBlock.blockRoot; // TODO DENEB: forkChoice should mantain an array of canonical blocks, and change only on reorg - const headChain = chain.forkChoice.getAllAncestorBlocks(headRoot); + const headChain = chain.forkChoice.getAllAncestorBlocks(headRoot, headBlock.payloadStatus); // getAllAncestorBlocks response includes the head node, so it's the full chain. // Iterate head chain with ascending block numbers diff --git a/packages/beacon-node/src/network/reqresp/handlers/blobSidecarsByRange.ts b/packages/beacon-node/src/network/reqresp/handlers/blobSidecarsByRange.ts index 03628b403352..ef6a59a2b5ce 100644 --- a/packages/beacon-node/src/network/reqresp/handlers/blobSidecarsByRange.ts +++ b/packages/beacon-node/src/network/reqresp/handlers/blobSidecarsByRange.ts @@ -34,9 +34,10 @@ export async function* onBlobSidecarsByRange( // Non-finalized range of blobs if (endSlot > finalizedSlot) { - const headRoot = chain.forkChoice.getHeadRoot(); + const headBlock = chain.forkChoice.getHead(); + const headRoot = headBlock.blockRoot; // TODO DENEB: forkChoice should mantain an array of canonical blocks, and change only on reorg - const headChain = chain.forkChoice.getAllAncestorBlocks(headRoot); + const headChain = chain.forkChoice.getAllAncestorBlocks(headRoot, headBlock.payloadStatus); // Iterate head chain with ascending block numbers for (let i = headChain.length - 1; i >= 0; i--) { diff --git a/packages/beacon-node/src/network/reqresp/handlers/dataColumnSidecarsByRange.ts b/packages/beacon-node/src/network/reqresp/handlers/dataColumnSidecarsByRange.ts index 3610b0136e8b..db5fef74b86b 100644 --- a/packages/beacon-node/src/network/reqresp/handlers/dataColumnSidecarsByRange.ts +++ b/packages/beacon-node/src/network/reqresp/handlers/dataColumnSidecarsByRange.ts @@ -78,8 +78,9 @@ export async function* onDataColumnSidecarsByRange( // Non-finalized range of columns if (endSlot > finalizedSlot) { - const headRoot = chain.forkChoice.getHeadRoot(); - const headChain = chain.forkChoice.getAllAncestorBlocks(headRoot); + const headBlock = chain.forkChoice.getHead(); + const headRoot = headBlock.blockRoot; + const headChain = chain.forkChoice.getAllAncestorBlocks(headRoot, headBlock.payloadStatus); // Iterate head chain with ascending block numbers for (let i = headChain.length - 1; i >= 0; i--) { diff --git a/packages/beacon-node/test/unit/chain/archiveStore/blockArchiver.test.ts b/packages/beacon-node/test/unit/chain/archiveStore/blockArchiver.test.ts index 2803cf03b995..ea4ff653c1e9 100644 --- a/packages/beacon-node/test/unit/chain/archiveStore/blockArchiver.test.ts +++ b/packages/beacon-node/test/unit/chain/archiveStore/blockArchiver.test.ts @@ -2,6 +2,7 @@ import {afterEach, beforeEach, describe, expect, it, vi} from "vitest"; import {fromHexString, toHexString} from "@chainsafe/ssz"; import {createChainForkConfig} from "@lodestar/config"; import {config as defaultConfig} from "@lodestar/config/default"; +import {PayloadStatus} from "@lodestar/fork-choice"; import {computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {ssz} from "@lodestar/types"; import {archiveBlocks} from "../../../../src/chain/archiveStore/utils/archiveBlocks.js"; @@ -68,7 +69,7 @@ describe("block archiver task", () => { forkChoiceStub, lightclientServer, logger, - {epoch: 5, rootHex: ZERO_HASH_HEX}, + {epoch: 5, root: fromHexString(ZERO_HASH_HEX), rootHex: ZERO_HASH_HEX, payloadStatus: PayloadStatus.FULL}, currentEpoch ); @@ -141,7 +142,12 @@ describe("block archiver task", () => { forkChoiceStub, lightclientServer, logger, - {epoch: config.FULU_FORK_EPOCH + 1, rootHex: ZERO_HASH_HEX}, + { + epoch: config.FULU_FORK_EPOCH + 1, + root: fromHexString(ZERO_HASH_HEX), + rootHex: ZERO_HASH_HEX, + payloadStatus: PayloadStatus.FULL, + }, currentEpoch ); diff --git a/packages/beacon-node/test/unit/chain/forkChoice/forkChoice.test.ts b/packages/beacon-node/test/unit/chain/forkChoice/forkChoice.test.ts index b9ca0f658226..6aeef48dbf23 100644 --- a/packages/beacon-node/test/unit/chain/forkChoice/forkChoice.test.ts +++ b/packages/beacon-node/test/unit/chain/forkChoice/forkChoice.test.ts @@ -1,7 +1,7 @@ import {beforeAll, beforeEach, describe, expect, it, vi} from "vitest"; import {toHexString} from "@chainsafe/ssz"; import {config} from "@lodestar/config/default"; -import {CheckpointWithHex, ExecutionStatus, ForkChoice} from "@lodestar/fork-choice"; +import {CheckpointWithHex, ExecutionStatus, ForkChoice, PayloadStatus} from "@lodestar/fork-choice"; import {FAR_FUTURE_EPOCH, MAX_EFFECTIVE_BALANCE} from "@lodestar/params"; import { CachedBeaconStateAllForks, @@ -200,19 +200,19 @@ describe("LodestarForkChoice", () => { forkChoice.onBlock(block20.message, state20, blockDelaySec, currentSlot, executionStatus, dataAvailabilityStatus); forkChoice.onBlock(block24.message, state24, blockDelaySec, currentSlot, executionStatus, dataAvailabilityStatus); forkChoice.onBlock(block28.message, state28, blockDelaySec, currentSlot, executionStatus, dataAvailabilityStatus); - expect(forkChoice.getAllAncestorBlocks(hashBlock(block16.message))).toHaveLength(3); - expect(forkChoice.getAllAncestorBlocks(hashBlock(block24.message))).toHaveLength(5); + expect(forkChoice.getAllAncestorBlocks(hashBlock(block16.message), PayloadStatus.FULL)).toHaveLength(3); + expect(forkChoice.getAllAncestorBlocks(hashBlock(block24.message), PayloadStatus.FULL)).toHaveLength(5); expect(forkChoice.getBlockHexDefaultStatus(hashBlock(block08.message))).not.toBeNull(); expect(forkChoice.getBlockHexDefaultStatus(hashBlock(block12.message))).not.toBeNull(); expect(forkChoice.hasBlockHex(hashBlock(block08.message))).toBe(true); expect(forkChoice.hasBlockHex(hashBlock(block12.message))).toBe(true); forkChoice.onBlock(block32.message, state32, blockDelaySec, currentSlot, executionStatus, dataAvailabilityStatus); forkChoice.prune(hashBlock(block16.message)); - expect(forkChoice.getAllAncestorBlocks(hashBlock(block16.message)).length).toBeWithMessage( + expect(forkChoice.getAllAncestorBlocks(hashBlock(block16.message), PayloadStatus.FULL).length).toBeWithMessage( 0, "getAllAncestorBlocks should not return finalized block" ); - expect(forkChoice.getAllAncestorBlocks(hashBlock(block24.message))).toHaveLength(2); + expect(forkChoice.getAllAncestorBlocks(hashBlock(block24.message), PayloadStatus.FULL)).toHaveLength(2); expect(forkChoice.getBlockHexDefaultStatus(hashBlock(block08.message))).toBe(null); expect(forkChoice.getBlockHexDefaultStatus(hashBlock(block12.message))).toBe(null); expect(forkChoice.hasBlockHex(hashBlock(block08.message))).toBe(false); @@ -274,10 +274,11 @@ describe("LodestarForkChoice", () => { .forwarditerateAncestorBlocks() .filter( (summary) => - summary.slot < childBlock.message.slot && !forkChoice.isDescendant(summary.blockRoot, childBlockRoot) + summary.slot < childBlock.message.slot && + !forkChoice.isDescendant(summary.blockRoot, summary.payloadStatus, childBlockRoot, PayloadStatus.FULL) ); // compare to getAllNonAncestorBlocks api - expect(forkChoice.getAllNonAncestorBlocks(childBlockRoot)).toEqual(nonCanonicalSummaries); + expect(forkChoice.getAllNonAncestorBlocks(childBlockRoot, PayloadStatus.FULL)).toEqual(nonCanonicalSummaries); }); /** diff --git a/packages/fork-choice/src/forkChoice/forkChoice.ts b/packages/fork-choice/src/forkChoice/forkChoice.ts index 4f8bab2ca01b..2d0cef22540c 100644 --- a/packages/fork-choice/src/forkChoice/forkChoice.ts +++ b/packages/fork-choice/src/forkChoice/forkChoice.ts @@ -54,7 +54,7 @@ import { NotReorgedReason, ShouldOverrideForkChoiceUpdateResult, } from "./interface.js"; -import {CheckpointWithPayload, IForkChoiceStore, JustifiedBalances, toCheckpointWithPayload} from "./store.js"; +import {CheckpointWithPayloadStatus, IForkChoiceStore, JustifiedBalances, toCheckpointWithPayload} from "./store.js"; export type ForkChoiceOpts = { proposerBoost?: boolean; @@ -557,11 +557,11 @@ export class ForkChoice implements IForkChoice { return this.protoArray.nodes; } - getFinalizedCheckpoint(): CheckpointWithPayload { + getFinalizedCheckpoint(): CheckpointWithPayloadStatus { return this.fcStore.finalizedCheckpoint; } - getJustifiedCheckpoint(): CheckpointWithPayload { + getJustifiedCheckpoint(): CheckpointWithPayloadStatus { return this.fcStore.justified.checkpoint; } @@ -700,8 +700,8 @@ export class ForkChoice implements IForkChoice { // This is an optimization. It should reduce the amount of times we run // `process_justification_and_finalization` by approximately 1/3rd when the chain is // performing optimally. - let unrealizedJustifiedCheckpoint: CheckpointWithPayload; - let unrealizedFinalizedCheckpoint: CheckpointWithPayload; + let unrealizedJustifiedCheckpoint: CheckpointWithPayloadStatus; + let unrealizedFinalizedCheckpoint: CheckpointWithPayloadStatus; if (this.opts?.computeUnrealized) { if ( parentBlock.unrealizedJustifiedEpoch === blockEpoch && @@ -1133,8 +1133,13 @@ export class ForkChoice implements IForkChoice { * Always returns `false` if either input roots are unknown. * Still returns `true` if `ancestorRoot===descendantRoot` (and the roots are known) */ - isDescendant(ancestorRoot: RootHex, descendantRoot: RootHex): boolean { - return this.protoArray.isDescendant(ancestorRoot, descendantRoot); + isDescendant( + ancestorRoot: RootHex, + ancestorPayloadStatus: PayloadStatus, + descendantRoot: RootHex, + descendantPayloadStatus: PayloadStatus + ): boolean { + return this.protoArray.isDescendant(ancestorRoot, ancestorPayloadStatus, descendantRoot, descendantPayloadStatus); } /** @@ -1177,16 +1182,16 @@ export class ForkChoice implements IForkChoice { * Iterates backwards through block summaries, starting from a block root. * Return only the non-finalized blocks. */ - iterateAncestorBlocks(blockRoot: RootHex): IterableIterator { - return this.protoArray.iterateAncestorNodes(blockRoot); + iterateAncestorBlocks(blockRoot: RootHex, payloadStatus: PayloadStatus): IterableIterator { + return this.protoArray.iterateAncestorNodes(blockRoot, payloadStatus); } /** * Returns all blocks backwards starting from a block root. * Return only the non-finalized blocks. */ - getAllAncestorBlocks(blockRoot: RootHex): ProtoBlock[] { - const blocks = this.protoArray.getAllAncestorNodes(blockRoot); + getAllAncestorBlocks(blockRoot: RootHex, payloadStatus: PayloadStatus): ProtoBlock[] { + const blocks = this.protoArray.getAllAncestorNodes(blockRoot, payloadStatus); // the last node is the previous finalized one, it's there to check onBlock finalized checkpoint only. return blocks.slice(0, blocks.length - 1); } @@ -1194,15 +1199,18 @@ export class ForkChoice implements IForkChoice { /** * The same to iterateAncestorBlocks but this gets non-ancestor nodes instead of ancestor nodes. */ - getAllNonAncestorBlocks(blockRoot: RootHex): ProtoBlock[] { - return this.protoArray.getAllNonAncestorNodes(blockRoot); + getAllNonAncestorBlocks(blockRoot: RootHex, payloadStatus: PayloadStatus): ProtoBlock[] { + return this.protoArray.getAllNonAncestorNodes(blockRoot, payloadStatus); } /** * Returns both ancestor and non-ancestor blocks in a single traversal. */ - getAllAncestorAndNonAncestorBlocks(blockRoot: RootHex): {ancestors: ProtoBlock[]; nonAncestors: ProtoBlock[]} { - const {ancestors, nonAncestors} = this.protoArray.getAllAncestorAndNonAncestorNodes(blockRoot); + getAllAncestorAndNonAncestorBlocks( + blockRoot: RootHex, + payloadStatus: PayloadStatus + ): {ancestors: ProtoBlock[]; nonAncestors: ProtoBlock[]} { + const {ancestors, nonAncestors} = this.protoArray.getAllAncestorAndNonAncestorNodes(blockRoot, payloadStatus); return { // the last node is the previous finalized one, it's there to check onBlock finalized checkpoint only. @@ -1217,7 +1225,7 @@ export class ForkChoice implements IForkChoice { return this.head; } - for (const block of this.protoArray.iterateAncestorNodes(this.head.blockRoot)) { + for (const block of this.protoArray.iterateAncestorNodes(this.head.blockRoot, this.head.payloadStatus)) { if (block.blockRoot === blockRootHex) { return block; } @@ -1235,7 +1243,7 @@ export class ForkChoice implements IForkChoice { return this.head; } - for (const block of this.protoArray.iterateAncestorNodes(this.head.blockRoot)) { + for (const block of this.protoArray.iterateAncestorNodes(this.head.blockRoot, this.head.payloadStatus)) { if (block.slot === slot) { return block; } @@ -1248,7 +1256,7 @@ export class ForkChoice implements IForkChoice { return this.head; } - for (const block of this.protoArray.iterateAncestorNodes(this.head.blockRoot)) { + for (const block of this.protoArray.iterateAncestorNodes(this.head.blockRoot, this.head.payloadStatus)) { if (slot >= block.slot) { return block; } @@ -1261,24 +1269,16 @@ export class ForkChoice implements IForkChoice { return this.protoArray.nodes; } - // TODO GLOAS: this function is ambiguous, consumer should also provide payload, or it should accept a ProtoBlock instead - // also consumer may want PENDING or EMPTY only - *forwardIterateDescendants(blockRoot: RootHex): IterableIterator { + *forwardIterateDescendants(blockRoot: RootHex, payloadStatus: PayloadStatus): IterableIterator { const rootsInChain = new Set([blockRoot]); - - const blockVariants = this.protoArray.indices.get(blockRoot); - if (blockVariants === undefined) { + const blockIndex = this.protoArray.getNodeIndexByRootAndStatus(blockRoot, payloadStatus); + if (blockIndex === undefined) { throw new ForkChoiceError({ code: ForkChoiceErrorCode.MISSING_PROTO_ARRAY_BLOCK, root: blockRoot, }); } - // Find the minimum index among all variants to start iteration - const blockIndex = Array.isArray(blockVariants) - ? Math.min(...blockVariants.filter((idx) => idx !== undefined)) - : blockVariants; - for (let i = blockIndex + 1; i < this.protoArray.nodes.length; i++) { const node = this.protoArray.nodes[i]; if (rootsInChain.has(node.parentRoot)) { @@ -1478,8 +1478,8 @@ export class ForkChoice implements IForkChoice { * Since this balances are already available the getter is just `() => balances`, without cache interaction */ private updateCheckpoints( - justifiedCheckpoint: CheckpointWithPayload, - finalizedCheckpoint: CheckpointWithPayload, + justifiedCheckpoint: CheckpointWithPayloadStatus, + finalizedCheckpoint: CheckpointWithPayloadStatus, getJustifiedBalances: () => JustifiedBalances ): void { // Update justified checkpoint. @@ -1499,8 +1499,8 @@ export class ForkChoice implements IForkChoice { * Update unrealized checkpoints in store if necessary */ private updateUnrealizedCheckpoints( - unrealizedJustifiedCheckpoint: CheckpointWithPayload, - unrealizedFinalizedCheckpoint: CheckpointWithPayload, + unrealizedJustifiedCheckpoint: CheckpointWithPayloadStatus, + unrealizedFinalizedCheckpoint: CheckpointWithPayloadStatus, getJustifiedBalances: () => JustifiedBalances ): void { if (unrealizedJustifiedCheckpoint.epoch > this.fcStore.unrealizedJustified.checkpoint.epoch) { diff --git a/packages/fork-choice/src/forkChoice/interface.ts b/packages/fork-choice/src/forkChoice/interface.ts index 66bb5f8de753..26e7f08c2be5 100644 --- a/packages/fork-choice/src/forkChoice/interface.ts +++ b/packages/fork-choice/src/forkChoice/interface.ts @@ -12,7 +12,7 @@ import { ProtoNode, } from "../protoArray/interface.js"; import {UpdateAndGetHeadOpt} from "./forkChoice.js"; -import {CheckpointWithHex, CheckpointWithPayload} from "./store.js"; +import {CheckpointWithHex, CheckpointWithPayloadStatus} from "./store.js"; export type CheckpointHex = { epoch: Epoch; @@ -25,7 +25,7 @@ export type CheckpointsWithHex = { }; export type CheckpointWithPayloadAndBalance = { - checkpoint: CheckpointWithPayload; + checkpoint: CheckpointWithPayloadStatus; balances: EffectiveBalanceIncrements; }; @@ -124,8 +124,8 @@ export interface IForkChoice { * Retrieve all nodes for the debug API. */ getAllNodes(): ProtoNode[]; - getFinalizedCheckpoint(): CheckpointWithPayload; - getJustifiedCheckpoint(): CheckpointWithPayload; + getFinalizedCheckpoint(): CheckpointWithPayloadStatus; + getJustifiedCheckpoint(): CheckpointWithPayloadStatus; /** * Add `block` to the fork choice DAG. * @@ -246,7 +246,12 @@ export interface IForkChoice { * Always returns `false` if either input roots are unknown. * Still returns `true` if `ancestorRoot===descendantRoot` (and the roots are known) */ - isDescendant(ancestorRoot: RootHex, descendantRoot: RootHex): boolean; + isDescendant( + ancestorRoot: RootHex, + ancestorPayloadStatus: PayloadStatus, + descendantRoot: RootHex, + descendantPayloadStatus: PayloadStatus + ): boolean; /** * Prune items up to a finalized root. */ @@ -255,16 +260,19 @@ export interface IForkChoice { /** * Iterates backwards through ancestor block summaries, starting from a block root */ - iterateAncestorBlocks(blockRoot: RootHex): IterableIterator; - getAllAncestorBlocks(blockRoot: RootHex): ProtoBlock[]; + iterateAncestorBlocks(blockRoot: RootHex, payloadStatus: PayloadStatus): IterableIterator; + getAllAncestorBlocks(blockRoot: RootHex, payloadStatus: PayloadStatus): ProtoBlock[]; /** * The same to iterateAncestorBlocks but this gets non-ancestor nodes instead of ancestor nodes. */ - getAllNonAncestorBlocks(blockRoot: RootHex): ProtoBlock[]; + getAllNonAncestorBlocks(blockRoot: RootHex, payloadStatus: PayloadStatus): ProtoBlock[]; /** * Returns both ancestor and non-ancestor blocks in a single traversal. */ - getAllAncestorAndNonAncestorBlocks(blockRoot: RootHex): {ancestors: ProtoBlock[]; nonAncestors: ProtoBlock[]}; + getAllAncestorAndNonAncestorBlocks( + blockRoot: RootHex, + payloadStatus: PayloadStatus + ): {ancestors: ProtoBlock[]; nonAncestors: ProtoBlock[]}; getCanonicalBlockByRoot(blockRoot: Root): ProtoBlock | null; getCanonicalBlockAtSlot(slot: Slot): ProtoBlock | null; getCanonicalBlockClosestLteSlot(slot: Slot): ProtoBlock | null; @@ -275,7 +283,7 @@ export interface IForkChoice { /** * Iterates forward descendants of blockRoot. Does not yield blockRoot itself */ - forwardIterateDescendants(blockRoot: RootHex): IterableIterator; + forwardIterateDescendants(blockRoot: RootHex, payloadStatus: PayloadStatus): IterableIterator; getBlockSummariesByParentRoot(parentRoot: RootHex): ProtoBlock[]; getBlockSummariesAtSlot(slot: Slot): ProtoBlock[]; /** Returns the distance of common ancestor of nodes to the max of the newNode and the prevNode. */ diff --git a/packages/fork-choice/src/forkChoice/store.ts b/packages/fork-choice/src/forkChoice/store.ts index 8e817987d7e6..6f2129d11f53 100644 --- a/packages/fork-choice/src/forkChoice/store.ts +++ b/packages/fork-choice/src/forkChoice/store.ts @@ -18,7 +18,7 @@ export type CheckpointWithHex = phase0.Checkpoint & {rootHex: RootHex}; * Pre-Gloas: payloadStatus is always FULL (payload embedded in block) * Gloas: determined by state.execution_payload_availability */ -export type CheckpointWithPayload = CheckpointWithHex & {payloadStatus: PayloadStatus}; +export type CheckpointWithPayloadStatus = CheckpointWithHex & {payloadStatus: PayloadStatus}; export type JustifiedBalances = EffectiveBalanceIncrements; @@ -29,7 +29,7 @@ export type JustifiedBalances = EffectiveBalanceIncrements; * @param blockState state that declares justified checkpoint `checkpoint` */ export type JustifiedBalancesGetter = ( - checkpoint: CheckpointWithPayload, + checkpoint: CheckpointWithPayloadStatus, blockState: CachedBeaconStateAllForks ) => JustifiedBalances; @@ -50,8 +50,8 @@ export interface IForkChoiceStore { get justified(): CheckpointWithPayloadAndTotalBalance; set justified(justified: CheckpointWithPayloadAndBalance); unrealizedJustified: CheckpointWithPayloadAndBalance; - finalizedCheckpoint: CheckpointWithPayload; - unrealizedFinalizedCheckpoint: CheckpointWithPayload; + finalizedCheckpoint: CheckpointWithPayloadStatus; + unrealizedFinalizedCheckpoint: CheckpointWithPayloadStatus; justifiedBalancesGetter: JustifiedBalancesGetter; equivocatingIndices: Set; } @@ -62,8 +62,8 @@ export interface IForkChoiceStore { export class ForkChoiceStore implements IForkChoiceStore { private _justified: CheckpointWithPayloadAndTotalBalance; unrealizedJustified: CheckpointWithPayloadAndBalance; - private _finalizedCheckpoint: CheckpointWithPayload; - unrealizedFinalizedCheckpoint: CheckpointWithPayload; + private _finalizedCheckpoint: CheckpointWithPayloadStatus; + unrealizedFinalizedCheckpoint: CheckpointWithPayloadStatus; equivocatingIndices = new Set(); justifiedBalancesGetter: JustifiedBalancesGetter; currentSlot: Slot; @@ -87,8 +87,8 @@ export class ForkChoiceStore implements IForkChoiceStore { */ finalizedPayloadStatus: PayloadStatus, private readonly events?: { - onJustified: (cp: CheckpointWithPayload) => void; - onFinalized: (cp: CheckpointWithPayload) => void; + onJustified: (cp: CheckpointWithPayloadStatus) => void; + onFinalized: (cp: CheckpointWithPayloadStatus) => void; } ) { this.justifiedBalancesGetter = justifiedBalancesGetter; @@ -112,10 +112,10 @@ export class ForkChoiceStore implements IForkChoiceStore { this.events?.onJustified(justified.checkpoint); } - get finalizedCheckpoint(): CheckpointWithPayload { + get finalizedCheckpoint(): CheckpointWithPayloadStatus { return this._finalizedCheckpoint; } - set finalizedCheckpoint(checkpoint: CheckpointWithPayload) { + set finalizedCheckpoint(checkpoint: CheckpointWithPayloadStatus) { const cp = toCheckpointWithPayload(checkpoint, checkpoint.payloadStatus); this._finalizedCheckpoint = cp; this.events?.onFinalized(cp); @@ -136,7 +136,7 @@ export function toCheckpointWithHex(checkpoint: phase0.Checkpoint): CheckpointWi export function toCheckpointWithPayload( checkpoint: phase0.Checkpoint, payloadStatus: PayloadStatus -): CheckpointWithPayload { +): CheckpointWithPayloadStatus { return { ...toCheckpointWithHex(checkpoint), payloadStatus, diff --git a/packages/fork-choice/src/index.ts b/packages/fork-choice/src/index.ts index 99246e5225f8..fd141bf7d70e 100644 --- a/packages/fork-choice/src/index.ts +++ b/packages/fork-choice/src/index.ts @@ -24,7 +24,7 @@ export { export * from "./forkChoice/safeBlocks.js"; export { type CheckpointWithHex, - type CheckpointWithPayload, + type CheckpointWithPayloadStatus, ForkChoiceStore, type IForkChoiceStore, type JustifiedBalancesGetter, diff --git a/packages/fork-choice/src/protoArray/protoArray.ts b/packages/fork-choice/src/protoArray/protoArray.ts index a8d13c3e5da1..fc55e9e1f388 100644 --- a/packages/fork-choice/src/protoArray/protoArray.ts +++ b/packages/fork-choice/src/protoArray/protoArray.ts @@ -1477,11 +1477,8 @@ export class ProtoArray { * For Gloas blocks: returns EMPTY/FULL variants (not PENDING) based on parent payload status * For pre-Gloas blocks: returns FULL variants */ - *iterateAncestorNodes(blockRoot: RootHex): IterableIterator { - // Get canonical node: FULL for pre-Gloas, PENDING for Gloas - const defaultStatus = this.getDefaultVariant(blockRoot); - const startIndex = - defaultStatus !== undefined ? this.getNodeIndexByRootAndStatus(blockRoot, defaultStatus) : undefined; + *iterateAncestorNodes(blockRoot: RootHex, payloadStatus: PayloadStatus): IterableIterator { + const startIndex = this.getNodeIndexByRootAndStatus(blockRoot, payloadStatus); if (startIndex === undefined) { return; } @@ -1520,11 +1517,8 @@ export class ProtoArray { * For Gloas blocks: returns EMPTY/FULL variants (not PENDING) based on parent payload status * For pre-Gloas blocks: returns FULL variants */ - getAllAncestorNodes(blockRoot: RootHex): ProtoNode[] { - // Get canonical node: FULL for pre-Gloas, PENDING for Gloas - const defaultStatus = this.getDefaultVariant(blockRoot); - const startIndex = - defaultStatus !== undefined ? this.getNodeIndexByRootAndStatus(blockRoot, defaultStatus) : undefined; + getAllAncestorNodes(blockRoot: RootHex, payloadStatus: PayloadStatus): ProtoNode[] { + const startIndex = this.getNodeIndexByRootAndStatus(blockRoot, payloadStatus); if (startIndex === undefined) { return []; } @@ -1537,12 +1531,10 @@ export class ProtoArray { }); } - // Include starting node if node is pre-gloas - // Reason why we exclude post-gloas is because node is always default variant (PENDING) - // which we want to exclude. + // Exclude PENDING variant from returned ancestors. const nodes: ProtoNode[] = []; - if (!isGloasBlock(node)) { + if (node.payloadStatus !== PayloadStatus.PENDING) { nodes.push(node); } @@ -1567,13 +1559,8 @@ export class ProtoArray { * For Gloas blocks: returns EMPTY/FULL variants (not PENDING) based on parent payload status * For pre-Gloas blocks: returns FULL variants */ - getAllNonAncestorNodes(blockRoot: RootHex): ProtoNode[] { - // Get canonical node: FULL for pre-Gloas, PENDING for Gloas - const defaultStatus = this.getDefaultVariant(blockRoot); - if (defaultStatus === undefined) { - return []; - } - const startIndex = this.getNodeIndexByRootAndStatus(blockRoot, defaultStatus); + getAllNonAncestorNodes(blockRoot: RootHex, payloadStatus: PayloadStatus): ProtoNode[] { + const startIndex = this.getNodeIndexByRootAndStatus(blockRoot, payloadStatus); if (startIndex === undefined) { return []; } @@ -1613,11 +1600,11 @@ export class ProtoArray { * For Gloas blocks: returns EMPTY/FULL variants (not PENDING) based on parent payload status * For pre-Gloas blocks: returns FULL variants */ - getAllAncestorAndNonAncestorNodes(blockRoot: RootHex): {ancestors: ProtoNode[]; nonAncestors: ProtoNode[]} { - // Get canonical node: FULL for pre-Gloas, PENDING for Gloas - const defaultStatus = this.getDefaultVariant(blockRoot); - const startIndex = - defaultStatus !== undefined ? this.getNodeIndexByRootAndStatus(blockRoot, defaultStatus) : undefined; + getAllAncestorAndNonAncestorNodes( + blockRoot: RootHex, + payloadStatus: PayloadStatus + ): {ancestors: ProtoNode[]; nonAncestors: ProtoNode[]} { + const startIndex = this.getNodeIndexByRootAndStatus(blockRoot, payloadStatus); if (startIndex === undefined) { return {ancestors: [], nonAncestors: []}; } @@ -1735,26 +1722,28 @@ export class ProtoArray { /** * Returns `true` if the `descendantRoot` has an ancestor with `ancestorRoot`. * Always returns `false` if either input roots are unknown. - * Still returns `true` if `ancestorRoot` === `descendantRoot` (and the roots are known) + * Still returns `true` if `ancestorRoot` === `descendantRoot` and payload statuses match. */ - isDescendant(ancestorRoot: RootHex, descendantRoot: RootHex): boolean { - // We use the default variant (PENDING for Gloas, FULL for pre-Gloas) - // We cannot use FULL/EMPTY variants for Gloas because they may not be canonical - const defaultStatus = this.getDefaultVariant(ancestorRoot); - const ancestorNode = defaultStatus !== undefined ? this.getNode(ancestorRoot, defaultStatus) : undefined; + isDescendant( + ancestorRoot: RootHex, + ancestorPayloadStatus: PayloadStatus, + descendantRoot: RootHex, + descendantPayloadStatus: PayloadStatus + ): boolean { + const ancestorNode = this.getNode(ancestorRoot, ancestorPayloadStatus); if (!ancestorNode) { return false; } - if (ancestorRoot === descendantRoot) { + if (ancestorRoot === descendantRoot && ancestorPayloadStatus === descendantPayloadStatus) { return true; } - for (const node of this.iterateAncestorNodes(descendantRoot)) { + for (const node of this.iterateAncestorNodes(descendantRoot, descendantPayloadStatus)) { if (node.slot < ancestorNode.slot) { return false; } - if (node.blockRoot === ancestorNode.blockRoot) { + if (node.blockRoot === ancestorNode.blockRoot && node.payloadStatus === ancestorNode.payloadStatus) { return true; } } diff --git a/packages/fork-choice/test/unit/forkChoice/forkChoice.test.ts b/packages/fork-choice/test/unit/forkChoice/forkChoice.test.ts index bb81100795f5..c2ab249d3093 100644 --- a/packages/fork-choice/test/unit/forkChoice/forkChoice.test.ts +++ b/packages/fork-choice/test/unit/forkChoice/forkChoice.test.ts @@ -152,7 +152,7 @@ describe("Forkchoice", () => { const block = getBlock(genesisSlot + 1); protoArr.onBlock(block, block.slot, null); const forkchoice = new ForkChoice(config, fcStore, protoArr, validatorCount, null); - const summaries = forkchoice.getAllAncestorBlocks(getBlockRoot(genesisSlot + 1)); + const summaries = forkchoice.getAllAncestorBlocks(getBlockRoot(genesisSlot + 1), PayloadStatus.FULL); // there are 2 blocks in protoArray but iterateAncestorBlocks should only return non-finalized blocks expect(summaries).toHaveLength(1); expect(summaries[0]).toEqual({ @@ -180,18 +180,18 @@ describe("Forkchoice", () => { // Test with a block from the canonical chain const canonicalBlockRoot = getBlockRoot(genesisSlot + 3); - const canonicalAncestorBlocks = forkchoice.getAllAncestorBlocks(canonicalBlockRoot); - const canonicalNonAncestorBlocks = forkchoice.getAllNonAncestorBlocks(canonicalBlockRoot); - const canonicalCombined = forkchoice.getAllAncestorAndNonAncestorBlocks(canonicalBlockRoot); + const canonicalAncestorBlocks = forkchoice.getAllAncestorBlocks(canonicalBlockRoot, PayloadStatus.FULL); + const canonicalNonAncestorBlocks = forkchoice.getAllNonAncestorBlocks(canonicalBlockRoot, PayloadStatus.FULL); + const canonicalCombined = forkchoice.getAllAncestorAndNonAncestorBlocks(canonicalBlockRoot, PayloadStatus.FULL); expect(canonicalCombined.ancestors).toEqual(canonicalAncestorBlocks); expect(canonicalCombined.nonAncestors).toEqual(canonicalNonAncestorBlocks); // Test with a block from the fork chain const forkBlockRoot = getBlockRoot(genesisSlot + 10); - const forkAncestorBlocks = forkchoice.getAllAncestorBlocks(forkBlockRoot); - const forkNonAncestorBlocks = forkchoice.getAllNonAncestorBlocks(forkBlockRoot); - const forkCombined = forkchoice.getAllAncestorAndNonAncestorBlocks(forkBlockRoot); + const forkAncestorBlocks = forkchoice.getAllAncestorBlocks(forkBlockRoot, PayloadStatus.FULL); + const forkNonAncestorBlocks = forkchoice.getAllNonAncestorBlocks(forkBlockRoot, PayloadStatus.FULL); + const forkCombined = forkchoice.getAllAncestorAndNonAncestorBlocks(forkBlockRoot, PayloadStatus.FULL); expect(forkCombined.ancestors).toEqual(forkAncestorBlocks); expect(forkCombined.nonAncestors).toEqual(forkNonAncestorBlocks); diff --git a/packages/fork-choice/test/unit/protoArray/protoArray.test.ts b/packages/fork-choice/test/unit/protoArray/protoArray.test.ts index bbfa95e59aa4..61567ea73204 100644 --- a/packages/fork-choice/test/unit/protoArray/protoArray.test.ts +++ b/packages/fork-choice/test/unit/protoArray/protoArray.test.ts @@ -133,7 +133,7 @@ describe("ProtoArray", () => { ]; for (const [ancestorRoot, descendantRoot, isDescendant] of assertions) { - expect(fc.isDescendant(ancestorRoot, descendantRoot)).toBeWithMessage( + expect(fc.isDescendant(ancestorRoot, PayloadStatus.FULL, descendantRoot, PayloadStatus.FULL)).toBeWithMessage( isDescendant, `${descendantRoot} must be ${isDescendant ? "descendant" : "not descendant"} of ${ancestorRoot}` ); From bb273175f2e35d1a5d2b615149c60728dbdf03d4 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Mon, 9 Mar 2026 19:40:20 +0000 Subject: [PATCH 25/63] chore: pin pnpm with sha512 integrity in package.json (#9008) --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 462aceaf183c..4a6840994720 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "engines": { "node": "^24.13.0" }, - "packageManager": "pnpm@10.24.0", + "packageManager": "pnpm@10.24.0+sha512.01ff8ae71b4419903b65c60fb2dc9d34cf8bb6e06d03bde112ef38f7a34d6904c424ba66bea5cdcf12890230bf39f9580473140ed9c946fef328b6e5238a345a", "workspaces": [ "packages/*" ], From 0df187678b8103f2aeb3baf10791281d29a8d914 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Tue, 10 Mar 2026 14:03:43 +0000 Subject: [PATCH 26/63] chore: pin github actions by commit hash (#9017) Closes https://github.com/ChainSafe/lodestar/issues/7618 --- .github/actions/core-dump/action.yml | 2 +- .github/actions/setup-and-build/action.yml | 8 ++++---- .github/dependabot.yml | 10 ++++++++++ .github/workflows/benchmark.yml | 4 ++-- .github/workflows/binaries.yml | 6 +++--- .github/workflows/build-debug-node.yml | 4 ++-- .github/workflows/check-specrefs.yml | 2 +- .github/workflows/codeql-analysis.yml | 8 ++++---- .github/workflows/docker.yml | 10 +++++----- .github/workflows/docs-backfill.yml | 6 +++--- .github/workflows/docs-check.yml | 4 ++-- .github/workflows/docs-version.yml | 6 +++--- .github/workflows/docs.yml | 10 +++++----- .github/workflows/kurtosis.yml | 6 +++--- .github/workflows/lint-pr-title.yml | 2 +- .github/workflows/publish-dev.yml | 8 ++++---- .github/workflows/publish-manual.yml | 2 +- .github/workflows/publish-nextfork.yml | 8 ++++---- .github/workflows/publish-rc.yml | 12 ++++++------ .github/workflows/publish-stable.yml | 14 +++++++------- .github/workflows/test-sim.yml | 18 +++++++++--------- .github/workflows/test.yml | 22 +++++++++++----------- 22 files changed, 91 insertions(+), 81 deletions(-) create mode 100644 .github/dependabot.yml diff --git a/.github/actions/core-dump/action.yml b/.github/actions/core-dump/action.yml index e2e0b1224912..910747888ab0 100644 --- a/.github/actions/core-dump/action.yml +++ b/.github/actions/core-dump/action.yml @@ -10,7 +10,7 @@ runs: shell: sh - name: Backup core dump - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: core-dump path: /cores/* diff --git a/.github/actions/setup-and-build/action.yml b/.github/actions/setup-and-build/action.yml index d19a8154bc2f..8071ca6b9f4b 100644 --- a/.github/actions/setup-and-build/action.yml +++ b/.github/actions/setup-and-build/action.yml @@ -8,10 +8,10 @@ inputs: runs: using: "composite" steps: - - uses: pnpm/action-setup@v4 + - uses: pnpm/action-setup@c5ba7f7862a0f64c1b1a05fbac13e0b8e86ba08c # v4 - name: Setup Node - uses: actions/setup-node@v6 + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version: ${{inputs.node}} check-latest: true @@ -32,7 +32,7 @@ runs: run: echo "key=build-cache-${{ runner.os }}-${{ runner.arch }}-node-${{ inputs.node }}-${{ github.sha }}" >> $GITHUB_OUTPUT - name: Restore build - uses: actions/cache/restore@v4 + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 id: cache-build-restore with: path: | @@ -58,7 +58,7 @@ runs: run: pnpm check-bundle - name: Cache build artifacts - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: | lib/ diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000000..05c25f68de31 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + groups: + actions: + patterns: + - "*" diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 6781922b5304..b3067d991ce8 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -30,7 +30,7 @@ jobs: steps: # - Uses YAML anchors in the future - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - uses: "./.github/actions/setup-and-build" with: node: 24 @@ -38,7 +38,7 @@ jobs: # Restore performance downloaded states - name: Restore performance state cache - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: packages/state-transition/test-cache key: perf-states-${{ hashFiles('packages/state-transition/test/perf/params.ts') }} diff --git a/.github/workflows/binaries.yml b/.github/workflows/binaries.yml index 2bbab6ccdc50..15bc25bd42eb 100644 --- a/.github/workflows/binaries.yml +++ b/.github/workflows/binaries.yml @@ -27,7 +27,7 @@ jobs: arch: arm64 runs-on: ${{matrix.os}} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Install arm64 specifics if: matrix.arch == 'arm64' run: |- @@ -45,13 +45,13 @@ jobs: npx caxa -m "Unpacking Lodestar binary, please wait..." -e "dashboards/**" -e "docs/**" -D -p "pnpm clean:nm && pnpm install --frozen-lockfile --prod" --input . --output "lodestar" -- "{{caxa}}/node_modules/.bin/node" "--max-old-space-size=8192" "{{caxa}}/packages/cli/bin/lodestar.js" tar -czf "dist/lodestar-${{ inputs.version }}-${{ matrix.platform }}-${{ matrix.arch }}.tar.gz" "lodestar" - name: Upload binaries - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: binaries-${{ matrix.os }} path: dist/ if-no-files-found: error - name: Sanity check binary - uses: actions/github-script@v7 + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 with: script: | exec.exec('./lodestar dev'); diff --git a/.github/workflows/build-debug-node.yml b/.github/workflows/build-debug-node.yml index f9c4c5802a2e..fc1b03b0b2f5 100644 --- a/.github/workflows/build-debug-node.yml +++ b/.github/workflows/build-debug-node.yml @@ -18,7 +18,7 @@ jobs: run: apt-get install python3 g++ make python3-pip - name: Download Node.js source - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: repository: "nodejs/node" ref: "v${{ github.event.inputs.version }}" @@ -44,7 +44,7 @@ jobs: working-directory: "nodejs" - name: Upload build to artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: nodejs-debug-build-${{ github.event.inputs.version }} path: nodejs-debug-build-${{ github.event.inputs.version }} diff --git a/.github/workflows/check-specrefs.yml b/.github/workflows/check-specrefs.yml index 229b832b4f97..51fd320e497a 100644 --- a/.github/workflows/check-specrefs.yml +++ b/.github/workflows/check-specrefs.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Check version consistency run: | diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 2d635afe2688..0c9d3f33e522 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -47,11 +47,11 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@8dca8a82e2fa1a2c8908956f711300f9c4a4f4f6 # v2 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -64,7 +64,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@8dca8a82e2fa1a2c8908956f711300f9c4a4f4f6 # v2 # ℹ️ Command-line programs to run using the OS shell. # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun @@ -77,4 +77,4 @@ jobs: # ./location_of_script_within_repo/buildscript.sh - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@8dca8a82e2fa1a2c8908956f711300f9c4a4f4f6 # v2 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d8056e7f03e5..db5d3f607be5 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -29,13 +29,13 @@ jobs: - arch: arm64 runner: buildjet-4vcpu-ubuntu-2204-arm steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: ref: ${{ inputs.ref || github.sha }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - name: Login to Docker Hub - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -74,9 +74,9 @@ jobs: needs: docker steps: - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - name: Login to Docker Hub - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.github/workflows/docs-backfill.yml b/.github/workflows/docs-backfill.yml index a68f8e99507e..d40d03311ea5 100644 --- a/.github/workflows/docs-backfill.yml +++ b/.github/workflows/docs-backfill.yml @@ -22,7 +22,7 @@ jobs: name: Backfill versioned docs runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: ref: unstable fetch-depth: 0 @@ -73,9 +73,9 @@ jobs: - name: Install pnpm if: steps.versions.outputs.missing != '' && github.event.inputs.dry_run != 'true' - uses: pnpm/action-setup@v4 + uses: pnpm/action-setup@c5ba7f7862a0f64c1b1a05fbac13e0b8e86ba08c # v4 - - uses: actions/setup-node@v6 + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 if: steps.versions.outputs.missing != '' && github.event.inputs.dry_run != 'true' with: node-version: 24 diff --git a/.github/workflows/docs-check.yml b/.github/workflows/docs-check.yml index 7969c198615d..50d810b2727c 100644 --- a/.github/workflows/docs-check.yml +++ b/.github/workflows/docs-check.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: # - Uses YAML anchors in the future - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - uses: "./.github/actions/setup-and-build" with: node: 24 @@ -29,4 +29,4 @@ jobs: # Run spellcheck AFTER building docs, in case the CLI reference has issues - name: Spellcheck - uses: rojopolis/spellcheck-github-actions@0.32.0 + uses: rojopolis/spellcheck-github-actions@531492f4dd27f0593ca398f2ed225d1b92eec828 # 0.32.0 diff --git a/.github/workflows/docs-version.yml b/.github/workflows/docs-version.yml index 3535fd32975b..30c0ad799cf4 100644 --- a/.github/workflows/docs-version.yml +++ b/.github/workflows/docs-version.yml @@ -33,14 +33,14 @@ jobs: fi # Checkout the tagged release to build its docs - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: ref: ${{ github.ref }} - name: Install pnpm - uses: pnpm/action-setup@v4 + uses: pnpm/action-setup@c5ba7f7862a0f64c1b1a05fbac13e0b8e86ba08c # v4 - - uses: actions/setup-node@v6 + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version: 24 check-latest: true diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 4171b2ee116d..8f73dd7e1c0e 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -30,14 +30,14 @@ jobs: - name: Log Deployment Ref run: echo "Deploying docs from ref $DEPLOY_REF" - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: ref: ${{ env.DEPLOY_REF }} - name: Install pnpm before setup node - uses: pnpm/action-setup@v4 + uses: pnpm/action-setup@c5ba7f7862a0f64c1b1a05fbac13e0b8e86ba08c # v4 - - uses: actions/setup-node@v6 + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version: 24 check-latest: true @@ -48,7 +48,7 @@ jobs: run: echo "v8CppApiVersion=$(node --print "process.versions.modules")" >> $GITHUB_OUTPUT - name: Restore dependencies - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 id: cache-deps with: path: | @@ -91,7 +91,7 @@ jobs: run: pnpm install && pnpm build - name: Deploy - uses: peaceiris/actions-gh-pages@v3 + uses: peaceiris/actions-gh-pages@4a2e02b36f31d8974a0d09d3bb9f3172aa2d0d0d # v3 with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: ./docs/build diff --git a/.github/workflows/kurtosis.yml b/.github/workflows/kurtosis.yml index bf1ad642a76f..4d17d9c50d66 100644 --- a/.github/workflows/kurtosis.yml +++ b/.github/workflows/kurtosis.yml @@ -11,17 +11,17 @@ jobs: name: Build and run Kurtosis runs-on: buildjet-4vcpu-ubuntu-2204 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: fetch-depth: 0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - name: Build Docker image - local only run: > docker buildx build . --load \ --tag chainsafe/lodestar:kurtosis-ci \ --build-arg COMMIT=$(git rev-parse HEAD) - name: Run test - uses: ethpandaops/kurtosis-assertoor-github-action@v1 + uses: ethpandaops/kurtosis-assertoor-github-action@5932604b244dbd2ddb811516b516a9094f4d2c2f # v1 with: ethereum_package_args: ".github/workflows/assets/kurtosis_sim_test_config.yaml" diff --git a/.github/workflows/lint-pr-title.yml b/.github/workflows/lint-pr-title.yml index 873343c8d799..49270f091a95 100644 --- a/.github/workflows/lint-pr-title.yml +++ b/.github/workflows/lint-pr-title.yml @@ -14,7 +14,7 @@ jobs: name: Validate PR title runs-on: ubuntu-latest steps: - - uses: amannn/action-semantic-pull-request@v5 + - uses: amannn/action-semantic-pull-request@e32d7e603df1aa1ba07e981f2a23455dee596825 # v5 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: diff --git a/.github/workflows/publish-dev.yml b/.github/workflows/publish-dev.yml index 0abec636425a..479b0495125e 100644 --- a/.github/workflows/publish-dev.yml +++ b/.github/workflows/publish-dev.yml @@ -19,14 +19,14 @@ jobs: runs-on: ubuntu-latest steps: # - Uses YAML anchors in the future - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - name: Install pnpm before setup node - uses: pnpm/action-setup@v4 + uses: pnpm/action-setup@c5ba7f7862a0f64c1b1a05fbac13e0b8e86ba08c # v4 - - uses: actions/setup-node@v6 + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version: 24 registry-url: "https://registry.npmjs.org" @@ -36,7 +36,7 @@ jobs: id: node run: echo "v8CppApiVersion=$(node --print "process.versions.modules")" >> $GITHUB_OUTPUT - name: Restore dependencies - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 id: cache-deps with: path: | diff --git a/.github/workflows/publish-manual.yml b/.github/workflows/publish-manual.yml index ad7bc6a31321..34a279719dd5 100644 --- a/.github/workflows/publish-manual.yml +++ b/.github/workflows/publish-manual.yml @@ -23,7 +23,7 @@ jobs: name: Validate inputs runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: ref: ${{ inputs.ref }} fetch-depth: 0 diff --git a/.github/workflows/publish-nextfork.yml b/.github/workflows/publish-nextfork.yml index 84eabc546728..80413ab1df0c 100644 --- a/.github/workflows/publish-nextfork.yml +++ b/.github/workflows/publish-nextfork.yml @@ -22,12 +22,12 @@ jobs: runs-on: ubuntu-latest steps: # - Uses YAML anchors in the future - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - name: Install pnpm before setup node - uses: pnpm/action-setup@v4 - - uses: actions/setup-node@v6 + uses: pnpm/action-setup@c5ba7f7862a0f64c1b1a05fbac13e0b8e86ba08c # v4 + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version: 24 registry-url: "https://registry.npmjs.org" @@ -37,7 +37,7 @@ jobs: id: node run: echo "v8CppApiVersion=$(node --print "process.versions.modules")" >> $GITHUB_OUTPUT - name: Restore dependencies - uses: actions/cache@master + uses: actions/cache@0769f2e44373d687c841be56f618397100943c5e # master id: cache-deps with: path: | diff --git a/.github/workflows/publish-rc.yml b/.github/workflows/publish-rc.yml index aabcde7ee3d5..3a7b0058cc90 100644 --- a/.github/workflows/publish-rc.yml +++ b/.github/workflows/publish-rc.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: fetch-depth: 0 @@ -59,7 +59,7 @@ jobs: needs: [tag, binaries] if: needs.tag.outputs.is_rc == 'true' steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 # Needs full depth for changelog generation @@ -71,14 +71,14 @@ jobs: run: node scripts/generate_changelog.mjs ${{ needs.tag.outputs.prev_tag }} ${{ needs.tag.outputs.tag }} CHANGELOG.md - name: Get binaries - uses: actions/download-artifact@v5 + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 with: path: dist/ merge-multiple: true - name: Create Release id: create_release - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2.5.0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: @@ -111,7 +111,7 @@ jobs: # In case of failure - name: Rollback on failure if: failure() - uses: author/action-rollback@1.0.4 + uses: author/action-rollback@f4473931b1155b601092ec00eb1fa4882b80219f # 1.0.4 with: release_id: ${{ steps.create_release.outputs.id }} tag: ${{ needs.tag.outputs.tag }} @@ -125,7 +125,7 @@ jobs: needs: [tag, npm] if: needs.tag.outputs.is_rc == 'true' steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - run: scripts/await-release.sh ${{ needs.tag.outputs.tag }} rc 900 docker: diff --git a/.github/workflows/publish-stable.yml b/.github/workflows/publish-stable.yml index 7bf82a6fe45d..2c4dbf7d6221 100644 --- a/.github/workflows/publish-stable.yml +++ b/.github/workflows/publish-stable.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 @@ -65,7 +65,7 @@ jobs: needs: [tag, binaries] if: needs.tag.outputs.is_stable == 'true' steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: fetch-depth: 0 # Needs full depth for changelog generation @@ -77,14 +77,14 @@ jobs: run: node scripts/generate_changelog.mjs ${{ needs.tag.outputs.prev_tag }} ${{ needs.tag.outputs.tag }} CHANGELOG.md - name: Get binaries - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: path: dist/ merge-multiple: true - name: Create Release id: create_release - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2.5.0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: @@ -101,7 +101,7 @@ jobs: # In case of failure - name: Rollback on failure if: failure() - uses: author/action-rollback@1.0.4 + uses: author/action-rollback@f4473931b1155b601092ec00eb1fa4882b80219f # 1.0.4 with: release_id: ${{ steps.create_release.outputs.id }} tag: ${{ needs.tag.outputs.tag }} @@ -115,7 +115,7 @@ jobs: needs: [tag, npm] if: needs.tag.outputs.is_stable == 'true' steps: - - uses: nflaig/release-comment-on-pr@v1 + - uses: nflaig/release-comment-on-pr@addf0108e7a60a9aaef4eee2900d69b126a75a5d # v1 with: token: ${{ secrets.GH_PAGES_TOKEN }} @@ -125,7 +125,7 @@ jobs: needs: [tag, npm] if: needs.tag.outputs.is_stable == 'true' steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - run: scripts/await-release.sh ${{ needs.tag.outputs.tag }} latest 900 docker: diff --git a/.github/workflows/test-sim.yml b/.github/workflows/test-sim.yml index 6cfaf1be3c27..1020b272bafc 100644 --- a/.github/workflows/test-sim.yml +++ b/.github/workflows/test-sim.yml @@ -28,7 +28,7 @@ jobs: runs-on: buildjet-4vcpu-ubuntu-2204 steps: # - Uses YAML anchors in the future - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - uses: "./.github/actions/setup-and-build" with: node: 24 @@ -39,7 +39,7 @@ jobs: runs-on: buildjet-4vcpu-ubuntu-2204 steps: # - Uses YAML anchors in the future - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - uses: "./.github/actions/setup-and-build" with: node: 24 @@ -57,7 +57,7 @@ jobs: GENESIS_DELAY_SLOTS: ${{github.event.inputs.genesisDelaySlots}} - name: Upload debug log test files for "packages/cli" if: ${{ always() }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: sim-test-multifork-logs path: packages/cli/test-logs @@ -68,7 +68,7 @@ jobs: runs-on: buildjet-4vcpu-ubuntu-2204 steps: # - Uses YAML anchors in the future - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - uses: "./.github/actions/setup-and-build" with: node: 24 @@ -86,7 +86,7 @@ jobs: GENESIS_DELAY_SLOTS: ${{github.event.inputs.genesisDelaySlots}} - name: Upload debug log test files for "packages/cli" if: ${{ always() }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: sim-test-endpoints-logs path: packages/cli/test-logs @@ -97,7 +97,7 @@ jobs: runs-on: buildjet-4vcpu-ubuntu-2204 steps: # - Uses YAML anchors in the future - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - uses: "./.github/actions/setup-and-build" with: node: 24 @@ -115,7 +115,7 @@ jobs: GENESIS_DELAY_SLOTS: ${{github.event.inputs.genesisDelaySlots}} - name: Upload debug log test files for "packages/cli" if: ${{ always() }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: sim-test-eth-backup-provider-logs path: packages/cli/test-logs @@ -126,7 +126,7 @@ jobs: runs-on: buildjet-4vcpu-ubuntu-2204 steps: # - Uses YAML anchors in the future - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - uses: "./.github/actions/setup-and-build" with: node: 24 @@ -144,7 +144,7 @@ jobs: GENESIS_DELAY_SLOTS: ${{github.event.inputs.genesisDelaySlots}} - name: Upload debug log test files for "packages/cli" if: ${{ always() }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: sim-test-mixed-clients-logs path: packages/cli/test-logs diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8b0e2d1dbd50..11fe85bf3bcb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,7 +22,7 @@ jobs: node: [24] steps: # - Uses YAML anchors in the future - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - uses: "./.github/actions/setup-and-build" with: node: ${{ matrix.node }} @@ -44,7 +44,7 @@ jobs: matrix: node: [24] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - uses: "./.github/actions/setup-and-build" with: node: ${{ matrix.node }} @@ -70,7 +70,7 @@ jobs: matrix: node: [24] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - uses: "./.github/actions/setup-and-build" with: @@ -91,14 +91,14 @@ jobs: matrix: node: [24] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - uses: "./.github/actions/setup-and-build" with: node: ${{ matrix.node }} # Cache validator slashing protection data tests - name: Restore spec tests cache - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: packages/validator/spec-tests key: spec-test-data-${{ hashFiles('packages/validator/test/spec/params.ts') }} @@ -114,7 +114,7 @@ jobs: # if: ${{ failure() && steps.unit_tests.conclusion == 'failure' }} - name: Upload coverage data - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0 with: token: ${{ secrets.CODECOV_TOKEN }} fail_ci_if_error: true @@ -131,7 +131,7 @@ jobs: node: [24] steps: # - Uses YAML anchors in the future - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - uses: "./.github/actions/setup-and-build" with: @@ -153,7 +153,7 @@ jobs: - name: Upload debug log test for test env if: ${{ always() }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: debug-e2e-test-logs-node-${{matrix.node}} path: test-logs/e2e-test-env @@ -168,7 +168,7 @@ jobs: node: [24] steps: # - Uses YAML anchors in the future - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - uses: "./.github/actions/setup-and-build" with: node: ${{ matrix.node }} @@ -189,14 +189,14 @@ jobs: matrix: node: [24] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - uses: "./.github/actions/setup-and-build" with: node: ${{ matrix.node }} # Download spec tests with cache - name: Restore spec tests cache - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: packages/beacon-node/spec-tests key: spec-test-data-${{ hashFiles('packages/beacon-node/test/spec/specTestVersioning.ts') }} From e6c853da0ed15d4bd6f2c183fc9077f770c449b2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Mar 2026 15:08:41 +0000 Subject: [PATCH 27/63] chore(deps): bump the actions group with 13 updates (#9018) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps the actions group with 13 updates: | Package | From | To | | --- | --- | --- | | [actions/checkout](https://github.com/actions/checkout) | `4.3.1` | `6.0.2` | | [actions/cache](https://github.com/actions/cache) | `4.3.0` | `5.0.3` | | [actions/upload-artifact](https://github.com/actions/upload-artifact) | `4.6.2` | `7.0.0` | | [actions/github-script](https://github.com/actions/github-script) | `7.1.0` | `8.0.0` | | [github/codeql-action](https://github.com/github/codeql-action) | `8dca8a82e2fa1a2c8908956f711300f9c4a4f4f6` | `b8d3b6e8af63cde30bdc382c0bc28114f4346c88` | | [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) | `3.12.0` | `4.0.0` | | [docker/login-action](https://github.com/docker/login-action) | `3.7.0` | `4.0.0` | | [pnpm/action-setup](https://github.com/pnpm/action-setup) | `c5ba7f7862a0f64c1b1a05fbac13e0b8e86ba08c` | `41ff72655975bd51cab0327fa583b6e92b6d3061` | | [rojopolis/spellcheck-github-actions](https://github.com/rojopolis/spellcheck-github-actions) | `531492f4dd27f0593ca398f2ed225d1b92eec828` | `ac9fbc2d852c70d7e9d305a62bdd7b1517c50e3d` | | [peaceiris/actions-gh-pages](https://github.com/peaceiris/actions-gh-pages) | `4a2e02b36f31d8974a0d09d3bb9f3172aa2d0d0d` | `373f7f263a76c20808c831209c920827a82a2847` | | [amannn/action-semantic-pull-request](https://github.com/amannn/action-semantic-pull-request) | `5` | `6` | | [actions/download-artifact](https://github.com/actions/download-artifact) | `4.3.0` | `8.0.0` | | [codecov/codecov-action](https://github.com/codecov/codecov-action) | `4.6.0` | `5.5.2` | Updates `actions/checkout` from 4.3.1 to 6.0.2
Release notes

Sourced from actions/checkout's releases.

v6.0.2

What's Changed

Full Changelog: https://github.com/actions/checkout/compare/v6.0.1...v6.0.2

v6.0.1

What's Changed

Full Changelog: https://github.com/actions/checkout/compare/v6...v6.0.1

v6.0.0

What's Changed

Full Changelog: https://github.com/actions/checkout/compare/v5.0.0...v6.0.0

v6-beta

What's Changed

Updated persist-credentials to store the credentials under $RUNNER_TEMP instead of directly in the local git config.

This requires a minimum Actions Runner version of v2.329.0 to access the persisted credentials for Docker container action scenarios.

v5.0.1

What's Changed

Full Changelog: https://github.com/actions/checkout/compare/v5...v5.0.1

v5.0.0

What's Changed

⚠️ Minimum Compatible Runner Version

v2.327.1
Release Notes

... (truncated)

Changelog

Sourced from actions/checkout's changelog.

Changelog

v6.0.2

v6.0.1

v6.0.0

v5.0.1

v5.0.0

v4.3.1

v4.3.0

v4.2.2

v4.2.1

v4.2.0

v4.1.7

v4.1.6

... (truncated)

Commits

Updates `actions/cache` from 4.3.0 to 5.0.3
Release notes

Sourced from actions/cache's releases.

v5.0.3

What's Changed

Full Changelog: https://github.com/actions/cache/compare/v5...v5.0.3

v.5.0.2

v5.0.2

What's Changed

When creating cache entries, 429s returned from the cache service will not be retried.

v5.0.1

[!IMPORTANT] actions/cache@v5 runs on the Node.js 24 runtime and requires a minimum Actions Runner version of 2.327.1.

If you are using self-hosted runners, ensure they are updated before upgrading.

v5.0.1

What's Changed

v5.0.0

What's Changed

Full Changelog: https://github.com/actions/cache/compare/v5...v5.0.1

v5.0.0

[!IMPORTANT] actions/cache@v5 runs on the Node.js 24 runtime and requires a minimum Actions Runner version of 2.327.1.

If you are using self-hosted runners, ensure they are updated before upgrading.

What's Changed

... (truncated)

Commits

Updates `actions/upload-artifact` from 4.6.2 to 7.0.0
Release notes

Sourced from actions/upload-artifact's releases.

v7.0.0

v7 What's new

Direct Uploads

Adds support for uploading single files directly (unzipped). Callers can set the new archive parameter to false to skip zipping the file during upload. Right now, we only support single files. The action will fail if the glob passed resolves to multiple files. The name parameter is also ignored with this setting. Instead, the name of the artifact will be the name of the uploaded file.

ESM

To support new versions of the @actions/* packages, we've upgraded the package to ESM.

What's Changed

New Contributors

Full Changelog: https://github.com/actions/upload-artifact/compare/v6...v7.0.0

v6.0.0

v6 - What's new

[!IMPORTANT] actions/upload-artifact@v6 now runs on Node.js 24 (runs.using: node24) and requires a minimum Actions Runner version of 2.327.1. If you are using self-hosted runners, ensure they are updated before upgrading.

Node.js 24

This release updates the runtime to Node.js 24. v5 had preliminary support for Node.js 24, however this action was by default still running on Node.js 20. Now this action by default will run on Node.js 24.

What's Changed

Full Changelog: https://github.com/actions/upload-artifact/compare/v5.0.0...v6.0.0

v5.0.0

What's Changed

BREAKING CHANGE: this update supports Node v24.x. This is not a breaking change per-se but we're treating it as such.

... (truncated)

Commits
  • bbbca2d Support direct file uploads (#764)
  • 589182c Upgrade the module to ESM and bump dependencies (#762)
  • 47309c9 Merge pull request #754 from actions/Link-/add-proxy-integration-tests
  • 02a8460 Add proxy integration test
  • b7c566a Merge pull request #745 from actions/upload-artifact-v6-release
  • e516bc8 docs: correct description of Node.js 24 support in README
  • ddc45ed docs: update README to correct action name for Node.js 24 support
  • 615b319 chore: release v6.0.0 for Node.js 24 support
  • 017748b Merge pull request #744 from actions/fix-storage-blob
  • 38d4c79 chore: rebuild dist
  • Additional commits viewable in compare view

Updates `actions/github-script` from 7.1.0 to 8.0.0
Release notes

Sourced from actions/github-script's releases.

v8.0.0

What's Changed

⚠️ Minimum Compatible Runner Version

v2.327.1
Release Notes

Make sure your runner is updated to this version or newer to use this release.

New Contributors

Full Changelog: https://github.com/actions/github-script/compare/v7.1.0...v8.0.0

Commits

Updates `github/codeql-action` from 8dca8a82e2fa1a2c8908956f711300f9c4a4f4f6 to b8d3b6e8af63cde30bdc382c0bc28114f4346c88
Changelog

Sourced from github/codeql-action's changelog.

CodeQL Action Changelog

See the releases page for the relevant changes to the CodeQL CLI and language packs.

[UNRELEASED]

  • Fixed a bug which caused the CodeQL Action to fail loading repository properties if a "Multi select" repository property was configured for the repository. #3557
  • The CodeQL Action now loads custom repository properties on GitHub Enterprise Server, enabling the customization of features such as github-codeql-disable-overlay that was previously only available on GitHub.com. #3559

4.32.6 - 05 Mar 2026

  • Update default CodeQL bundle version to 2.24.3. #3548

4.32.5 - 02 Mar 2026

  • Repositories owned by an organization can now set up the github-codeql-disable-overlay custom repository property to disable improved incremental analysis for CodeQL. First, create a custom repository property with the name github-codeql-disable-overlay and the type "True/false" in the organization's settings. Then in the repository's settings, set this property to true to disable improved incremental analysis. For more information, see Managing custom properties for repositories in your organization. This feature is not yet available on GitHub Enterprise Server. #3507
  • Added an experimental change so that when improved incremental analysis fails on a runner — potentially due to insufficient disk space — the failure is recorded in the Actions cache so that subsequent runs will automatically skip improved incremental analysis until something changes (e.g. a larger runner is provisioned or a new CodeQL version is released). We expect to roll this change out to everyone in March. #3487
  • The minimum memory check for improved incremental analysis is now skipped for CodeQL 2.24.3 and later, which has reduced peak RAM usage. #3515
  • Reduced log levels for best-effort private package registry connection check failures to reduce noise from workflow annotations. #3516
  • Added an experimental change which lowers the minimum disk space requirement for improved incremental analysis, enabling it to run on standard GitHub Actions runners. We expect to roll this change out to everyone in March. #3498
  • Added an experimental change which allows the start-proxy action to resolve the CodeQL CLI version from feature flags instead of using the linked CLI bundle version. We expect to roll this change out to everyone in March. #3512
  • The previously experimental changes from versions 4.32.3, 4.32.4, 3.32.3 and 3.32.4 are now enabled by default. #3503, #3504

4.32.4 - 20 Feb 2026

  • Update default CodeQL bundle version to 2.24.2. #3493
  • Added an experimental change which improves how certificates are generated for the authentication proxy that is used by the CodeQL Action in Default Setup when private package registries are configured. This is expected to generate more widely compatible certificates and should have no impact on analyses which are working correctly already. We expect to roll this change out to everyone in February. #3473
  • When the CodeQL Action is run with debugging enabled in Default Setup and private package registries are configured, the "Setup proxy for registries" step will output additional diagnostic information that can be used for troubleshooting. #3486
  • Added a setting which allows the CodeQL Action to enable network debugging for Java programs. This will help GitHub staff support customers with troubleshooting issues in GitHub-managed CodeQL workflows, such as Default Setup. This setting can only be enabled by GitHub staff. #3485
  • Added a setting which enables GitHub-managed workflows, such as Default Setup, to use a nightly CodeQL CLI release instead of the latest, stable release that is used by default. This will help GitHub staff support customers whose analyses for a given repository or organization require early access to a change in an upcoming CodeQL CLI release. This setting can only be enabled by GitHub staff. #3484

4.32.3 - 13 Feb 2026

  • Added experimental support for testing connections to private package registries. This feature is not currently enabled for any analysis. In the future, it may be enabled by default for Default Setup. #3466

4.32.2 - 05 Feb 2026

  • Update default CodeQL bundle version to 2.24.1. #3460

4.32.1 - 02 Feb 2026

  • A warning is now shown in Default Setup workflow logs if a private package registry is configured using a GitHub Personal Access Token (PAT), but no username is configured. #3422
  • Fixed a bug which caused the CodeQL Action to fail when repository properties cannot successfully be retrieved. #3421

4.32.0 - 26 Jan 2026

  • Update default CodeQL bundle version to 2.24.0. #3425

4.31.11 - 23 Jan 2026

... (truncated)

Commits

Updates `docker/setup-buildx-action` from 3.12.0 to 4.0.0
Release notes

Sourced from docker/setup-buildx-action's releases.

v4.0.0

Full Changelog: https://github.com/docker/setup-buildx-action/compare/v3.12.0...v4.0.0

Commits
  • 4d04d5d Merge pull request #485 from docker/dependabot/npm_and_yarn/docker/actions-to...
  • cd74e05 chore: update generated content
  • eee38ec build(deps): bump @​docker/actions-toolkit from 0.77.0 to 0.79.0
  • 7a83f65 Merge pull request #484 from docker/dependabot/github_actions/docker/setup-qe...
  • a5aa967 Merge pull request #464 from crazy-max/rm-deprecated
  • e73d53f build(deps): bump docker/setup-qemu-action from 3 to 4
  • 28a438e Merge pull request #483 from crazy-max/node24
  • 034e9d3 chore: update generated content
  • b4664d8 remove deprecated inputs/outputs
  • a8257de node 24 as default runtime
  • Additional commits viewable in compare view

Updates `docker/login-action` from 3.7.0 to 4.0.0
Release notes

Sourced from docker/login-action's releases.

v4.0.0

Full Changelog: https://github.com/docker/login-action/compare/v3.7.0...v4.0.0

Commits
  • b45d80f Merge pull request #929 from crazy-max/node24
  • 176cb9c node 24 as default runtime
  • cad8984 Merge pull request #920 from docker/dependabot/npm_and_yarn/aws-sdk-dependenc...
  • 92cbcb2 chore: update generated content
  • 5a2d6a7 build(deps): bump the aws-sdk-dependencies group with 2 updates
  • 44512b6 Merge pull request #928 from docker/dependabot/npm_and_yarn/docker/actions-to...
  • 28737a5 chore: update generated content
  • dac0793 build(deps): bump @​docker/actions-toolkit from 0.76.0 to 0.77.0
  • 62029f3 Merge pull request #919 from docker/dependabot/npm_and_yarn/actions/core-3.0.0
  • 08c8f06 chore: update generated content
  • Additional commits viewable in compare view

Updates `pnpm/action-setup` from c5ba7f7862a0f64c1b1a05fbac13e0b8e86ba08c to 41ff72655975bd51cab0327fa583b6e92b6d3061
Commits

Updates `rojopolis/spellcheck-github-actions` from 531492f4dd27f0593ca398f2ed225d1b92eec828 to ac9fbc2d852c70d7e9d305a62bdd7b1517c50e3d
Changelog

Sourced from rojopolis/spellcheck-github-actions's changelog.

Change Log for spellcheck-github-actions

0.59.0, 2026-03-02, feature release, update recommended

  • Improvements have been added to the docker entrypoint, based on a PR from @​akohout-hai which fixes an issue with handling of spaces in files names and directories, see PR #322 for details. This is his first contribution to the project and I want to thank him for his contribution, which is highly appreciated.

  • Docker based image updated to Python 3.14.3 slim trixie via PR #320 from Dependabot.

0.58.0, 2026-01-20, security release, update not required

0.57.0, 2026-01-14, maintenance release, update not required

  • Docker based image updated to Python 3.14.2 slim trixie via PR #310 from Dependabot. The version is the same, the image has had updates.

0.56.0, 2025-12-27, feature and maintenance release, update not required

0.55.0, 2025-11-27, maintenance release, update not required

  • Via an issue #293 from @​shoverbj, an update to the core component PySpelling from version 2.12.0 to version 2.12.1 was made, this allows for use of large dictionaries with Aspell

0.54.0, 2025-11-05, feature release, update not required

0.53.0, 2025-10-25, maintenance release, update not required

  • Docker image updated to Python 3.14.0 trixie slim Release notes for Python 3.14.0, this originated from the PR mentioned below, however updated to Trixie from Bookworm and as always the slim variant is used

  • Bumped the requirement for cython to 3.0.11 or above, addressing a build issue with lxml, located when testing the PR : #274 from @​dependabot, the above update of Python

  • In general the Docker build file had a few updates since the above changes required some tweaking of the Dockerfile

    • Order of installation of dependencies adjusted to ensure that lxml can build correctly
    • Installation of:
      • build-essential
      • pkg-config
      • libxml2-dev
      • libxslt1-dev
      • zlib1g-dev

0.52.0, 2025-09-10, feature release, update not required

... (truncated)

Commits

Updates `peaceiris/actions-gh-pages` from 4a2e02b36f31d8974a0d09d3bb9f3172aa2d0d0d to 373f7f263a76c20808c831209c920827a82a2847
Changelog

Sourced from peaceiris/actions-gh-pages's changelog.

Changelog

All notable changes to this project will be documented in this file. See standard-version for commit guidelines.

4.0.0 (2024-04-08)

build

chore

ci

docs

3.9.3 (2023-03-30)

docs

fix

3.9.2 (2023-01-17)

chore

... (truncated)

Commits

Updates `amannn/action-semantic-pull-request` from 5 to 6
Release notes

Sourced from amannn/action-semantic-pull-request's releases.

v6.0.0

6.0.0 (2025-08-13)

⚠ BREAKING CHANGES

  • Upgrade action to use Node.js 24 and ESM (#287)

Features

  • Upgrade action to use Node.js 24 and ESM (#287) (bc0c9a7)

v5.5.3

5.5.3 (2024-06-28)

Bug Fixes

v5.5.2

5.5.2 (2024-04-24)

Bug Fixes

v5.5.1

5.5.1 (2024-04-24)

Bug Fixes

v5.5.0

5.5.0 (2024-04-23)

Features

v5.4.0

5.4.0 (2023-11-03)

... (truncated)

Changelog

Sourced from amannn/action-semantic-pull-request's changelog.

Changelog

6.1.1 (2025-08-22)

Bug Fixes

  • Parse headerPatternCorrespondence properly (#295) (800da4c)

6.1.0 (2025-08-19)

Features

Bug Fixes

  • Remove trailing whitespace from "unknown release type" error message (#291) (afa4edb)

6.0.1 (2025-08-13)

Bug Fixes

6.0.0 (2025-08-13)

⚠ BREAKING CHANGES

  • Upgrade action to use Node.js 24 and ESM (#287)

Features

  • Upgrade action to use Node.js 24 and ESM (#287) (bc0c9a7)

5.5.3 (2024-06-28)

Bug Fixes

5.5.2 (2024-04-24)

Bug Fixes

- Uses YAML anchors in the future - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: "./.github/actions/setup-and-build" with: node: 24 @@ -29,4 +29,4 @@ jobs: # Run spellcheck AFTER building docs, in case the CLI reference has issues - name: Spellcheck - uses: rojopolis/spellcheck-github-actions@531492f4dd27f0593ca398f2ed225d1b92eec828 # 0.32.0 + uses: rojopolis/spellcheck-github-actions@ac9fbc2d852c70d7e9d305a62bdd7b1517c50e3d # 0.32.0 diff --git a/.github/workflows/docs-version.yml b/.github/workflows/docs-version.yml index 30c0ad799cf4..0d5f36940cf7 100644 --- a/.github/workflows/docs-version.yml +++ b/.github/workflows/docs-version.yml @@ -33,12 +33,12 @@ jobs: fi # Checkout the tagged release to build its docs - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ github.ref }} - name: Install pnpm - uses: pnpm/action-setup@c5ba7f7862a0f64c1b1a05fbac13e0b8e86ba08c # v4 + uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4 - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 8f73dd7e1c0e..c0dd4c7ad9a8 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -30,12 +30,12 @@ jobs: - name: Log Deployment Ref run: echo "Deploying docs from ref $DEPLOY_REF" - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ env.DEPLOY_REF }} - name: Install pnpm before setup node - uses: pnpm/action-setup@c5ba7f7862a0f64c1b1a05fbac13e0b8e86ba08c # v4 + uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4 - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: @@ -48,7 +48,7 @@ jobs: run: echo "v8CppApiVersion=$(node --print "process.versions.modules")" >> $GITHUB_OUTPUT - name: Restore dependencies - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 id: cache-deps with: path: | @@ -91,7 +91,7 @@ jobs: run: pnpm install && pnpm build - name: Deploy - uses: peaceiris/actions-gh-pages@4a2e02b36f31d8974a0d09d3bb9f3172aa2d0d0d # v3 + uses: peaceiris/actions-gh-pages@373f7f263a76c20808c831209c920827a82a2847 # v3 with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: ./docs/build diff --git a/.github/workflows/kurtosis.yml b/.github/workflows/kurtosis.yml index 4d17d9c50d66..673d0a36b31a 100644 --- a/.github/workflows/kurtosis.yml +++ b/.github/workflows/kurtosis.yml @@ -11,11 +11,11 @@ jobs: name: Build and run Kurtosis runs-on: buildjet-4vcpu-ubuntu-2204 steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Build Docker image - local only run: > docker buildx build . --load \ diff --git a/.github/workflows/lint-pr-title.yml b/.github/workflows/lint-pr-title.yml index 49270f091a95..4157ee34d802 100644 --- a/.github/workflows/lint-pr-title.yml +++ b/.github/workflows/lint-pr-title.yml @@ -14,7 +14,7 @@ jobs: name: Validate PR title runs-on: ubuntu-latest steps: - - uses: amannn/action-semantic-pull-request@e32d7e603df1aa1ba07e981f2a23455dee596825 # v5 + - uses: amannn/action-semantic-pull-request@48f256284bd46cdaab1048c3721360e808335d50 # v6.1.1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: diff --git a/.github/workflows/publish-dev.yml b/.github/workflows/publish-dev.yml index 479b0495125e..e11b96ba08a9 100644 --- a/.github/workflows/publish-dev.yml +++ b/.github/workflows/publish-dev.yml @@ -24,7 +24,7 @@ jobs: fetch-depth: 0 - name: Install pnpm before setup node - uses: pnpm/action-setup@c5ba7f7862a0f64c1b1a05fbac13e0b8e86ba08c # v4 + uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4 - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: @@ -36,7 +36,7 @@ jobs: id: node run: echo "v8CppApiVersion=$(node --print "process.versions.modules")" >> $GITHUB_OUTPUT - name: Restore dependencies - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 id: cache-deps with: path: | diff --git a/.github/workflows/publish-manual.yml b/.github/workflows/publish-manual.yml index 34a279719dd5..db7954978ae3 100644 --- a/.github/workflows/publish-manual.yml +++ b/.github/workflows/publish-manual.yml @@ -23,7 +23,7 @@ jobs: name: Validate inputs runs-on: ubuntu-latest steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ inputs.ref }} fetch-depth: 0 diff --git a/.github/workflows/publish-nextfork.yml b/.github/workflows/publish-nextfork.yml index 80413ab1df0c..8ccac06fcc93 100644 --- a/.github/workflows/publish-nextfork.yml +++ b/.github/workflows/publish-nextfork.yml @@ -26,7 +26,7 @@ jobs: with: fetch-depth: 0 - name: Install pnpm before setup node - uses: pnpm/action-setup@c5ba7f7862a0f64c1b1a05fbac13e0b8e86ba08c # v4 + uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4 - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version: 24 @@ -37,7 +37,7 @@ jobs: id: node run: echo "v8CppApiVersion=$(node --print "process.versions.modules")" >> $GITHUB_OUTPUT - name: Restore dependencies - uses: actions/cache@0769f2e44373d687c841be56f618397100943c5e # master + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # master id: cache-deps with: path: | diff --git a/.github/workflows/publish-rc.yml b/.github/workflows/publish-rc.yml index 3a7b0058cc90..c2dc5537b954 100644 --- a/.github/workflows/publish-rc.yml +++ b/.github/workflows/publish-rc.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 @@ -71,7 +71,7 @@ jobs: run: node scripts/generate_changelog.mjs ${{ needs.tag.outputs.prev_tag }} ${{ needs.tag.outputs.tag }} CHANGELOG.md - name: Get binaries - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: path: dist/ merge-multiple: true @@ -125,7 +125,7 @@ jobs: needs: [tag, npm] if: needs.tag.outputs.is_rc == 'true' steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - run: scripts/await-release.sh ${{ needs.tag.outputs.tag }} rc 900 docker: diff --git a/.github/workflows/publish-stable.yml b/.github/workflows/publish-stable.yml index 2c4dbf7d6221..02fa0e627dbb 100644 --- a/.github/workflows/publish-stable.yml +++ b/.github/workflows/publish-stable.yml @@ -65,7 +65,7 @@ jobs: needs: [tag, binaries] if: needs.tag.outputs.is_stable == 'true' steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 # Needs full depth for changelog generation @@ -77,7 +77,7 @@ jobs: run: node scripts/generate_changelog.mjs ${{ needs.tag.outputs.prev_tag }} ${{ needs.tag.outputs.tag }} CHANGELOG.md - name: Get binaries - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: path: dist/ merge-multiple: true @@ -125,7 +125,7 @@ jobs: needs: [tag, npm] if: needs.tag.outputs.is_stable == 'true' steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - run: scripts/await-release.sh ${{ needs.tag.outputs.tag }} latest 900 docker: diff --git a/.github/workflows/test-sim.yml b/.github/workflows/test-sim.yml index 1020b272bafc..cede48a6ede7 100644 --- a/.github/workflows/test-sim.yml +++ b/.github/workflows/test-sim.yml @@ -28,7 +28,7 @@ jobs: runs-on: buildjet-4vcpu-ubuntu-2204 steps: # - Uses YAML anchors in the future - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: "./.github/actions/setup-and-build" with: node: 24 @@ -39,7 +39,7 @@ jobs: runs-on: buildjet-4vcpu-ubuntu-2204 steps: # - Uses YAML anchors in the future - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: "./.github/actions/setup-and-build" with: node: 24 @@ -57,7 +57,7 @@ jobs: GENESIS_DELAY_SLOTS: ${{github.event.inputs.genesisDelaySlots}} - name: Upload debug log test files for "packages/cli" if: ${{ always() }} - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: sim-test-multifork-logs path: packages/cli/test-logs @@ -68,7 +68,7 @@ jobs: runs-on: buildjet-4vcpu-ubuntu-2204 steps: # - Uses YAML anchors in the future - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: "./.github/actions/setup-and-build" with: node: 24 @@ -86,7 +86,7 @@ jobs: GENESIS_DELAY_SLOTS: ${{github.event.inputs.genesisDelaySlots}} - name: Upload debug log test files for "packages/cli" if: ${{ always() }} - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: sim-test-endpoints-logs path: packages/cli/test-logs @@ -97,7 +97,7 @@ jobs: runs-on: buildjet-4vcpu-ubuntu-2204 steps: # - Uses YAML anchors in the future - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: "./.github/actions/setup-and-build" with: node: 24 @@ -115,7 +115,7 @@ jobs: GENESIS_DELAY_SLOTS: ${{github.event.inputs.genesisDelaySlots}} - name: Upload debug log test files for "packages/cli" if: ${{ always() }} - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: sim-test-eth-backup-provider-logs path: packages/cli/test-logs @@ -126,7 +126,7 @@ jobs: runs-on: buildjet-4vcpu-ubuntu-2204 steps: # - Uses YAML anchors in the future - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: "./.github/actions/setup-and-build" with: node: 24 @@ -144,7 +144,7 @@ jobs: GENESIS_DELAY_SLOTS: ${{github.event.inputs.genesisDelaySlots}} - name: Upload debug log test files for "packages/cli" if: ${{ always() }} - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: sim-test-mixed-clients-logs path: packages/cli/test-logs diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 11fe85bf3bcb..af3f3e1b7e7e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,7 +22,7 @@ jobs: node: [24] steps: # - Uses YAML anchors in the future - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: "./.github/actions/setup-and-build" with: node: ${{ matrix.node }} @@ -44,7 +44,7 @@ jobs: matrix: node: [24] steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: "./.github/actions/setup-and-build" with: node: ${{ matrix.node }} @@ -70,7 +70,7 @@ jobs: matrix: node: [24] steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: "./.github/actions/setup-and-build" with: @@ -91,14 +91,14 @@ jobs: matrix: node: [24] steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: "./.github/actions/setup-and-build" with: node: ${{ matrix.node }} # Cache validator slashing protection data tests - name: Restore spec tests cache - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 with: path: packages/validator/spec-tests key: spec-test-data-${{ hashFiles('packages/validator/test/spec/params.ts') }} @@ -114,7 +114,7 @@ jobs: # if: ${{ failure() && steps.unit_tests.conclusion == 'failure' }} - name: Upload coverage data - uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0 + uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 with: token: ${{ secrets.CODECOV_TOKEN }} fail_ci_if_error: true @@ -131,7 +131,7 @@ jobs: node: [24] steps: # - Uses YAML anchors in the future - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: "./.github/actions/setup-and-build" with: @@ -153,7 +153,7 @@ jobs: - name: Upload debug log test for test env if: ${{ always() }} - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: debug-e2e-test-logs-node-${{matrix.node}} path: test-logs/e2e-test-env @@ -168,7 +168,7 @@ jobs: node: [24] steps: # - Uses YAML anchors in the future - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: "./.github/actions/setup-and-build" with: node: ${{ matrix.node }} @@ -189,14 +189,14 @@ jobs: matrix: node: [24] steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: "./.github/actions/setup-and-build" with: node: ${{ matrix.node }} # Download spec tests with cache - name: Restore spec tests cache - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 with: path: packages/beacon-node/spec-tests key: spec-test-data-${{ hashFiles('packages/beacon-node/test/spec/specTestVersioning.ts') }} From b847afb846a163fd35c097ddfd40da56325e0e6d Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Tue, 10 Mar 2026 18:59:57 +0000 Subject: [PATCH 28/63] fix: prune serialized cache by block input keys (#9007) - add `getSerializedCacheKeys()` to block input variants - more closely couple `SerializedCache` and `SeenBlockInput` - delete only relevant serialized cache object references after block input is written - remove global `SerializedCache.clear()` usage to avoid premature cache eviction --- dashboards/lodestar_beacon_chain.json | 12 +++++ .../src/chain/blocks/blockInput/blockInput.ts | 35 +++++++++++++ .../src/chain/blocks/blockInput/types.ts | 5 ++ .../src/chain/blocks/writeBlockInputToDb.ts | 19 +------ packages/beacon-node/src/chain/chain.ts | 4 +- .../chain/seenCache/seenGossipBlockInput.ts | 48 +++++++++++++---- .../src/metrics/metrics/lodestar.ts | 4 ++ packages/beacon-node/src/network/network.ts | 3 +- .../beacon-node/src/util/serializedCache.ts | 12 +++-- .../chain/seenCache/seenBlockInput.test.ts | 52 +++++++++++++++++++ packages/beacon-node/test/utils/blockInput.ts | 4 ++ 11 files changed, 162 insertions(+), 36 deletions(-) diff --git a/dashboards/lodestar_beacon_chain.json b/dashboards/lodestar_beacon_chain.json index d65beb99c217..0d738393de53 100644 --- a/dashboards/lodestar_beacon_chain.json +++ b/dashboards/lodestar_beacon_chain.json @@ -388,6 +388,18 @@ "legendFormat": "created_by_blob", "range": true, "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "lodestar_seen_block_input_cache_serialized_object_count", + "instant": false, + "legendFormat": "serialized_objects", + "range": true, + "refId": "G" } ], "title": "Block Input", diff --git a/packages/beacon-node/src/chain/blocks/blockInput/blockInput.ts b/packages/beacon-node/src/chain/blocks/blockInput/blockInput.ts index 875b55b1ac12..65eed2f85437 100644 --- a/packages/beacon-node/src/chain/blocks/blockInput/blockInput.ts +++ b/packages/beacon-node/src/chain/blocks/blockInput/blockInput.ts @@ -105,6 +105,7 @@ abstract class AbstractBlockInput): void; + abstract getSerializedCacheKeys(): object[]; hasBlock(): boolean { return this.state.hasBlock; @@ -243,6 +244,10 @@ export class BlockInputPreData extends AbstractBlockInput { ); } } + + getSerializedCacheKeys(): object[] { + return [this.state.block]; + } } // Blobs DA @@ -526,6 +531,20 @@ export class BlockInputBlobs extends AbstractBlockInput blobSidecar); } + + getSerializedCacheKeys(): object[] { + const objects: object[] = []; + + if (this.state.hasBlock) { + objects.push(this.state.block); + } + + for (const {blobSidecar} of this.blobsCache.values()) { + objects.push(blobSidecar); + } + + return objects; + } } function blockAndBlobArePaired(block: SignedBeaconBlock, blobSidecar: deneb.BlobSidecar): boolean { @@ -906,6 +925,18 @@ export class BlockInputColumns extends AbstractBlockInput { return (this.state.block.message.body as gloas.BeaconBlockBody).signedExecutionPayloadBid.message .blobKzgCommitments; } + + getSerializedCacheKeys(): object[] { + return [this.state.block]; + } } diff --git a/packages/beacon-node/src/chain/blocks/blockInput/types.ts b/packages/beacon-node/src/chain/blocks/blockInput/types.ts index d6acd478fc1a..e932360a9269 100644 --- a/packages/beacon-node/src/chain/blocks/blockInput/types.ts +++ b/packages/beacon-node/src/chain/blocks/blockInput/types.ts @@ -151,6 +151,11 @@ export interface IBlockInput>; waitForAllData(timeout: number, signal?: AbortSignal): Promise; diff --git a/packages/beacon-node/src/chain/blocks/writeBlockInputToDb.ts b/packages/beacon-node/src/chain/blocks/writeBlockInputToDb.ts index 4a8ec0f7dbe8..173044a2824f 100644 --- a/packages/beacon-node/src/chain/blocks/writeBlockInputToDb.ts +++ b/packages/beacon-node/src/chain/blocks/writeBlockInputToDb.ts @@ -3,13 +3,7 @@ import {SignedBeaconBlock} from "@lodestar/types"; import {fromHex, toRootHex} from "@lodestar/utils"; import {getBlobKzgCommitments} from "../../util/dataColumns.js"; import {BeaconChain} from "../chain.js"; -import { - IBlockInput, - IDataColumnsInput, - isBlockInputBlobs, - isBlockInputColumns, - isBlockInputNoData, -} from "./blockInput/index.js"; +import {IBlockInput, IDataColumnsInput, isBlockInputBlobs, isBlockInputColumns} from "./blockInput/index.js"; import {BLOB_AVAILABILITY_TIMEOUT} from "./verifyBlocksDataAvailability.js"; /** @@ -137,17 +131,6 @@ export async function persistBlockInput(this: BeaconChain, blockInput: IBlockInp }) .finally(() => { this.seenBlockInputCache.prune(blockInput.blockRootHex); - // Without forcefully clearing this cache, we would rely on WeakMap to evict memory which is not reliable. - // Clear here (after the DB write) so that writeBlockInputToDb can still use the cached serialized bytes. - // - // For Gloas (BlockInputNoData), the execution payload and columns arrive separately after the beacon block. - // Do NOT clear the cache here — it must remain available for writeDataColumnsToDb when the payload arrives. - // The cache is cleared in the Gloas payload persistence path instead. - if (!isBlockInputNoData(blockInput)) { - // TODO: enhance this SerializedCache for Gloas because payload may not come - // see https://github.com/ChainSafe/lodestar/pull/8974#discussion_r2885598229 - this.serializedCache.clear(); - } this.logger.debug("Pruned block input", { slot: blockInput.slot, root: blockInput.blockRootHex, diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 4aefd4b9aef3..5fbcf7534cec 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -320,12 +320,14 @@ export class BeaconChain implements IBeaconChain { this.beaconProposerCache = new BeaconProposerCache(opts); this.checkpointBalancesCache = new CheckpointBalancesCache(); + this.serializedCache = new SerializedCache(); this.seenBlockInputCache = new SeenBlockInput({ config, custodyConfig: this.custodyConfig, clock, chainEvents: emitter, signal, + serializedCache: this.serializedCache, metrics, logger, }); @@ -412,8 +414,6 @@ export class BeaconChain implements IBeaconChain { this.bls = bls; this.emitter = emitter; - this.serializedCache = new SerializedCache(); - this.getBlobsTracker = new GetBlobsTracker({ logger, executionEngine: this.executionEngine, diff --git a/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts b/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts index a11f2e2a03b0..3d739d265d2b 100644 --- a/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts +++ b/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts @@ -17,6 +17,7 @@ import {Metrics} from "../../metrics/metrics.js"; import {MAX_LOOK_AHEAD_EPOCHS} from "../../sync/constants.js"; import {IClock} from "../../util/clock.js"; import {CustodyConfig} from "../../util/dataColumns.js"; +import {SerializedCache} from "../../util/serializedCache.js"; import { BlockInput, BlockInputBlobs, @@ -55,6 +56,7 @@ export type SeenBlockInputCacheModules = { chainEvents: ChainEventEmitter; signal: AbortSignal; custodyConfig: CustodyConfig; + serializedCache: SerializedCache; metrics: Metrics | null; logger?: Logger; }; @@ -101,6 +103,7 @@ export class SeenBlockInput { private readonly clock: IClock; private readonly chainEvents: ChainEventEmitter; private readonly signal: AbortSignal; + private readonly serializedCache: SerializedCache; private readonly metrics: Metrics | null; private readonly logger?: Logger; private blockInputs = new Map(); @@ -109,19 +112,35 @@ export class SeenBlockInput { // and the signature to ensure we only skip verification if both match private verifiedProposerSignatures = new Map>(); - constructor({config, custodyConfig, clock, chainEvents, signal, metrics, logger}: SeenBlockInputCacheModules) { + constructor({ + config, + custodyConfig, + clock, + chainEvents, + signal, + serializedCache, + metrics, + logger, + }: SeenBlockInputCacheModules) { this.config = config; this.custodyConfig = custodyConfig; this.clock = clock; this.chainEvents = chainEvents; this.signal = signal; + this.serializedCache = serializedCache; this.metrics = metrics; this.logger = logger; if (metrics) { - metrics.seenCache.blockInput.blockInputCount.addCollect(() => - metrics.seenCache.blockInput.blockInputCount.set(this.blockInputs.size) - ); + metrics.seenCache.blockInput.blockInputCount.addCollect(() => { + metrics.seenCache.blockInput.blockInputCount.set(this.blockInputs.size); + metrics.seenCache.blockInput.serializedObjectCount.set( + Array.from(this.blockInputs.values()).reduce( + (count, blockInput) => count + blockInput.getSerializedCacheKeys().length, + 0 + ) + ); + }); } this.chainEvents.on(ChainEvent.forkChoiceFinalized, this.onFinalized); @@ -142,7 +161,10 @@ export class SeenBlockInput { * Removes the single BlockInput from the cache */ remove(rootHex: RootHex): void { - this.blockInputs.delete(rootHex); + const blockInput = this.blockInputs.get(rootHex); + if (blockInput) { + this.evictBlockInput(blockInput); + } } /** @@ -154,7 +176,7 @@ export class SeenBlockInput { let deletedCount = 0; while (blockInput) { deletedCount++; - this.blockInputs.delete(blockInput.blockRootHex); + this.evictBlockInput(blockInput); blockInput = this.blockInputs.get(parentRootHex ?? ""); parentRootHex = blockInput?.parentRootHex; } @@ -165,10 +187,10 @@ export class SeenBlockInput { onFinalized = (checkpoint: CheckpointWithHex) => { let deletedCount = 0; const cutoffSlot = computeStartSlotAtEpoch(checkpoint.epoch); - for (const [rootHex, blockInput] of this.blockInputs) { + for (const [, blockInput] of this.blockInputs) { if (blockInput.slot < cutoffSlot) { deletedCount++; - this.blockInputs.delete(rootHex); + this.evictBlockInput(blockInput); } } this.logger?.debug(`BlockInputCache.onFinalized deleted ${deletedCount} cached BlockInputs`); @@ -408,14 +430,20 @@ export class SeenBlockInput { if (itemsToDelete > 0) { const sorted = [...this.blockInputs.entries()].sort((a, b) => a[1].slot - b[1].slot); - for (const [rootHex] of sorted) { - this.blockInputs.delete(rootHex); + for (const [, blockInput] of sorted) { + this.evictBlockInput(blockInput); itemsToDelete--; if (itemsToDelete <= 0) return; } } pruneSetToMax(this.verifiedProposerSignatures, MAX_BLOCK_INPUT_CACHE_SIZE); } + + private evictBlockInput(blockInput: IBlockInput): void { + // Without forcefully clearing this cache, we would rely on WeakMap to evict memory which is not reliable + this.serializedCache.delete(blockInput.getSerializedCacheKeys()); + this.blockInputs.delete(blockInput.blockRootHex); + } } enum SeenBlockInputCacheErrorCode { diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index f6abe18eb96a..b87befc62e22 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -1461,6 +1461,10 @@ export function createLodestarMetrics( name: "lodestar_seen_block_input_cache_size", help: "Number of cached BlockInputs", }), + serializedObjectCount: register.gauge({ + name: "lodestar_seen_block_input_cache_serialized_object_count", + help: "Number of serialized objects retained by cached BlockInputs", + }), duplicateBlockCount: register.gauge<{source: BlockInputSource}>({ name: "lodestar_seen_block_input_cache_duplicate_block_count", help: "Total number of duplicate blocks that pass validation and attempt to be cached but are known", diff --git a/packages/beacon-node/src/network/network.ts b/packages/beacon-node/src/network/network.ts index 340ba2f82a73..6afe85dbad2e 100644 --- a/packages/beacon-node/src/network/network.ts +++ b/packages/beacon-node/src/network/network.ts @@ -539,7 +539,8 @@ export class Network implements INetwork { this.config.getForkSeq(this.clock.currentSlot) >= ForkSeq.altair ? [Version.V2] : [Version.V2, Version.V1], request ), - request + request, + this.chain.serializedCache ); } diff --git a/packages/beacon-node/src/util/serializedCache.ts b/packages/beacon-node/src/util/serializedCache.ts index 45785eb807fb..777ac4dd0c3b 100644 --- a/packages/beacon-node/src/util/serializedCache.ts +++ b/packages/beacon-node/src/util/serializedCache.ts @@ -4,7 +4,7 @@ * This is a thin wrapper around WeakMap */ export class SerializedCache { - map: WeakMap = new WeakMap(); + private map: WeakMap = new WeakMap(); get(obj: object): Uint8Array | undefined { return this.map.get(obj); @@ -15,11 +15,13 @@ export class SerializedCache { } /** - * Replace the internal WeakMap to force GC of all cached entries. - * Must only be called after all DB writes that may read from this cache have completed, + * Delete cached serialized entries for the provided object references. + * Must only be called after all DB writes that read from this cache for these objects have completed, * otherwise cached serialized bytes will be unavailable and data will be re-serialized unnecessarily. */ - clear(): void { - this.map = new WeakMap(); + delete(objs: object[]): void { + for (const obj of objs) { + this.map.delete(obj); + } } } diff --git a/packages/beacon-node/test/unit/chain/seenCache/seenBlockInput.test.ts b/packages/beacon-node/test/unit/chain/seenCache/seenBlockInput.test.ts index 21f92dd32d4e..08338d992c7e 100644 --- a/packages/beacon-node/test/unit/chain/seenCache/seenBlockInput.test.ts +++ b/packages/beacon-node/test/unit/chain/seenCache/seenBlockInput.test.ts @@ -17,6 +17,7 @@ import {SeenBlockInput} from "../../../../src/chain/seenCache/seenGossipBlockInp import {computeNodeIdFromPrivateKey} from "../../../../src/network/subnets/index.js"; import {Clock} from "../../../../src/util/clock.js"; import {CustodyConfig} from "../../../../src/util/dataColumns.js"; +import {SerializedCache} from "../../../../src/util/serializedCache.js"; import { config, generateBlock, @@ -29,6 +30,7 @@ describe("SeenBlockInputCache", async () => { let cache: SeenBlockInput; let abortController: AbortController; let chainEvents: ChainEventEmitter; + let serializedCache: SerializedCache; const privateKey = await generateKeyPair("secp256k1"); const nodeId = computeNodeIdFromPrivateKey(privateKey); @@ -38,6 +40,7 @@ describe("SeenBlockInputCache", async () => { beforeEach(() => { chainEvents = new ChainEventEmitter(); abortController = new AbortController(); + serializedCache = new SerializedCache(); const signal = abortController.signal; const genesisTime = Math.floor(Date.now() / 1000); cache = new SeenBlockInput({ @@ -46,6 +49,7 @@ describe("SeenBlockInputCache", async () => { clock: new Clock({config, genesisTime, signal}), chainEvents, signal, + serializedCache, logger, metrics: null, }); @@ -121,6 +125,21 @@ describe("SeenBlockInputCache", async () => { expect(cache.get(rootHex)).toBeUndefined(); }); + it("should remove serialized cache entries for the evicted BlockInput", () => { + const {block, rootHex} = generateBlock({forkName: ForkName.capella}); + cache.getByBlock({ + block, + blockRootHex: rootHex, + source: BlockInputSource.gossip, + seenTimestampSec: Date.now() / 1000, + }); + serializedCache.set(block, new Uint8Array([1])); + + expect(serializedCache.get(block)).toBeDefined(); + cache.remove(rootHex); + expect(serializedCache.get(block)).toBeUndefined(); + }); + it("should not throw an error if BlockInput not in cache", () => { const {block, blockRoot, rootHex} = generateBlock({forkName: ForkName.capella}); const blockInput = cache.getByBlock({ @@ -179,6 +198,34 @@ describe("SeenBlockInputCache", async () => { expect(cache.get(childRootHex)).toBeUndefined(); expect(cache.get(parentRootHex)).toBeUndefined(); }); + + it("should remove serialized cache entries for the pruned BlockInputs", () => { + const blocks = generateChainOfBlocks({forkName: ForkName.capella, count: 2}); + const parentBlock = blocks[0].block; + const parentRootHex = blocks[0].rootHex; + const childBlock = blocks[1].block; + const childRootHex = blocks[1].rootHex; + + cache.getByBlock({ + block: parentBlock, + blockRootHex: parentRootHex, + source: BlockInputSource.gossip, + seenTimestampSec: Date.now() / 1000, + }); + cache.getByBlock({ + block: childBlock, + blockRootHex: childRootHex, + source: BlockInputSource.gossip, + seenTimestampSec: Date.now() / 1000, + }); + serializedCache.set(parentBlock, new Uint8Array([1])); + serializedCache.set(childBlock, new Uint8Array([2])); + + cache.prune(childRootHex); + + expect(serializedCache.get(parentBlock)).toBeUndefined(); + expect(serializedCache.get(childBlock)).toBeUndefined(); + }); }); describe("onFinalized()", () => { @@ -215,6 +262,9 @@ describe("SeenBlockInputCache", async () => { }); it("should remove all BlockInputs in slots before the checkpoint", () => { + serializedCache.set(parentBlockInput.getBlock(), new Uint8Array([1])); + serializedCache.set(childBlockInput.getBlock(), new Uint8Array([2])); + chainEvents.emit(ChainEvent.forkChoiceFinalized, { epoch: config.DENEB_FORK_EPOCH, root, @@ -223,6 +273,8 @@ describe("SeenBlockInputCache", async () => { }); expect(cache.get(childRootHex)).toBeUndefined(); expect(cache.get(parentRootHex)).toBeUndefined(); + expect(serializedCache.get(parentBlockInput.getBlock())).toBeUndefined(); + expect(serializedCache.get(childBlockInput.getBlock())).toBeUndefined(); }); it("should not remove BlockInputs in slots after the checkpoint", () => { diff --git a/packages/beacon-node/test/utils/blockInput.ts b/packages/beacon-node/test/utils/blockInput.ts index 34b9d0747f07..23e97c555a30 100644 --- a/packages/beacon-node/test/utils/blockInput.ts +++ b/packages/beacon-node/test/utils/blockInput.ts @@ -91,6 +91,10 @@ export class MockBlockInput implements IBlockInput { return this._timeCompleted ?? 0; } + getSerializedCacheKeys(): object[] { + return this._block ? [this._block] : []; + } + waitForAllData(_timeout: number, _signal?: AbortSignal): Promise { return Promise.resolve(null); } From 165a02f873c667a3bf9a7c921f327c2636c2663b Mon Sep 17 00:00:00 2001 From: NC <17676176+ensi321@users.noreply.github.com> Date: Tue, 10 Mar 2026 19:44:35 -0700 Subject: [PATCH 29/63] refactor: clone state in processExecutionPayloadEnvelope (#9015) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Refactor `processExecutionPayloadEnvelope` to clone the beacon state before mutating it and return the post-state, matching the pattern used by `stateTransition()` - Add `dontTransferCache` option to control SSZ cache transfer behavior during clone - Update `computeEnvelopeStateRoot` and spec test runner to use the returned post-state ## Test plan - [x] `pnpm build` passes - [x] `pnpm check-types` passes 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 4.6 --- .../chain/produceBlock/computeNewStateRoot.ts | 7 +-- .../test/spec/presets/operations.test.ts | 24 +++++----- .../block/processExecutionPayloadEnvelope.ts | 44 ++++++++++++------- 3 files changed, 47 insertions(+), 28 deletions(-) diff --git a/packages/beacon-node/src/chain/produceBlock/computeNewStateRoot.ts b/packages/beacon-node/src/chain/produceBlock/computeNewStateRoot.ts index 55c44501db74..466b42d6e0f9 100644 --- a/packages/beacon-node/src/chain/produceBlock/computeNewStateRoot.ts +++ b/packages/beacon-node/src/chain/produceBlock/computeNewStateRoot.ts @@ -61,7 +61,6 @@ export function computeNewStateRoot( * Compute the state root after processing an execution payload envelope. * Similar to `computeNewStateRoot` but for payload envelope processing. * - * The `postBlockState` is mutated in place, callers must ensure it is not needed afterward. */ export function computeEnvelopeStateRoot( metrics: Metrics | null, @@ -74,13 +73,15 @@ export function computeEnvelopeStateRoot( }; const processEnvelopeTimer = metrics?.blockPayload.executionPayloadEnvelopeProcessingTime.startTimer(); - processExecutionPayloadEnvelope(postBlockState, signedEnvelope, false); + const postEnvelopeState = processExecutionPayloadEnvelope(postBlockState, signedEnvelope, false, { + dontTransferCache: true, + }); processEnvelopeTimer?.(); const hashTreeRootTimer = metrics?.stateHashTreeRootTime.startTimer({ source: StateHashTreeRootSource.computeEnvelopeStateRoot, }); - const stateRoot = postBlockState.hashTreeRoot(); + const stateRoot = postEnvelopeState.hashTreeRoot(); hashTreeRootTimer?.(); return stateRoot; diff --git a/packages/beacon-node/test/spec/presets/operations.test.ts b/packages/beacon-node/test/spec/presets/operations.test.ts index 99019c685272..300092667960 100644 --- a/packages/beacon-node/test/spec/presets/operations.test.ts +++ b/packages/beacon-node/test/spec/presets/operations.test.ts @@ -75,17 +75,16 @@ const operationFns: Record> = signed_envelope: gloas.SignedExecutionPayloadEnvelope; execution: {execution_valid: boolean}; } - ) => { + ): CachedBeaconStateAllForks | void => { const fork = state.config.getForkSeq(state.slot); if (fork >= ForkSeq.gloas) { - blockFns.processExecutionPayloadEnvelope(state as CachedBeaconStateGloas, testCase.signed_envelope, true); - } else { - blockFns.processExecutionPayload(fork, state as CachedBeaconStateBellatrix, testCase.body, { - executionPayloadStatus: testCase.execution.execution_valid - ? ExecutionPayloadStatus.valid - : ExecutionPayloadStatus.invalid, - }); + return blockFns.processExecutionPayloadEnvelope(state as CachedBeaconStateGloas, testCase.signed_envelope, true); } + blockFns.processExecutionPayload(fork, state as CachedBeaconStateBellatrix, testCase.body, { + executionPayloadStatus: testCase.execution.execution_valid + ? ExecutionPayloadStatus.valid + : ExecutionPayloadStatus.invalid, + }); }, bls_to_execution_change: (state, testCase: {address_change: capella.SignedBLSToExecutionChange}) => { @@ -120,7 +119,7 @@ const operationFns: Record> = }, }; -export type BlockProcessFn = (state: T, testCase: any) => void; +export type BlockProcessFn = (state: T, testCase: any) => T | void; export type OperationsTestCase = { meta?: {bls_setting?: bigint}; @@ -141,7 +140,12 @@ const operations: TestRunnerFn = (fork, const epoch = (state.fork as phase0.Fork).epoch; const cachedState = createCachedBeaconStateTest(state, getConfig(fork, epoch)); - operationFn(cachedState, testcase); + const postState = operationFn(cachedState, testcase); + // processExecutionPayloadEnvelope returns the postState, other operations mutate the state in-place and return void + if (postState !== undefined) { + postState.commit(); + return postState; + } state.commit(); return state; }, diff --git a/packages/state-transition/src/block/processExecutionPayloadEnvelope.ts b/packages/state-transition/src/block/processExecutionPayloadEnvelope.ts index 04188dafb979..10aba9b5b1b8 100644 --- a/packages/state-transition/src/block/processExecutionPayloadEnvelope.ts +++ b/packages/state-transition/src/block/processExecutionPayloadEnvelope.ts @@ -13,12 +13,19 @@ import {processConsolidationRequest} from "./processConsolidationRequest.ts"; import {processDepositRequest} from "./processDepositRequest.ts"; import {processWithdrawalRequest} from "./processWithdrawalRequest.ts"; -// This function does not call execution engine to verify payload. Need to call it from other place +export type ProcessExecutionPayloadEnvelopeOpts = { + dontTransferCache?: boolean; +}; + +// Unlike other block processing functions which mutate state in-place, this function +// clones the state and returns the post-state, similar to stateTransition(). +// This function does not call execution engine to verify payload. Need to call it from other place. export function processExecutionPayloadEnvelope( state: CachedBeaconStateGloas, signedEnvelope: gloas.SignedExecutionPayloadEnvelope, - verify: boolean -): void { + verify: boolean, + opts?: ProcessExecutionPayloadEnvelopeOpts +): CachedBeaconStateGloas { const envelope = signedEnvelope.message; const payload = envelope.payload; const fork = state.config.getForkSeq(envelope.slot); @@ -27,42 +34,49 @@ export function processExecutionPayloadEnvelope( throw Error(`Execution payload envelope has invalid signature builderIndex=${envelope.builderIndex}`); } - validateExecutionPayloadEnvelope(state, envelope); + // .clone() before mutating state, similar to stateTransition() + const postState = state.clone(opts?.dontTransferCache) as CachedBeaconStateGloas; + + validateExecutionPayloadEnvelope(postState, envelope); const requests = envelope.executionRequests; for (const deposit of requests.deposits) { - processDepositRequest(fork, state, deposit); + processDepositRequest(fork, postState, deposit); } for (const withdrawal of requests.withdrawals) { - processWithdrawalRequest(fork, state, withdrawal); + processWithdrawalRequest(fork, postState, withdrawal); } for (const consolidation of requests.consolidations) { - processConsolidationRequest(state, consolidation); + processConsolidationRequest(postState, consolidation); } // Queue the builder payment - const paymentIndex = SLOTS_PER_EPOCH + (state.slot % SLOTS_PER_EPOCH); - const payment = state.builderPendingPayments.get(paymentIndex).clone(); + const paymentIndex = SLOTS_PER_EPOCH + (postState.slot % SLOTS_PER_EPOCH); + const payment = postState.builderPendingPayments.get(paymentIndex).clone(); const amount = payment.withdrawal.amount; if (amount > 0) { - state.builderPendingWithdrawals.push(payment.withdrawal); + postState.builderPendingWithdrawals.push(payment.withdrawal); } - state.builderPendingPayments.set(paymentIndex, ssz.gloas.BuilderPendingPayment.defaultViewDU()); + postState.builderPendingPayments.set(paymentIndex, ssz.gloas.BuilderPendingPayment.defaultViewDU()); // Cache the execution payload hash - state.executionPayloadAvailability.set(state.slot % SLOTS_PER_HISTORICAL_ROOT, true); - state.latestBlockHash = payload.blockHash; + postState.executionPayloadAvailability.set(postState.slot % SLOTS_PER_HISTORICAL_ROOT, true); + postState.latestBlockHash = payload.blockHash; - if (verify && !byteArrayEquals(envelope.stateRoot, state.hashTreeRoot())) { + postState.commit(); + + if (verify && !byteArrayEquals(envelope.stateRoot, postState.hashTreeRoot())) { throw new Error( - `Envelope's state root does not match state envelope=${toRootHex(envelope.stateRoot)} state=${toRootHex(state.hashTreeRoot())}` + `Envelope's state root does not match state envelope=${toRootHex(envelope.stateRoot)} state=${toRootHex(postState.hashTreeRoot())}` ); } + + return postState; } function validateExecutionPayloadEnvelope( From be5acbb8f7c93e694e2fbb763c78d5d4c0b39202 Mon Sep 17 00:00:00 2001 From: Nazar Hussain Date: Wed, 11 Mar 2026 14:25:12 +0100 Subject: [PATCH 30/63] chore: fix private imports (#8990) **Motivation** Do not use private imports **Description** - Move shared test utils to individual packages. - Fix private imports --- .github/workflows/check-specrefs.yml | 6 +- .github/workflows/test.yml | 2 +- packages/beacon-node/src/chain/chain.ts | 8 +- .../src/chain/errors/executionPayloadBid.ts | 2 +- .../src/chain/errors/payloadAttestation.ts | 2 +- packages/beacon-node/src/db/index.ts | 1 + .../src/network/processor/gossipHandlers.ts | 2 +- .../network/processor/gossipValidatorFn.ts | 2 +- .../src/network/reqresp/ReqRespBeaconNode.ts | 2 +- .../reqresp/handlers/beaconBlocksByRange.ts | 2 +- .../handlers/dataColumnSidecarsByRange.ts | 2 +- .../handlers/dataColumnSidecarsByRoot.ts | 2 +- .../src/network/reqresp/utils/collect.ts | 2 +- .../utils/collectSequentialBlocksInRange.ts | 2 +- .../src/sync/utils/downloadByRoot.ts | 2 +- .../src/sync/utils/remoteSyncType.ts | 2 +- .../api/impl/beacon/block/endpoint.test.ts | 4 +- .../api/impl/beacon/node/endpoints.test.ts | 2 +- .../api/impl/beacon/state/endpoint.test.ts | 2 +- .../test/e2e/api/impl/config.test.ts | 4 +- .../e2e/api/impl/lightclient/endpoint.test.ts | 2 +- .../test/e2e/api/lodestar/lodestar.test.ts | 2 +- .../test/e2e/chain/bls/multithread.test.ts | 2 +- .../test/e2e/chain/lightclient.test.ts | 2 +- .../test/e2e/chain/proposerBoostReorg.test.ts | 2 +- .../stateCache/nHistoricalStates.test.ts | 2 +- .../e2e/doppelganger/doppelganger.test.ts | 2 +- .../beacon-node/test/e2e/network/mdns.test.ts | 2 +- .../e2e/network/peers/peerManager.test.ts | 2 +- .../test/e2e/network/reqrespEncode.test.ts | 2 +- .../test/e2e/sync/finalizedSync.test.ts | 2 +- .../test/e2e/sync/unknownBlockSync.test.ts | 2 +- .../perf/api/impl/validator/attester.test.ts | 2 +- .../opPools/aggregatedAttestationPool.test.ts | 2 +- .../test/perf/chain/opPools/opPool.test.ts | 2 +- .../produceBlock/produceBlockBody.test.ts | 4 +- .../validation/aggregateAndProof.test.ts | 2 +- .../perf/chain/validation/attestation.test.ts | 2 +- .../perf/chain/verifyImportBlocks.test.ts | 7 +- .../beacon-node/test/perf/util/blobs.test.ts | 8 +- .../test/sim/electra-interop.test.ts | 2 +- .../beacon-node/test/spec-tests-version.json | 1 + .../spec/presets/epoch_processing.test.ts | 2 +- .../test/spec/presets/finality.test.ts | 2 +- .../test/spec/presets/fork_choice.test.ts | 4 +- .../test/spec/presets/genesis.test.ts | 2 +- .../test/spec/presets/light_client/sync.ts | 2 +- .../test/spec/presets/operations.test.ts | 2 +- .../test/spec/presets/rewards.test.ts | 2 +- .../test/spec/presets/sanity.test.ts | 2 +- .../test/spec/specTestVersioning.ts | 29 +-- .../test/spec/utils/gossipValidation.ts | 4 +- .../persistentCheckpointsCache.test.ts | 2 +- .../chain/archiveStore/blockArchiver.test.ts | 2 +- .../test/unit/chain/bls/bls.test.ts | 2 +- .../test/unit/chain/regen/regen.test.ts | 2 +- .../chain/seenCache/seenBlockInput.test.ts | 2 +- .../test/unit/chain/shufflingCache.test.ts | 2 +- .../validation/aggregateAndProof.test.ts | 2 +- .../attestation/validateAttestation.test.ts | 2 +- .../test/unit/chain/validatorMonitor.test.ts | 2 +- .../db/api/repositories/blockArchive.test.ts | 2 +- .../db/api/repositories/dataColumn.test.ts | 2 +- .../test/unit/metrics/server/http.test.ts | 2 +- .../test/unit/network/metadata.test.ts | 4 +- .../network/subnets/attnetsService.test.ts | 2 +- .../test/unit/sync/range/chain.test.ts | 2 +- .../test/unit/sync/unknownBlock.test.ts | 2 +- .../test/utils/beforeValueBenchmark.ts | 36 +++ packages/beacon-node/test/utils/db.ts | 2 +- .../test/utils/networkWithMockDb.ts | 2 +- .../beacon-node/test/utils/node/beacon.ts | 2 +- .../beacon-node/test/utils/node/validator.ts | 2 +- packages/beacon-node/test/utils/state.ts | 2 +- .../utils/validationData/aggregateAndProof.ts | 2 +- .../test/utils/validationData/attestation.ts | 6 +- packages/cli/src/cmds/dev/options.ts | 2 +- packages/cli/test/unit/db.test.ts | 4 +- .../cli/test/utils/mockBeaconApiServer.ts | 2 +- packages/config/package.json | 8 +- .../utils => config/src/testUtils}/config.ts | 7 +- packages/config/src/testUtils/index.ts | 1 + .../test/e2e/ensure-config-is-synced.test.ts | 12 +- packages/config/test/spec-tests-version.json | 1 + packages/era/src/e2s.ts | 2 +- packages/era/src/era/reader.ts | 6 +- packages/era/src/era/util.ts | 4 +- packages/era/src/era/writer.ts | 6 +- .../era.readwrite.integration.test.ts | 2 +- packages/era/test/unit/era.unit.test.ts | 2 +- packages/logger/package.json | 5 + .../logger.ts => logger/src/testUtils.ts} | 4 +- .../test/e2e/ensure-config-is-synced.test.ts | 6 +- packages/params/test/spec-tests-version.json | 1 + .../reqresp/src/encoders/requestDecode.ts | 2 +- packages/state-transition/package.json | 5 + packages/state-transition/src/block/index.ts | 6 +- .../block/isValidIndexedPayloadAttestation.ts | 4 +- .../src/block/processAttestationsAltair.ts | 2 +- .../src/block/processExecutionPayloadBid.ts | 8 +- .../block/processExecutionPayloadEnvelope.ts | 10 +- .../src/block/processOperations.ts | 2 +- .../src/block/processPayloadAttestation.ts | 4 +- .../src/block/processWithdrawals.ts | 2 +- packages/state-transition/src/epoch/index.ts | 2 +- .../epoch/processBuilderPendingPayments.ts | 4 +- .../{test => src/testUtils}/cache.ts | 2 +- .../state-transition/src/testUtils/index.ts | 5 + .../{test/utils => src/testUtils}/infura.ts | 0 .../{test/utils => src/testUtils}/interop.ts | 4 +- .../{test/perf => src/testUtils}/params.ts | 0 .../state-transition/src/testUtils/state.ts | 110 ++++++++++ .../utils => src/testUtils}/testFileCache.ts | 7 +- .../{test/perf => src/testUtils}/util.ts | 12 +- .../test/perf/analyzeBlocks.ts | 2 +- .../test/perf/analyzeEpochs.ts | 4 +- .../perf/block/processAttestation.test.ts | 2 +- .../perf/block/processBlockAltair.test.ts | 6 +- .../perf/block/processBlockPhase0.test.ts | 2 +- .../test/perf/block/processEth1Data.test.ts | 2 +- .../perf/block/processWithdrawals.test.ts | 2 +- .../test/perf/epoch/afterProcessEpoch.test.ts | 2 +- .../perf/epoch/beforeProcessEpoch.test.ts | 2 +- .../test/perf/epoch/epochAltair.test.ts | 4 +- .../test/perf/epoch/epochCapella.test.ts | 4 +- .../test/perf/epoch/epochPhase0.test.ts | 4 +- .../processEffectiveBalanceUpdates.test.ts | 4 +- .../epoch/processInactivityUpdates.test.ts | 2 +- .../perf/epoch/processRegistryUpdates.test.ts | 2 +- .../epoch/processRewardsAndPenalties.test.ts | 2 +- .../processRewardsAndPenaltiesPhase0.test.ts | 2 +- .../epoch/processSlashingsAllForks.test.ts | 2 +- .../epoch/processSyncCommitteeUpdates.test.ts | 2 +- .../test/perf/hashing.test.ts | 2 +- .../test/perf/misc/byteArrayEquals.test.ts | 2 +- .../test/perf/slot/slots.test.ts | 2 +- .../test/perf/util/balance.test.ts | 2 +- .../test/perf/util/epochContext.test.ts | 2 +- .../loadState/findModifiedValidators.test.ts | 2 +- .../perf/util/loadState/loadState.test.ts | 2 +- .../test/perf/util/rootCache.test.ts | 2 +- .../test/perf/util/seed.test.ts | 2 +- .../test/perf/util/serializeState.test.ts | 2 +- .../test/perf/util/shufflings.test.ts | 2 +- .../block/isValidIndexedAttestation.test.ts | 2 +- .../block/processConsolidationRequest.test.ts | 2 +- .../unit/block/processWithdrawals.test.ts | 2 +- .../test/unit/cachedBeaconState.test.ts | 4 +- .../test/unit/rewards/blockRewards.test.ts | 2 +- .../test/unit/sanityCheck.test.ts | 6 +- .../unit/signatureSets/signatureSets.test.ts | 2 +- .../test/unit/util/balance.test.ts | 2 +- .../test/unit/util/deposit.test.ts | 2 +- .../test/unit/util/epoch.test.ts | 2 +- .../loadState/findModifiedValidators.test.ts | 2 +- .../unit/util/loadState/loadValidator.test.ts | 2 +- .../test/unit/util/misc.test.ts | 2 +- .../test/unit/util/seed.test.ts | 2 +- .../test/unit/util/shuffling.test.ts | 2 +- .../test/unit/util/validator.test.ts | 2 +- .../state-transition/test/utils/capella.ts | 4 +- packages/state-transition/test/utils/state.ts | 207 +++++++++++------- packages/test-utils/src/externalSigner.ts | 2 +- packages/utils/src/bytes/nodejs.ts | 2 +- packages/utils/test/perf/bytes.test.ts | 4 +- packages/validator/src/index.ts | 1 + pnpm-lock.yaml | 3 + spec-tests-version.json | 20 ++ 168 files changed, 553 insertions(+), 316 deletions(-) create mode 120000 packages/beacon-node/test/spec-tests-version.json create mode 100644 packages/beacon-node/test/utils/beforeValueBenchmark.ts rename packages/{beacon-node/test/utils => config/src/testUtils}/config.ts (92%) create mode 100644 packages/config/src/testUtils/index.ts create mode 120000 packages/config/test/spec-tests-version.json rename packages/{beacon-node/test/utils/logger.ts => logger/src/testUtils.ts} (80%) create mode 120000 packages/params/test/spec-tests-version.json rename packages/state-transition/{test => src/testUtils}/cache.ts (80%) create mode 100644 packages/state-transition/src/testUtils/index.ts rename packages/state-transition/{test/utils => src/testUtils}/infura.ts (100%) rename packages/state-transition/{test/utils => src/testUtils}/interop.ts (91%) rename packages/state-transition/{test/perf => src/testUtils}/params.ts (100%) create mode 100644 packages/state-transition/src/testUtils/state.ts rename packages/state-transition/{test/utils => src/testUtils}/testFileCache.ts (94%) rename packages/state-transition/{test/perf => src/testUtils}/util.ts (97%) create mode 100644 spec-tests-version.json diff --git a/.github/workflows/check-specrefs.yml b/.github/workflows/check-specrefs.yml index 996b9318e80d..a0576bbd87e6 100644 --- a/.github/workflows/check-specrefs.yml +++ b/.github/workflows/check-specrefs.yml @@ -20,11 +20,11 @@ jobs: - name: Check version consistency run: | - SPEC_TEST_VERSION=$(grep 'specVersion:' packages/beacon-node/test/spec/specTestVersioning.ts | head -1 | sed 's/.*specVersion: "\(.*\)".*/\1/') + SPEC_TEST_VERSION=$(grep '"specVersion":' spec-tests-version.json | head -1 | sed 's/.*"specVersion": "\(.*\)".*/\1/') ETHSPECIFY_VERSION=$(grep '^version:' specrefs/.ethspecify.yml | sed 's/version: //') if [ "$SPEC_TEST_VERSION" != "$ETHSPECIFY_VERSION" ]; then - echo "Version mismatch between specTestVersioning.ts and ethspecify" - echo " packages/beacon-node/test/spec/specTestVersioning.ts: $SPEC_TEST_VERSION" + echo "Version mismatch between spec-teests-version.json and ethspecify" + echo " spec-tests-version.json: $SPEC_TEST_VERSION" echo " specrefs/.ethspecify.yml: $ETHSPECIFY_VERSION" exit 1 else diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index af3f3e1b7e7e..5f5df7441041 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -199,7 +199,7 @@ jobs: uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 with: path: packages/beacon-node/spec-tests - key: spec-test-data-${{ hashFiles('packages/beacon-node/test/spec/specTestVersioning.ts') }} + key: spec-test-data-${{ hashFiles('spec-tests-version.json') }} - name: Download spec tests run: pnpm download-spec-tests # Run them in different steps to quickly identifying which command failed diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 5fbcf7534cec..5338a6bee134 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -64,7 +64,7 @@ import {Logger, fromHex, gweiToWei, isErrorAborted, pruneSetToMax, sleep, toRoot import {ProcessShutdownCallback} from "@lodestar/validator"; import {GENESIS_EPOCH, ZERO_HASH} from "../constants/index.js"; import {IBeaconDb} from "../db/index.js"; -import {BLOB_SIDECARS_IN_WRAPPER_INDEX} from "../db/repositories/blobSidecars.ts"; +import {BLOB_SIDECARS_IN_WRAPPER_INDEX} from "../db/repositories/blobSidecars.js"; import {BuilderStatus} from "../execution/builder/http.js"; import {IExecutionBuilder, IExecutionEngine} from "../execution/index.js"; import {Metrics} from "../metrics/index.js"; @@ -75,15 +75,15 @@ import {CustodyConfig, getValidatorsCustodyRequirement} from "../util/dataColumn import {callInNextEventLoop} from "../util/eventLoop.js"; import {ensureDir, writeIfNotExist} from "../util/file.js"; import {isOptimisticBlock} from "../util/forkChoice.js"; -import {JobItemQueue} from "../util/queue/itemQueue.ts"; +import {JobItemQueue} from "../util/queue/itemQueue.js"; import {SerializedCache} from "../util/serializedCache.js"; -import {getSlotFromSignedBeaconBlockSerialized} from "../util/sszBytes.ts"; +import {getSlotFromSignedBeaconBlockSerialized} from "../util/sszBytes.js"; import {ArchiveStore} from "./archiveStore/archiveStore.js"; import {CheckpointBalancesCache} from "./balancesCache.js"; import {BeaconProposerCache} from "./beaconProposerCache.js"; import {IBlockInput, isBlockInputBlobs, isBlockInputColumns} from "./blocks/blockInput/index.js"; import {BlockProcessor, ImportBlockOpts} from "./blocks/index.js"; -import {persistBlockInput} from "./blocks/writeBlockInputToDb.ts"; +import {persistBlockInput} from "./blocks/writeBlockInputToDb.js"; import {BlsMultiThreadWorkerPool, BlsSingleThreadVerifier, IBlsVerifier} from "./bls/index.js"; import {ColumnReconstructionTracker} from "./ColumnReconstructionTracker.js"; import {ChainEvent, ChainEventEmitter} from "./emitter.js"; diff --git a/packages/beacon-node/src/chain/errors/executionPayloadBid.ts b/packages/beacon-node/src/chain/errors/executionPayloadBid.ts index 5770d5efc045..6abb90896c60 100644 --- a/packages/beacon-node/src/chain/errors/executionPayloadBid.ts +++ b/packages/beacon-node/src/chain/errors/executionPayloadBid.ts @@ -1,5 +1,5 @@ import {BuilderIndex, RootHex, Slot} from "@lodestar/types"; -import {GossipActionError} from "./gossipValidation.ts"; +import {GossipActionError} from "./gossipValidation.js"; export enum ExecutionPayloadBidErrorCode { BUILDER_NOT_ELIGIBLE = "EXECUTION_PAYLOAD_BID_ERROR_BUILDER_NOT_ELIGIBLE", diff --git a/packages/beacon-node/src/chain/errors/payloadAttestation.ts b/packages/beacon-node/src/chain/errors/payloadAttestation.ts index 6acbed4a47f7..c9f15a2b657d 100644 --- a/packages/beacon-node/src/chain/errors/payloadAttestation.ts +++ b/packages/beacon-node/src/chain/errors/payloadAttestation.ts @@ -1,5 +1,5 @@ import {RootHex, Slot, ValidatorIndex} from "@lodestar/types"; -import {GossipActionError} from "./gossipValidation.ts"; +import {GossipActionError} from "./gossipValidation.js"; export enum PayloadAttestationErrorCode { NOT_CURRENT_SLOT = "PAYLOAD_ATTESTATION_ERROR_NOT_CURRENT_SLOT", diff --git a/packages/beacon-node/src/db/index.ts b/packages/beacon-node/src/db/index.ts index df83fb16e08d..ecd83a045154 100644 --- a/packages/beacon-node/src/db/index.ts +++ b/packages/beacon-node/src/db/index.ts @@ -1,2 +1,3 @@ export {BeaconDb} from "./beacon.js"; +export {Bucket} from "./buckets.js"; export type {IBeaconDb} from "./interface.js"; diff --git a/packages/beacon-node/src/network/processor/gossipHandlers.ts b/packages/beacon-node/src/network/processor/gossipHandlers.ts index d681e49b9c80..b022a1866470 100644 --- a/packages/beacon-node/src/network/processor/gossipHandlers.ts +++ b/packages/beacon-node/src/network/processor/gossipHandlers.ts @@ -71,7 +71,7 @@ import {validateGossipPayloadAttestationMessage} from "../../chain/validation/pa import {OpSource} from "../../chain/validatorMonitor.js"; import {Metrics} from "../../metrics/index.js"; import {kzgCommitmentToVersionedHash} from "../../util/blobs.js"; -import {getBlobKzgCommitments} from "../../util/dataColumns.ts"; +import {getBlobKzgCommitments} from "../../util/dataColumns.js"; import {INetworkCore} from "../core/index.js"; import {NetworkEventBus} from "../events.js"; import { diff --git a/packages/beacon-node/src/network/processor/gossipValidatorFn.ts b/packages/beacon-node/src/network/processor/gossipValidatorFn.ts index 2ee57fc693ed..19442c7173c4 100644 --- a/packages/beacon-node/src/network/processor/gossipValidatorFn.ts +++ b/packages/beacon-node/src/network/processor/gossipValidatorFn.ts @@ -11,7 +11,7 @@ import { GossipValidatorBatchFn, GossipValidatorFn, } from "../gossip/interface.js"; -import {prettyPrintPeerIdStr} from "../util.ts"; +import {prettyPrintPeerIdStr} from "../util.js"; export type ValidatorFnModules = { config: ChainForkConfig; diff --git a/packages/beacon-node/src/network/reqresp/ReqRespBeaconNode.ts b/packages/beacon-node/src/network/reqresp/ReqRespBeaconNode.ts index 248c4b391b0c..a59a5b5026cf 100644 --- a/packages/beacon-node/src/network/reqresp/ReqRespBeaconNode.ts +++ b/packages/beacon-node/src/network/reqresp/ReqRespBeaconNode.ts @@ -19,7 +19,7 @@ import {callInNextEventLoop} from "../../util/eventLoop.js"; import {NetworkCoreMetrics} from "../core/metrics.js"; import {INetworkEventBus, NetworkEvent} from "../events.js"; import {MetadataController} from "../metadata.js"; -import {ClientKind} from "../peers/client.ts"; +import {ClientKind} from "../peers/client.js"; import {PeersData} from "../peers/peersData.js"; import {IPeerRpcScoreStore, PeerAction} from "../peers/score/index.js"; import {StatusCache} from "../statusCache.js"; diff --git a/packages/beacon-node/src/network/reqresp/handlers/beaconBlocksByRange.ts b/packages/beacon-node/src/network/reqresp/handlers/beaconBlocksByRange.ts index e659266c23a3..33709cbbbc7f 100644 --- a/packages/beacon-node/src/network/reqresp/handlers/beaconBlocksByRange.ts +++ b/packages/beacon-node/src/network/reqresp/handlers/beaconBlocksByRange.ts @@ -6,7 +6,7 @@ import {computeEpochAtSlot} from "@lodestar/state-transition"; import {deneb, phase0} from "@lodestar/types"; import {IBeaconChain} from "../../../chain/index.js"; import {IBeaconDb} from "../../../db/index.js"; -import {prettyPrintPeerId} from "../../util.ts"; +import {prettyPrintPeerId} from "../../util.js"; // TODO: Unit test diff --git a/packages/beacon-node/src/network/reqresp/handlers/dataColumnSidecarsByRange.ts b/packages/beacon-node/src/network/reqresp/handlers/dataColumnSidecarsByRange.ts index db5fef74b86b..ebb1d62d68fd 100644 --- a/packages/beacon-node/src/network/reqresp/handlers/dataColumnSidecarsByRange.ts +++ b/packages/beacon-node/src/network/reqresp/handlers/dataColumnSidecarsByRange.ts @@ -7,7 +7,7 @@ import {ColumnIndex, fulu} from "@lodestar/types"; import {fromHex} from "@lodestar/utils"; import {IBeaconChain} from "../../../chain/index.js"; import {IBeaconDb} from "../../../db/index.js"; -import {prettyPrintPeerId} from "../../util.ts"; +import {prettyPrintPeerId} from "../../util.js"; import { handleColumnSidecarUnavailability, validateRequestedDataColumns, diff --git a/packages/beacon-node/src/network/reqresp/handlers/dataColumnSidecarsByRoot.ts b/packages/beacon-node/src/network/reqresp/handlers/dataColumnSidecarsByRoot.ts index aa51c73b1d97..7748c940e236 100644 --- a/packages/beacon-node/src/network/reqresp/handlers/dataColumnSidecarsByRoot.ts +++ b/packages/beacon-node/src/network/reqresp/handlers/dataColumnSidecarsByRoot.ts @@ -6,7 +6,7 @@ import {toRootHex} from "@lodestar/utils"; import {IBeaconChain} from "../../../chain/index.js"; import {IBeaconDb} from "../../../db/index.js"; import {DataColumnSidecarsByRootRequest} from "../../../util/types.js"; -import {prettyPrintPeerId} from "../../util.ts"; +import {prettyPrintPeerId} from "../../util.js"; import { handleColumnSidecarUnavailability, validateRequestedDataColumns, diff --git a/packages/beacon-node/src/network/reqresp/utils/collect.ts b/packages/beacon-node/src/network/reqresp/utils/collect.ts index 6e528cbf545d..5f80a28b42b5 100644 --- a/packages/beacon-node/src/network/reqresp/utils/collect.ts +++ b/packages/beacon-node/src/network/reqresp/utils/collect.ts @@ -1,6 +1,6 @@ import {Type} from "@chainsafe/ssz"; import {RequestError, RequestErrorCode, ResponseIncoming} from "@lodestar/reqresp"; -import {SerializedCache} from "../../../util/serializedCache.ts"; +import {SerializedCache} from "../../../util/serializedCache.js"; import {ResponseTypeGetter} from "../types.js"; /** diff --git a/packages/beacon-node/src/network/reqresp/utils/collectSequentialBlocksInRange.ts b/packages/beacon-node/src/network/reqresp/utils/collectSequentialBlocksInRange.ts index 5e8df776eb6c..b44fb07ca843 100644 --- a/packages/beacon-node/src/network/reqresp/utils/collectSequentialBlocksInRange.ts +++ b/packages/beacon-node/src/network/reqresp/utils/collectSequentialBlocksInRange.ts @@ -1,7 +1,7 @@ import {ResponseIncoming} from "@lodestar/reqresp"; import {SignedBeaconBlock, phase0} from "@lodestar/types"; import {LodestarError} from "@lodestar/utils"; -import {SerializedCache} from "../../../util/serializedCache.ts"; +import {SerializedCache} from "../../../util/serializedCache.js"; import {ReqRespMethod, responseSszTypeByMethod} from "../types.js"; import {sszDeserializeResponse} from "./collect.js"; diff --git a/packages/beacon-node/src/sync/utils/downloadByRoot.ts b/packages/beacon-node/src/sync/utils/downloadByRoot.ts index 6e5c6e551afe..40cd33a56fba 100644 --- a/packages/beacon-node/src/sync/utils/downloadByRoot.ts +++ b/packages/beacon-node/src/sync/utils/downloadByRoot.ts @@ -6,7 +6,7 @@ import {LodestarError, byteArrayEquals, fromHex, prettyPrintIndices, toHex, toRo import {isBlockInputBlobs, isBlockInputColumns} from "../../chain/blocks/blockInput/blockInput.js"; import {BlockInputSource, IBlockInput} from "../../chain/blocks/blockInput/types.js"; import {ChainEventEmitter} from "../../chain/emitter.js"; -import {IBeaconChain} from "../../chain/interface.ts"; +import {IBeaconChain} from "../../chain/interface.js"; import {validateBlockBlobSidecars} from "../../chain/validation/blobSidecar.js"; import {validateBlockDataColumnSidecars} from "../../chain/validation/dataColumnSidecar.js"; import {INetwork} from "../../network/interface.js"; diff --git a/packages/beacon-node/src/sync/utils/remoteSyncType.ts b/packages/beacon-node/src/sync/utils/remoteSyncType.ts index 9024d8cb7367..68f958a203a6 100644 --- a/packages/beacon-node/src/sync/utils/remoteSyncType.ts +++ b/packages/beacon-node/src/sync/utils/remoteSyncType.ts @@ -1,7 +1,7 @@ import {IForkChoice} from "@lodestar/fork-choice"; import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {Slot, Status} from "@lodestar/types"; -import {IBeaconChain} from "../../chain/interface.ts"; +import {IBeaconChain} from "../../chain/interface.js"; import {ChainTarget} from "../range/utils/index.js"; /** The type of peer relative to our current state */ diff --git a/packages/beacon-node/test/e2e/api/impl/beacon/block/endpoint.test.ts b/packages/beacon-node/test/e2e/api/impl/beacon/block/endpoint.test.ts index de4e0a693c0b..b80dd73b9b63 100644 --- a/packages/beacon-node/test/e2e/api/impl/beacon/block/endpoint.test.ts +++ b/packages/beacon-node/test/e2e/api/impl/beacon/block/endpoint.test.ts @@ -1,6 +1,8 @@ import {afterAll, beforeAll, describe, expect, it, vi} from "vitest"; import {ApiClient, WireFormat, getClient} from "@lodestar/api"; import {createBeaconConfig} from "@lodestar/config"; +import {getConfig} from "@lodestar/config/test-utils"; +import {LogLevel, testLogger} from "@lodestar/logger/test-utils"; import {ForkName} from "@lodestar/params"; import { SignedBeaconBlock, @@ -10,8 +12,6 @@ import { isExecutionPayloadHeader, } from "@lodestar/types"; import {BeaconNode} from "../../../../../../src/node/nodejs.js"; -import {getConfig} from "../../../../../utils/config.js"; -import {LogLevel, testLogger} from "../../../../../utils/logger.js"; import {getDevBeaconNode} from "../../../../../utils/node/beacon.js"; describe("beacon block api", () => { diff --git a/packages/beacon-node/test/e2e/api/impl/beacon/node/endpoints.test.ts b/packages/beacon-node/test/e2e/api/impl/beacon/node/endpoints.test.ts index 52508af1ea66..183595dcfee6 100644 --- a/packages/beacon-node/test/e2e/api/impl/beacon/node/endpoints.test.ts +++ b/packages/beacon-node/test/e2e/api/impl/beacon/node/endpoints.test.ts @@ -3,9 +3,9 @@ import {routes} from "@lodestar/api"; import {ApiClient, getClient} from "@lodestar/api/beacon"; import {createBeaconConfig} from "@lodestar/config"; import {chainConfig as chainConfigDef} from "@lodestar/config/default"; +import {LogLevel, testLogger} from "@lodestar/logger/test-utils"; import {sleep} from "@lodestar/utils"; import {BeaconNode} from "../../../../../../src/node/nodejs.js"; -import {LogLevel, testLogger} from "../../../../../utils/logger.js"; import {getDevBeaconNode} from "../../../../../utils/node/beacon.js"; import {getAndInitDevValidators} from "../../../../../utils/node/validator.js"; diff --git a/packages/beacon-node/test/e2e/api/impl/beacon/state/endpoint.test.ts b/packages/beacon-node/test/e2e/api/impl/beacon/state/endpoint.test.ts index 25d4dee79ee0..1338780cf59d 100644 --- a/packages/beacon-node/test/e2e/api/impl/beacon/state/endpoint.test.ts +++ b/packages/beacon-node/test/e2e/api/impl/beacon/state/endpoint.test.ts @@ -2,10 +2,10 @@ import {afterAll, beforeAll, describe, expect, it} from "vitest"; import {ApiClient, getClient} from "@lodestar/api"; import {createBeaconConfig} from "@lodestar/config"; import {chainConfig as chainConfigDef} from "@lodestar/config/default"; +import {LogLevel, testLogger} from "@lodestar/logger/test-utils"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {computeCommitteeCount} from "@lodestar/state-transition"; import {BeaconNode} from "../../../../../../src/node/nodejs.js"; -import {LogLevel, testLogger} from "../../../../../utils/logger.js"; import {getDevBeaconNode} from "../../../../../utils/node/beacon.js"; describe("beacon state api", () => { diff --git a/packages/beacon-node/test/e2e/api/impl/config.test.ts b/packages/beacon-node/test/e2e/api/impl/config.test.ts index 2bb042f1da32..c9f208a41266 100644 --- a/packages/beacon-node/test/e2e/api/impl/config.test.ts +++ b/packages/beacon-node/test/e2e/api/impl/config.test.ts @@ -3,7 +3,7 @@ import {chainConfig} from "@lodestar/config/default"; import {ForkName, activePreset} from "@lodestar/params"; import {fetch} from "@lodestar/utils"; import {specConstants} from "../../../../src/api/impl/config/constants.js"; -import {ethereumConsensusSpecsTests} from "../../../spec/specTestVersioning.js"; +import specTestVersion from "../../../spec-tests-version.json" with {type: "json"}; const CONSTANT_NAMES_SKIP_LIST = new Set([ // This constant is an array, so it's skipped due to not being just a string. @@ -14,7 +14,7 @@ const CONSTANT_NAMES_SKIP_LIST = new Set([ describe("api / impl / config", () => { it("Ensure all constants are exposed", async () => { - const constantNames = await downloadRemoteConstants(ethereumConsensusSpecsTests.specVersion); + const constantNames = await downloadRemoteConstants(specTestVersion.ethereumConsensusSpecsTests.specVersion); const constantsInCode = new Set([ // Constants for API only diff --git a/packages/beacon-node/test/e2e/api/impl/lightclient/endpoint.test.ts b/packages/beacon-node/test/e2e/api/impl/lightclient/endpoint.test.ts index 50b8e219b824..b968ed34fdc5 100644 --- a/packages/beacon-node/test/e2e/api/impl/lightclient/endpoint.test.ts +++ b/packages/beacon-node/test/e2e/api/impl/lightclient/endpoint.test.ts @@ -1,6 +1,7 @@ import {afterEach, beforeEach, describe, expect, it} from "vitest"; import {HttpHeader, getClient, routes} from "@lodestar/api"; import {ChainConfig, createBeaconConfig} from "@lodestar/config"; +import {LogLevel, TestLoggerOpts, testLogger} from "@lodestar/logger/test-utils"; import {ForkName} from "@lodestar/params"; import {CachedBeaconStateAltair} from "@lodestar/state-transition"; import {phase0} from "@lodestar/types"; @@ -8,7 +9,6 @@ import {sleep} from "@lodestar/utils"; import {Validator} from "@lodestar/validator"; import {BeaconNode} from "../../../../../src/node/nodejs.js"; import {waitForEvent} from "../../../../utils/events/resolver.js"; -import {LogLevel, TestLoggerOpts, testLogger} from "../../../../utils/logger.js"; import {getDevBeaconNode} from "../../../../utils/node/beacon.js"; import {getAndInitDevValidators} from "../../../../utils/node/validator.js"; diff --git a/packages/beacon-node/test/e2e/api/lodestar/lodestar.test.ts b/packages/beacon-node/test/e2e/api/lodestar/lodestar.test.ts index 53389b38f917..774ba8e1c569 100644 --- a/packages/beacon-node/test/e2e/api/lodestar/lodestar.test.ts +++ b/packages/beacon-node/test/e2e/api/lodestar/lodestar.test.ts @@ -2,12 +2,12 @@ import {afterEach, describe, expect, it, vi} from "vitest"; import {getClient} from "@lodestar/api"; import {ChainConfig, createBeaconConfig} from "@lodestar/config"; import {chainConfig as chainConfigDef} from "@lodestar/config/default"; +import {LogLevel, TestLoggerOpts, testLogger} from "@lodestar/logger/test-utils"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {phase0} from "@lodestar/types"; import {BeaconNode} from "../../../../src/index.js"; import {ClockEvent} from "../../../../src/util/clock.js"; import {waitForEvent} from "../../../utils/events/resolver.js"; -import {LogLevel, TestLoggerOpts, testLogger} from "../../../utils/logger.js"; import {getDevBeaconNode} from "../../../utils/node/beacon.js"; describe("api / impl / validator", () => { diff --git a/packages/beacon-node/test/e2e/chain/bls/multithread.test.ts b/packages/beacon-node/test/e2e/chain/bls/multithread.test.ts index 0428389a3df1..aa3fb09998ed 100644 --- a/packages/beacon-node/test/e2e/chain/bls/multithread.test.ts +++ b/packages/beacon-node/test/e2e/chain/bls/multithread.test.ts @@ -1,9 +1,9 @@ import {afterEach, beforeAll, beforeEach, describe, expect, it} from "vitest"; import {PublicKey, SecretKey} from "@chainsafe/blst"; +import {testLogger} from "@lodestar/logger/test-utils"; import {ISignatureSet, SignatureSetType, createPubkeyCache} from "@lodestar/state-transition"; import {VerifySignatureOpts} from "../../../../src/chain/bls/interface.js"; import {BlsMultiThreadWorkerPool} from "../../../../src/chain/bls/multithread/index.js"; -import {testLogger} from "../../../utils/logger.js"; describe("chain / bls / multithread queue", () => { const logger = testLogger(); diff --git a/packages/beacon-node/test/e2e/chain/lightclient.test.ts b/packages/beacon-node/test/e2e/chain/lightclient.test.ts index 8f9c5e7f0612..9f20a2a904a6 100644 --- a/packages/beacon-node/test/e2e/chain/lightclient.test.ts +++ b/packages/beacon-node/test/e2e/chain/lightclient.test.ts @@ -6,11 +6,11 @@ import {BeaconConfig, ChainConfig} from "@lodestar/config"; import {Lightclient} from "@lodestar/light-client"; import {LightClientRestTransport} from "@lodestar/light-client/transport"; import {TimestampFormatCode} from "@lodestar/logger"; +import {LogLevel, TestLoggerOpts, testLogger} from "@lodestar/logger/test-utils"; import {EPOCHS_PER_SYNC_COMMITTEE_PERIOD, SLOTS_PER_EPOCH} from "@lodestar/params"; import {computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {LightClientHeader} from "@lodestar/types"; import {HeadEventData} from "../../../src/chain/index.js"; -import {LogLevel, TestLoggerOpts, testLogger} from "../../utils/logger.js"; import {getDevBeaconNode} from "../../utils/node/beacon.js"; import {getAndInitDevValidators} from "../../utils/node/validator.js"; diff --git a/packages/beacon-node/test/e2e/chain/proposerBoostReorg.test.ts b/packages/beacon-node/test/e2e/chain/proposerBoostReorg.test.ts index 51e7a0ac5198..460fcc13314f 100644 --- a/packages/beacon-node/test/e2e/chain/proposerBoostReorg.test.ts +++ b/packages/beacon-node/test/e2e/chain/proposerBoostReorg.test.ts @@ -2,13 +2,13 @@ import {afterEach, describe, expect, it, vi} from "vitest"; import {routes} from "@lodestar/api"; import {ChainConfig} from "@lodestar/config"; import {TimestampFormatCode} from "@lodestar/logger"; +import {LogLevel, TestLoggerOpts, testLogger} from "@lodestar/logger/test-utils"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {RootHex, Slot} from "@lodestar/types"; import {toHexString} from "@lodestar/utils"; import {ReorgEventData} from "../../../src/chain/emitter.js"; import {TimelinessForkChoice} from "../../mocks/fork-choice/timeliness.js"; import {waitForEvent} from "../../utils/events/resolver.js"; -import {LogLevel, TestLoggerOpts, testLogger} from "../../utils/logger.js"; import {getDevBeaconNode} from "../../utils/node/beacon.js"; import {getAndInitDevValidators} from "../../utils/node/validator.js"; diff --git a/packages/beacon-node/test/e2e/chain/stateCache/nHistoricalStates.test.ts b/packages/beacon-node/test/e2e/chain/stateCache/nHistoricalStates.test.ts index 3b23a35a81fc..40182ecb5d9a 100644 --- a/packages/beacon-node/test/e2e/chain/stateCache/nHistoricalStates.test.ts +++ b/packages/beacon-node/test/e2e/chain/stateCache/nHistoricalStates.test.ts @@ -3,13 +3,13 @@ import {afterEach, describe, expect, it, vi} from "vitest"; import {routes} from "@lodestar/api"; import {ChainConfig} from "@lodestar/config"; import {TimestampFormatCode} from "@lodestar/logger"; +import {LogLevel, TestLoggerOpts, testLogger} from "@lodestar/logger/test-utils"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {Slot, phase0} from "@lodestar/types"; import {ChainEvent, ReorgEventData} from "../../../../src/chain/emitter.js"; import {CacheItemType} from "../../../../src/chain/stateCache/types.js"; import {ReorgedForkChoice} from "../../../mocks/fork-choice/reorg.js"; import {waitForEvent} from "../../../utils/events/resolver.js"; -import {LogLevel, TestLoggerOpts, testLogger} from "../../../utils/logger.js"; import {connect, onPeerConnect} from "../../../utils/network.js"; import {getDevBeaconNode} from "../../../utils/node/beacon.js"; import {getAndInitDevValidators} from "../../../utils/node/validator.js"; diff --git a/packages/beacon-node/test/e2e/doppelganger/doppelganger.test.ts b/packages/beacon-node/test/e2e/doppelganger/doppelganger.test.ts index 063d9a44b326..2ddc38507d91 100644 --- a/packages/beacon-node/test/e2e/doppelganger/doppelganger.test.ts +++ b/packages/beacon-node/test/e2e/doppelganger/doppelganger.test.ts @@ -2,13 +2,13 @@ import {afterEach, describe, expect, it} from "vitest"; import {fromHexString} from "@chainsafe/ssz"; import {routes} from "@lodestar/api/beacon"; import {ChainConfig} from "@lodestar/config"; +import {LogLevel, TestLoggerOpts, testLogger} from "@lodestar/logger/test-utils"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {BLSPubkey, Epoch, Slot, phase0, ssz} from "@lodestar/types"; import {Validator} from "@lodestar/validator"; import {BeaconNode} from "../../../src/node/index.js"; import {ClockEvent} from "../../../src/util/clock.js"; import {waitForEvent} from "../../utils/events/resolver.js"; -import {LogLevel, TestLoggerOpts, testLogger} from "../../utils/logger.js"; import {connect} from "../../utils/network.js"; import {getDevBeaconNode} from "../../utils/node/beacon.js"; import {getAndInitDevValidators} from "../../utils/node/validator.js"; diff --git a/packages/beacon-node/test/e2e/network/mdns.test.ts b/packages/beacon-node/test/e2e/network/mdns.test.ts index d5403369b03c..66023433d16f 100644 --- a/packages/beacon-node/test/e2e/network/mdns.test.ts +++ b/packages/beacon-node/test/e2e/network/mdns.test.ts @@ -5,6 +5,7 @@ import {afterEach, beforeEach, describe, expect, it, vi} from "vitest"; import {SignableENR} from "@chainsafe/enr"; import {createBeaconConfig} from "@lodestar/config"; import {config} from "@lodestar/config/default"; +import {testLogger} from "@lodestar/logger/test-utils"; import {computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {ssz} from "@lodestar/types"; import {GossipHandlers} from "../../../src/network/gossip/index.js"; @@ -13,7 +14,6 @@ import {NetworkOptions, defaultNetworkOptions} from "../../../src/network/option import {getMockedBeaconChain} from "../../mocks/mockedBeaconChain.js"; import {getMockedBeaconDb} from "../../mocks/mockedBeaconDb.js"; import {memoOnce} from "../../utils/cache.js"; -import {testLogger} from "../../utils/logger.js"; import {createNetworkModules, onPeerConnect} from "../../utils/network.js"; import {generateState, zeroProtoBlock} from "../../utils/state.js"; diff --git a/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts b/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts index deecd6e6ee25..64f2e87f47fc 100644 --- a/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts +++ b/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts @@ -4,6 +4,7 @@ import {afterEach, describe, expect, it, vi} from "vitest"; import {BitArray} from "@chainsafe/ssz"; import {createBeaconConfig} from "@lodestar/config"; import {config} from "@lodestar/config/default"; +import {testLogger} from "@lodestar/logger/test-utils"; import {phase0, ssz} from "@lodestar/types"; import {sleep} from "@lodestar/utils"; import {Eth2Gossipsub, NetworkEvent, NetworkEventBus, getConnectionsMap} from "../../../../src/network/index.js"; @@ -17,7 +18,6 @@ import {IAttnetsService, computeNodeId} from "../../../../src/network/subnets/in import {Clock} from "../../../../src/util/clock.js"; import {CustodyConfig, getCustodyGroups} from "../../../../src/util/dataColumns.js"; import {waitForEvent} from "../../../utils/events/resolver.js"; -import {testLogger} from "../../../utils/logger.js"; import {createNode, getAttnets, getSyncnets} from "../../../utils/network.js"; import {getValidPeerId} from "../../../utils/peer.js"; import {generateState} from "../../../utils/state.js"; diff --git a/packages/beacon-node/test/e2e/network/reqrespEncode.test.ts b/packages/beacon-node/test/e2e/network/reqrespEncode.test.ts index 5d82185666b7..23d9c04bbcf7 100644 --- a/packages/beacon-node/test/e2e/network/reqrespEncode.test.ts +++ b/packages/beacon-node/test/e2e/network/reqrespEncode.test.ts @@ -11,6 +11,7 @@ import {afterEach, describe, expect, it} from "vitest"; import {noise} from "@chainsafe/libp2p-noise"; import {createBeaconConfig} from "@lodestar/config"; import {config} from "@lodestar/config/default"; +import {testLogger} from "@lodestar/logger/test-utils"; import {ForkName, GENESIS_EPOCH} from "@lodestar/params"; import {ssz} from "@lodestar/types"; import {fromHex, sleep, toHex} from "@lodestar/utils"; @@ -28,7 +29,6 @@ import {GetReqRespHandlerFn} from "../../../src/network/reqresp/types.js"; import {LocalStatusCache} from "../../../src/network/statusCache.js"; import {computeNodeId} from "../../../src/network/subnets/index.js"; import {CustodyConfig} from "../../../src/util/dataColumns.js"; -import {testLogger} from "../../utils/logger.js"; describe("reqresp encoder", () => { const afterEachCallbacks: (() => Promise | void)[] = []; diff --git a/packages/beacon-node/test/e2e/sync/finalizedSync.test.ts b/packages/beacon-node/test/e2e/sync/finalizedSync.test.ts index c492974d3ec7..ab4d508f0e5e 100644 --- a/packages/beacon-node/test/e2e/sync/finalizedSync.test.ts +++ b/packages/beacon-node/test/e2e/sync/finalizedSync.test.ts @@ -3,11 +3,11 @@ import {fromHexString} from "@chainsafe/ssz"; import {routes} from "@lodestar/api"; import {ChainConfig} from "@lodestar/config"; import {TimestampFormatCode} from "@lodestar/logger"; +import {LogLevel, TestLoggerOpts, testLogger} from "@lodestar/logger/test-utils"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {phase0} from "@lodestar/types"; import {ChainEvent} from "../../../src/chain/index.js"; import {waitForEvent} from "../../utils/events/resolver.js"; -import {LogLevel, TestLoggerOpts, testLogger} from "../../utils/logger.js"; import {connect, onPeerConnect} from "../../utils/network.js"; import {getDevBeaconNode} from "../../utils/node/beacon.js"; import {getAndInitDevValidators} from "../../utils/node/validator.js"; diff --git a/packages/beacon-node/test/e2e/sync/unknownBlockSync.test.ts b/packages/beacon-node/test/e2e/sync/unknownBlockSync.test.ts index 7403cb5d7c11..b69c34db3ca9 100644 --- a/packages/beacon-node/test/e2e/sync/unknownBlockSync.test.ts +++ b/packages/beacon-node/test/e2e/sync/unknownBlockSync.test.ts @@ -3,6 +3,7 @@ import {fromHexString} from "@chainsafe/ssz"; import {routes} from "@lodestar/api"; import {ChainConfig} from "@lodestar/config"; import {TimestampFormatCode} from "@lodestar/logger"; +import {LogLevel, TestLoggerOpts, testLogger} from "@lodestar/logger/test-utils"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {fulu} from "@lodestar/types"; import {retry} from "@lodestar/utils"; @@ -12,7 +13,6 @@ import {ChainEvent} from "../../../src/chain/emitter.js"; import {BlockError, BlockErrorCode} from "../../../src/chain/errors/index.js"; import {INTEROP_BLOCK_HASH} from "../../../src/node/utils/interop/state.js"; import {waitForEvent} from "../../utils/events/resolver.js"; -import {LogLevel, TestLoggerOpts, testLogger} from "../../utils/logger.js"; import {connect, onPeerConnect} from "../../utils/network.js"; import {getDevBeaconNode} from "../../utils/node/beacon.js"; import {getAndInitDevValidators} from "../../utils/node/validator.js"; diff --git a/packages/beacon-node/test/perf/api/impl/validator/attester.test.ts b/packages/beacon-node/test/perf/api/impl/validator/attester.test.ts index 0366692dd32a..eeba4bd72083 100644 --- a/packages/beacon-node/test/perf/api/impl/validator/attester.test.ts +++ b/packages/beacon-node/test/perf/api/impl/validator/attester.test.ts @@ -1,5 +1,5 @@ import {beforeAll, bench, describe} from "@chainsafe/benchmark"; -import {generatePerfTestCachedStatePhase0, numValidators} from "../../../../../../state-transition/test/perf/util.js"; +import {generatePerfTestCachedStatePhase0, numValidators} from "@lodestar/state-transition/test-utils"; import {getPubkeysForIndices} from "../../../../../src/api/impl/validator/utils.js"; import {linspace} from "../../../../../src/util/numpy.js"; diff --git a/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts b/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts index 785c92fc44ef..1136d16ab590 100644 --- a/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts +++ b/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts @@ -12,8 +12,8 @@ import { getBlockRootAtSlot, newFilledArray, } from "@lodestar/state-transition"; +import {generatePerfTestCachedStateAltair} from "@lodestar/state-transition/test-utils"; import {ssz} from "@lodestar/types"; -import {generatePerfTestCachedStateAltair} from "../../../../../state-transition/test/perf/util.js"; import {AggregatedAttestationPool} from "../../../../src/chain/opPools/aggregatedAttestationPool.js"; import {ShufflingCache} from "../../../../src/chain/shufflingCache.js"; diff --git a/packages/beacon-node/test/perf/chain/opPools/opPool.test.ts b/packages/beacon-node/test/perf/chain/opPools/opPool.test.ts index ff96e430199d..814dad30e27b 100644 --- a/packages/beacon-node/test/perf/chain/opPools/opPool.test.ts +++ b/packages/beacon-node/test/perf/chain/opPools/opPool.test.ts @@ -9,8 +9,8 @@ import { MAX_VOLUNTARY_EXITS, } from "@lodestar/params"; import {CachedBeaconStateAltair, PubkeyCache} from "@lodestar/state-transition"; +import {generatePerfTestCachedStateAltair} from "@lodestar/state-transition/test-utils"; import {ssz} from "@lodestar/types"; -import {generatePerfTestCachedStateAltair} from "../../../../../state-transition/test/perf/util.js"; import {BlockType} from "../../../../src/chain/interface.js"; import {OpPool} from "../../../../src/chain/opPools/opPool.js"; import {generateBlsToExecutionChanges} from "../../../fixtures/capella.js"; diff --git a/packages/beacon-node/test/perf/chain/produceBlock/produceBlockBody.test.ts b/packages/beacon-node/test/perf/chain/produceBlock/produceBlockBody.test.ts index 77425276e216..4f615e6c6a49 100644 --- a/packages/beacon-node/test/perf/chain/produceBlock/produceBlockBody.test.ts +++ b/packages/beacon-node/test/perf/chain/produceBlock/produceBlockBody.test.ts @@ -2,14 +2,14 @@ import {generateKeyPair} from "@libp2p/crypto/keys"; import {afterAll, beforeAll, bench, describe} from "@chainsafe/benchmark"; import {config} from "@lodestar/config/default"; import {LevelDbController} from "@lodestar/db/controller/level"; +import {testLogger} from "@lodestar/logger/test-utils"; import {CachedBeaconStateAltair} from "@lodestar/state-transition"; +import {generatePerfTestCachedStateAltair} from "@lodestar/state-transition/test-utils"; import {defaultOptions as defaultValidatorOptions} from "@lodestar/validator"; -import {generatePerfTestCachedStateAltair} from "../../../../../state-transition/test/perf/util.js"; import {BeaconChain} from "../../../../src/chain/index.js"; import {BlockType, produceBlockBody} from "../../../../src/chain/produceBlock/produceBlockBody.js"; import {ExecutionEngineDisabled} from "../../../../src/execution/engine/index.js"; import {ArchiveMode, BeaconDb} from "../../../../src/index.js"; -import {testLogger} from "../../../utils/logger.js"; const logger = testLogger(); diff --git a/packages/beacon-node/test/perf/chain/validation/aggregateAndProof.test.ts b/packages/beacon-node/test/perf/chain/validation/aggregateAndProof.test.ts index 35dfcc589225..cdd23b07ea73 100644 --- a/packages/beacon-node/test/perf/chain/validation/aggregateAndProof.test.ts +++ b/packages/beacon-node/test/perf/chain/validation/aggregateAndProof.test.ts @@ -1,6 +1,6 @@ import {bench, describe} from "@chainsafe/benchmark"; +import {generateTestCachedBeaconStateOnlyValidators} from "@lodestar/state-transition/test-utils"; import {ssz} from "@lodestar/types"; -import {generateTestCachedBeaconStateOnlyValidators} from "../../../../../state-transition/test/perf/util.js"; import {validateApiAggregateAndProof, validateGossipAggregateAndProof} from "../../../../src/chain/validation/index.js"; import {getAggregateAndProofValidData} from "../../../utils/validationData/aggregateAndProof.js"; diff --git a/packages/beacon-node/test/perf/chain/validation/attestation.test.ts b/packages/beacon-node/test/perf/chain/validation/attestation.test.ts index 94c6c381b21c..2ddc6ba7e67f 100644 --- a/packages/beacon-node/test/perf/chain/validation/attestation.test.ts +++ b/packages/beacon-node/test/perf/chain/validation/attestation.test.ts @@ -1,7 +1,7 @@ import assert from "node:assert"; import {bench, describe} from "@chainsafe/benchmark"; +import {generateTestCachedBeaconStateOnlyValidators} from "@lodestar/state-transition/test-utils"; import {ssz} from "@lodestar/types"; -import {generateTestCachedBeaconStateOnlyValidators} from "../../../../../state-transition/test/perf/util.js"; import {validateGossipAttestationsSameAttData} from "../../../../src/chain/validation/index.js"; import {getAttDataFromAttestationSerialized} from "../../../../src/util/sszBytes.js"; import {getAttestationValidData} from "../../../utils/validationData/attestation.js"; diff --git a/packages/beacon-node/test/perf/chain/verifyImportBlocks.test.ts b/packages/beacon-node/test/perf/chain/verifyImportBlocks.test.ts index a4a062300a30..41972532be91 100644 --- a/packages/beacon-node/test/perf/chain/verifyImportBlocks.test.ts +++ b/packages/beacon-node/test/perf/chain/verifyImportBlocks.test.ts @@ -2,12 +2,11 @@ import {generateKeyPair} from "@libp2p/crypto/keys"; import {afterAll, beforeAll, bench, describe, setBenchOpts} from "@chainsafe/benchmark"; import {config} from "@lodestar/config/default"; import {LevelDbController} from "@lodestar/db/controller/level"; +import {testLogger} from "@lodestar/logger/test-utils"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; +import {getNetworkCachedBlock, getNetworkCachedState, rangeSyncTest} from "@lodestar/state-transition/test-utils"; import {sleep, toHex} from "@lodestar/utils"; import {defaultOptions as defaultValidatorOptions} from "@lodestar/validator"; -import {rangeSyncTest} from "../../../../state-transition/test/perf/params.js"; -import {beforeValue} from "../../../../state-transition/test/utils/beforeValueBenchmark.js"; -import {getNetworkCachedBlock, getNetworkCachedState} from "../../../../state-transition/test/utils/testFileCache.js"; import {BlockInputPreData} from "../../../src/chain/blocks/blockInput/blockInput.js"; import {BlockInputSource} from "../../../src/chain/blocks/blockInput/types.js"; import {AttestationImportOpt} from "../../../src/chain/blocks/types.js"; @@ -15,7 +14,7 @@ import {BeaconChain} from "../../../src/chain/index.js"; import {ExecutionEngineDisabled} from "../../../src/execution/engine/index.js"; import {ArchiveMode, BeaconDb} from "../../../src/index.js"; import {linspace} from "../../../src/util/numpy.js"; -import {testLogger} from "../../utils/logger.js"; +import {beforeValue} from "../../utils/beforeValueBenchmark.js"; // Define this params in `packages/state-transition/test/perf/params.ts` // to trigger Github actions CI cache diff --git a/packages/beacon-node/test/perf/util/blobs.test.ts b/packages/beacon-node/test/perf/util/blobs.test.ts index 8072667161c8..b430612be7de 100644 --- a/packages/beacon-node/test/perf/util/blobs.test.ts +++ b/packages/beacon-node/test/perf/util/blobs.test.ts @@ -2,10 +2,10 @@ import {bench, describe} from "@chainsafe/benchmark"; import {createChainForkConfig} from "@lodestar/config"; import {NUMBER_OF_COLUMNS} from "@lodestar/params"; import {ssz} from "@lodestar/types"; -import {reconstructBlobs} from "../../../src/util/blobs.ts"; -import {getDataColumnSidecarsFromBlock} from "../../../src/util/dataColumns.ts"; -import {kzg} from "../../../src/util/kzg.ts"; -import {generateRandomBlob} from "../../utils/kzg.ts"; +import {reconstructBlobs} from "../../../src/util/blobs.js"; +import {getDataColumnSidecarsFromBlock} from "../../../src/util/dataColumns.js"; +import {kzg} from "../../../src/util/kzg.js"; +import {generateRandomBlob} from "../../utils/kzg.js"; describe("reconstructBlobs", () => { const config = createChainForkConfig({ diff --git a/packages/beacon-node/test/sim/electra-interop.test.ts b/packages/beacon-node/test/sim/electra-interop.test.ts index 7497926cd1fd..b44ce153297c 100644 --- a/packages/beacon-node/test/sim/electra-interop.test.ts +++ b/packages/beacon-node/test/sim/electra-interop.test.ts @@ -3,6 +3,7 @@ import fs from "node:fs"; import {afterAll, afterEach, describe, it, vi} from "vitest"; import {ChainConfig} from "@lodestar/config"; import {TimestampFormatCode} from "@lodestar/logger"; +import {TestLoggerOpts, testLogger} from "@lodestar/logger/test-utils"; import {ForkName, SLOTS_PER_EPOCH, UNSET_DEPOSIT_REQUESTS_START_INDEX} from "@lodestar/params"; import {CachedBeaconStateElectra} from "@lodestar/state-transition"; import {Epoch, Slot, electra} from "@lodestar/types"; @@ -15,7 +16,6 @@ import {bytesToData, dataToBytes} from "../../src/execution/engine/utils.js"; import {initializeExecutionEngine} from "../../src/execution/index.js"; import {BeaconNode} from "../../src/index.js"; import {ClockEvent} from "../../src/util/clock.js"; -import {TestLoggerOpts, testLogger} from "../utils/logger.js"; import {getDevBeaconNode} from "../utils/node/beacon.js"; import {simTestInfoTracker} from "../utils/node/simTest.js"; import {getAndInitDevValidators} from "../utils/node/validator.js"; diff --git a/packages/beacon-node/test/spec-tests-version.json b/packages/beacon-node/test/spec-tests-version.json new file mode 120000 index 000000000000..bc2e3fd621b2 --- /dev/null +++ b/packages/beacon-node/test/spec-tests-version.json @@ -0,0 +1 @@ +../../../spec-tests-version.json \ No newline at end of file diff --git a/packages/beacon-node/test/spec/presets/epoch_processing.test.ts b/packages/beacon-node/test/spec/presets/epoch_processing.test.ts index 37c0e421688e..d6ba7efefe39 100644 --- a/packages/beacon-node/test/spec/presets/epoch_processing.test.ts +++ b/packages/beacon-node/test/spec/presets/epoch_processing.test.ts @@ -1,5 +1,6 @@ import path from "node:path"; import {expect} from "vitest"; +import {getConfig} from "@lodestar/config/test-utils"; import {ACTIVE_PRESET} from "@lodestar/params"; import { BeaconStateAllForks, @@ -12,7 +13,6 @@ import { import * as epochFns from "@lodestar/state-transition/epoch"; import {ssz} from "@lodestar/types"; import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState.js"; -import {getConfig} from "../../utils/config.js"; import {assertCorrectProgressiveBalances} from "../config.js"; import {ethereumConsensusSpecsTests} from "../specTestVersioning.js"; import {expectEqualBeaconState, inputTypeSszTreeViewDU} from "../utils/expectEqualBeaconState.js"; diff --git a/packages/beacon-node/test/spec/presets/finality.test.ts b/packages/beacon-node/test/spec/presets/finality.test.ts index 3c3e91062a23..ea3a60e212bd 100644 --- a/packages/beacon-node/test/spec/presets/finality.test.ts +++ b/packages/beacon-node/test/spec/presets/finality.test.ts @@ -1,4 +1,5 @@ import path from "node:path"; +import {getConfig} from "@lodestar/config/test-utils"; import {ACTIVE_PRESET, ForkName} from "@lodestar/params"; import { BeaconStateAllForks, @@ -8,7 +9,6 @@ import { } from "@lodestar/state-transition"; import {altair, bellatrix, ssz} from "@lodestar/types"; import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState.js"; -import {getConfig} from "../../utils/config.js"; import {assertCorrectProgressiveBalances} from "../config.js"; import {ethereumConsensusSpecsTests} from "../specTestVersioning.js"; import {expectEqualBeaconState, inputTypeSszTreeViewDU} from "../utils/expectEqualBeaconState.js"; diff --git a/packages/beacon-node/test/spec/presets/fork_choice.test.ts b/packages/beacon-node/test/spec/presets/fork_choice.test.ts index 8908296c6dd2..dc3e9dcf5a33 100644 --- a/packages/beacon-node/test/spec/presets/fork_choice.test.ts +++ b/packages/beacon-node/test/spec/presets/fork_choice.test.ts @@ -3,7 +3,9 @@ import {generateKeyPair} from "@libp2p/crypto/keys"; import {expect} from "vitest"; import {toHexString} from "@chainsafe/ssz"; import {createBeaconConfig} from "@lodestar/config"; +import {getConfig} from "@lodestar/config/test-utils"; import {CheckpointWithHex, ForkChoice} from "@lodestar/fork-choice"; +import {testLogger} from "@lodestar/logger/test-utils"; import { ACTIVE_PRESET, ForkPostDeneb, @@ -52,8 +54,6 @@ import {computePreFuluKzgCommitmentsInclusionProof} from "../../../src/util/blob import {ClockEvent} from "../../../src/util/clock.js"; import {ClockStopped} from "../../mocks/clock.js"; import {getMockedBeaconDb} from "../../mocks/mockedBeaconDb.js"; -import {getConfig} from "../../utils/config.js"; -import {testLogger} from "../../utils/logger.js"; import {assertCorrectProgressiveBalances} from "../config.js"; import {ethereumConsensusSpecsTests} from "../specTestVersioning.js"; import {specTestIterator} from "../utils/specTestIterator.js"; diff --git a/packages/beacon-node/test/spec/presets/genesis.test.ts b/packages/beacon-node/test/spec/presets/genesis.test.ts index 3f2a76851a7e..530666207646 100644 --- a/packages/beacon-node/test/spec/presets/genesis.test.ts +++ b/packages/beacon-node/test/spec/presets/genesis.test.ts @@ -1,5 +1,6 @@ import path from "node:path"; import {expect} from "vitest"; +import {getConfig} from "@lodestar/config/test-utils"; import {ACTIVE_PRESET, ForkName} from "@lodestar/params"; import {InputType} from "@lodestar/spec-test-util"; import { @@ -10,7 +11,6 @@ import { } from "@lodestar/state-transition"; import {ExecutionPayloadHeader, Root, TimeSeconds, phase0, ssz, sszTypesFor} from "@lodestar/types"; import {bnToNum} from "@lodestar/utils"; -import {getConfig} from "../../utils/config.js"; import {ethereumConsensusSpecsTests} from "../specTestVersioning.js"; import {expectEqualBeaconState} from "../utils/expectEqualBeaconState.js"; import {specTestIterator} from "../utils/specTestIterator.js"; diff --git a/packages/beacon-node/test/spec/presets/light_client/sync.ts b/packages/beacon-node/test/spec/presets/light_client/sync.ts index 3b788fecd4e8..96f1d219c6ae 100644 --- a/packages/beacon-node/test/spec/presets/light_client/sync.ts +++ b/packages/beacon-node/test/spec/presets/light_client/sync.ts @@ -1,12 +1,12 @@ import {expect} from "vitest"; import {ChainConfig, createBeaconConfig} from "@lodestar/config"; import {LightclientSpec, toLightClientUpdateSummary} from "@lodestar/light-client/spec"; +import {testLogger} from "@lodestar/logger/test-utils"; import {isForkPostAltair} from "@lodestar/params"; import {InputType} from "@lodestar/spec-test-util"; import {computeSyncPeriodAtSlot} from "@lodestar/state-transition"; import {RootHex, Slot, altair, phase0, ssz, sszTypesFor} from "@lodestar/types"; import {fromHex, toHex} from "@lodestar/utils"; -import {testLogger} from "../../../utils/logger.js"; import {TestRunnerFn} from "../../utils/types.js"; // https://github.com/ethereum/consensus-specs/blob/da3f5af919be4abb5a6db5a80b235deb8b4b5cba/tests/formats/light_client/single_merkle_proof.md diff --git a/packages/beacon-node/test/spec/presets/operations.test.ts b/packages/beacon-node/test/spec/presets/operations.test.ts index 300092667960..6b8d7a90300a 100644 --- a/packages/beacon-node/test/spec/presets/operations.test.ts +++ b/packages/beacon-node/test/spec/presets/operations.test.ts @@ -1,4 +1,5 @@ import path from "node:path"; +import {getConfig} from "@lodestar/config/test-utils"; import {ACTIVE_PRESET, ForkName, ForkSeq} from "@lodestar/params"; import {InputType} from "@lodestar/spec-test-util"; import { @@ -14,7 +15,6 @@ import { import * as blockFns from "@lodestar/state-transition/block"; import {AttesterSlashing, altair, bellatrix, capella, electra, gloas, phase0, ssz, sszTypesFor} from "@lodestar/types"; import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState.js"; -import {getConfig} from "../../utils/config.js"; import {ethereumConsensusSpecsTests} from "../specTestVersioning.js"; import {expectEqualBeaconState, inputTypeSszTreeViewDU} from "../utils/expectEqualBeaconState.js"; import {specTestIterator} from "../utils/specTestIterator.js"; diff --git a/packages/beacon-node/test/spec/presets/rewards.test.ts b/packages/beacon-node/test/spec/presets/rewards.test.ts index eda1837b9e2d..1ca70f297159 100644 --- a/packages/beacon-node/test/spec/presets/rewards.test.ts +++ b/packages/beacon-node/test/spec/presets/rewards.test.ts @@ -1,12 +1,12 @@ import path from "node:path"; import {expect} from "vitest"; import {VectorCompositeType} from "@chainsafe/ssz"; +import {getConfig} from "@lodestar/config/test-utils"; import {ACTIVE_PRESET} from "@lodestar/params"; import {BeaconStateAllForks, beforeProcessEpoch} from "@lodestar/state-transition"; import {getRewardsAndPenalties} from "@lodestar/state-transition/epoch"; import {ssz} from "@lodestar/types"; import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState.js"; -import {getConfig} from "../../utils/config.js"; import {assertCorrectProgressiveBalances} from "../config.js"; import {ethereumConsensusSpecsTests} from "../specTestVersioning.js"; import {inputTypeSszTreeViewDU} from "../utils/expectEqualBeaconState.js"; diff --git a/packages/beacon-node/test/spec/presets/sanity.test.ts b/packages/beacon-node/test/spec/presets/sanity.test.ts index 4731bdc22cd9..1d35ab48ecf8 100644 --- a/packages/beacon-node/test/spec/presets/sanity.test.ts +++ b/packages/beacon-node/test/spec/presets/sanity.test.ts @@ -1,4 +1,5 @@ import path from "node:path"; +import {getConfig} from "@lodestar/config/test-utils"; import {ACTIVE_PRESET, ForkName} from "@lodestar/params"; import {InputType} from "@lodestar/spec-test-util"; import { @@ -11,7 +12,6 @@ import { import {SignedBeaconBlock, deneb, ssz} from "@lodestar/types"; import {bnToNum} from "@lodestar/utils"; import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState.js"; -import {getConfig} from "../../utils/config.js"; import {assertCorrectProgressiveBalances} from "../config.js"; import {ethereumConsensusSpecsTests} from "../specTestVersioning.js"; import {expectEqualBeaconState, inputTypeSszTreeViewDU} from "../utils/expectEqualBeaconState.js"; diff --git a/packages/beacon-node/test/spec/specTestVersioning.ts b/packages/beacon-node/test/spec/specTestVersioning.ts index 93833af14b17..d4726ca26e23 100644 --- a/packages/beacon-node/test/spec/specTestVersioning.ts +++ b/packages/beacon-node/test/spec/specTestVersioning.ts @@ -1,30 +1,15 @@ import path from "node:path"; import {fileURLToPath} from "node:url"; -import {DownloadTestsOptions} from "@lodestar/spec-test-util/downloadTests"; +import specTestVersions from "../spec-tests-version.json" with {type: "json"}; -// WARNING! Don't move or rename this file !!! -// -// This file is used to generate the cache ID for spec tests download in Github Actions CI -// It's path is hardcoded in: `.github/workflows/test-spec.yml` -// -// The contents of this file MUST include the URL, version and target path, and nothing else. - -// Global variable __dirname no longer available in ES6 modules. -// Solutions: https://stackoverflow.com/questions/46745014/alternative-for-dirname-in-node-js-when-using-es6-modules const __dirname = path.dirname(fileURLToPath(import.meta.url)); -export const ethereumConsensusSpecsTests: DownloadTestsOptions = { - specVersion: "v1.7.0-alpha.2", - // Target directory is the host package root: 'packages/*/spec-tests' - outputDir: path.join(__dirname, "../../spec-tests"), - specTestsRepoUrl: "https://github.com/ethereum/consensus-specs", - testsToDownload: ["general", "mainnet", "minimal"], +export const ethereumConsensusSpecsTests = { + ...specTestVersions.ethereumConsensusSpecsTests, + outputDir: path.join(__dirname, "../../", specTestVersions.ethereumConsensusSpecsTests.outputDirBase), }; -export const blsSpecTests: DownloadTestsOptions = { - specVersion: "v0.1.1", - // Target directory is the host package root: 'packages/*/spec-tests-bls' - outputDir: path.join(__dirname, "../../spec-tests-bls"), - specTestsRepoUrl: "https://github.com/ethereum/bls12-381-tests", - testsToDownload: ["bls_tests_yaml"], +export const blsSpecTests = { + ...specTestVersions.blsSpecTests, + outputDir: path.join(__dirname, "../../", specTestVersions.blsSpecTests.outputDirBase), }; diff --git a/packages/beacon-node/test/spec/utils/gossipValidation.ts b/packages/beacon-node/test/spec/utils/gossipValidation.ts index fdea09cc23d8..3228838ca11a 100644 --- a/packages/beacon-node/test/spec/utils/gossipValidation.ts +++ b/packages/beacon-node/test/spec/utils/gossipValidation.ts @@ -5,6 +5,8 @@ import {generateKeyPair} from "@libp2p/crypto/keys"; import snappy from "snappy"; import {expect} from "vitest"; import {createBeaconConfig} from "@lodestar/config"; +import {getConfig} from "@lodestar/config/test-utils"; +import {testLogger} from "@lodestar/logger/test-utils"; import {ForkName} from "@lodestar/params"; import { BeaconStateAllForks, @@ -35,8 +37,6 @@ import {GossipType} from "../../../src/network/gossip/interface.js"; import type {IClock} from "../../../src/util/clock.js"; import {getBeaconAttestationGossipIndex, getSlotFromBeaconAttestationSerialized} from "../../../src/util/sszBytes.js"; import {getMockedBeaconDb} from "../../mocks/mockedBeaconDb.js"; -import {getConfig} from "../../utils/config.js"; -import {testLogger} from "../../utils/logger.js"; import {assertCorrectProgressiveBalances} from "../config.js"; /** diff --git a/packages/beacon-node/test/unit-minimal/chain/stateCache/persistentCheckpointsCache.test.ts b/packages/beacon-node/test/unit-minimal/chain/stateCache/persistentCheckpointsCache.test.ts index aa503a0836fd..9dd74e170f29 100644 --- a/packages/beacon-node/test/unit-minimal/chain/stateCache/persistentCheckpointsCache.test.ts +++ b/packages/beacon-node/test/unit-minimal/chain/stateCache/persistentCheckpointsCache.test.ts @@ -1,6 +1,7 @@ import {beforeAll, beforeEach, describe, expect, it} from "vitest"; import {createBeaconConfig} from "@lodestar/config"; import {chainConfig as chainConfigDef} from "@lodestar/config/default"; +import {testLogger} from "@lodestar/logger/test-utils"; import {ACTIVE_PRESET, PresetName, SLOTS_PER_EPOCH, SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params"; import {CachedBeaconStateAllForks, computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {RootHex, phase0} from "@lodestar/types"; @@ -13,7 +14,6 @@ import { } from "../../../../src/chain/stateCache/persistentCheckpointsCache.js"; import {CheckpointHex} from "../../../../src/chain/stateCache/types.js"; import {getTestDatastore} from "../../../utils/chain/stateCache/datastore.js"; -import {testLogger} from "../../../utils/logger.js"; import {generateCachedState} from "../../../utils/state.js"; describe("PersistentCheckpointStateCache", () => { diff --git a/packages/beacon-node/test/unit/chain/archiveStore/blockArchiver.test.ts b/packages/beacon-node/test/unit/chain/archiveStore/blockArchiver.test.ts index ea4ff653c1e9..cc8730ba24b8 100644 --- a/packages/beacon-node/test/unit/chain/archiveStore/blockArchiver.test.ts +++ b/packages/beacon-node/test/unit/chain/archiveStore/blockArchiver.test.ts @@ -3,13 +3,13 @@ import {fromHexString, toHexString} from "@chainsafe/ssz"; import {createChainForkConfig} from "@lodestar/config"; import {config as defaultConfig} from "@lodestar/config/default"; import {PayloadStatus} from "@lodestar/fork-choice"; +import {testLogger} from "@lodestar/logger/test-utils"; import {computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {ssz} from "@lodestar/types"; import {archiveBlocks} from "../../../../src/chain/archiveStore/utils/archiveBlocks.js"; import {ZERO_HASH_HEX} from "../../../../src/constants/index.js"; import {MockedBeaconChain, getMockedBeaconChain} from "../../../mocks/mockedBeaconChain.js"; import {MockedBeaconDb, getMockedBeaconDb} from "../../../mocks/mockedBeaconDb.js"; -import {testLogger} from "../../../utils/logger.js"; import {generateProtoBlock} from "../../../utils/typeGenerator.js"; function toAsyncIterable(items: T[]): AsyncIterable { diff --git a/packages/beacon-node/test/unit/chain/bls/bls.test.ts b/packages/beacon-node/test/unit/chain/bls/bls.test.ts index c91831f74b11..1e749d91e3b9 100644 --- a/packages/beacon-node/test/unit/chain/bls/bls.test.ts +++ b/packages/beacon-node/test/unit/chain/bls/bls.test.ts @@ -1,9 +1,9 @@ import {beforeEach, describe, expect, it} from "vitest"; import {PublicKey, SecretKey, Signature} from "@chainsafe/blst"; +import {testLogger} from "@lodestar/logger/test-utils"; import {ISignatureSet, SignatureSetType, createPubkeyCache} from "@lodestar/state-transition"; import {BlsMultiThreadWorkerPool} from "../../../../src/chain/bls/multithread/index.js"; import {BlsSingleThreadVerifier} from "../../../../src/chain/bls/singleThread.js"; -import {testLogger} from "../../../utils/logger.js"; describe("BlsVerifier ", () => { // take time for creating thread pool diff --git a/packages/beacon-node/test/unit/chain/regen/regen.test.ts b/packages/beacon-node/test/unit/chain/regen/regen.test.ts index 941b9adcb8ec..b2e860d1a17c 100644 --- a/packages/beacon-node/test/unit/chain/regen/regen.test.ts +++ b/packages/beacon-node/test/unit/chain/regen/regen.test.ts @@ -1,6 +1,7 @@ import {beforeEach, describe, expect, it} from "vitest"; import {createBeaconConfig} from "@lodestar/config"; import {chainConfig as chainConfigDef} from "@lodestar/config/default"; +import {testLogger} from "@lodestar/logger/test-utils"; import {SLOTS_PER_EPOCH, SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params"; import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {RegenCaller} from "../../../../src/chain/regen/interface.js"; @@ -8,7 +9,6 @@ import {processSlotsToNearestCheckpoint} from "../../../../src/chain/regen/regen import {FIFOBlockStateCache} from "../../../../src/chain/stateCache/fifoBlockStateCache.js"; import {PersistentCheckpointStateCache} from "../../../../src/chain/stateCache/persistentCheckpointsCache.js"; import {getTestDatastore} from "../../../utils/chain/stateCache/datastore.js"; -import {testLogger} from "../../../utils/logger.js"; import {generateCachedState} from "../../../utils/state.js"; describe("regen", () => { diff --git a/packages/beacon-node/test/unit/chain/seenCache/seenBlockInput.test.ts b/packages/beacon-node/test/unit/chain/seenCache/seenBlockInput.test.ts index 08338d992c7e..1a121e70bf8e 100644 --- a/packages/beacon-node/test/unit/chain/seenCache/seenBlockInput.test.ts +++ b/packages/beacon-node/test/unit/chain/seenCache/seenBlockInput.test.ts @@ -1,6 +1,7 @@ import {generateKeyPair} from "@libp2p/crypto/keys"; import {beforeEach, describe, expect, it} from "vitest"; import {PayloadStatus} from "@lodestar/fork-choice"; +import {testLogger} from "@lodestar/logger/test-utils"; import {ForkName} from "@lodestar/params"; import {signedBlockToSignedHeader} from "@lodestar/state-transition"; import {toRootHex} from "@lodestar/utils"; @@ -24,7 +25,6 @@ import { generateBlockWithBlobSidecars, generateChainOfBlocks, } from "../../../utils/blocksAndData.js"; -import {testLogger} from "../../../utils/logger.js"; describe("SeenBlockInputCache", async () => { let cache: SeenBlockInput; diff --git a/packages/beacon-node/test/unit/chain/shufflingCache.test.ts b/packages/beacon-node/test/unit/chain/shufflingCache.test.ts index b8a9602ea6ce..63cf2f53a77c 100644 --- a/packages/beacon-node/test/unit/chain/shufflingCache.test.ts +++ b/packages/beacon-node/test/unit/chain/shufflingCache.test.ts @@ -1,5 +1,5 @@ import {beforeEach, describe, expect, it} from "vitest"; -import {generateTestCachedBeaconStateOnlyValidators} from "../../../../state-transition/test/perf/util.js"; +import {generateTestCachedBeaconStateOnlyValidators} from "@lodestar/state-transition/test-utils"; import {ShufflingCache} from "../../../src/chain/shufflingCache.js"; describe("ShufflingCache", () => { diff --git a/packages/beacon-node/test/unit/chain/validation/aggregateAndProof.test.ts b/packages/beacon-node/test/unit/chain/validation/aggregateAndProof.test.ts index e25fda1a14bb..6cf39e084adf 100644 --- a/packages/beacon-node/test/unit/chain/validation/aggregateAndProof.test.ts +++ b/packages/beacon-node/test/unit/chain/validation/aggregateAndProof.test.ts @@ -1,8 +1,8 @@ import {describe, it} from "vitest"; import {BitArray, toHexString} from "@chainsafe/ssz"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; +import {generateTestCachedBeaconStateOnlyValidators} from "@lodestar/state-transition/test-utils"; import {phase0, ssz} from "@lodestar/types"; -import {generateTestCachedBeaconStateOnlyValidators} from "../../../../../state-transition/test/perf/util.js"; import {AttestationErrorCode} from "../../../../src/chain/errors/index.js"; import {IBeaconChain} from "../../../../src/chain/index.js"; import {validateApiAggregateAndProof, validateGossipAggregateAndProof} from "../../../../src/chain/validation/index.js"; diff --git a/packages/beacon-node/test/unit/chain/validation/attestation/validateAttestation.test.ts b/packages/beacon-node/test/unit/chain/validation/attestation/validateAttestation.test.ts index fcfa71655fa7..99c5d63c9bf2 100644 --- a/packages/beacon-node/test/unit/chain/validation/attestation/validateAttestation.test.ts +++ b/packages/beacon-node/test/unit/chain/validation/attestation/validateAttestation.test.ts @@ -1,9 +1,9 @@ import {describe, expect, it} from "vitest"; import {BitArray} from "@chainsafe/ssz"; import {ForkName, SLOTS_PER_EPOCH} from "@lodestar/params"; +import {generateTestCachedBeaconStateOnlyValidators} from "@lodestar/state-transition/test-utils"; import {ssz} from "@lodestar/types"; import {LodestarError} from "@lodestar/utils"; -import {generateTestCachedBeaconStateOnlyValidators} from "../../../../../../state-transition/test/perf/util.js"; import {AttestationErrorCode, GossipErrorCode} from "../../../../../src/chain/errors/index.js"; import {IBeaconChain} from "../../../../../src/chain/index.js"; import { diff --git a/packages/beacon-node/test/unit/chain/validatorMonitor.test.ts b/packages/beacon-node/test/unit/chain/validatorMonitor.test.ts index 8b4a5687b43e..fd194ab1ee51 100644 --- a/packages/beacon-node/test/unit/chain/validatorMonitor.test.ts +++ b/packages/beacon-node/test/unit/chain/validatorMonitor.test.ts @@ -1,8 +1,8 @@ import {describe, expect, it, vi} from "vitest"; import {createChainForkConfig, defaultChainConfig} from "@lodestar/config"; +import {testLogger} from "@lodestar/logger/test-utils"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {createValidatorMonitor} from "../../../src/chain/validatorMonitor.js"; -import {testLogger} from "../../utils/logger.js"; describe("ValidatorMonitor", () => { // Use phase0 config (no altair) to avoid needing full state with block roots diff --git a/packages/beacon-node/test/unit/db/api/repositories/blockArchive.test.ts b/packages/beacon-node/test/unit/db/api/repositories/blockArchive.test.ts index 86c8416eaf13..eeab033952af 100644 --- a/packages/beacon-node/test/unit/db/api/repositories/blockArchive.test.ts +++ b/packages/beacon-node/test/unit/db/api/repositories/blockArchive.test.ts @@ -3,11 +3,11 @@ import {afterEach, beforeEach, describe, expect, it, vi} from "vitest"; import {config} from "@lodestar/config/default"; import {encodeKey} from "@lodestar/db"; import {LevelDbController} from "@lodestar/db/controller/level"; +import {testLogger} from "@lodestar/logger/test-utils"; import {ssz} from "@lodestar/types"; import {intToBytes} from "@lodestar/utils"; import {Bucket, getBucketNameByValue} from "../../../../../src/db/buckets.js"; import {BlockArchiveRepository} from "../../../../../src/db/repositories/index.js"; -import {testLogger} from "../../../../utils/logger.js"; describe("block archive repository", () => { const testDir = "./.tmp_block_archive_unit_test"; diff --git a/packages/beacon-node/test/unit/db/api/repositories/dataColumn.test.ts b/packages/beacon-node/test/unit/db/api/repositories/dataColumn.test.ts index 150e79bbe2a6..a2252a5638a1 100644 --- a/packages/beacon-node/test/unit/db/api/repositories/dataColumn.test.ts +++ b/packages/beacon-node/test/unit/db/api/repositories/dataColumn.test.ts @@ -2,6 +2,7 @@ import {rimraf} from "rimraf"; import {afterEach, beforeEach, describe, expect, it} from "vitest"; import {createChainForkConfig} from "@lodestar/config"; import {LevelDbController} from "@lodestar/db/controller/level"; +import {testLogger} from "@lodestar/logger/test-utils"; import {NUMBER_OF_COLUMNS} from "@lodestar/params"; import {Root, fulu, ssz} from "@lodestar/types"; import {fromAsync, toHex} from "@lodestar/utils"; @@ -9,7 +10,6 @@ import {DataColumnSidecarRepository} from "../../../../../src/db/repositories/da import {DataColumnSidecarArchiveRepository} from "../../../../../src/db/repositories/dataColumnSidecarArchive.js"; import {getDataColumnSidecarsFromBlock} from "../../../../../src/util/dataColumns.js"; import {kzg} from "../../../../../src/util/kzg.js"; -import {testLogger} from "../../../../utils/logger.js"; /* eslint-disable @typescript-eslint/naming-convention */ const config = createChainForkConfig({ diff --git a/packages/beacon-node/test/unit/metrics/server/http.test.ts b/packages/beacon-node/test/unit/metrics/server/http.test.ts index 3172ae72cd0e..247d9cfbfff5 100644 --- a/packages/beacon-node/test/unit/metrics/server/http.test.ts +++ b/packages/beacon-node/test/unit/metrics/server/http.test.ts @@ -1,7 +1,7 @@ import {afterAll, describe, it} from "vitest"; +import {testLogger} from "@lodestar/logger/test-utils"; import {fetch} from "@lodestar/utils"; import {HttpMetricsServer, getHttpMetricsServer} from "../../../../src/metrics/index.js"; -import {testLogger} from "../../../utils/logger.js"; import {createMetricsTest} from "../utils.js"; describe("HttpMetricsServer", () => { diff --git a/packages/beacon-node/test/unit/network/metadata.test.ts b/packages/beacon-node/test/unit/network/metadata.test.ts index b2b63687de9d..e1ae191d05f8 100644 --- a/packages/beacon-node/test/unit/network/metadata.test.ts +++ b/packages/beacon-node/test/unit/network/metadata.test.ts @@ -1,5 +1,7 @@ import {describe, expect, it, vi} from "vitest"; import {createBeaconConfig, createChainForkConfig} from "@lodestar/config"; +import {config} from "@lodestar/config/test-utils"; +import {testLogger} from "@lodestar/logger/test-utils"; import {ZERO_HASH} from "@lodestar/params"; import {ssz} from "@lodestar/types"; import {toHex} from "@lodestar/utils"; @@ -8,8 +10,6 @@ import {NetworkConfig} from "../../../src/network/networkConfig.js"; import {computeNodeId} from "../../../src/network/subnets/index.js"; import {CustodyConfig} from "../../../src/util/dataColumns.js"; import {serializeCgc} from "../../../src/util/metadata.js"; -import {config} from "../../utils/config.js"; -import {testLogger} from "../../utils/logger.js"; import {getValidPeerId} from "../../utils/peer.js"; describe("network / metadata", () => { diff --git a/packages/beacon-node/test/unit/network/subnets/attnetsService.test.ts b/packages/beacon-node/test/unit/network/subnets/attnetsService.test.ts index 1442d5098471..d729a8d01022 100644 --- a/packages/beacon-node/test/unit/network/subnets/attnetsService.test.ts +++ b/packages/beacon-node/test/unit/network/subnets/attnetsService.test.ts @@ -1,5 +1,6 @@ import {MockedObject, afterEach, beforeEach, describe, expect, it, vi} from "vitest"; import {createBeaconConfig} from "@lodestar/config"; +import {testLogger} from "@lodestar/logger/test-utils"; import {ATTESTATION_SUBNET_COUNT, ForkName, GENESIS_EPOCH, SLOTS_PER_EPOCH} from "@lodestar/params"; import {ZERO_HASH, getCurrentSlot} from "@lodestar/state-transition"; import {SubnetID} from "@lodestar/types"; @@ -11,7 +12,6 @@ import {AttnetsService} from "../../../../src/network/subnets/attnetsService.js" import {CommitteeSubscription} from "../../../../src/network/subnets/interface.js"; import {Clock, IClock} from "../../../../src/util/clock.js"; import {CustodyConfig} from "../../../../src/util/dataColumns.js"; -import {testLogger} from "../../../utils/logger.js"; vi.mock("../../../../src/network/gossip/gossipsub.js"); diff --git a/packages/beacon-node/test/unit/sync/range/chain.test.ts b/packages/beacon-node/test/unit/sync/range/chain.test.ts index 4146202cf034..246db737b897 100644 --- a/packages/beacon-node/test/unit/sync/range/chain.test.ts +++ b/packages/beacon-node/test/unit/sync/range/chain.test.ts @@ -1,5 +1,6 @@ import {afterEach, describe, it} from "vitest"; import {config} from "@lodestar/config/default"; +import {testLogger} from "@lodestar/logger/test-utils"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {Epoch, Slot, phase0, ssz} from "@lodestar/types"; @@ -12,7 +13,6 @@ import {RangeSyncType} from "../../../../src/sync/utils/remoteSyncType.js"; import {Clock} from "../../../../src/util/clock.js"; import {CustodyConfig} from "../../../../src/util/dataColumns.js"; import {linspace} from "../../../../src/util/numpy.js"; -import {testLogger} from "../../../utils/logger.js"; import {validPeerIdStr} from "../../../utils/peer.js"; describe("sync / range / chain", () => { diff --git a/packages/beacon-node/test/unit/sync/unknownBlock.test.ts b/packages/beacon-node/test/unit/sync/unknownBlock.test.ts index 8873e1bcbedf..53dbf6cbc589 100644 --- a/packages/beacon-node/test/unit/sync/unknownBlock.test.ts +++ b/packages/beacon-node/test/unit/sync/unknownBlock.test.ts @@ -4,6 +4,7 @@ import {toHexString} from "@chainsafe/ssz"; import {createChainForkConfig} from "@lodestar/config"; import {config as minimalConfig} from "@lodestar/config/default"; import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; +import {testLogger} from "@lodestar/logger/test-utils"; import {ForkName} from "@lodestar/params"; import {ssz} from "@lodestar/types"; import {notNullish, sleep} from "@lodestar/utils"; @@ -22,7 +23,6 @@ import {PeerIdStr} from "../../../src/util/peerId.js"; import {ClockStopped} from "../../mocks/clock.js"; import {MockedBeaconChain, getMockedBeaconChain} from "../../mocks/mockedBeaconChain.js"; import {generateBlockWithColumnSidecars} from "../../utils/blocksAndData.js"; -import {testLogger} from "../../utils/logger.js"; import {getRandPeerIdStr, getRandPeerSyncMeta} from "../../utils/peer.js"; describe("sync by UnknownBlockSync", {timeout: 20_000}, () => { diff --git a/packages/beacon-node/test/utils/beforeValueBenchmark.ts b/packages/beacon-node/test/utils/beforeValueBenchmark.ts new file mode 100644 index 000000000000..5bca88fe4df2 --- /dev/null +++ b/packages/beacon-node/test/utils/beforeValueBenchmark.ts @@ -0,0 +1,36 @@ +import {beforeAll} from "@chainsafe/benchmark"; + +export type LazyValue = {value: T}; + +/** + * Register a callback to compute a value in the before() block of benchmark tests + * ```ts + * const state = beforeValue(() => getState()) + * it("test", () => { + * doTest(state.value) + * }) + * ``` + */ +export function beforeValue(fn: () => T | Promise, timeout?: number): LazyValue { + let value: T = null as unknown as T; + + beforeAll(async () => { + value = await fn(); + }, timeout ?? 300_000); + + return new Proxy<{value: T}>( + {value}, + { + get: (_target, prop) => { + if (prop === "value") { + if (value === null) { + throw Error("beforeValue has not yet run the before() block"); + } + return value; + } + + return undefined; + }, + } + ); +} diff --git a/packages/beacon-node/test/utils/db.ts b/packages/beacon-node/test/utils/db.ts index eb0f546f702b..9206d9ca657b 100644 --- a/packages/beacon-node/test/utils/db.ts +++ b/packages/beacon-node/test/utils/db.ts @@ -2,8 +2,8 @@ import childProcess from "node:child_process"; import {ChainForkConfig} from "@lodestar/config"; import {FilterOptions} from "@lodestar/db"; import {LevelDbController} from "@lodestar/db/controller/level"; +import {testLogger} from "@lodestar/logger/test-utils"; import {BeaconDb} from "../../src/index.js"; -import {testLogger} from "./logger.js"; export const TEMP_DB_LOCATION = ".tmpdb"; diff --git a/packages/beacon-node/test/utils/networkWithMockDb.ts b/packages/beacon-node/test/utils/networkWithMockDb.ts index fc927bd18145..6bef00294cbf 100644 --- a/packages/beacon-node/test/utils/networkWithMockDb.ts +++ b/packages/beacon-node/test/utils/networkWithMockDb.ts @@ -1,5 +1,6 @@ import {generateKeyPair} from "@libp2p/crypto/keys"; import {ChainForkConfig, createBeaconConfig} from "@lodestar/config"; +import {testLogger} from "@lodestar/logger/test-utils"; import {createCachedBeaconState, createPubkeyCache, syncPubkeys} from "@lodestar/state-transition"; import {ssz} from "@lodestar/types"; import {sleep} from "@lodestar/utils"; @@ -11,7 +12,6 @@ import {NetworkOptions, defaultNetworkOptions} from "../../src/network/options.j import {GetReqRespHandlerFn} from "../../src/network/reqresp/types.js"; import {getMockedBeaconDb} from "../mocks/mockedBeaconDb.js"; import {ClockStatic} from "./clock.js"; -import {testLogger} from "./logger.js"; import {generateState} from "./state.js"; export type NetworkForTestOpts = { diff --git a/packages/beacon-node/test/utils/node/beacon.ts b/packages/beacon-node/test/utils/node/beacon.ts index 3543ac7dd2ab..4f9b369f3635 100644 --- a/packages/beacon-node/test/utils/node/beacon.ts +++ b/packages/beacon-node/test/utils/node/beacon.ts @@ -8,6 +8,7 @@ import {ChainConfig, createBeaconConfig, createChainForkConfig} from "@lodestar/ import {config as minimalConfig} from "@lodestar/config/default"; import {LevelDbController} from "@lodestar/db/controller/level"; import {LoggerNode} from "@lodestar/logger/node"; +import {testLogger} from "@lodestar/logger/test-utils"; import {ForkSeq, GENESIS_SLOT} from "@lodestar/params"; import { BeaconStateAllForks, @@ -26,7 +27,6 @@ import {defaultNetworkOptions} from "../../../src/network/options.js"; import {IBeaconNodeOptions, defaultOptions} from "../../../src/node/options.js"; import {InteropStateOpts} from "../../../src/node/utils/interop/state.js"; import {initDevState} from "../../../src/node/utils/state.js"; -import {testLogger} from "../logger.js"; export async function getDevBeaconNode( opts: { diff --git a/packages/beacon-node/test/utils/node/validator.ts b/packages/beacon-node/test/utils/node/validator.ts index e77fb1083f01..4bc28beef3b3 100644 --- a/packages/beacon-node/test/utils/node/validator.ts +++ b/packages/beacon-node/test/utils/node/validator.ts @@ -4,11 +4,11 @@ import {SecretKey} from "@chainsafe/blst"; import {ApiClient, ApiError, ApiResponse, HttpStatusCode} from "@lodestar/api"; import {BeaconApiMethods} from "@lodestar/api/beacon/server"; import {LevelDbController} from "@lodestar/db/controller/level"; +import {TestLoggerOpts, testLogger} from "@lodestar/logger/test-utils"; import {interopSecretKey} from "@lodestar/state-transition"; import {mapValues} from "@lodestar/utils"; import {Signer, SignerType, SlashingProtection, Validator, ValidatorProposerConfig} from "@lodestar/validator"; import {BeaconNode} from "../../../src/index.js"; -import {TestLoggerOpts, testLogger} from "../logger.js"; export async function getAndInitDevValidators({ node, diff --git a/packages/beacon-node/test/utils/state.ts b/packages/beacon-node/test/utils/state.ts index bab23523b29b..af7791611087 100644 --- a/packages/beacon-node/test/utils/state.ts +++ b/packages/beacon-node/test/utils/state.ts @@ -1,6 +1,7 @@ import {SecretKey} from "@chainsafe/blst"; import {ChainForkConfig, createBeaconConfig} from "@lodestar/config"; import {config as minimalConfig} from "@lodestar/config/default"; +import {getConfig} from "@lodestar/config/test-utils"; import {ExecutionStatus, PayloadStatus, ProtoBlock} from "@lodestar/fork-choice"; import {FAR_FUTURE_EPOCH, ForkName, ForkSeq, MAX_EFFECTIVE_BALANCE, SYNC_COMMITTEE_SIZE} from "@lodestar/params"; import { @@ -16,7 +17,6 @@ import { } from "@lodestar/state-transition"; import {BeaconState, altair, bellatrix, electra, ssz} from "@lodestar/types"; import {ZERO_HASH_HEX} from "../../src/constants/constants.js"; -import {getConfig} from "./config.js"; import {generateValidator, generateValidators} from "./validator.js"; /** diff --git a/packages/beacon-node/test/utils/validationData/aggregateAndProof.ts b/packages/beacon-node/test/utils/validationData/aggregateAndProof.ts index a8ac5833fbd5..be91ae47c35c 100644 --- a/packages/beacon-node/test/utils/validationData/aggregateAndProof.ts +++ b/packages/beacon-node/test/utils/validationData/aggregateAndProof.ts @@ -1,7 +1,7 @@ import {DOMAIN_AGGREGATE_AND_PROOF, DOMAIN_SELECTION_PROOF} from "@lodestar/params"; import {computeSigningRoot} from "@lodestar/state-transition"; +import {getSecretKeyFromIndexCached} from "@lodestar/state-transition/test-utils"; import {phase0, ssz} from "@lodestar/types"; -import {getSecretKeyFromIndexCached} from "../../../../state-transition/test/perf/util.js"; import {IBeaconChain} from "../../../src/chain/index.js"; import {SeenAggregators} from "../../../src/chain/seenCache/index.js"; import {signCached} from "../cache.js"; diff --git a/packages/beacon-node/test/utils/validationData/attestation.ts b/packages/beacon-node/test/utils/validationData/attestation.ts index 18175f596b19..1e57551cd8e8 100644 --- a/packages/beacon-node/test/utils/validationData/attestation.ts +++ b/packages/beacon-node/test/utils/validationData/attestation.ts @@ -1,5 +1,6 @@ import {BitArray, toHexString} from "@chainsafe/ssz"; import {ExecutionStatus, IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; +import {testLogger} from "@lodestar/logger/test-utils"; import {DOMAIN_BEACON_ATTESTER} from "@lodestar/params"; import { DataAvailabilityStatus, @@ -7,11 +8,11 @@ import { computeSigningRoot, computeStartSlotAtEpoch, } from "@lodestar/state-transition"; -import {Slot, SubnetID, phase0, ssz} from "@lodestar/types"; import { generateTestCachedBeaconStateOnlyValidators, getSecretKeyFromIndexCached, -} from "../../../../state-transition/test/perf/util.js"; +} from "@lodestar/state-transition/test-utils"; +import {Slot, SubnetID, phase0, ssz} from "@lodestar/types"; import {BlsMultiThreadWorkerPool, BlsSingleThreadVerifier} from "../../../src/chain/bls/index.js"; import {IBeaconChain} from "../../../src/chain/index.js"; import {defaultChainOptions} from "../../../src/chain/options.js"; @@ -23,7 +24,6 @@ import {ShufflingCache} from "../../../src/chain/shufflingCache.js"; import {ZERO_HASH, ZERO_HASH_HEX} from "../../../src/constants/index.js"; import {signCached} from "../cache.js"; import {ClockStatic} from "../clock.js"; -import {testLogger} from "../logger.js"; export type AttestationValidDataOpts = { currentSlot?: Slot; diff --git a/packages/cli/src/cmds/dev/options.ts b/packages/cli/src/cmds/dev/options.ts index a79aff6e6707..0df90862be44 100644 --- a/packages/cli/src/cmds/dev/options.ts +++ b/packages/cli/src/cmds/dev/options.ts @@ -1,7 +1,7 @@ import {CliCommandOptions, CliOptionDefinition} from "@lodestar/utils"; import {NetworkName} from "../../networks/index.js"; import {beaconNodeOptions, globalOptions} from "../../options/index.js"; -import {parseRange} from "../../util/format.ts"; +import {parseRange} from "../../util/format.js"; import {BeaconArgs, beaconOptions} from "../beacon/options.js"; import {IValidatorCliArgs, validatorOptions} from "../validator/options.js"; diff --git a/packages/cli/test/unit/db.test.ts b/packages/cli/test/unit/db.test.ts index b39c9492c9cb..3735ff5a255a 100644 --- a/packages/cli/test/unit/db.test.ts +++ b/packages/cli/test/unit/db.test.ts @@ -1,6 +1,6 @@ import {describe, it} from "vitest"; -import {Bucket as BeaconBucket} from "../../../beacon-node/src/db/buckets.js"; -import {Bucket as ValidatorBucket} from "../../../validator/src/buckets.js"; +import {Bucket as BeaconBucket} from "@lodestar/beacon-node/db"; +import {Bucket as ValidatorBucket} from "@lodestar/validator"; describe("no db bucket overlap", () => { it("beacon and validator dn buckets do not overlap", () => { diff --git a/packages/cli/test/utils/mockBeaconApiServer.ts b/packages/cli/test/utils/mockBeaconApiServer.ts index 24b8b9c7473b..cac8bb3f90b4 100644 --- a/packages/cli/test/utils/mockBeaconApiServer.ts +++ b/packages/cli/test/utils/mockBeaconApiServer.ts @@ -3,9 +3,9 @@ import {BeaconApiMethods, registerRoutes} from "@lodestar/api/beacon/server"; import {RestApiServer, RestApiServerModules, RestApiServerOpts} from "@lodestar/beacon-node"; import {ChainForkConfig} from "@lodestar/config"; import {config} from "@lodestar/config/default"; +import {testLogger} from "@lodestar/logger/test-utils"; import {ssz} from "@lodestar/types"; import {fromHex, toHex} from "@lodestar/utils"; -import {testLogger} from "../../../beacon-node/test/utils/logger.js"; const ZERO_HASH_HEX = toHex(Buffer.alloc(32, 0)); diff --git a/packages/config/package.json b/packages/config/package.json index 2c296fef32be..14464b5e65d4 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -29,6 +29,11 @@ "bun": "./src/configs.ts", "types": "./lib/configs.d.ts", "import": "./lib/configs.js" + }, + "./test-utils": { + "bun": "./src/testUtils/index.ts", + "types": "./lib/testUtils/index.d.ts", + "import": "./lib/testUtils/index.js" } }, "files": [ @@ -64,6 +69,7 @@ "@chainsafe/ssz": "^1.2.2", "@lodestar/params": "workspace:^", "@lodestar/types": "workspace:^", - "@lodestar/utils": "workspace:^" + "@lodestar/utils": "workspace:^", + "@lodestar/spec-test-util": "workspace:^" } } diff --git a/packages/beacon-node/test/utils/config.ts b/packages/config/src/testUtils/config.ts similarity index 92% rename from packages/beacon-node/test/utils/config.ts rename to packages/config/src/testUtils/config.ts index c9bbf215e0fa..a3768bb9adf4 100644 --- a/packages/beacon-node/test/utils/config.ts +++ b/packages/config/src/testUtils/config.ts @@ -1,7 +1,8 @@ -import {ChainForkConfig, createBeaconConfig, createChainForkConfig} from "@lodestar/config"; -import {config as chainConfig} from "@lodestar/config/default"; import {ForkName} from "@lodestar/params"; -import {ZERO_HASH} from "../../src/constants/index.js"; +import {config as chainConfig} from "../default.js"; +import {ChainForkConfig, createBeaconConfig, createChainForkConfig} from "../index.js"; + +export const ZERO_HASH = Buffer.alloc(32, 0); /** default config with ZERO_HASH as genesisValidatorsRoot */ export const config = createBeaconConfig(chainConfig, ZERO_HASH); diff --git a/packages/config/src/testUtils/index.ts b/packages/config/src/testUtils/index.ts new file mode 100644 index 000000000000..d1bc97e80940 --- /dev/null +++ b/packages/config/src/testUtils/index.ts @@ -0,0 +1 @@ +export * from "./config.js"; diff --git a/packages/config/test/e2e/ensure-config-is-synced.test.ts b/packages/config/test/e2e/ensure-config-is-synced.test.ts index 96afac82d784..9380e1e308f6 100644 --- a/packages/config/test/e2e/ensure-config-is-synced.test.ts +++ b/packages/config/test/e2e/ensure-config-is-synced.test.ts @@ -1,9 +1,9 @@ import {describe, expect, it, vi} from "vitest"; import {fetch, fromHex} from "@lodestar/utils"; -import {ethereumConsensusSpecsTests} from "../../../beacon-node/test/spec/specTestVersioning.js"; import {chainConfig as mainnetChainConfig} from "../../src/chainConfig/configs/mainnet.js"; import {chainConfig as minimalChainConfig} from "../../src/chainConfig/configs/minimal.js"; import {ChainConfig} from "../../src/chainConfig/types.js"; +import specTestsVersions from "../spec-tests-version.json" with {type: "json"}; // Not e2e, but slow. Run with e2e tests @@ -61,12 +61,18 @@ describe("Ensure chainConfig is synced", () => { vi.setConfig({testTimeout: 60 * 1000}); it("mainnet chainConfig values match spec", async () => { - const remoteConfig = await downloadRemoteConfig("mainnet", ethereumConsensusSpecsTests.specVersion); + const remoteConfig = await downloadRemoteConfig( + "mainnet", + specTestsVersions.ethereumConsensusSpecsTests.specVersion + ); assertCorrectConfig({...mainnetChainConfig}, remoteConfig); }); it("minimal chainConfig values match spec", async () => { - const remoteConfig = await downloadRemoteConfig("minimal", ethereumConsensusSpecsTests.specVersion); + const remoteConfig = await downloadRemoteConfig( + "minimal", + specTestsVersions.ethereumConsensusSpecsTests.specVersion + ); assertCorrectConfig({...minimalChainConfig}, remoteConfig); }); }); diff --git a/packages/config/test/spec-tests-version.json b/packages/config/test/spec-tests-version.json new file mode 120000 index 000000000000..bc2e3fd621b2 --- /dev/null +++ b/packages/config/test/spec-tests-version.json @@ -0,0 +1 @@ +../../../spec-tests-version.json \ No newline at end of file diff --git a/packages/era/src/e2s.ts b/packages/era/src/e2s.ts index 0060cd167d7f..58bd97f0cc4e 100644 --- a/packages/era/src/e2s.ts +++ b/packages/era/src/e2s.ts @@ -1,7 +1,7 @@ import type {FileHandle} from "node:fs/promises"; import {Slot} from "@lodestar/types"; import {byteArrayEquals} from "@lodestar/utils"; -import {readInt48, readUint16, readUint32, writeInt48, writeUint16, writeUint32} from "./util.ts"; +import {readInt48, readUint16, readUint32, writeInt48, writeUint16, writeUint32} from "./util.js"; /** * Known entry types in an E2Store (.e2s) file along with their exact 2-byte codes. diff --git a/packages/era/src/era/reader.ts b/packages/era/src/era/reader.ts index bf7f8df3460b..675a835a9aa3 100644 --- a/packages/era/src/era/reader.ts +++ b/packages/era/src/era/reader.ts @@ -5,15 +5,15 @@ import {ChainForkConfig, createCachedGenesis} from "@lodestar/config"; import {DOMAIN_BEACON_PROPOSER, GENESIS_SLOT, SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params"; import {BeaconState, SignedBeaconBlock, Slot, ssz} from "@lodestar/types"; import {byteArrayEquals} from "@lodestar/utils"; -import {E2STORE_HEADER_SIZE, EntryType, readEntry, readVersion} from "../e2s.ts"; -import {snappyUncompress} from "../util.ts"; +import {E2STORE_HEADER_SIZE, EntryType, readEntry, readVersion} from "../e2s.js"; +import {snappyUncompress} from "../util.js"; import { EraIndices, computeEraNumberFromBlockSlot, parseEraName, readAllEraIndices, readSlotFromBeaconStateBytes, -} from "./util.ts"; +} from "./util.js"; /** * EraReader is responsible for reading and validating ERA files. diff --git a/packages/era/src/era/util.ts b/packages/era/src/era/util.ts index b49f7fdb4aee..554fba589b97 100644 --- a/packages/era/src/era/util.ts +++ b/packages/era/src/era/util.ts @@ -2,8 +2,8 @@ import type {FileHandle} from "node:fs/promises"; import {ChainForkConfig} from "@lodestar/config"; import {SLOTS_PER_HISTORICAL_ROOT, isForkPostCapella} from "@lodestar/params"; import {BeaconState, Slot, capella, ssz} from "@lodestar/types"; -import {E2STORE_HEADER_SIZE, SlotIndex, readSlotIndex} from "../e2s.ts"; -import {readUint48} from "../util.ts"; +import {E2STORE_HEADER_SIZE, SlotIndex, readSlotIndex} from "../e2s.js"; +import {readUint48} from "../util.js"; /** * Parsed components of an .era file name. diff --git a/packages/era/src/era/writer.ts b/packages/era/src/era/writer.ts index d0aeaf95763b..d377ce9f0ac9 100644 --- a/packages/era/src/era/writer.ts +++ b/packages/era/src/era/writer.ts @@ -3,14 +3,14 @@ import {format, parse} from "node:path"; import {ChainForkConfig} from "@lodestar/config"; import {SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params"; import {BeaconState, SignedBeaconBlock, Slot} from "@lodestar/types"; -import {E2STORE_HEADER_SIZE, EntryType, SlotIndex, serializeSlotIndex, writeEntry} from "../e2s.ts"; -import {snappyCompress} from "../util.ts"; +import {E2STORE_HEADER_SIZE, EntryType, SlotIndex, serializeSlotIndex, writeEntry} from "../e2s.js"; +import {snappyCompress} from "../util.js"; import { computeStartBlockSlotFromEraNumber, getShortHistoricalRoot, isSlotInRange, isValidEraStateSlot, -} from "./util.ts"; +} from "./util.js"; enum WriterStateType { InitGroup, diff --git a/packages/era/test/e2e-mainnet/era.readwrite.integration.test.ts b/packages/era/test/e2e-mainnet/era.readwrite.integration.test.ts index e7b783e252df..f8ab9dc42691 100644 --- a/packages/era/test/e2e-mainnet/era.readwrite.integration.test.ts +++ b/packages/era/test/e2e-mainnet/era.readwrite.integration.test.ts @@ -5,7 +5,7 @@ import {beforeAll, describe, expect, it} from "vitest"; import {ChainForkConfig, createChainForkConfig} from "@lodestar/config"; import {mainnetChainConfig} from "@lodestar/config/networks"; import {SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params"; -import {EraReader, EraWriter} from "../../src/era/index.ts"; +import {EraReader, EraWriter} from "../../src/era/index.js"; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); diff --git a/packages/era/test/unit/era.unit.test.ts b/packages/era/test/unit/era.unit.test.ts index ebb927e9fa48..1b6c0006b344 100644 --- a/packages/era/test/unit/era.unit.test.ts +++ b/packages/era/test/unit/era.unit.test.ts @@ -1,5 +1,5 @@ import {assert, describe, it} from "vitest"; -import {E2STORE_HEADER_SIZE, EntryType, parseEntryHeader} from "../../src/e2s.ts"; +import {E2STORE_HEADER_SIZE, EntryType, parseEntryHeader} from "../../src/e2s.js"; function header(type: EntryType, dataLen: number): Uint8Array { const h = new Uint8Array(8); diff --git a/packages/logger/package.json b/packages/logger/package.json index 1350175546a0..51c06219e1a2 100644 --- a/packages/logger/package.json +++ b/packages/logger/package.json @@ -38,6 +38,11 @@ "bun": "./src/empty.ts", "types": "./lib/empty.d.ts", "import": "./lib/empty.js" + }, + "./test-utils": { + "bun": "./src/testUtils.ts", + "types": "./lib/testUtils.d.ts", + "import": "./lib/testUtils.js" } }, "files": [ diff --git a/packages/beacon-node/test/utils/logger.ts b/packages/logger/src/testUtils.ts similarity index 80% rename from packages/beacon-node/test/utils/logger.ts rename to packages/logger/src/testUtils.ts index 7d67dd6fbd16..abf4925d105f 100644 --- a/packages/beacon-node/test/utils/logger.ts +++ b/packages/logger/src/testUtils.ts @@ -1,6 +1,6 @@ -import {getEnvLogLevel} from "@lodestar/logger/env"; -import {LoggerNode, LoggerNodeOpts, getNodeLogger} from "@lodestar/logger/node"; import {LogLevel} from "@lodestar/utils"; +import {getEnvLogLevel} from "./env.js"; +import {LoggerNode, LoggerNodeOpts, getNodeLogger} from "./node.js"; export {LogLevel}; export type TestLoggerOpts = LoggerNodeOpts; diff --git a/packages/params/test/e2e/ensure-config-is-synced.test.ts b/packages/params/test/e2e/ensure-config-is-synced.test.ts index ba9849eff410..b1bab52a76f8 100644 --- a/packages/params/test/e2e/ensure-config-is-synced.test.ts +++ b/packages/params/test/e2e/ensure-config-is-synced.test.ts @@ -1,9 +1,9 @@ import axios from "axios"; import {describe, expect, it, vi} from "vitest"; -import {ethereumConsensusSpecsTests} from "../../../beacon-node/test/spec/specTestVersioning.js"; import {BeaconPreset, ForkName} from "../../src/index.js"; import {mainnetPreset} from "../../src/presets/mainnet.js"; import {minimalPreset} from "../../src/presets/minimal.js"; +import specTests from "../spec-tests-version.json" with {type: "json"}; import {loadConfigYaml} from "../yaml.js"; // Not e2e, but slow. Run with e2e tests @@ -17,12 +17,12 @@ describe("Ensure config is synced", () => { vi.setConfig({testTimeout: 60 * 1000}); it("mainnet", async () => { - const remotePreset = await downloadRemoteConfig("mainnet", ethereumConsensusSpecsTests.specVersion); + const remotePreset = await downloadRemoteConfig("mainnet", specTests.ethereumConsensusSpecsTests.specVersion); assertCorrectPreset({...mainnetPreset}, remotePreset); }); it("minimal", async () => { - const remotePreset = await downloadRemoteConfig("minimal", ethereumConsensusSpecsTests.specVersion); + const remotePreset = await downloadRemoteConfig("minimal", specTests.ethereumConsensusSpecsTests.specVersion); assertCorrectPreset({...minimalPreset}, remotePreset); }); }); diff --git a/packages/params/test/spec-tests-version.json b/packages/params/test/spec-tests-version.json new file mode 120000 index 000000000000..bc2e3fd621b2 --- /dev/null +++ b/packages/params/test/spec-tests-version.json @@ -0,0 +1 @@ +../../../spec-tests-version.json \ No newline at end of file diff --git a/packages/reqresp/src/encoders/requestDecode.ts b/packages/reqresp/src/encoders/requestDecode.ts index 1fb102a2a4f5..d134ac47f4ea 100644 --- a/packages/reqresp/src/encoders/requestDecode.ts +++ b/packages/reqresp/src/encoders/requestDecode.ts @@ -2,7 +2,7 @@ import type {Stream} from "@libp2p/interface"; import {byteStream} from "@libp2p/utils"; import {readEncodedPayload} from "../encodingStrategies/index.js"; import {MixedProtocol} from "../types.js"; -import {drainByteStream} from "../utils/stream.ts"; +import {drainByteStream} from "../utils/stream.js"; const EMPTY_DATA = new Uint8Array(); diff --git a/packages/state-transition/package.json b/packages/state-transition/package.json index ebf94eb284fa..64b24209d797 100644 --- a/packages/state-transition/package.json +++ b/packages/state-transition/package.json @@ -33,6 +33,11 @@ "bun": "./src/slot/index.ts", "types": "./lib/slot/index.d.ts", "import": "./lib/slot/index.js" + }, + "./test-utils": { + "bun": "./src/testUtils/index.ts", + "types": "./lib/testUtils/index.d.ts", + "import": "./lib/testUtils/index.js" } }, "files": [ diff --git a/packages/state-transition/src/block/index.ts b/packages/state-transition/src/block/index.ts index 1104a52e2fa9..2dc24d48bd5b 100644 --- a/packages/state-transition/src/block/index.ts +++ b/packages/state-transition/src/block/index.ts @@ -13,10 +13,10 @@ import {processBlobKzgCommitments} from "./processBlobKzgCommitments.js"; import {processBlockHeader} from "./processBlockHeader.js"; import {processEth1Data} from "./processEth1Data.js"; import {processExecutionPayload} from "./processExecutionPayload.js"; -import {processExecutionPayloadBid} from "./processExecutionPayloadBid.ts"; -import {processExecutionPayloadEnvelope} from "./processExecutionPayloadEnvelope.ts"; +import {processExecutionPayloadBid} from "./processExecutionPayloadBid.js"; +import {processExecutionPayloadEnvelope} from "./processExecutionPayloadEnvelope.js"; import {processOperations} from "./processOperations.js"; -import {processPayloadAttestation} from "./processPayloadAttestation.ts"; +import {processPayloadAttestation} from "./processPayloadAttestation.js"; import {processRandao} from "./processRandao.js"; import {processSyncAggregate} from "./processSyncCommittee.js"; import {processWithdrawals} from "./processWithdrawals.js"; diff --git a/packages/state-transition/src/block/isValidIndexedPayloadAttestation.ts b/packages/state-transition/src/block/isValidIndexedPayloadAttestation.ts index 058f22660c2c..3e733e1922e7 100644 --- a/packages/state-transition/src/block/isValidIndexedPayloadAttestation.ts +++ b/packages/state-transition/src/block/isValidIndexedPayloadAttestation.ts @@ -1,7 +1,7 @@ import {gloas} from "@lodestar/types"; -import {getIndexedPayloadAttestationSignatureSet} from "../signatureSets/index.ts"; +import {getIndexedPayloadAttestationSignatureSet} from "../signatureSets/index.js"; import {CachedBeaconStateGloas} from "../types.js"; -import {verifySignatureSet} from "../util/index.ts"; +import {verifySignatureSet} from "../util/index.js"; export function isValidIndexedPayloadAttestation( state: CachedBeaconStateGloas, diff --git a/packages/state-transition/src/block/processAttestationsAltair.ts b/packages/state-transition/src/block/processAttestationsAltair.ts index b28b22020a1e..e6cac307c70f 100644 --- a/packages/state-transition/src/block/processAttestationsAltair.ts +++ b/packages/state-transition/src/block/processAttestationsAltair.ts @@ -18,7 +18,7 @@ import {byteArrayEquals, intSqrt} from "@lodestar/utils"; import {BeaconStateTransitionMetrics} from "../metrics.js"; import {getAttestationWithIndicesSignatureSet} from "../signatureSets/indexedAttestation.js"; import {CachedBeaconStateAltair, CachedBeaconStateGloas} from "../types.js"; -import {isAttestationSameSlot, isAttestationSameSlotRootCache} from "../util/gloas.ts"; +import {isAttestationSameSlot, isAttestationSameSlotRootCache} from "../util/gloas.js"; import {increaseBalance, verifySignatureSet} from "../util/index.js"; import {RootCache} from "../util/rootCache.js"; import {checkpointToStr, isTimelyTarget, validateAttestation} from "./processAttestationPhase0.js"; diff --git a/packages/state-transition/src/block/processExecutionPayloadBid.ts b/packages/state-transition/src/block/processExecutionPayloadBid.ts index c906200fecde..c03513d00727 100644 --- a/packages/state-transition/src/block/processExecutionPayloadBid.ts +++ b/packages/state-transition/src/block/processExecutionPayloadBid.ts @@ -2,11 +2,11 @@ import {PublicKey, Signature, verify} from "@chainsafe/blst"; import {BUILDER_INDEX_SELF_BUILD, ForkPostGloas, SLOTS_PER_EPOCH} from "@lodestar/params"; import {BeaconBlock, gloas, ssz} from "@lodestar/types"; import {byteArrayEquals, toHex, toRootHex} from "@lodestar/utils"; -import {G2_POINT_AT_INFINITY} from "../constants/constants.ts"; +import {G2_POINT_AT_INFINITY} from "../constants/constants.js"; import {getExecutionPayloadBidSigningRoot} from "../signatureSets/executionPayloadBid.js"; -import {CachedBeaconStateGloas} from "../types.ts"; -import {canBuilderCoverBid, isActiveBuilder} from "../util/gloas.ts"; -import {getCurrentEpoch, getRandaoMix} from "../util/index.ts"; +import {CachedBeaconStateGloas} from "../types.js"; +import {canBuilderCoverBid, isActiveBuilder} from "../util/gloas.js"; +import {getCurrentEpoch, getRandaoMix} from "../util/index.js"; export function processExecutionPayloadBid(state: CachedBeaconStateGloas, block: BeaconBlock): void { const signedBid = block.body.signedExecutionPayloadBid; diff --git a/packages/state-transition/src/block/processExecutionPayloadEnvelope.ts b/packages/state-transition/src/block/processExecutionPayloadEnvelope.ts index 10aba9b5b1b8..50fd56edcc7b 100644 --- a/packages/state-transition/src/block/processExecutionPayloadEnvelope.ts +++ b/packages/state-transition/src/block/processExecutionPayloadEnvelope.ts @@ -7,11 +7,11 @@ import { } from "@lodestar/params"; import {gloas, ssz} from "@lodestar/types"; import {byteArrayEquals, toHex, toRootHex} from "@lodestar/utils"; -import {CachedBeaconStateGloas} from "../types.ts"; -import {computeSigningRoot, computeTimeAtSlot} from "../util/index.ts"; -import {processConsolidationRequest} from "./processConsolidationRequest.ts"; -import {processDepositRequest} from "./processDepositRequest.ts"; -import {processWithdrawalRequest} from "./processWithdrawalRequest.ts"; +import {CachedBeaconStateGloas} from "../types.js"; +import {computeSigningRoot, computeTimeAtSlot} from "../util/index.js"; +import {processConsolidationRequest} from "./processConsolidationRequest.js"; +import {processDepositRequest} from "./processDepositRequest.js"; +import {processWithdrawalRequest} from "./processWithdrawalRequest.js"; export type ProcessExecutionPayloadEnvelopeOpts = { dontTransferCache?: boolean; diff --git a/packages/state-transition/src/block/processOperations.ts b/packages/state-transition/src/block/processOperations.ts index 34b21984c124..b4455f5315d5 100644 --- a/packages/state-transition/src/block/processOperations.ts +++ b/packages/state-transition/src/block/processOperations.ts @@ -14,7 +14,7 @@ import {processBlsToExecutionChange} from "./processBlsToExecutionChange.js"; import {processConsolidationRequest} from "./processConsolidationRequest.js"; import {processDeposit} from "./processDeposit.js"; import {processDepositRequest} from "./processDepositRequest.js"; -import {processPayloadAttestation} from "./processPayloadAttestation.ts"; +import {processPayloadAttestation} from "./processPayloadAttestation.js"; import {processProposerSlashing} from "./processProposerSlashing.js"; import {processVoluntaryExit} from "./processVoluntaryExit.js"; import {processWithdrawalRequest} from "./processWithdrawalRequest.js"; diff --git a/packages/state-transition/src/block/processPayloadAttestation.ts b/packages/state-transition/src/block/processPayloadAttestation.ts index d58cf704dc00..389fbe318017 100644 --- a/packages/state-transition/src/block/processPayloadAttestation.ts +++ b/packages/state-transition/src/block/processPayloadAttestation.ts @@ -1,7 +1,7 @@ import {gloas} from "@lodestar/types"; import {byteArrayEquals} from "@lodestar/utils"; -import {CachedBeaconStateGloas} from "../types.ts"; -import {isValidIndexedPayloadAttestation} from "./isValidIndexedPayloadAttestation.ts"; +import {CachedBeaconStateGloas} from "../types.js"; +import {isValidIndexedPayloadAttestation} from "./isValidIndexedPayloadAttestation.js"; export function processPayloadAttestation( state: CachedBeaconStateGloas, diff --git a/packages/state-transition/src/block/processWithdrawals.ts b/packages/state-transition/src/block/processWithdrawals.ts index 768987b76631..e2b54f5e183a 100644 --- a/packages/state-transition/src/block/processWithdrawals.ts +++ b/packages/state-transition/src/block/processWithdrawals.ts @@ -16,7 +16,7 @@ import { convertValidatorIndexToBuilderIndex, isBuilderIndex, isParentBlockFull, -} from "../util/gloas.ts"; +} from "../util/gloas.js"; import { decreaseBalance, getMaxEffectiveBalance, diff --git a/packages/state-transition/src/epoch/index.ts b/packages/state-transition/src/epoch/index.ts index 562613b44b05..47aa812ef792 100644 --- a/packages/state-transition/src/epoch/index.ts +++ b/packages/state-transition/src/epoch/index.ts @@ -16,7 +16,7 @@ import { CachedBeaconStatePhase0, EpochTransitionCache, } from "../types.js"; -import {processBuilderPendingPayments} from "./processBuilderPendingPayments.ts"; +import {processBuilderPendingPayments} from "./processBuilderPendingPayments.js"; import {processEffectiveBalanceUpdates} from "./processEffectiveBalanceUpdates.js"; import {processEth1DataReset} from "./processEth1DataReset.js"; import {processHistoricalRootsUpdate} from "./processHistoricalRootsUpdate.js"; diff --git a/packages/state-transition/src/epoch/processBuilderPendingPayments.ts b/packages/state-transition/src/epoch/processBuilderPendingPayments.ts index 947b1666be95..a1a95013a63e 100644 --- a/packages/state-transition/src/epoch/processBuilderPendingPayments.ts +++ b/packages/state-transition/src/epoch/processBuilderPendingPayments.ts @@ -1,7 +1,7 @@ import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {ssz} from "@lodestar/types"; -import {CachedBeaconStateGloas} from "../types.ts"; -import {getBuilderPaymentQuorumThreshold} from "../util/gloas.ts"; +import {CachedBeaconStateGloas} from "../types.js"; +import {getBuilderPaymentQuorumThreshold} from "../util/gloas.js"; /** * Processes the builder pending payments from the previous epoch. diff --git a/packages/state-transition/test/cache.ts b/packages/state-transition/src/testUtils/cache.ts similarity index 80% rename from packages/state-transition/test/cache.ts rename to packages/state-transition/src/testUtils/cache.ts index 59ddfe40b7c1..5d880338aeab 100644 --- a/packages/state-transition/test/cache.ts +++ b/packages/state-transition/src/testUtils/cache.ts @@ -5,4 +5,4 @@ import {fileURLToPath} from "node:url"; // Solutions: https://stackoverflow.com/questions/46745014/alternative-for-dirname-in-node-js-when-using-es6-modules const __dirname = path.dirname(fileURLToPath(import.meta.url)); -export const testCachePath = path.join(__dirname, "../test-cache"); +export const testCachePath = path.join(__dirname, "../../test/test-cache"); diff --git a/packages/state-transition/src/testUtils/index.ts b/packages/state-transition/src/testUtils/index.ts new file mode 100644 index 000000000000..2443c291b0b9 --- /dev/null +++ b/packages/state-transition/src/testUtils/index.ts @@ -0,0 +1,5 @@ +export * from "./cache.js"; +export * from "./interop.js"; +export * from "./params.js"; +export * from "./testFileCache.js"; +export * from "./util.js"; diff --git a/packages/state-transition/test/utils/infura.ts b/packages/state-transition/src/testUtils/infura.ts similarity index 100% rename from packages/state-transition/test/utils/infura.ts rename to packages/state-transition/src/testUtils/infura.ts diff --git a/packages/state-transition/test/utils/interop.ts b/packages/state-transition/src/testUtils/interop.ts similarity index 91% rename from packages/state-transition/test/utils/interop.ts rename to packages/state-transition/src/testUtils/interop.ts index 5a0dca4e0018..16bd63dbe5ac 100644 --- a/packages/state-transition/test/utils/interop.ts +++ b/packages/state-transition/src/testUtils/interop.ts @@ -1,8 +1,8 @@ import fs from "node:fs"; import path from "node:path"; import {fromHexString, toHexString} from "@chainsafe/ssz"; -import {interopSecretKey} from "../../src/index.js"; -import {testCachePath} from "../cache.js"; +import {interopSecretKey} from "../index.js"; +import {testCachePath} from "./cache.js"; const interopPubkeysCachedPath = path.join(testCachePath, "interop-pubkeys.json"); diff --git a/packages/state-transition/test/perf/params.ts b/packages/state-transition/src/testUtils/params.ts similarity index 100% rename from packages/state-transition/test/perf/params.ts rename to packages/state-transition/src/testUtils/params.ts diff --git a/packages/state-transition/src/testUtils/state.ts b/packages/state-transition/src/testUtils/state.ts new file mode 100644 index 000000000000..e6cbac014e73 --- /dev/null +++ b/packages/state-transition/src/testUtils/state.ts @@ -0,0 +1,110 @@ +import {ChainForkConfig, createBeaconConfig} from "@lodestar/config"; +import {config, config as minimalConfig} from "@lodestar/config/default"; +import { + EPOCHS_PER_HISTORICAL_VECTOR, + EPOCHS_PER_SLASHINGS_VECTOR, + GENESIS_EPOCH, + GENESIS_SLOT, + SLOTS_PER_HISTORICAL_ROOT, +} from "@lodestar/params"; +import {phase0, ssz} from "@lodestar/types"; +import {EpochCacheOpts} from "../cache/epochCache.js"; +import {BeaconStateCache} from "../cache/stateCache.js"; +import {ZERO_HASH} from "../constants/index.js"; +import { + BeaconStateAllForks, + BeaconStatePhase0, + CachedBeaconStateAllForks, + createCachedBeaconState, + createPubkeyCache, +} from "../index.js"; +import {newZeroedArray} from "../util/index.js"; + +/** + * Copy of BeaconState, but all fields are marked optional to allow for swapping out variables as needed. + */ +type TestBeaconState = Partial; + +/** + * Generate beaconState, by default it will use the initial state defined when the `ChainStart` log is emitted. + * NOTE: All fields can be overridden through `opts`. + * @param {TestBeaconState} opts + * @returns {BeaconState} + */ +export function generateState(opts?: TestBeaconState): BeaconStatePhase0 { + return ssz.phase0.BeaconState.toViewDU({ + genesisTime: Math.floor(Date.now() / 1000), + genesisValidatorsRoot: ZERO_HASH, + slot: GENESIS_SLOT, + fork: { + previousVersion: config.GENESIS_FORK_VERSION, + currentVersion: config.GENESIS_FORK_VERSION, + epoch: GENESIS_EPOCH, + }, + latestBlockHeader: { + slot: 0, + proposerIndex: 0, + parentRoot: Buffer.alloc(32), + stateRoot: Buffer.alloc(32), + bodyRoot: ssz.phase0.BeaconBlockBody.hashTreeRoot(ssz.phase0.BeaconBlockBody.defaultValue()), + }, + blockRoots: Array.from({length: SLOTS_PER_HISTORICAL_ROOT}, () => ZERO_HASH), + stateRoots: Array.from({length: SLOTS_PER_HISTORICAL_ROOT}, () => ZERO_HASH), + historicalRoots: [], + eth1Data: { + depositRoot: Buffer.alloc(32), + blockHash: Buffer.alloc(32), + depositCount: 0, + }, + eth1DataVotes: [], + eth1DepositIndex: 0, + validators: [], + balances: [], + randaoMixes: Array.from({length: EPOCHS_PER_HISTORICAL_VECTOR}, () => ZERO_HASH), + slashings: newZeroedArray(EPOCHS_PER_SLASHINGS_VECTOR), + previousEpochAttestations: [], + currentEpochAttestations: [], + justificationBits: ssz.phase0.JustificationBits.defaultValue(), + previousJustifiedCheckpoint: { + epoch: GENESIS_EPOCH, + root: ZERO_HASH, + }, + currentJustifiedCheckpoint: { + epoch: GENESIS_EPOCH, + root: ZERO_HASH, + }, + finalizedCheckpoint: { + epoch: GENESIS_EPOCH, + root: ZERO_HASH, + }, + ...opts, + }); +} + +export function generateCachedState( + config: ChainForkConfig = minimalConfig, + opts: TestBeaconState = {} +): CachedBeaconStateAllForks { + const state = generateState(opts); + return createCachedBeaconState(state, { + config: createBeaconConfig(config, state.genesisValidatorsRoot), + // This is a test state, there's no need to have a global shared cache of keys + pubkeyCache: createPubkeyCache(), + }); +} + +export function createCachedBeaconStateTest( + state: T, + configCustom: ChainForkConfig = config, + opts?: EpochCacheOpts +): T & BeaconStateCache { + return createCachedBeaconState( + state, + { + config: createBeaconConfig(configCustom, state.genesisValidatorsRoot), + // This is a test state, there's no need to have a global shared cache of keys + pubkeyCache: createPubkeyCache(), + }, + opts + ); +} diff --git a/packages/state-transition/test/utils/testFileCache.ts b/packages/state-transition/src/testUtils/testFileCache.ts similarity index 94% rename from packages/state-transition/test/utils/testFileCache.ts rename to packages/state-transition/src/testUtils/testFileCache.ts index 1cf381b33beb..e2b98bb64820 100644 --- a/packages/state-transition/test/utils/testFileCache.ts +++ b/packages/state-transition/src/testUtils/testFileCache.ts @@ -5,10 +5,10 @@ import {ChainForkConfig, createChainForkConfig} from "@lodestar/config"; import {NetworkName, networksChainConfig} from "@lodestar/config/networks"; import {SignedBeaconBlock} from "@lodestar/types"; import {fetch} from "@lodestar/utils"; -import {CachedBeaconStateAllForks} from "../../src/index.js"; -import {testCachePath} from "../cache.js"; -import {createCachedBeaconStateTest} from "../utils/state.js"; +import {CachedBeaconStateAllForks} from "../index.js"; +import {testCachePath} from "./cache.js"; import {getInfuraBeaconUrl} from "./infura.js"; +import {createCachedBeaconStateTest} from "./state.js"; /** * Full link example: @@ -96,6 +96,7 @@ export async function getNetworkCachedBlock( async function downloadTestFile(fileId: string): Promise { const fileUrl = `${TEST_FILES_BASE_URL}/${fileId}`; + // biome-ignore lint/suspicious/noConsole: We explicity need to log to console console.log(`Downloading file ${fileUrl}`); try { diff --git a/packages/state-transition/test/perf/util.ts b/packages/state-transition/src/testUtils/util.ts similarity index 97% rename from packages/state-transition/test/perf/util.ts rename to packages/state-transition/src/testUtils/util.ts index fc644379c2f3..2c0daa87117d 100644 --- a/packages/state-transition/test/perf/util.ts +++ b/packages/state-transition/src/testUtils/util.ts @@ -13,7 +13,7 @@ import { SLOTS_PER_HISTORICAL_ROOT, } from "@lodestar/params"; import {BeaconState, Slot, phase0, ssz} from "@lodestar/types"; -import {getEffectiveBalanceIncrements} from "../../src/cache/effectiveBalanceIncrements.js"; +import {getEffectiveBalanceIncrements} from "../cache/effectiveBalanceIncrements.js"; import { computeCommitteeCount, computeEpochAtSlot, @@ -22,17 +22,17 @@ import { interopSecretKey, newFilledArray, processSlots, -} from "../../src/index.js"; +} from "../index.js"; import { BeaconStateAltair, BeaconStatePhase0, CachedBeaconStateAllForks, CachedBeaconStateAltair, CachedBeaconStatePhase0, -} from "../../src/types.js"; -import {getNextSyncCommittee} from "../../src/util/syncCommittee.js"; -import {getActiveValidatorIndices} from "../../src/util/validator.js"; -import {interopPubkeysCached} from "../utils/interop.js"; +} from "../types.js"; +import {getNextSyncCommittee} from "../util/syncCommittee.js"; +import {getActiveValidatorIndices} from "../util/validator.js"; +import {interopPubkeysCached} from "./interop.js"; let phase0State: BeaconStatePhase0 | null = null; let phase0CachedState23637: CachedBeaconStatePhase0 | null = null; diff --git a/packages/state-transition/test/perf/analyzeBlocks.ts b/packages/state-transition/test/perf/analyzeBlocks.ts index 1b6b7d592fbd..167ba5378f1f 100644 --- a/packages/state-transition/test/perf/analyzeBlocks.ts +++ b/packages/state-transition/test/perf/analyzeBlocks.ts @@ -1,6 +1,6 @@ import {getClient} from "@lodestar/api"; import {config} from "@lodestar/config/default"; -import {getInfuraBeaconUrl} from "../utils/infura.js"; +import {getInfuraBeaconUrl} from "../../src/testUtils/infura.js"; // Analyze how Ethereum Consensus blocks are in a target network to prepare accurate performance states and blocks diff --git a/packages/state-transition/test/perf/analyzeEpochs.ts b/packages/state-transition/test/perf/analyzeEpochs.ts index c25fa0bcb703..5aed6c010ce9 100644 --- a/packages/state-transition/test/perf/analyzeEpochs.ts +++ b/packages/state-transition/test/perf/analyzeEpochs.ts @@ -11,8 +11,8 @@ import { parseAttesterFlags, processSlots, } from "../../src/index.js"; -import {getInfuraBeaconUrl} from "../utils/infura.js"; -import {createCachedBeaconStateTest} from "../utils/state.js"; +import {getInfuraBeaconUrl} from "../../src/testUtils/infura.js"; +import {createCachedBeaconStateTest} from "../../src/testUtils/state.js"; import {csvAppend, readCsv} from "./csv.js"; // Understand the real network characteristics regarding epoch transitions to accurately produce performance test data. diff --git a/packages/state-transition/test/perf/block/processAttestation.test.ts b/packages/state-transition/test/perf/block/processAttestation.test.ts index 46de84c2566e..733c9891c3af 100644 --- a/packages/state-transition/test/perf/block/processAttestation.test.ts +++ b/packages/state-transition/test/perf/block/processAttestation.test.ts @@ -13,7 +13,7 @@ import { import {phase0} from "@lodestar/types"; import {processAttestationsAltair} from "../../../src/block/processAttestationsAltair.js"; import {CachedBeaconStateAllForks, CachedBeaconStateAltair} from "../../../src/index.js"; -import {generatePerfTestCachedStateAltair, perfStateId} from "../util.js"; +import {generatePerfTestCachedStateAltair, perfStateId} from "../../../src/testUtils/util.js"; import {BlockAltairOpts, getBlockAltair} from "./util.js"; type StateAttestations = { diff --git a/packages/state-transition/test/perf/block/processBlockAltair.test.ts b/packages/state-transition/test/perf/block/processBlockAltair.test.ts index b3d4fea3e3b7..eabceed23874 100644 --- a/packages/state-transition/test/perf/block/processBlockAltair.test.ts +++ b/packages/state-transition/test/perf/block/processBlockAltair.test.ts @@ -16,8 +16,12 @@ import { ExecutionPayloadStatus, stateTransition, } from "../../../src/index.js"; +import { + cachedStateAltairPopulateCaches, + generatePerfTestCachedStateAltair, + perfStateId, +} from "../../../src/testUtils/util.js"; import {StateBlock} from "../types.js"; -import {cachedStateAltairPopulateCaches, generatePerfTestCachedStateAltair, perfStateId} from "../util.js"; import {BlockAltairOpts, getBlockAltair} from "./util.js"; // As of Jun 12 2021 diff --git a/packages/state-transition/test/perf/block/processBlockPhase0.test.ts b/packages/state-transition/test/perf/block/processBlockPhase0.test.ts index 58478a1258fd..b3ea8d3459ec 100644 --- a/packages/state-transition/test/perf/block/processBlockPhase0.test.ts +++ b/packages/state-transition/test/perf/block/processBlockPhase0.test.ts @@ -9,8 +9,8 @@ import { PresetName, } from "@lodestar/params"; import {DataAvailabilityStatus, ExecutionPayloadStatus, stateTransition} from "../../../src/index.js"; +import {generatePerfTestCachedStatePhase0, perfStateId} from "../../../src/testUtils/util.js"; import {StateBlock} from "../types.js"; -import {generatePerfTestCachedStatePhase0, perfStateId} from "../util.js"; import {BlockOpts, getBlockPhase0} from "./util.js"; // As of Jun 12 2021 diff --git a/packages/state-transition/test/perf/block/processEth1Data.test.ts b/packages/state-transition/test/perf/block/processEth1Data.test.ts index b70a63fd8310..f13fd60e0b56 100644 --- a/packages/state-transition/test/perf/block/processEth1Data.test.ts +++ b/packages/state-transition/test/perf/block/processEth1Data.test.ts @@ -3,7 +3,7 @@ import {ACTIVE_PRESET, PresetName, SYNC_COMMITTEE_SIZE} from "@lodestar/params"; import {phase0} from "@lodestar/types"; import {processEth1Data} from "../../../src/block/processEth1Data.js"; import {CachedBeaconStateAllForks} from "../../../src/index.js"; -import {generatePerfTestCachedStateAltair, perfStateId} from "../util.js"; +import {generatePerfTestCachedStateAltair, perfStateId} from "../../../src/testUtils/util.js"; import {getBlockAltair} from "./util.js"; type StateEth1Data = { diff --git a/packages/state-transition/test/perf/block/processWithdrawals.test.ts b/packages/state-transition/test/perf/block/processWithdrawals.test.ts index a95c9f5d7ae2..13cbd8e35dfe 100644 --- a/packages/state-transition/test/perf/block/processWithdrawals.test.ts +++ b/packages/state-transition/test/perf/block/processWithdrawals.test.ts @@ -2,8 +2,8 @@ import {bench, describe} from "@chainsafe/benchmark"; import {ForkSeq} from "@lodestar/params"; import {getExpectedWithdrawals} from "../../../src/block/processWithdrawals.js"; import {CachedBeaconStateCapella} from "../../../src/index.js"; +import {numValidators} from "../../../src/testUtils/util.js"; import {WithdrawalOpts, getExpectedWithdrawalsTestData} from "../../utils/capella.js"; -import {numValidators} from "../util.js"; // PERF: Fixed cost for MAX_WITHDRAWALS_PER_PAYLOAD probes // + cost 'proportional' to $VALIDATOR_COUNT with balances under MAX_EFFECTIVE_BALANCE or diff --git a/packages/state-transition/test/perf/epoch/afterProcessEpoch.test.ts b/packages/state-transition/test/perf/epoch/afterProcessEpoch.test.ts index 9a1f7a645251..2da043c8a51b 100644 --- a/packages/state-transition/test/perf/epoch/afterProcessEpoch.test.ts +++ b/packages/state-transition/test/perf/epoch/afterProcessEpoch.test.ts @@ -1,7 +1,7 @@ import {bench, describe} from "@chainsafe/benchmark"; import {beforeProcessEpoch} from "../../../src/index.js"; +import {generatePerfTestCachedStatePhase0, perfStateId} from "../../../src/testUtils/util.js"; import {StateEpoch} from "../types.js"; -import {generatePerfTestCachedStatePhase0, perfStateId} from "../util.js"; // PERF: Cost = compute attester and proposer shufflings ~ 'proportional' to $VALIDATOR_COUNT, but independent of // network conditions. See also individual benchmarks for shuffling computations. diff --git a/packages/state-transition/test/perf/epoch/beforeProcessEpoch.test.ts b/packages/state-transition/test/perf/epoch/beforeProcessEpoch.test.ts index 3123e7f5a806..abe21f514a32 100644 --- a/packages/state-transition/test/perf/epoch/beforeProcessEpoch.test.ts +++ b/packages/state-transition/test/perf/epoch/beforeProcessEpoch.test.ts @@ -1,7 +1,7 @@ import {bench, describe} from "@chainsafe/benchmark"; import {beforeProcessEpoch} from "../../../src/index.js"; +import {generatePerfTestCachedStatePhase0, perfStateId} from "../../../src/testUtils/util.js"; import {State} from "../types.js"; -import {generatePerfTestCachedStatePhase0, perfStateId} from "../util.js"; // PERF: Two major steps: // 1. Iterate over state.validators. Cost = 'proportional' to $VALIDATOR_COUNT not network conditions diff --git a/packages/state-transition/test/perf/epoch/epochAltair.test.ts b/packages/state-transition/test/perf/epoch/epochAltair.test.ts index 24635acfdb7f..1315dae8a2be 100644 --- a/packages/state-transition/test/perf/epoch/epochAltair.test.ts +++ b/packages/state-transition/test/perf/epoch/epochAltair.test.ts @@ -19,9 +19,9 @@ import { beforeProcessEpoch, computeStartSlotAtEpoch, } from "../../../src/index.js"; +import {altairState} from "../../../src/testUtils/params.js"; +import {getNetworkCachedState} from "../../../src/testUtils/testFileCache.js"; import {LazyValue, beforeValue} from "../../utils/beforeValueBenchmark.js"; -import {getNetworkCachedState} from "../../utils/testFileCache.js"; -import {altairState} from "../params.js"; import {StateEpoch} from "../types.js"; const slot = computeStartSlotAtEpoch(altairState.epoch) - 1; diff --git a/packages/state-transition/test/perf/epoch/epochCapella.test.ts b/packages/state-transition/test/perf/epoch/epochCapella.test.ts index c34d6861aaaf..7d40727f8647 100644 --- a/packages/state-transition/test/perf/epoch/epochCapella.test.ts +++ b/packages/state-transition/test/perf/epoch/epochCapella.test.ts @@ -19,9 +19,9 @@ import { beforeProcessEpoch, computeStartSlotAtEpoch, } from "../../../src/index.js"; +import {capellaState} from "../../../src/testUtils/params.js"; +import {getNetworkCachedState} from "../../../src/testUtils/testFileCache.js"; import {LazyValue, beforeValue} from "../../utils/beforeValueBenchmark.js"; -import {getNetworkCachedState} from "../../utils/testFileCache.js"; -import {capellaState} from "../params.js"; import {StateEpoch} from "../types.js"; const slot = computeStartSlotAtEpoch(capellaState.epoch) - 1; diff --git a/packages/state-transition/test/perf/epoch/epochPhase0.test.ts b/packages/state-transition/test/perf/epoch/epochPhase0.test.ts index 733fb9dd9922..95e18f63dc10 100644 --- a/packages/state-transition/test/perf/epoch/epochPhase0.test.ts +++ b/packages/state-transition/test/perf/epoch/epochPhase0.test.ts @@ -17,9 +17,9 @@ import { beforeProcessEpoch, computeStartSlotAtEpoch, } from "../../../src/index.js"; +import {phase0State} from "../../../src/testUtils/params.js"; +import {getNetworkCachedState} from "../../../src/testUtils/testFileCache.js"; import {LazyValue, beforeValue} from "../../utils/beforeValueBenchmark.js"; -import {getNetworkCachedState} from "../../utils/testFileCache.js"; -import {phase0State} from "../params.js"; import {StateEpoch} from "../types.js"; const slot = computeStartSlotAtEpoch(phase0State.epoch) - 1; diff --git a/packages/state-transition/test/perf/epoch/processEffectiveBalanceUpdates.test.ts b/packages/state-transition/test/perf/epoch/processEffectiveBalanceUpdates.test.ts index 384c1e2cf520..5f624809d82a 100644 --- a/packages/state-transition/test/perf/epoch/processEffectiveBalanceUpdates.test.ts +++ b/packages/state-transition/test/perf/epoch/processEffectiveBalanceUpdates.test.ts @@ -4,9 +4,9 @@ import {ForkSeq} from "@lodestar/params"; import {ssz} from "@lodestar/types"; import {processEffectiveBalanceUpdates} from "../../../src/epoch/processEffectiveBalanceUpdates.js"; import {CachedBeaconStateAllForks, EpochTransitionCache, beforeProcessEpoch} from "../../../src/index.js"; -import {createCachedBeaconStateTest} from "../../utils/state.js"; +import {createCachedBeaconStateTest} from "../../../src/testUtils/state.js"; +import {numValidators} from "../../../src/testUtils/util.js"; import {StateEpoch} from "../types.js"; -import {numValidators} from "../util.js"; // PERF: Cost 'proportional' to $VALIDATOR_COUNT, to iterate over all balances. Then cost is proportional to the amount // of validators whose effectiveBalance changed. Worst case is a massive network leak or a big slashing event which diff --git a/packages/state-transition/test/perf/epoch/processInactivityUpdates.test.ts b/packages/state-transition/test/perf/epoch/processInactivityUpdates.test.ts index 8db7e1d72929..15a09e58fcde 100644 --- a/packages/state-transition/test/perf/epoch/processInactivityUpdates.test.ts +++ b/packages/state-transition/test/perf/epoch/processInactivityUpdates.test.ts @@ -1,7 +1,7 @@ import {bench, describe} from "@chainsafe/benchmark"; import {processInactivityUpdates} from "../../../src/epoch/processInactivityUpdates.js"; +import {generatePerfTestCachedStateAltair, numValidators} from "../../../src/testUtils/util.js"; import {StateAltairEpoch} from "../types.js"; -import {generatePerfTestCachedStateAltair, numValidators} from "../util.js"; import {mutateInactivityScores} from "./util.js"; import {FlagFactors, generateBalanceDeltasEpochTransitionCache} from "./utilPhase0.js"; diff --git a/packages/state-transition/test/perf/epoch/processRegistryUpdates.test.ts b/packages/state-transition/test/perf/epoch/processRegistryUpdates.test.ts index 307fe1de386d..f43fdab6d3a1 100644 --- a/packages/state-transition/test/perf/epoch/processRegistryUpdates.test.ts +++ b/packages/state-transition/test/perf/epoch/processRegistryUpdates.test.ts @@ -2,8 +2,8 @@ import {bench, describe} from "@chainsafe/benchmark"; import {ForkSeq} from "@lodestar/params"; import {processRegistryUpdates} from "../../../src/epoch/processRegistryUpdates.js"; import {CachedBeaconStateAllForks, EpochTransitionCache, beforeProcessEpoch} from "../../../src/index.js"; +import {generatePerfTestCachedStatePhase0, numValidators} from "../../../src/testUtils/util.js"; import {StateEpoch} from "../types.js"; -import {generatePerfTestCachedStatePhase0, numValidators} from "../util.js"; // PERF: Cost 'proportional' to only validators that active + exit. For mainnet conditions: // - indicesEligibleForActivationQueue: Maxing deposits triggers 512 validator mutations diff --git a/packages/state-transition/test/perf/epoch/processRewardsAndPenalties.test.ts b/packages/state-transition/test/perf/epoch/processRewardsAndPenalties.test.ts index 61579256b606..f7596c97508b 100644 --- a/packages/state-transition/test/perf/epoch/processRewardsAndPenalties.test.ts +++ b/packages/state-transition/test/perf/epoch/processRewardsAndPenalties.test.ts @@ -1,7 +1,7 @@ import {bench, describe} from "@chainsafe/benchmark"; import {processRewardsAndPenalties} from "../../../src/epoch/processRewardsAndPenalties.js"; +import {generatePerfTestCachedStateAltair, numValidators} from "../../../src/testUtils/util.js"; import {StateAltairEpoch} from "../types.js"; -import {generatePerfTestCachedStateAltair, numValidators} from "../util.js"; import {mutateInactivityScores} from "./util.js"; import {FlagFactors, generateBalanceDeltasEpochTransitionCache} from "./utilPhase0.js"; diff --git a/packages/state-transition/test/perf/epoch/processRewardsAndPenaltiesPhase0.test.ts b/packages/state-transition/test/perf/epoch/processRewardsAndPenaltiesPhase0.test.ts index 3a7e12f32c8e..cebf95fd2da1 100644 --- a/packages/state-transition/test/perf/epoch/processRewardsAndPenaltiesPhase0.test.ts +++ b/packages/state-transition/test/perf/epoch/processRewardsAndPenaltiesPhase0.test.ts @@ -1,7 +1,7 @@ import {bench, describe} from "@chainsafe/benchmark"; import {getAttestationDeltas} from "../../../src/epoch/getAttestationDeltas.js"; +import {generatePerfTestCachedStatePhase0, numValidators} from "../../../src/testUtils/util.js"; import {StatePhase0Epoch} from "../types.js"; -import {generatePerfTestCachedStatePhase0, numValidators} from "../util.js"; import {FlagFactors, generateBalanceDeltasEpochTransitionCache} from "./utilPhase0.js"; // - On normal mainnet conditions diff --git a/packages/state-transition/test/perf/epoch/processSlashingsAllForks.test.ts b/packages/state-transition/test/perf/epoch/processSlashingsAllForks.test.ts index aa1190f6364a..1dd878abf95c 100644 --- a/packages/state-transition/test/perf/epoch/processSlashingsAllForks.test.ts +++ b/packages/state-transition/test/perf/epoch/processSlashingsAllForks.test.ts @@ -7,8 +7,8 @@ import { EpochTransitionCache, beforeProcessEpoch, } from "../../../src/index.js"; +import {generatePerfTestCachedStatePhase0, numValidators} from "../../../src/testUtils/util.js"; import {StateEpoch} from "../types.js"; -import {generatePerfTestCachedStatePhase0, numValidators} from "../util.js"; // PERF: Cost 'proportional' to only validators that are slashed. For mainnet conditions: // - indicesToSlash: max len is 8704. But it's very unlikely since it would require all validators on the same diff --git a/packages/state-transition/test/perf/epoch/processSyncCommitteeUpdates.test.ts b/packages/state-transition/test/perf/epoch/processSyncCommitteeUpdates.test.ts index 794bcdf510d1..869c04849c29 100644 --- a/packages/state-transition/test/perf/epoch/processSyncCommitteeUpdates.test.ts +++ b/packages/state-transition/test/perf/epoch/processSyncCommitteeUpdates.test.ts @@ -1,8 +1,8 @@ import {bench, describe} from "@chainsafe/benchmark"; import {EPOCHS_PER_SYNC_COMMITTEE_PERIOD, ForkSeq} from "@lodestar/params"; import {processSyncCommitteeUpdates} from "../../../src/epoch/processSyncCommitteeUpdates.js"; +import {generatePerfTestCachedStateAltair, numValidators} from "../../../src/testUtils/util.js"; import {StateAltair} from "../types.js"; -import {generatePerfTestCachedStateAltair, numValidators} from "../util.js"; // PERF: Cost = once per epoch compute committee, proportional to $VALIDATOR_COUNT diff --git a/packages/state-transition/test/perf/hashing.test.ts b/packages/state-transition/test/perf/hashing.test.ts index 21e7c1112d96..ea5aa1a81f34 100644 --- a/packages/state-transition/test/perf/hashing.test.ts +++ b/packages/state-transition/test/perf/hashing.test.ts @@ -2,7 +2,7 @@ import {beforeAll, bench, describe} from "@chainsafe/benchmark"; import {unshuffleList} from "@chainsafe/swap-or-not-shuffle"; import {SHUFFLE_ROUND_COUNT} from "@lodestar/params"; import {ssz} from "@lodestar/types"; -import {generatePerfTestCachedStatePhase0, numValidators} from "./util.js"; +import {generatePerfTestCachedStatePhase0, numValidators} from "../../src/testUtils/util.js"; // Test cost of hashing state after some modifications diff --git a/packages/state-transition/test/perf/misc/byteArrayEquals.test.ts b/packages/state-transition/test/perf/misc/byteArrayEquals.test.ts index 43a62be44ad3..7b904151a51b 100644 --- a/packages/state-transition/test/perf/misc/byteArrayEquals.test.ts +++ b/packages/state-transition/test/perf/misc/byteArrayEquals.test.ts @@ -1,7 +1,7 @@ import crypto from "node:crypto"; import {bench, describe} from "@chainsafe/benchmark"; import {byteArrayEquals} from "@lodestar/utils"; -import {generateState} from "../../utils/state.js"; +import {generateState} from "../../../src/testUtils/state.js"; import {generateValidators} from "../../utils/validator.js"; /** diff --git a/packages/state-transition/test/perf/slot/slots.test.ts b/packages/state-transition/test/perf/slot/slots.test.ts index ee8288e20030..c920ddc622ae 100644 --- a/packages/state-transition/test/perf/slot/slots.test.ts +++ b/packages/state-transition/test/perf/slot/slots.test.ts @@ -1,8 +1,8 @@ import {bench, describe} from "@chainsafe/benchmark"; import {ForkSeq} from "@lodestar/params"; import {processSlot} from "../../../src/slot/index.js"; +import {generatePerfTestCachedStatePhase0} from "../../../src/testUtils/util.js"; import {State} from "../types.js"; -import {generatePerfTestCachedStatePhase0} from "../util.js"; // Test advancing through an empty slot, without any epoch transition diff --git a/packages/state-transition/test/perf/util/balance.test.ts b/packages/state-transition/test/perf/util/balance.test.ts index bca42db717ec..9ed917d63104 100644 --- a/packages/state-transition/test/perf/util/balance.test.ts +++ b/packages/state-transition/test/perf/util/balance.test.ts @@ -1,7 +1,7 @@ import {bench, describe} from "@chainsafe/benchmark"; +import {generatePerfTestCachedStatePhase0, perfStateId} from "../../../src/testUtils/util.js"; import {getEffectiveBalanceIncrementsZeroInactive} from "../../../src/util/index.js"; import {State} from "../types.js"; -import {generatePerfTestCachedStatePhase0, perfStateId} from "../util.js"; describe("getEffectiveBalanceIncrementsZeroInactive", () => { bench({ diff --git a/packages/state-transition/test/perf/util/epochContext.test.ts b/packages/state-transition/test/perf/util/epochContext.test.ts index 520c7987b706..29193045029f 100644 --- a/packages/state-transition/test/perf/util/epochContext.test.ts +++ b/packages/state-transition/test/perf/util/epochContext.test.ts @@ -1,7 +1,7 @@ import {beforeAll, bench, describe} from "@chainsafe/benchmark"; import {Epoch} from "@lodestar/types"; import {CachedBeaconStateAllForks, computeEpochAtSlot} from "../../../src/index.js"; -import {generatePerfTestCachedStatePhase0, numValidators} from "../util.js"; +import {generatePerfTestCachedStatePhase0, numValidators} from "../../../src/testUtils/util.js"; // Current implementation scales very well with number of requested validators // Benchmark data from Wed Jun 30 2021 - Intel(R) Core(TM) i5-8250U CPU @ 1.60GHz diff --git a/packages/state-transition/test/perf/util/loadState/findModifiedValidators.test.ts b/packages/state-transition/test/perf/util/loadState/findModifiedValidators.test.ts index a7071aed52e9..d2df5c335702 100644 --- a/packages/state-transition/test/perf/util/loadState/findModifiedValidators.test.ts +++ b/packages/state-transition/test/perf/util/loadState/findModifiedValidators.test.ts @@ -3,9 +3,9 @@ import {bench, describe} from "@chainsafe/benchmark"; import {CompositeViewDU} from "@chainsafe/ssz"; import {ssz} from "@lodestar/types"; import {byteArrayEquals, bytesToInt} from "@lodestar/utils"; +import {generateState} from "../../../../src/testUtils/state.js"; import {findModifiedValidators} from "../../../../src/util/loadState/findModifiedValidators.js"; import {VALIDATOR_BYTES_SIZE} from "../../../../src/util/sszBytes.js"; -import {generateState} from "../../../utils/state.js"; import {generateValidators} from "../../../utils/validator.js"; /** diff --git a/packages/state-transition/test/perf/util/loadState/loadState.test.ts b/packages/state-transition/test/perf/util/loadState/loadState.test.ts index 29b15e6d975d..5396f03b0c15 100644 --- a/packages/state-transition/test/perf/util/loadState/loadState.test.ts +++ b/packages/state-transition/test/perf/util/loadState/loadState.test.ts @@ -1,8 +1,8 @@ import {bench, describe} from "@chainsafe/benchmark"; import {PubkeyCache, createPubkeyCache} from "../../../../src/cache/pubkeyCache.js"; import {createCachedBeaconState} from "../../../../src/cache/stateCache.js"; +import {generatePerfTestCachedStateAltair} from "../../../../src/testUtils/util.js"; import {loadState} from "../../../../src/util/loadState/loadState.js"; -import {generatePerfTestCachedStateAltair} from "../../util.js"; /** * This benchmark shows a stable performance from 2s to 3s on a Mac M1. And it does not really depend on the seed validators, diff --git a/packages/state-transition/test/perf/util/rootCache.test.ts b/packages/state-transition/test/perf/util/rootCache.test.ts index 015019e3545f..88ab2ef830bb 100644 --- a/packages/state-transition/test/perf/util/rootCache.test.ts +++ b/packages/state-transition/test/perf/util/rootCache.test.ts @@ -1,7 +1,7 @@ import {bench, describe} from "@chainsafe/benchmark"; +import {generatePerfTestCachedStatePhase0, perfStateEpoch, perfStateId} from "../../../src/testUtils/util.js"; import {RootCache, computeStartSlotAtEpoch, getBlockRootAtSlot} from "../../../src/util/index.js"; import {State} from "../types.js"; -import {generatePerfTestCachedStatePhase0, perfStateEpoch, perfStateId} from "../util.js"; const slot = computeStartSlotAtEpoch(perfStateEpoch) - 1; diff --git a/packages/state-transition/test/perf/util/seed.test.ts b/packages/state-transition/test/perf/util/seed.test.ts index 65b196d7c7f1..ad9211fb0faa 100644 --- a/packages/state-transition/test/perf/util/seed.test.ts +++ b/packages/state-transition/test/perf/util/seed.test.ts @@ -1,6 +1,7 @@ import {bench, describe} from "@chainsafe/benchmark"; import {ForkSeq} from "@lodestar/params"; import {fromHex} from "@lodestar/utils"; +import {generatePerfTestCachedStateAltair} from "../../../src/testUtils/util.js"; import { computeProposerIndex, computeShuffledIndex, @@ -9,7 +10,6 @@ import { naiveComputeProposerIndex, naiveGetNextSyncCommitteeIndices, } from "../../../src/util/seed.js"; -import {generatePerfTestCachedStateAltair} from "../util.js"; // I'm not sure how to populate a good test data for this benchmark describe("computeProposerIndex", () => { diff --git a/packages/state-transition/test/perf/util/serializeState.test.ts b/packages/state-transition/test/perf/util/serializeState.test.ts index 7d144882d767..d22f76b83e94 100644 --- a/packages/state-transition/test/perf/util/serializeState.test.ts +++ b/packages/state-transition/test/perf/util/serializeState.test.ts @@ -1,6 +1,6 @@ import {bench, describe, setBenchOpts} from "@chainsafe/benchmark"; import {ssz} from "@lodestar/types"; -import {cachedStateAltairPopulateCaches, generatePerfTestCachedStateAltair} from "../util.js"; +import {cachedStateAltairPopulateCaches, generatePerfTestCachedStateAltair} from "../../../src/testUtils/util.js"; /** * This shows different statistics between allocating memory once vs every time. diff --git a/packages/state-transition/test/perf/util/shufflings.test.ts b/packages/state-transition/test/perf/util/shufflings.test.ts index d0b82ac94b77..bb7c6f2fc313 100644 --- a/packages/state-transition/test/perf/util/shufflings.test.ts +++ b/packages/state-transition/test/perf/util/shufflings.test.ts @@ -9,7 +9,7 @@ import { getNextSyncCommittee, getSeed, } from "../../../src/index.js"; -import {generatePerfTestCachedStatePhase0, numValidators} from "../util.js"; +import {generatePerfTestCachedStatePhase0, numValidators} from "../../../src/testUtils/util.js"; describe("epoch shufflings", () => { let state: CachedBeaconStateAllForks; diff --git a/packages/state-transition/test/unit/block/isValidIndexedAttestation.test.ts b/packages/state-transition/test/unit/block/isValidIndexedAttestation.test.ts index 3943c825cc56..2ee5197e61e3 100644 --- a/packages/state-transition/test/unit/block/isValidIndexedAttestation.test.ts +++ b/packages/state-transition/test/unit/block/isValidIndexedAttestation.test.ts @@ -4,7 +4,7 @@ import {FAR_FUTURE_EPOCH, MAX_EFFECTIVE_BALANCE} from "@lodestar/params"; import {IndexedAttestation, ssz} from "@lodestar/types"; import {isValidIndexedAttestation} from "../../../src/block/isValidIndexedAttestation.js"; import {EMPTY_SIGNATURE} from "../../../src/index.js"; -import {generateCachedState} from "../../utils/state.js"; +import {generateCachedState} from "../../../src/testUtils/state.js"; import {generateValidators} from "../../utils/validator.js"; describe("validate indexed attestation", () => { diff --git a/packages/state-transition/test/unit/block/processConsolidationRequest.test.ts b/packages/state-transition/test/unit/block/processConsolidationRequest.test.ts index 86b9c646b33e..57db5fc6a0ab 100644 --- a/packages/state-transition/test/unit/block/processConsolidationRequest.test.ts +++ b/packages/state-transition/test/unit/block/processConsolidationRequest.test.ts @@ -8,8 +8,8 @@ import { SLOTS_PER_EPOCH, } from "@lodestar/params"; import {ssz} from "@lodestar/types"; -import {generateCachedElectraState} from "../../../../beacon-node/test/utils/state.js"; import {processConsolidationRequest} from "../../../src/block/processConsolidationRequest.js"; +import {generateCachedElectraState} from "../../utils/state.js"; import {generateValidators} from "../../utils/validator.js"; describe("processConsolidationRequest", () => { diff --git a/packages/state-transition/test/unit/block/processWithdrawals.test.ts b/packages/state-transition/test/unit/block/processWithdrawals.test.ts index 939a29809470..ba1ef057c3af 100644 --- a/packages/state-transition/test/unit/block/processWithdrawals.test.ts +++ b/packages/state-transition/test/unit/block/processWithdrawals.test.ts @@ -1,7 +1,7 @@ import {describe, expect, it} from "vitest"; import {ForkSeq} from "@lodestar/params"; import {getExpectedWithdrawals} from "../../../src/block/processWithdrawals.js"; -import {numValidators} from "../../perf/util.js"; +import {numValidators} from "../../../src/testUtils/util.js"; import {beforeValue} from "../../utils/beforeValue.js"; import {WithdrawalOpts, getExpectedWithdrawalsTestData} from "../../utils/capella.js"; diff --git a/packages/state-transition/test/unit/cachedBeaconState.test.ts b/packages/state-transition/test/unit/cachedBeaconState.test.ts index d3c9c59b4243..9f887634da1d 100644 --- a/packages/state-transition/test/unit/cachedBeaconState.test.ts +++ b/packages/state-transition/test/unit/cachedBeaconState.test.ts @@ -6,10 +6,10 @@ import {Epoch, RootHex, ssz} from "@lodestar/types"; import {toHexString} from "@lodestar/utils"; import {createPubkeyCache} from "../../src/cache/pubkeyCache.js"; import {createCachedBeaconState, loadCachedBeaconState} from "../../src/cache/stateCache.js"; +import {interopPubkeysCached} from "../../src/testUtils/interop.js"; +import {createCachedBeaconStateTest} from "../../src/testUtils/state.js"; import {EpochShuffling, calculateShufflingDecisionRoot} from "../../src/util/epochShuffling.js"; import {modifyStateSameValidator, newStateWithValidators} from "../utils/capella.js"; -import {interopPubkeysCached} from "../utils/interop.js"; -import {createCachedBeaconStateTest} from "../utils/state.js"; describe("CachedBeaconState", () => { vi.setConfig({testTimeout: 30_000, hookTimeout: 30_000}); diff --git a/packages/state-transition/test/unit/rewards/blockRewards.test.ts b/packages/state-transition/test/unit/rewards/blockRewards.test.ts index dfdf8a4ee506..e3d4347fceb2 100644 --- a/packages/state-transition/test/unit/rewards/blockRewards.test.ts +++ b/packages/state-transition/test/unit/rewards/blockRewards.test.ts @@ -6,9 +6,9 @@ import {ssz} from "@lodestar/types"; import {DataAvailabilityStatus, ExecutionPayloadStatus} from "../../../src/block/externalData.js"; import {computeBlockRewards} from "../../../src/rewards/blockRewards.js"; import {stateTransition} from "../../../src/stateTransition.js"; +import {cachedStateAltairPopulateCaches, generatePerfTestCachedStateAltair} from "../../../src/testUtils/util.js"; import {CachedBeaconStateAllForks} from "../../../src/types.js"; import {BlockAltairOpts, getBlockAltair} from "../../perf/block/util.js"; -import {cachedStateAltairPopulateCaches, generatePerfTestCachedStateAltair} from "../../perf/util.js"; describe("chain / rewards / blockRewards", () => { const config = createBeaconConfig({...chainConfigDef, ALTAIR_FORK_EPOCH: 0}, Buffer.alloc(32, 0xaa)); diff --git a/packages/state-transition/test/unit/sanityCheck.test.ts b/packages/state-transition/test/unit/sanityCheck.test.ts index 9d62cbaa0b12..70fa4809b60d 100644 --- a/packages/state-transition/test/unit/sanityCheck.test.ts +++ b/packages/state-transition/test/unit/sanityCheck.test.ts @@ -1,7 +1,11 @@ import {beforeAll, describe, expect, it, vi} from "vitest"; import {ACTIVE_PRESET, EFFECTIVE_BALANCE_INCREMENT, PresetName} from "@lodestar/params"; import {beforeProcessEpoch} from "../../src/index.js"; -import {generatePerfTestCachedStateAltair, generatePerfTestCachedStatePhase0, perfStateId} from "../perf/util.js"; +import { + generatePerfTestCachedStateAltair, + generatePerfTestCachedStatePhase0, + perfStateId, +} from "../../src/testUtils/util.js"; describe("Perf test sanity check", () => { vi.setConfig({testTimeout: 90 * 1000}); diff --git a/packages/state-transition/test/unit/signatureSets/signatureSets.test.ts b/packages/state-transition/test/unit/signatureSets/signatureSets.test.ts index 14f97e9dcf9a..4d79f409e660 100644 --- a/packages/state-transition/test/unit/signatureSets/signatureSets.test.ts +++ b/packages/state-transition/test/unit/signatureSets/signatureSets.test.ts @@ -7,7 +7,7 @@ import {FAR_FUTURE_EPOCH, MAX_EFFECTIVE_BALANCE} from "@lodestar/params"; import {BLSSignature, ValidatorIndex, capella, phase0, ssz} from "@lodestar/types"; import {ZERO_HASH} from "../../../src/constants/index.js"; import {getBlockSignatureSets} from "../../../src/signatureSets/index.js"; -import {generateCachedState} from "../../utils/state.js"; +import {generateCachedState} from "../../../src/testUtils/state.js"; import {generateValidators} from "../../utils/validator.js"; const EMPTY_SIGNATURE = Buffer.alloc(96); diff --git a/packages/state-transition/test/unit/util/balance.test.ts b/packages/state-transition/test/unit/util/balance.test.ts index bd61424452ab..959adc9d52aa 100644 --- a/packages/state-transition/test/unit/util/balance.test.ts +++ b/packages/state-transition/test/unit/util/balance.test.ts @@ -3,8 +3,8 @@ import {config as minimalConfig} from "@lodestar/config/default"; import {EFFECTIVE_BALANCE_INCREMENT} from "@lodestar/params"; import {ValidatorIndex} from "@lodestar/types"; import {getEffectiveBalanceIncrementsZeroInactive, getEffectiveBalanceIncrementsZeroed} from "../../../src/index.js"; +import {generateCachedState, generateState} from "../../../src/testUtils/state.js"; import {decreaseBalance, getTotalBalance, increaseBalance, isActiveValidator} from "../../../src/util/index.js"; -import {generateCachedState, generateState} from "../../utils/state.js"; import {generateValidators} from "../../utils/validator.js"; describe("getTotalBalance", () => { diff --git a/packages/state-transition/test/unit/util/deposit.test.ts b/packages/state-transition/test/unit/util/deposit.test.ts index 14ae0ef08357..0ed2ab6d4f6e 100644 --- a/packages/state-transition/test/unit/util/deposit.test.ts +++ b/packages/state-transition/test/unit/util/deposit.test.ts @@ -2,8 +2,8 @@ import {describe, expect, it} from "vitest"; import {createChainForkConfig} from "@lodestar/config"; import {MAX_DEPOSITS} from "@lodestar/params"; import {ssz} from "@lodestar/types"; +import {createCachedBeaconStateTest} from "../../../src/testUtils/state.js"; import {getEth1DepositCount} from "../../../src/util/deposit.js"; -import {createCachedBeaconStateTest} from "../../utils/state.js"; describe("getEth1DepositCount", () => { it("Pre Electra", () => { diff --git a/packages/state-transition/test/unit/util/epoch.test.ts b/packages/state-transition/test/unit/util/epoch.test.ts index 54742e4e2ab0..98c60ac0b1ee 100644 --- a/packages/state-transition/test/unit/util/epoch.test.ts +++ b/packages/state-transition/test/unit/util/epoch.test.ts @@ -1,13 +1,13 @@ import {describe, expect, it} from "vitest"; import {GENESIS_SLOT, MAX_SEED_LOOKAHEAD} from "@lodestar/params"; import {Epoch, Slot} from "@lodestar/types"; +import {generateState} from "../../../src/testUtils/state.js"; import { computeActivationExitEpoch, computeEpochAtSlot, computeStartSlotAtEpoch, getPreviousEpoch, } from "../../../src/util/index.js"; -import {generateState} from "../../utils/state.js"; describe("computeEpochAtSlot", () => { it.each([ diff --git a/packages/state-transition/test/unit/util/loadState/findModifiedValidators.test.ts b/packages/state-transition/test/unit/util/loadState/findModifiedValidators.test.ts index a504e5ea15d2..6575e9dd1925 100644 --- a/packages/state-transition/test/unit/util/loadState/findModifiedValidators.test.ts +++ b/packages/state-transition/test/unit/util/loadState/findModifiedValidators.test.ts @@ -1,7 +1,7 @@ import {describe, expect, it} from "vitest"; import {fromHexString} from "@chainsafe/ssz"; +import {generateState} from "../../../../src/testUtils/state.js"; import {findModifiedValidators} from "../../../../src/util/loadState/findModifiedValidators.js"; -import {generateState} from "../../../utils/state.js"; import {generateValidators} from "../../../utils/validator.js"; describe("findModifiedValidators", () => { diff --git a/packages/state-transition/test/unit/util/loadState/loadValidator.test.ts b/packages/state-transition/test/unit/util/loadState/loadValidator.test.ts index cccdec2d5d73..f9b48442385b 100644 --- a/packages/state-transition/test/unit/util/loadState/loadValidator.test.ts +++ b/packages/state-transition/test/unit/util/loadState/loadValidator.test.ts @@ -3,8 +3,8 @@ import {CompositeViewDU} from "@chainsafe/ssz"; import {config} from "@lodestar/config/default"; import {phase0, ssz} from "@lodestar/types"; import {byteArrayEquals} from "@lodestar/utils"; +import {generateState} from "../../../../src/testUtils/state.js"; import {getEffectiveBalancesFromStateBytes, loadValidator} from "../../../../src/util/loadState/loadValidator.js"; -import {generateState} from "../../../utils/state.js"; import {generateValidators} from "../../../utils/validator.js"; describe("loadValidator", () => { diff --git a/packages/state-transition/test/unit/util/misc.test.ts b/packages/state-transition/test/unit/util/misc.test.ts index 9b6bc20c6931..5c8ff44208f8 100644 --- a/packages/state-transition/test/unit/util/misc.test.ts +++ b/packages/state-transition/test/unit/util/misc.test.ts @@ -1,8 +1,8 @@ import {toBigIntLE} from "@vekexasia/bigint-buffer2"; import {describe, expect, it} from "vitest"; import {GENESIS_SLOT, SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params"; +import {generateState} from "../../../src/testUtils/state.js"; import {getBlockRoot} from "../../../src/util/index.js"; -import {generateState} from "../../utils/state.js"; describe("getBlockRoot", () => { it("should return first block root for genesis slot", () => { diff --git a/packages/state-transition/test/unit/util/seed.test.ts b/packages/state-transition/test/unit/util/seed.test.ts index 701ed95c1077..07bac422780c 100644 --- a/packages/state-transition/test/unit/util/seed.test.ts +++ b/packages/state-transition/test/unit/util/seed.test.ts @@ -3,6 +3,7 @@ import {describe, expect, it} from "vitest"; import {toHexString} from "@chainsafe/ssz"; import {ForkSeq, GENESIS_EPOCH, GENESIS_SLOT, SLOTS_PER_EPOCH} from "@lodestar/params"; import {bytesToInt} from "@lodestar/utils"; +import {generateState} from "../../../src/testUtils/state.js"; import { computePayloadTimelinessCommitteeIndices, computeProposerIndex, @@ -14,7 +15,6 @@ import { naiveComputeProposerIndex, naiveGetNextSyncCommitteeIndices, } from "../../../src/util/index.js"; -import {generateState} from "../../utils/state.js"; import {generateValidators} from "../../utils/validator.js"; describe("getRandaoMix", () => { diff --git a/packages/state-transition/test/unit/util/shuffling.test.ts b/packages/state-transition/test/unit/util/shuffling.test.ts index 0f37735215e1..9f01deee1c03 100644 --- a/packages/state-transition/test/unit/util/shuffling.test.ts +++ b/packages/state-transition/test/unit/util/shuffling.test.ts @@ -1,8 +1,8 @@ import {describe, expect, it} from "vitest"; import {ssz} from "@lodestar/types"; import {computeEpochAtSlot} from "../../../src/index.js"; +import {generateState} from "../../../src/testUtils/state.js"; import {computeEpochShuffling, computeEpochShufflingAsync} from "../../../src/util/epochShuffling.js"; -import {generateState} from "../../utils/state.js"; describe("EpochShuffling", () => { it("async and sync versions should be identical", async () => { diff --git a/packages/state-transition/test/unit/util/validator.test.ts b/packages/state-transition/test/unit/util/validator.test.ts index 0cb341d3e6ee..e24e79e7c919 100644 --- a/packages/state-transition/test/unit/util/validator.test.ts +++ b/packages/state-transition/test/unit/util/validator.test.ts @@ -1,8 +1,8 @@ import {beforeEach, describe, expect, it} from "vitest"; import {phase0, ssz} from "@lodestar/types"; +import {generateState} from "../../../src/testUtils/state.js"; import {getActiveValidatorIndices, isActiveValidator, isSlashableValidator} from "../../../src/util/index.js"; import {randBetween} from "../../utils/misc.js"; -import {generateState} from "../../utils/state.js"; import {generateValidator} from "../../utils/validator.js"; describe("getActiveValidatorIndices", () => { diff --git a/packages/state-transition/test/utils/capella.ts b/packages/state-transition/test/utils/capella.ts index 249053e2a22a..8a02dc7de11b 100644 --- a/packages/state-transition/test/utils/capella.ts +++ b/packages/state-transition/test/utils/capella.ts @@ -8,9 +8,9 @@ import { } from "@lodestar/params"; import {ssz} from "@lodestar/types"; import {BeaconStateCapella, CachedBeaconStateCapella} from "../../src/index.js"; -import {interopPubkeysCached} from "./interop.js"; +import {interopPubkeysCached} from "../../src/testUtils/interop.js"; +import {createCachedBeaconStateTest} from "../../src/testUtils/state.js"; import {mulberry32} from "./rand.js"; -import {createCachedBeaconStateTest} from "./state.js"; export interface WithdrawalOpts { excessBalance: number; diff --git a/packages/state-transition/test/utils/state.ts b/packages/state-transition/test/utils/state.ts index d8161bf69e37..1541c9df636a 100644 --- a/packages/state-transition/test/utils/state.ts +++ b/packages/state-transition/test/utils/state.ts @@ -1,110 +1,151 @@ import {ChainForkConfig, createBeaconConfig} from "@lodestar/config"; -import {config, config as minimalConfig} from "@lodestar/config/default"; -import { - EPOCHS_PER_HISTORICAL_VECTOR, - EPOCHS_PER_SLASHINGS_VECTOR, - GENESIS_EPOCH, - GENESIS_SLOT, - SLOTS_PER_HISTORICAL_ROOT, -} from "@lodestar/params"; -import {phase0, ssz} from "@lodestar/types"; -import {EpochCacheOpts} from "../../src/cache/epochCache.js"; -import {BeaconStateCache} from "../../src/cache/stateCache.js"; -import {ZERO_HASH} from "../../src/constants/index.js"; +import {config as minimalConfig} from "@lodestar/config/default"; +import {getConfig} from "@lodestar/config/test-utils"; +import {FAR_FUTURE_EPOCH, ForkName, ForkSeq, MAX_EFFECTIVE_BALANCE, SYNC_COMMITTEE_SIZE} from "@lodestar/params"; +import {BeaconState, altair, bellatrix, electra, ssz} from "@lodestar/types"; import { BeaconStateAllForks, - BeaconStatePhase0, + BeaconStateBellatrix, + BeaconStateElectra, CachedBeaconStateAllForks, + CachedBeaconStateBellatrix, + CachedBeaconStateElectra, createCachedBeaconState, createPubkeyCache, } from "../../src/index.js"; -import {newZeroedArray} from "../../src/util/index.js"; +import {generateValidator, generateValidators} from "./validator.js"; /** * Copy of BeaconState, but all fields are marked optional to allow for swapping out variables as needed. */ -type TestBeaconState = Partial; +type TestBeaconState = Partial; /** - * Generate beaconState, by default it will use the initial state defined when the `ChainStart` log is emitted. + * Generate beaconState, by default it will generate a mostly empty state with "just enough" to be valid-ish * NOTE: All fields can be overridden through `opts`. + * should allow 1st test calling generateState more time since creating a new instance is expensive. + * * @param {TestBeaconState} opts + * @param config * @returns {BeaconState} */ -export function generateState(opts?: TestBeaconState): BeaconStatePhase0 { - return ssz.phase0.BeaconState.toViewDU({ - genesisTime: Math.floor(Date.now() / 1000), - genesisValidatorsRoot: ZERO_HASH, - slot: GENESIS_SLOT, - fork: { - previousVersion: config.GENESIS_FORK_VERSION, - currentVersion: config.GENESIS_FORK_VERSION, - epoch: GENESIS_EPOCH, - }, - latestBlockHeader: { - slot: 0, - proposerIndex: 0, - parentRoot: Buffer.alloc(32), - stateRoot: Buffer.alloc(32), - bodyRoot: ssz.phase0.BeaconBlockBody.hashTreeRoot(ssz.phase0.BeaconBlockBody.defaultValue()), - }, - blockRoots: Array.from({length: SLOTS_PER_HISTORICAL_ROOT}, () => ZERO_HASH), - stateRoots: Array.from({length: SLOTS_PER_HISTORICAL_ROOT}, () => ZERO_HASH), - historicalRoots: [], - eth1Data: { - depositRoot: Buffer.alloc(32), - blockHash: Buffer.alloc(32), - depositCount: 0, - }, - eth1DataVotes: [], - eth1DepositIndex: 0, - validators: [], - balances: [], - randaoMixes: Array.from({length: EPOCHS_PER_HISTORICAL_VECTOR}, () => ZERO_HASH), - slashings: newZeroedArray(EPOCHS_PER_SLASHINGS_VECTOR), - previousEpochAttestations: [], - currentEpochAttestations: [], - justificationBits: ssz.phase0.JustificationBits.defaultValue(), - previousJustifiedCheckpoint: { - epoch: GENESIS_EPOCH, - root: ZERO_HASH, - }, - currentJustifiedCheckpoint: { - epoch: GENESIS_EPOCH, - root: ZERO_HASH, - }, - finalizedCheckpoint: { - epoch: GENESIS_EPOCH, - root: ZERO_HASH, - }, - ...opts, +export function generateState( + opts: TestBeaconState = {}, + config: ChainForkConfig = minimalConfig, + withPubkey = false +): BeaconStateAllForks { + const stateSlot = opts.slot ?? 0; + const state = config.getForkTypes(stateSlot).BeaconState.defaultValue(); + const forkSeq = config.getForkSeq(stateSlot); + + // Mutate state adding properties + Object.assign(state, opts); + + const validatorOpts = { + activationEpoch: 0, + effectiveBalance: MAX_EFFECTIVE_BALANCE, + withdrawableEpoch: FAR_FUTURE_EPOCH, + exitEpoch: FAR_FUTURE_EPOCH, + }; + const numValidators = 16; + + const validators = + opts.validators ?? + (withPubkey + ? Array.from({length: numValidators}, (_, i) => { + return generateValidator({ + ...validatorOpts, + }); + }) + : generateValidators(numValidators, validatorOpts)); + + state.genesisTime = Math.floor(Date.now() / 1000); + state.slot = stateSlot; + state.fork.previousVersion = config.GENESIS_FORK_VERSION; + state.fork.currentVersion = config.GENESIS_FORK_VERSION; + state.latestBlockHeader.bodyRoot = ssz.phase0.BeaconBlockBody.hashTreeRoot(ssz.phase0.BeaconBlockBody.defaultValue()); + state.validators = validators; + state.balances = Array.from({length: validators.length}, () => MAX_EFFECTIVE_BALANCE); + + if (forkSeq >= ForkSeq.altair) { + const stateAltair = state as altair.BeaconState; + stateAltair.previousEpochParticipation = [...[0xff, 0xff], ...Array.from({length: validators.length - 2}, () => 0)]; + stateAltair.currentEpochParticipation = [...[0xff, 0xff], ...Array.from({length: validators.length - 2}, () => 0)]; + stateAltair.currentSyncCommittee = { + pubkeys: Array.from({length: SYNC_COMMITTEE_SIZE}, (_, i) => validators[i % validators.length].pubkey), + aggregatePubkey: ssz.BLSPubkey.defaultValue(), + }; + stateAltair.nextSyncCommittee = { + pubkeys: Array.from({length: SYNC_COMMITTEE_SIZE}, (_, i) => validators[i % validators.length].pubkey), + aggregatePubkey: ssz.BLSPubkey.defaultValue(), + }; + } + + if (forkSeq >= ForkSeq.bellatrix) { + const stateBellatrix = state as bellatrix.BeaconState; + stateBellatrix.latestExecutionPayloadHeader = { + ...ssz.bellatrix.ExecutionPayloadHeader.defaultValue(), + blockNumber: 2022, + }; + } + + if (forkSeq >= ForkSeq.electra) { + const stateElectra = state as electra.BeaconState; + stateElectra.depositRequestsStartIndex = 2023n; + stateElectra.latestExecutionPayloadHeader = ssz.electra.ExecutionPayloadHeader.defaultValue(); + } + + return config.getForkTypes(stateSlot).BeaconState.toViewDU(state); +} + +/** + * This generates state with default pubkey + * TODO: (@matthewkeil) - this is duplicated and exists in state-transition as well + */ +export function generateCachedState(opts?: TestBeaconState): CachedBeaconStateAllForks { + const config = getConfig(ForkName.phase0); + const state = generateState(opts, config); + return createCachedBeaconState(state, { + config: createBeaconConfig(config, state.genesisValidatorsRoot), + // This is a performance test, there's no need to have a global shared cache of keys + pubkeyCache: createPubkeyCache(), }); } -export function generateCachedState( - config: ChainForkConfig = minimalConfig, - opts: TestBeaconState = {} -): CachedBeaconStateAllForks { - const state = generateState(opts); +/** + * This generates state with default pubkey + */ +export function generateCachedAltairState(opts?: TestBeaconState, altairForkEpoch = 0): CachedBeaconStateAllForks { + const config = getConfig(ForkName.altair, altairForkEpoch); + const state = generateState(opts, config); return createCachedBeaconState(state, { config: createBeaconConfig(config, state.genesisValidatorsRoot), - // This is a test state, there's no need to have a global shared cache of keys + // This is a performance test, there's no need to have a global shared cache of keys + pubkeyCache: createPubkeyCache(), + }); +} + +/** + * This generates state with default pubkey + */ +export function generateCachedBellatrixState(opts?: TestBeaconState): CachedBeaconStateBellatrix { + const config = getConfig(ForkName.bellatrix); + const state = generateState(opts, config); + return createCachedBeaconState(state as BeaconStateBellatrix, { + config: createBeaconConfig(config, state.genesisValidatorsRoot), + // This is a performance test, there's no need to have a global shared cache of keys pubkeyCache: createPubkeyCache(), }); } -export function createCachedBeaconStateTest( - state: T, - configCustom: ChainForkConfig = config, - opts?: EpochCacheOpts -): T & BeaconStateCache { - return createCachedBeaconState( - state, - { - config: createBeaconConfig(configCustom, state.genesisValidatorsRoot), - // This is a test state, there's no need to have a global shared cache of keys - pubkeyCache: createPubkeyCache(), - }, - opts - ); +/** + * This generates state with default pubkey + */ +export function generateCachedElectraState(opts?: TestBeaconState, electraForkEpoch = 0): CachedBeaconStateElectra { + const config = getConfig(ForkName.electra, electraForkEpoch); + const state = generateState(opts, config); + return createCachedBeaconState(state as BeaconStateElectra, { + config: createBeaconConfig(config, state.genesisValidatorsRoot), + pubkeyCache: createPubkeyCache(), + }); } diff --git a/packages/test-utils/src/externalSigner.ts b/packages/test-utils/src/externalSigner.ts index b668566b3520..297a0eb53c0a 100644 --- a/packages/test-utils/src/externalSigner.ts +++ b/packages/test-utils/src/externalSigner.ts @@ -3,7 +3,7 @@ import path from "node:path"; import {dirSync as tmpDirSync} from "tmp"; import {ForkSeq} from "@lodestar/params"; import {fetch, retry, withTimeout} from "@lodestar/utils"; -import {runDockerContainer} from "./dockercontainer.ts"; +import {runDockerContainer} from "./dockercontainer.js"; const web3signerVersion = "25.11.0"; const web3signerImage = `consensys/web3signer:${web3signerVersion}`; diff --git a/packages/utils/src/bytes/nodejs.ts b/packages/utils/src/bytes/nodejs.ts index c98066c74cbd..106fbf6563a8 100644 --- a/packages/utils/src/bytes/nodejs.ts +++ b/packages/utils/src/bytes/nodejs.ts @@ -89,4 +89,4 @@ export function byteArrayEquals(a: Uint8Array, b: Uint8Array): boolean { return Buffer.compare(a, b) === 0; } -export {bigIntToBytes, bytesToBigInt, bytesToInt, fromHexInto, intToBytes, toHexString, xor} from "./browser.ts"; +export {bigIntToBytes, bytesToBigInt, bytesToInt, fromHexInto, intToBytes, toHexString, xor} from "./browser.js"; diff --git a/packages/utils/test/perf/bytes.test.ts b/packages/utils/test/perf/bytes.test.ts index 17f7153fef2d..3184158bfb03 100644 --- a/packages/utils/test/perf/bytes.test.ts +++ b/packages/utils/test/perf/bytes.test.ts @@ -1,6 +1,6 @@ import {bench, describe} from "@chainsafe/benchmark"; -import * as browser from "../../src/bytes/browser.ts"; -import * as nodejs from "../../src/bytes/nodejs.ts"; +import * as browser from "../../src/bytes/browser.js"; +import * as nodejs from "../../src/bytes/nodejs.js"; describe("bytes utils", async () => { const runsFactor = 1000; diff --git a/packages/validator/src/index.ts b/packages/validator/src/index.ts index 7e46cbb11ad5..06c2ec972198 100644 --- a/packages/validator/src/index.ts +++ b/packages/validator/src/index.ts @@ -1,3 +1,4 @@ +export {Bucket} from "./buckets.js"; export {waitForGenesis} from "./genesis.js"; export {type Metrics, getMetrics} from "./metrics.js"; export * from "./repositories/index.js"; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 273409581ac8..862096884edb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -581,6 +581,9 @@ importers: '@lodestar/params': specifier: workspace:^ version: link:../params + '@lodestar/spec-test-util': + specifier: workspace:^ + version: link:../spec-test-util '@lodestar/types': specifier: workspace:^ version: link:../types diff --git a/spec-tests-version.json b/spec-tests-version.json new file mode 100644 index 000000000000..74088e7fb373 --- /dev/null +++ b/spec-tests-version.json @@ -0,0 +1,20 @@ +{ + "ethereumConsensusSpecsTests": { + "specVersion": "v1.7.0-alpha.2", + "specTestsRepoUrl": "https://github.com/ethereum/consensus-specs", + "outputDirBase": "spec-tests", + "testsToDownload": [ + "general", + "mainnet", + "minimal" + ] + }, + "blsSpecTests": { + "specVersion": "v0.1.1", + "specTestsRepoUrl": "https://github.com/ethereum/bls12-381-tests", + "outputDirBase": "spec-tests-bls", + "testsToDownload": [ + "bls_tests_yaml" + ] + } +} From 0087e7a664c8c1d77d22e5cf895746b62a50ca31 Mon Sep 17 00:00:00 2001 From: twoeths <10568965+twoeths@users.noreply.github.com> Date: Wed, 11 Mar 2026 21:22:04 +0700 Subject: [PATCH 31/63] chore: remove getComputeShuffledIndexFn() (#9011) **Motivation** `getComputeShuffledIndexFn()` is no longer used anywhere in production code. Production paths use the native implementation from `@chainsafe/swap-or-not-shuffle` instead. **Description** - Removes `getComputeShuffledIndexFn()` and its associated `ComputeShuffledIndexFn` type from `packages/state-transition/src/util/seed.ts` - Removes the unit test that only existed to verify parity with `computeShuffledIndex` - Removes the already-skipped perf bench **AI Assistance Disclosure** Implemented with Claude Code (claude-sonnet-4-6). --------- Co-authored-by: Tuyen Nguyen --- packages/state-transition/src/util/seed.ts | 71 ------------------- .../test/perf/util/seed.test.ts | 12 ---- .../test/unit/util/seed.test.ts | 14 ---- 3 files changed, 97 deletions(-) diff --git a/packages/state-transition/src/util/seed.ts b/packages/state-transition/src/util/seed.ts index 978f3e41f557..be01506f6cd4 100644 --- a/packages/state-transition/src/util/seed.ts +++ b/packages/state-transition/src/util/seed.ts @@ -417,8 +417,6 @@ export function naiveComputePayloadTimelinessCommitteeIndices( * https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf * * See the 'generalized domain' algorithm on page 3. - * This is the naive implementation just to make sure lodestar follows the spec, this is not for production. - * The optimized version is in `getComputeShuffledIndexFn`. */ export function computeShuffledIndex(index: number, indexCount: number, seed: Bytes32): number { let permuted = index; @@ -439,75 +437,6 @@ export function computeShuffledIndex(index: number, indexCount: number, seed: By return permuted; } -type ComputeShuffledIndexFn = (index: number) => number; - -/** - * An optimized version of `computeShuffledIndex`, this is for production. - */ -export function getComputeShuffledIndexFn(indexCount: number, seed: Bytes32): ComputeShuffledIndexFn { - // there are possibly SHUFFLE_ROUND_COUNT (90 for mainnet) values for this cache - // this cache will always hit after the 1st call - const pivotByIndex: Map = new Map(); - // given 2M active validators, there are 2 M / 256 = 8k possible positionDiv - // it means there are at most 8k different sources for each round - const sourceByPositionDivByIndex: Map> = new Map(); - // 32 bytes seed + 1 byte i - const pivotBuffer = Buffer.alloc(32 + 1); - pivotBuffer.set(seed, 0); - // 32 bytes seed + 1 byte i + 4 bytes positionDiv - const sourceBuffer = Buffer.alloc(32 + 1 + 4); - sourceBuffer.set(seed, 0); - - return (index): number => { - assert.lt(index, indexCount, "indexCount must be less than index"); - assert.lte(indexCount, 2 ** 40, "indexCount too big"); - let permuted = index; - // const _seed = seed; - for (let i = 0; i < SHUFFLE_ROUND_COUNT; i++) { - // optimized version of the below naive code - // const pivot = Number( - // bytesToBigInt(digest(Buffer.concat([_seed, intToBytes(i, 1)])).slice(0, 8)) % BigInt(indexCount) - // ); - - let pivot = pivotByIndex.get(i); - if (pivot == null) { - // naive version always creates a new buffer, we can reuse the buffer - // pivot = Number( - // bytesToBigInt(digest(Buffer.concat([_seed, intToBytes(i, 1)])).slice(0, 8)) % BigInt(indexCount) - // ); - pivotBuffer[32] = i % 256; - pivot = Number(bytesToBigInt(digest(pivotBuffer).subarray(0, 8)) % BigInt(indexCount)); - pivotByIndex.set(i, pivot); - } - - const flip = (pivot + indexCount - permuted) % indexCount; - const position = Math.max(permuted, flip); - - // optimized version of the below naive code - // const source = digest(Buffer.concat([_seed, intToBytes(i, 1), intToBytes(Math.floor(position / 256), 4)])); - let sourceByPositionDiv = sourceByPositionDivByIndex.get(i); - if (sourceByPositionDiv == null) { - sourceByPositionDiv = new Map(); - sourceByPositionDivByIndex.set(i, sourceByPositionDiv); - } - const positionDiv256 = Math.floor(position / 256); - let source = sourceByPositionDiv.get(positionDiv256); - if (source == null) { - // naive version always creates a new buffer, we can reuse the buffer - // don't want to go through intToBytes() to avoid BigInt - sourceBuffer[32] = i % 256; - sourceBuffer.writeUint32LE(positionDiv256, 33); - source = digest(sourceBuffer); - sourceByPositionDiv.set(positionDiv256, source); - } - const byte = source[Math.floor((position % 256) / 8)]; - const bit = (byte >> (position % 8)) % 2; - permuted = bit ? flip : permuted; - } - return permuted; - }; -} - /** * Return the randao mix at a recent [[epoch]]. */ diff --git a/packages/state-transition/test/perf/util/seed.test.ts b/packages/state-transition/test/perf/util/seed.test.ts index ad9211fb0faa..cd3aa18677a4 100644 --- a/packages/state-transition/test/perf/util/seed.test.ts +++ b/packages/state-transition/test/perf/util/seed.test.ts @@ -5,7 +5,6 @@ import {generatePerfTestCachedStateAltair} from "../../../src/testUtils/util.js" import { computeProposerIndex, computeShuffledIndex, - getComputeShuffledIndexFn, getNextSyncCommitteeIndices, naiveComputeProposerIndex, naiveGetNextSyncCommitteeIndices, @@ -88,16 +87,5 @@ describe("computeShuffledIndex", () => { } }, }); - - const shuffledIndexFn = getComputeShuffledIndexFn(vc, seed); - // getComputeShuffledIndexFn() is also not in prod anymore so no need to track it - bench.skip({ - id: `cached computeShuffledIndex ${vc} validators`, - fn: () => { - for (let i = 0; i < vc; i++) { - shuffledIndexFn(i); - } - }, - }); } }); diff --git a/packages/state-transition/test/unit/util/seed.test.ts b/packages/state-transition/test/unit/util/seed.test.ts index 07bac422780c..c208e4c73f0e 100644 --- a/packages/state-transition/test/unit/util/seed.test.ts +++ b/packages/state-transition/test/unit/util/seed.test.ts @@ -7,8 +7,6 @@ import {generateState} from "../../../src/testUtils/state.js"; import { computePayloadTimelinessCommitteeIndices, computeProposerIndex, - computeShuffledIndex, - getComputeShuffledIndexFn, getNextSyncCommitteeIndices, getRandaoMix, naiveComputePayloadTimelinessCommitteeIndices, @@ -58,18 +56,6 @@ describe("computeProposerIndex", () => { } }); -describe("computeShuffledIndex", () => { - const seed = crypto.randomBytes(32); - const vc = 1000; - const shuffledIndexFn = getComputeShuffledIndexFn(vc, seed); - it("should be the same to the naive version", () => { - for (let i = 0; i < vc; i++) { - const expectedIndex = computeShuffledIndex(i, vc, seed); - expect(shuffledIndexFn(i)).toBe(expectedIndex); - } - }); -}); - describe("electra getNextSyncCommitteeIndices", () => { const vc = 1000; const validators = generateValidators(vc); From f7a5f4ddda7cca154ef6b831610609f05a78f8d5 Mon Sep 17 00:00:00 2001 From: Nazar Hussain Date: Thu, 12 Mar 2026 00:06:51 +0100 Subject: [PATCH 32/63] feat: use tsgo for build and type checks (#8992) **Motivation** To improve the performance for agentic development need to migrate to `tsgo` **Description** - Update `typescript` to latest transition version - Add `tsgo` as dependency --- .vscode/settings.json | 1 + package.json | 5 +- packages/api/package.json | 4 +- packages/api/src/beacon/routes/node.ts | 4 + packages/api/src/utils/client/httpClient.ts | 4 + packages/api/src/utils/client/response.ts | 4 + packages/beacon-node/package.json | 4 +- packages/beacon-node/tsconfig.json | 5 +- packages/cli/package.json | 6 +- packages/config/package.json | 4 +- packages/config/tsconfig.json | 5 +- packages/db/package.json | 4 +- packages/era/package.json | 4 +- packages/flare/package.json | 4 +- packages/fork-choice/package.json | 4 +- packages/light-client/package.json | 4 +- packages/logger/package.json | 4 +- packages/params/package.json | 4 +- packages/params/tsconfig.json | 5 +- packages/prover/package.json | 4 +- packages/reqresp/package.json | 4 +- packages/spec-test-util/package.json | 4 +- packages/state-transition/package.json | 4 +- packages/state-transition/test/utils/state.ts | 2 +- packages/state-transition/tsconfig.json | 5 +- packages/test-utils/package.json | 4 +- packages/types/package.json | 4 +- packages/utils/package.json | 4 +- packages/validator/package.json | 4 +- pnpm-lock.yaml | 145 +++++++++++++----- tsconfig.build.json | 42 +---- tsconfig.common.json | 50 ++++++ tsconfig.json | 7 +- 33 files changed, 233 insertions(+), 129 deletions(-) create mode 100644 tsconfig.common.json diff --git a/.vscode/settings.json b/.vscode/settings.json index 7ae1039bcebc..4892bc4ce12f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,6 +2,7 @@ "window.title": "${activeEditorShort}${separator}${rootName}${separator}${profileName}${separator}[${activeRepositoryBranchName}]", // For `sysoev.vscode-open-in-github` extension "openInGitHub.defaultBranch": "unstable", + "js/ts.experimental.useTsgo": true, "editor.formatOnSave": true, "editor.codeActionsOnSave": { "source.fixAll.biome": "explicit", diff --git a/package.json b/package.json index 4a6840994720..5323cf833559 100644 --- a/package.json +++ b/package.json @@ -77,8 +77,9 @@ "stream-browserify": "^3.0.0", "stream-http": "^3.2.0", "supertest": "^6.3.3", - "ts-node": "^10.9.2", - "typescript": "^5.9.2", + "ts-node": "^11.0.0-beta.1", + "typescript": "^6.0.1-rc", + "@typescript/native-preview": "7.0.0-dev.20260303.1", "vite": "^6.0.11", "vite-plugin-dts": "^4.5.4", "vite-plugin-node-polyfills": "^0.24.0", diff --git a/packages/api/package.json b/packages/api/package.json index d8ed1fb2cebc..8272a90a3ab2 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -62,11 +62,11 @@ ], "scripts": { "clean": "rm -rf lib && rm -f *.tsbuildinfo", - "build": "tsc -p tsconfig.build.json", + "build": "tsgo -p tsconfig.build.json", "build:watch": "pnpm run build --watch", "build:release": "pnpm clean && pnpm run build", "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\"", - "check-types": "tsc", + "check-types": "tsgo", "lint": "biome check src/ test/", "lint:fix": "pnpm run lint --write", "test": "pnpm test:unit", diff --git a/packages/api/src/beacon/routes/node.ts b/packages/api/src/beacon/routes/node.ts index 1db16e33fede..f6b3d77d0bff 100644 --- a/packages/api/src/beacon/routes/node.ts +++ b/packages/api/src/beacon/routes/node.ts @@ -15,6 +15,10 @@ import {HttpStatusCode} from "../../utils/httpStatusCode.js"; import {Endpoint, RouteDefinitions, Schema} from "../../utils/index.js"; import {WireFormat} from "../../utils/wireFormat.js"; +// TODO: Workaround for tsgo import-elision bug: ensure this is treated as a runtime value. +// https://github.com/microsoft/typescript-go/issues/2212 +void HttpStatusCode; + export const NetworkIdentityType = new ContainerType( { /** Cryptographic hash of a peer’s public key. [Read more](https://docs.libp2p.io/concepts/peer-id/) */ diff --git a/packages/api/src/utils/client/httpClient.ts b/packages/api/src/utils/client/httpClient.ts index 1f1b97163309..4bedab36f657 100644 --- a/packages/api/src/utils/client/httpClient.ts +++ b/packages/api/src/utils/client/httpClient.ts @@ -25,6 +25,10 @@ import { } from "./request.js"; import {ApiResponse} from "./response.js"; +// TODO: Workaround for tsgo import-elision bug: ensure this is treated as a runtime value. +// https://github.com/microsoft/typescript-go/issues/2212 +void HttpStatusCode; + /** A higher default timeout, validator will set its own shorter timeoutMs */ const DEFAULT_TIMEOUT_MS = 60_000; const DEFAULT_RETRIES = 0; diff --git a/packages/api/src/utils/client/response.ts b/packages/api/src/utils/client/response.ts index 626252b9aaca..b5d63ec02603 100644 --- a/packages/api/src/utils/client/response.ts +++ b/packages/api/src/utils/client/response.ts @@ -5,6 +5,10 @@ import {WireFormat, getWireFormat} from "../wireFormat.js"; import {ApiError} from "./error.js"; import {RouteDefinitionExtra} from "./request.js"; +// TODO: Workaround for tsgo import-elision bug: ensure this is treated as a runtime value. +// https://github.com/microsoft/typescript-go/issues/2212 +void HttpStatusCode; + export type RawBody = | {type: WireFormat.json; value: unknown} | {type: WireFormat.ssz; value: Uint8Array} diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index 615889647afa..e3a34deef0d5 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -87,11 +87,11 @@ ], "scripts": { "clean": "rm -rf lib && rm -f *.tsbuildinfo", - "build": "tsc -p tsconfig.build.json", + "build": "tsgo -p tsconfig.build.json", "build:watch": "pnpm run build --watch", "build:release": "pnpm clean && pnpm run build", "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\"", - "check-types": "tsc", + "check-types": "tsgo", "lint": "biome check src/ test/", "lint:fix": "pnpm run lint --write", "test": "pnpm test:unit && pnpm test:e2e", diff --git a/packages/beacon-node/tsconfig.json b/packages/beacon-node/tsconfig.json index a0f4f2a31e93..19b9527ba0e3 100644 --- a/packages/beacon-node/tsconfig.json +++ b/packages/beacon-node/tsconfig.json @@ -1,6 +1,9 @@ { "extends": "../../tsconfig.json", - "include": ["src", "test"], + "include": [ + "src", + "test" + ], "compilerOptions": { "outDir": "lib" } diff --git a/packages/cli/package.json b/packages/cli/package.json index a879acf1f913..a41ae747d102 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -28,12 +28,12 @@ }, "scripts": { "clean": "rm -rf lib && rm -f *.tsbuildinfo", - "build": "tsc -p tsconfig.build.json && pnpm write-git-data", + "build": "tsgo -p tsconfig.build.json && pnpm write-git-data", "build:release": "pnpm clean && pnpm run build", - "build:watch": "tsc -p tsconfig.build.json --watch", + "build:watch": "tsgo -p tsconfig.build.json --watch", "write-git-data": "node lib/util/gitData/writeGitData.js", "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\" lodestar --help", - "check-types": "tsc", + "check-types": "tsgo", "docs:build": "node --loader ts-node/esm ./docsgen/index.ts", "lint": "biome check src/ test/", "lint:fix": "pnpm run lint --write", diff --git a/packages/config/package.json b/packages/config/package.json index 14464b5e65d4..784e6e2d1add 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -43,11 +43,11 @@ ], "scripts": { "clean": "rm -rf lib && rm -f *.tsbuildinfo", - "build": "tsc -p tsconfig.build.json", + "build": "tsgo -p tsconfig.build.json", "build:release": "pnpm run clean && pnpm run build", "build:watch": "pnpm run build --watch", "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\"", - "check-types": "tsc", + "check-types": "tsgo", "lint": "biome check src/", "lint:fix": "pnpm run lint --write", "test": "pnpm test:unit", diff --git a/packages/config/tsconfig.json b/packages/config/tsconfig.json index a0f4f2a31e93..19b9527ba0e3 100644 --- a/packages/config/tsconfig.json +++ b/packages/config/tsconfig.json @@ -1,6 +1,9 @@ { "extends": "../../tsconfig.json", - "include": ["src", "test"], + "include": [ + "src", + "test" + ], "compilerOptions": { "outDir": "lib" } diff --git a/packages/db/package.json b/packages/db/package.json index fabcf0365e35..6feaf4b9fb3d 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -30,11 +30,11 @@ ], "scripts": { "clean": "rm -rf lib && rm -f *.tsbuildinfo", - "build": "tsc -p tsconfig.build.json", + "build": "tsgo -p tsconfig.build.json", "build:watch": "pnpm run build --watch", "build:release": "pnpm clean && pnpm run build", "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\"", - "check-types": "tsc", + "check-types": "tsgo", "lint": "biome check src/ test/", "lint:fix": "pnpm run lint --write", "test": "pnpm test:unit", diff --git a/packages/era/package.json b/packages/era/package.json index 7947184eeb17..d60f46033e04 100644 --- a/packages/era/package.json +++ b/packages/era/package.json @@ -27,11 +27,11 @@ ], "scripts": { "clean": "rm -rf lib && rm -f *.tsbuildinfo", - "build": "tsc -p tsconfig.build.json", + "build": "tsgo -p tsconfig.build.json", "build:watch": "pnpm run build --watch", "build:release": "pnpm clean && pnpm run build", "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\"", - "check-types": "tsc", + "check-types": "tsgo", "lint": "biome check src/ test/", "lint:fix": "pnpm run lint --write", "test": "pnpm test:unit", diff --git a/packages/flare/package.json b/packages/flare/package.json index 45ada05108ce..ac4328cd4c76 100644 --- a/packages/flare/package.json +++ b/packages/flare/package.json @@ -26,11 +26,11 @@ }, "scripts": { "clean": "rm -rf lib && rm -f *.tsbuildinfo", - "build": "tsc -p tsconfig.build.json", + "build": "tsgo -p tsconfig.build.json", "build:release": "pnpm run clean && pnpm run build", "build:watch": "pnpm run build --watch", "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\" flare --help", - "check-types": "tsc", + "check-types": "tsgo", "lint": "biome check src/", "lint:fix": "pnpm run lint --write", "test": "pnpm test:unit", diff --git a/packages/fork-choice/package.json b/packages/fork-choice/package.json index cdea9f49076e..2efea39cfddf 100644 --- a/packages/fork-choice/package.json +++ b/packages/fork-choice/package.json @@ -27,11 +27,11 @@ ], "scripts": { "clean": "rm -rf lib && rm -f *.tsbuildinfo", - "build": "tsc -p tsconfig.build.json", + "build": "tsgo -p tsconfig.build.json", "build:watch": "pnpm run build --watch", "build:release": "pnpm clean && pnpm run build", "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\"", - "check-types": "tsc", + "check-types": "tsgo", "lint": "biome check src/ test/", "lint:fix": "pnpm run lint --write", "test": "pnpm test:unit", diff --git a/packages/light-client/package.json b/packages/light-client/package.json index aa88860bac9f..6746541272ed 100644 --- a/packages/light-client/package.json +++ b/packages/light-client/package.json @@ -48,13 +48,13 @@ ], "scripts": { "clean": "rm -rf lib && rm -rf dist && rm -f *.tsbuildinfo", - "build": "tsc -p tsconfig.build.json", + "build": "tsgo -p tsconfig.build.json", "build:watch": "pnpm run build --watch", "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\"", "build:bundle": "vite build", "check-bundle": "node -e \"(async function() { await import('./dist/lightclient.min.mjs') })()\"", "build:release": "pnpm clean && pnpm run build && pnpm run build:bundle", - "check-types": "tsc", + "check-types": "tsgo", "lint": "biome check src/ test/", "lint:fix": "pnpm run lint --write", "test": "pnpm test:unit", diff --git a/packages/logger/package.json b/packages/logger/package.json index 51c06219e1a2..38b241f94ec3 100644 --- a/packages/logger/package.json +++ b/packages/logger/package.json @@ -52,11 +52,11 @@ ], "scripts": { "clean": "rm -rf lib && rm -f *.tsbuildinfo", - "build": "tsc -p tsconfig.build.json", + "build": "tsgo -p tsconfig.build.json", "build:watch": "pnpm run build --watch", "build:release": "pnpm clean && pnpm build", "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\"", - "check-types": "tsc", + "check-types": "tsgo", "lint": "biome check src/ test/", "lint:fix": "pnpm run lint --write", "test": "pnpm test:unit && pnpm test:e2e", diff --git a/packages/params/package.json b/packages/params/package.json index 5e14929685e7..2f7fbbabfda0 100644 --- a/packages/params/package.json +++ b/packages/params/package.json @@ -43,11 +43,11 @@ ], "scripts": { "clean": "rm -rf lib && rm -f *.tsbuildinfo", - "build": "tsc -p tsconfig.build.json", + "build": "tsgo -p tsconfig.build.json", "build:release": "pnpm clean && pnpm build", "build:watch": "pnpm run build --watch", "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\"", - "check-types": "tsc", + "check-types": "tsgo", "lint": "biome check src/ test/", "lint:fix": "pnpm run lint --write", "test": "pnpm test:unit", diff --git a/packages/params/tsconfig.json b/packages/params/tsconfig.json index a0f4f2a31e93..19b9527ba0e3 100644 --- a/packages/params/tsconfig.json +++ b/packages/params/tsconfig.json @@ -1,6 +1,9 @@ { "extends": "../../tsconfig.json", - "include": ["src", "test"], + "include": [ + "src", + "test" + ], "compilerOptions": { "outDir": "lib" } diff --git a/packages/prover/package.json b/packages/prover/package.json index 3e3c6ed187ff..5c900d28d600 100644 --- a/packages/prover/package.json +++ b/packages/prover/package.json @@ -36,11 +36,11 @@ ], "scripts": { "clean": "rm -rf lib && rm -f *.tsbuildinfo", - "build": "tsc -p tsconfig.build.json", + "build": "tsgo -p tsconfig.build.json", "build:watch": "pnpm run build --watch", "build:release": "pnpm clean && pnpm run build", "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\"", - "check-types": "tsc", + "check-types": "tsgo", "lint": "biome check src/ test/", "lint:fix": "pnpm run lint --write", "test": "pnpm test:unit && pnpm test:e2e", diff --git a/packages/reqresp/package.json b/packages/reqresp/package.json index a6dd03b75597..a17eedcd812b 100644 --- a/packages/reqresp/package.json +++ b/packages/reqresp/package.json @@ -32,11 +32,11 @@ ], "scripts": { "clean": "rm -rf lib && rm -f *.tsbuildinfo", - "build": "tsc -p tsconfig.build.json", + "build": "tsgo -p tsconfig.build.json", "build:watch": "pnpm run build --watch", "build:release": "pnpm clean && pnpm run build", "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\"", - "check-types": "tsc", + "check-types": "tsgo", "lint": "biome check src/ test/", "lint:fix": "pnpm run lint --write", "test": "pnpm test:unit", diff --git a/packages/spec-test-util/package.json b/packages/spec-test-util/package.json index 14575761f351..5cad567cf9c3 100644 --- a/packages/spec-test-util/package.json +++ b/packages/spec-test-util/package.json @@ -31,11 +31,11 @@ }, "scripts": { "clean": "rm -rf lib && rm -f *.tsbuildinfo", - "build": "tsc -p tsconfig.build.json", + "build": "tsgo -p tsconfig.build.json", "build:release": "pnpm clean && pnpm build", "build:watch": "pnpm run build --watch", "check-build": "node -e \"(async function() { await import('./lib/downloadTests.js') })()\"", - "check-types": "tsc", + "check-types": "tsgo", "lint": "biome check src/ test/", "lint:fix": "pnpm run lint --write", "test": "pnpm test:unit && pnpm test:e2e", diff --git a/packages/state-transition/package.json b/packages/state-transition/package.json index 64b24209d797..e2c44d09a8e6 100644 --- a/packages/state-transition/package.json +++ b/packages/state-transition/package.json @@ -47,11 +47,11 @@ ], "scripts": { "clean": "rm -rf lib && rm -f *.tsbuildinfo", - "build": "tsc -p tsconfig.build.json", + "build": "tsgo -p tsconfig.build.json", "build:watch": "pnpm run build --watch", "build:release": "pnpm clean && pnpm build", "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\"", - "check-types": "tsc", + "check-types": "tsgo", "lint": "biome check src/ test/", "lint:fix": "pnpm run lint --write", "test": "pnpm test:unit", diff --git a/packages/state-transition/test/utils/state.ts b/packages/state-transition/test/utils/state.ts index 1541c9df636a..ef44dd070f01 100644 --- a/packages/state-transition/test/utils/state.ts +++ b/packages/state-transition/test/utils/state.ts @@ -52,7 +52,7 @@ export function generateState( const validators = opts.validators ?? (withPubkey - ? Array.from({length: numValidators}, (_, i) => { + ? Array.from({length: numValidators}, (_, _i) => { return generateValidator({ ...validatorOpts, }); diff --git a/packages/state-transition/tsconfig.json b/packages/state-transition/tsconfig.json index a0f4f2a31e93..19b9527ba0e3 100644 --- a/packages/state-transition/tsconfig.json +++ b/packages/state-transition/tsconfig.json @@ -1,6 +1,9 @@ { "extends": "../../tsconfig.json", - "include": ["src", "test"], + "include": [ + "src", + "test" + ], "compilerOptions": { "outDir": "lib" } diff --git a/packages/test-utils/package.json b/packages/test-utils/package.json index 1bc6dd34beea..4b267fccb591 100644 --- a/packages/test-utils/package.json +++ b/packages/test-utils/package.json @@ -29,11 +29,11 @@ ], "scripts": { "clean": "rm -rf lib && rm -f *.tsbuildinfo", - "build": "tsc -p tsconfig.build.json", + "build": "tsgo -p tsconfig.build.json", "build:release": "pnpm clean && pnpm build", "build:watch": "pnpm run build --watch", "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\"", - "check-types": "tsc", + "check-types": "tsgo", "lint": "biome check src/", "lint:fix": "pnpm run lint --write", "test:unit": "echo 'No unit tests for @lodestar/test-utils'", diff --git a/packages/types/package.json b/packages/types/package.json index 79b052e89058..c54677425b7a 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -62,11 +62,11 @@ ], "scripts": { "clean": "rm -rf lib && rm -f *.tsbuildinfo", - "build": "tsc -p tsconfig.build.json", + "build": "tsgo -p tsconfig.build.json", "build:watch": "pnpm run build --watch", "build:release": "pnpm clean && pnpm build", "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\"", - "check-types": "tsc", + "check-types": "tsgo", "lint": "biome check src/ test/", "lint:fix": "pnpm run lint --write", "test": "pnpm test:unit", diff --git a/packages/utils/package.json b/packages/utils/package.json index c468025d36a8..0e3d1e24a6eb 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -33,11 +33,11 @@ ], "scripts": { "clean": "rm -rf lib && rm -f *.tsbuildinfo", - "build": "tsc -p tsconfig.build.json", + "build": "tsgo -p tsconfig.build.json", "build:watch": "pnpm run build --watch", "build:release": "pnpm clean && pnpm build", "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\"", - "check-types": "tsc && vitest run --project types", + "check-types": "tsgo && vitest run --project types", "lint": "biome check src/ test/", "lint:fix": "pnpm run lint --write", "test:unit": "vitest run --project unit --project unit-minimal", diff --git a/packages/validator/package.json b/packages/validator/package.json index 4711ddac7a52..3364107ab993 100644 --- a/packages/validator/package.json +++ b/packages/validator/package.json @@ -23,11 +23,11 @@ ], "scripts": { "clean": "rm -rf lib && rm -f *.tsbuildinfo", - "build": "tsc -p tsconfig.build.json", + "build": "tsgo -p tsconfig.build.json", "build:release": "pnpm clean && pnpm run build", "build:watch": "pnpm run build --watch", "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\"", - "check-types": "tsc", + "check-types": "tsgo", "lint": "biome check src/ test/", "lint:fix": "pnpm run lint --write", "test:unit": "vitest run --project unit --project unit-minimal", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 862096884edb..c41042d0c243 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -71,6 +71,9 @@ importers: '@types/react': specifier: ^19.1.12 version: 19.1.12 + '@typescript/native-preview': + specifier: 7.0.0-dev.20260303.1 + version: 7.0.0-dev.20260303.1 '@vitest/browser': specifier: 'catalog:' version: 4.0.7(vite@6.1.6(@types/node@24.10.1)(yaml@2.8.2))(vitest@4.0.7) @@ -144,17 +147,17 @@ importers: specifier: ^6.3.3 version: 6.3.3 ts-node: - specifier: ^10.9.2 - version: 10.9.2(@swc/core@1.14.0)(@swc/wasm@1.14.0)(@types/node@24.10.1)(typescript@5.9.2) + specifier: ^11.0.0-beta.1 + version: 11.0.0-beta.1(@swc/core@1.14.0)(@swc/wasm@1.14.0)(@types/node@24.10.1)(typescript@6.0.1-rc) typescript: - specifier: ^5.9.2 - version: 5.9.2 + specifier: ^6.0.1-rc + version: 6.0.1-rc vite: specifier: ^6.0.11 version: 6.1.6(@types/node@24.10.1)(yaml@2.8.2) vite-plugin-dts: specifier: ^4.5.4 - version: 4.5.4(@types/node@24.10.1)(rollup@4.52.5)(typescript@5.9.2)(vite@6.1.6(@types/node@24.10.1)(yaml@2.8.2)) + version: 4.5.4(@types/node@24.10.1)(rollup@4.52.5)(typescript@6.0.1-rc)(vite@6.1.6(@types/node@24.10.1)(yaml@2.8.2)) vite-plugin-node-polyfills: specifier: ^0.24.0 version: 0.24.0(rollup@4.52.5)(vite@6.1.6(@types/node@24.10.1)(yaml@2.8.2)) @@ -3048,18 +3051,18 @@ packages: resolution: {integrity: sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==} engines: {node: '>= 10'} - '@tsconfig/node10@1.0.8': - resolution: {integrity: sha512-6XFfSQmMgq0CFLY1MslA/CPUfhIL919M1rMsa5lP2P097N2Wd1sSX0tx1u4olM16fLNhtHZpRhedZJphNJqmZg==} - - '@tsconfig/node12@1.0.9': - resolution: {integrity: sha512-/yBMcem+fbvhSREH+s14YJi18sp7J9jpuhYByADT2rypfajMZZN4WQ6zBGgBKp53NKmqI36wFYDb3yaMPurITw==} - '@tsconfig/node14@1.0.1': resolution: {integrity: sha512-509r2+yARFfHHE7T6Puu2jjkoycftovhXRqW328PDXTVGKihlb1P8Z9mMZH04ebyajfRY7dedfGynlrFHJUQCg==} '@tsconfig/node16@1.0.2': resolution: {integrity: sha512-eZxlbI8GZscaGS7kkc/trHTT5xgrjH3/1n2JDwusC9iahPKWMRvRjJSAN5mCXviuTGQ/lHnhvv8Q1YTpnfz9gA==} + '@tsconfig/node18@18.2.6': + resolution: {integrity: sha512-eAWQzAjPj18tKnDzmWstz4OyWewLUNBm9tdoN9LayzoboRktYx3Enk1ZXPmThj55L7c4VWYq/Bzq0A51znZfhw==} + + '@tsconfig/node20@20.1.9': + resolution: {integrity: sha512-IjlTv1RsvnPtUcjTqtVsZExKVq+KQx4g5pCP5tI7rAs6Xesl2qFwSz/tPDBC4JajkL/MlezBu3gPUwqRHl+RIg==} + '@tufjs/canonical-json@2.0.0': resolution: {integrity: sha512-yVtV8zsdo8qFHe+/3kw81dSLyF7D576A5cCFCi4X7B39tWT7SekaEFUnvnWJHz+9qO7qJTah1JbrDjWKqFtdWA==} engines: {node: ^16.14.0 || >=18.0.0} @@ -3183,6 +3186,45 @@ packages: '@types/yauzl@2.10.0': resolution: {integrity: sha512-Cn6WYCm0tXv8p6k+A8PvbDG763EDpBoTzHdA+Q/MF6H3sapGjCm9NzoaJncJS9tUKSuCoDs9XHxYYsQDgxR6kw==} + '@typescript/native-preview-darwin-arm64@7.0.0-dev.20260303.1': + resolution: {integrity: sha512-jIIQWmFi0bJY4ML8/7eyz1EGpkI6E0R1E5l4lxJdV/orpMr91vYfAajKICs7DUiMGEJX9HpeiA6TD2piw4DKPQ==} + cpu: [arm64] + os: [darwin] + + '@typescript/native-preview-darwin-x64@7.0.0-dev.20260303.1': + resolution: {integrity: sha512-UkaK+J3f185VXBiAGNG4UKHjGzn4R/nhAz5tArnCKHnIUI7rEnsIm4Xlo5YwmgIATFMU1sVwWUwRshVkMVeFAw==} + cpu: [x64] + os: [darwin] + + '@typescript/native-preview-linux-arm64@7.0.0-dev.20260303.1': + resolution: {integrity: sha512-ELJSV2Q4/mK+ampttssOl4H9s9ZBCc3k7y/u5ivJX8TdlMvZuH/JHqI6cS4Y00flt0R5wc70X+Nlcor4I4+rpw==} + cpu: [arm64] + os: [linux] + + '@typescript/native-preview-linux-arm@7.0.0-dev.20260303.1': + resolution: {integrity: sha512-H84jTRYqUfc/vhuVGQ6VKcBvJoZ4YmomWDx9U4uwYgW6eoUcRpDXqv3S3YqcNJcUmz22d/tTwIYz8ssXNLa/Qw==} + cpu: [arm] + os: [linux] + + '@typescript/native-preview-linux-x64@7.0.0-dev.20260303.1': + resolution: {integrity: sha512-HkddFrPJ0jcrohe+HnCqVTv8PunjqNs7FisRmtIAnc36+ccraDB6MVFEdPyAIL3PUID+TP/ESquqeKNnB7HdrQ==} + cpu: [x64] + os: [linux] + + '@typescript/native-preview-win32-arm64@7.0.0-dev.20260303.1': + resolution: {integrity: sha512-kTTMrBpWuxbHPt9hAFQSWeP//5Oa0KOdAEvceOfXUJhTS8RAA/kZSlFGE/Zw1EtrFLQx2J7uTHUZnYxH1hYXNw==} + cpu: [arm64] + os: [win32] + + '@typescript/native-preview-win32-x64@7.0.0-dev.20260303.1': + resolution: {integrity: sha512-UcVZbf4pra46Yx/eFV6m9F+awvihliPEud4Rq+A8Q3q3zI67VRaNH6R2/qeo4AqqKRahmiEdLM6Tnm+gPtLRQQ==} + cpu: [x64] + os: [win32] + + '@typescript/native-preview@7.0.0-dev.20260303.1': + resolution: {integrity: sha512-BDHJjXlPldInEogbzAc7OCLvT75p3rdkmb5YIA6Je0vjg+5z1UQp3moAvcBGvZQflO/gusOd9a74EfrMVUU/4g==} + hasBin: true + '@vekexasia/bigint-buffer2@1.1.0': resolution: {integrity: sha512-CB19/UHoop2Q+HaRU1lI3fWKUkwju7XtwGTvdDfJgZyPHI+lLXKDYOOkz+NOCLfcBSXpJXpue/vLN0PDZtBT/Q==} engines: {node: '>= 14.0.0'} @@ -6707,14 +6749,14 @@ packages: triple-beam@1.3.0: resolution: {integrity: sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw==} - ts-node@10.9.2: - resolution: {integrity: sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==} + ts-node@11.0.0-beta.1: + resolution: {integrity: sha512-WMSROP+1pU22Q/Tm40mjfRg130yD8i0g6ROST04ZpocfH8sl1zD75ON4XQMcBEVViXMVemJBH0alflE7xePdRA==} hasBin: true peerDependencies: - '@swc/core': '>=1.2.50' - '@swc/wasm': '>=1.2.50' + '@swc/core': '>=1.3.85' + '@swc/wasm': '>=1.3.85' '@types/node': '*' - typescript: '>=2.7' + typescript: '>=4.4' peerDependenciesMeta: '@swc/core': optional: true @@ -6800,8 +6842,8 @@ packages: engines: {node: '>=14.17'} hasBin: true - typescript@5.9.2: - resolution: {integrity: sha512-CWBzXQrc/qOkhidw1OzBTQuYRbfyxDXJMVJ1XNwUHGROVmuaeiEm3OslpZ1RV96d7SKKjZKrSJu3+t/xlw3R9A==} + typescript@6.0.1-rc: + resolution: {integrity: sha512-7XlzYb+p/7YxX6qSOzwB4mxVFRdAgWWkj1PgAZ+jzldeuFV6Z77vwFbNxHsUXAL/bhlWY2jCT8shLwDJR8337g==} engines: {node: '>=14.17'} hasBin: true @@ -7381,10 +7423,6 @@ packages: yauzl@2.10.0: resolution: {integrity: sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==} - yn@3.1.1: - resolution: {integrity: sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==} - engines: {node: '>=6'} - yocto-queue@0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} @@ -9835,14 +9873,14 @@ snapshots: '@tootallnate/once@2.0.0': {} - '@tsconfig/node10@1.0.8': {} - - '@tsconfig/node12@1.0.9': {} - '@tsconfig/node14@1.0.1': {} '@tsconfig/node16@1.0.2': {} + '@tsconfig/node18@18.2.6': {} + + '@tsconfig/node20@20.1.9': {} + '@tufjs/canonical-json@2.0.0': {} '@tufjs/models@4.0.0': @@ -9979,6 +10017,37 @@ snapshots: '@types/node': 24.10.1 optional: true + '@typescript/native-preview-darwin-arm64@7.0.0-dev.20260303.1': + optional: true + + '@typescript/native-preview-darwin-x64@7.0.0-dev.20260303.1': + optional: true + + '@typescript/native-preview-linux-arm64@7.0.0-dev.20260303.1': + optional: true + + '@typescript/native-preview-linux-arm@7.0.0-dev.20260303.1': + optional: true + + '@typescript/native-preview-linux-x64@7.0.0-dev.20260303.1': + optional: true + + '@typescript/native-preview-win32-arm64@7.0.0-dev.20260303.1': + optional: true + + '@typescript/native-preview-win32-x64@7.0.0-dev.20260303.1': + optional: true + + '@typescript/native-preview@7.0.0-dev.20260303.1': + optionalDependencies: + '@typescript/native-preview-darwin-arm64': 7.0.0-dev.20260303.1 + '@typescript/native-preview-darwin-x64': 7.0.0-dev.20260303.1 + '@typescript/native-preview-linux-arm': 7.0.0-dev.20260303.1 + '@typescript/native-preview-linux-arm64': 7.0.0-dev.20260303.1 + '@typescript/native-preview-linux-x64': 7.0.0-dev.20260303.1 + '@typescript/native-preview-win32-arm64': 7.0.0-dev.20260303.1 + '@typescript/native-preview-win32-x64': 7.0.0-dev.20260303.1 + '@vekexasia/bigint-buffer2@1.1.0': {} '@vitest/browser-playwright@4.0.7(playwright@1.56.1)(vite@6.1.6(@types/node@24.10.1)(yaml@2.8.2))(vitest@4.0.7)': @@ -10154,7 +10223,7 @@ snapshots: de-indent: 1.0.2 he: 1.2.0 - '@vue/language-core@2.2.0(typescript@5.9.2)': + '@vue/language-core@2.2.0(typescript@6.0.1-rc)': dependencies: '@volar/language-core': 2.4.11 '@vue/compiler-dom': 3.5.13 @@ -10165,7 +10234,7 @@ snapshots: muggle-string: 0.4.1 path-browserify: 1.0.1 optionalDependencies: - typescript: 5.9.2 + typescript: 6.0.1-rc '@vue/shared@3.5.13': {} @@ -14046,23 +14115,21 @@ snapshots: triple-beam@1.3.0: {} - ts-node@10.9.2(@swc/core@1.14.0)(@swc/wasm@1.14.0)(@types/node@24.10.1)(typescript@5.9.2): + ts-node@11.0.0-beta.1(@swc/core@1.14.0)(@swc/wasm@1.14.0)(@types/node@24.10.1)(typescript@6.0.1-rc): dependencies: '@cspotcode/source-map-support': 0.8.1 - '@tsconfig/node10': 1.0.8 - '@tsconfig/node12': 1.0.9 '@tsconfig/node14': 1.0.1 '@tsconfig/node16': 1.0.2 + '@tsconfig/node18': 18.2.6 + '@tsconfig/node20': 20.1.9 '@types/node': 24.10.1 acorn: 8.14.0 acorn-walk: 8.2.0 arg: 4.1.3 - create-require: 1.1.1 diff: 4.0.2 make-error: 1.3.6 - typescript: 5.9.2 + typescript: 6.0.1-rc v8-compile-cache-lib: 3.0.1 - yn: 3.1.1 optionalDependencies: '@swc/core': 1.14.0 '@swc/wasm': 1.14.0 @@ -14143,7 +14210,7 @@ snapshots: typescript@5.8.2: {} - typescript@5.9.2: {} + typescript@6.0.1-rc: {} ufo@1.5.4: {} @@ -14272,18 +14339,18 @@ snapshots: validate-npm-package-name@7.0.0: {} - vite-plugin-dts@4.5.4(@types/node@24.10.1)(rollup@4.52.5)(typescript@5.9.2)(vite@6.1.6(@types/node@24.10.1)(yaml@2.8.2)): + vite-plugin-dts@4.5.4(@types/node@24.10.1)(rollup@4.52.5)(typescript@6.0.1-rc)(vite@6.1.6(@types/node@24.10.1)(yaml@2.8.2)): dependencies: '@microsoft/api-extractor': 7.53.3(@types/node@24.10.1) '@rollup/pluginutils': 5.1.4(rollup@4.52.5) '@volar/typescript': 2.4.11 - '@vue/language-core': 2.2.0(typescript@5.9.2) + '@vue/language-core': 2.2.0(typescript@6.0.1-rc) compare-versions: 6.1.1 debug: 4.4.3 kolorist: 1.8.0 local-pkg: 1.1.2 magic-string: 0.30.21 - typescript: 5.9.2 + typescript: 6.0.1-rc optionalDependencies: vite: 6.1.6(@types/node@24.10.1)(yaml@2.8.2) transitivePeerDependencies: @@ -14878,8 +14945,6 @@ snapshots: buffer-crc32: 0.2.13 fd-slicer: 1.1.0 - yn@3.1.1: {} - yocto-queue@0.1.0: {} yocto-queue@1.0.0: {} diff --git a/tsconfig.build.json b/tsconfig.build.json index bcdd12f83369..75959f0924f9 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -1,44 +1,6 @@ { + "extends": "./tsconfig.common.json", "compilerOptions": { - "target": "es2023", - "lib": ["es2023", "dom"], - - "module": "nodenext", - "moduleResolution": "nodenext", - - "pretty": true, - "strict": true, - "sourceMap": true, - "alwaysStrict": true, - "strictNullChecks": true, - "strictFunctionTypes": true, - "strictBindCallApply": true, - "strictPropertyInitialization": true, - "useUnknownInCatchVariables": true, - "noImplicitAny": true, - "noImplicitThis": true, - "noImplicitReturns": true, - - "allowSyntheticDefaultImports": true, - "esModuleInterop": true, - "declaration": true, - "declarationMap": true, - "incremental": true, - "preserveWatchOutput": true, - "noUncheckedSideEffectImports": true, - "rewriteRelativeImportExtensions": true, - // TODO: Investigate following errors: - // - Cannot find module 'rollup/parseAst' or its corresponding type declarations - "skipLibCheck": true, - - // Required to run benchmark command from root directory - "typeRoots": [ - "node_modules/@types", - "./types", - "${configDir}/node_modules/@types", - "${configDir}/types", - "../../node_modules/@types", - "../../types", - ] + "rootDir": "${configDir}/src", } } diff --git a/tsconfig.common.json b/tsconfig.common.json new file mode 100644 index 000000000000..ab263d04fc6b --- /dev/null +++ b/tsconfig.common.json @@ -0,0 +1,50 @@ +{ + "compilerOptions": { + "target": "es2023", + "lib": [ + "es2023", + "dom" + ], + "module": "nodenext", + "moduleResolution": "nodenext", + "pretty": true, + "strict": true, + "sourceMap": true, + "alwaysStrict": true, + "strictNullChecks": true, + "strictFunctionTypes": true, + "strictBindCallApply": true, + "strictPropertyInitialization": true, + "useUnknownInCatchVariables": true, + "noImplicitAny": true, + "noImplicitThis": true, + "noImplicitReturns": true, + "allowSyntheticDefaultImports": true, + "esModuleInterop": true, + "declaration": true, + "declarationMap": true, + "incremental": true, + "preserveWatchOutput": true, + "noUncheckedSideEffectImports": true, + "rewriteRelativeImportExtensions": true, + // TODO: Investigate following errors: + // - Cannot find module 'rollup/parseAst' or its corresponding type declarations + "skipLibCheck": true, + "types": [ + "node", + "bun", + "snappyjs", + "mitt", + "vitest" + ], + // Required to run benchmark command from root directory + "typeRoots": [ + "node_modules/@types", + "./types", + "${configDir}/node_modules/@types", + "${configDir}/types", + "../../node_modules/@types", + "../../types", + ] + } +} diff --git a/tsconfig.json b/tsconfig.json index 50360530b8e9..b7328aca571e 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,17 +1,18 @@ { - "extends": "./tsconfig.build.json", + "extends": "./tsconfig.common.json", "compilerOptions": { + "rootDir": "${configDir}", "emitDeclarationOnly": false, "incremental": false, "noEmit": true, // To be used in the test fixtures "resolveJsonModule": true, // We want to speed up the CI run for all tests, which require us to use the - // `transpileOnly` mode for the `ts-node`. This change requires to treat types for each module + // `transpileOnly` mode for the `ts-node`. This change requires to treat types for each module // independently, which is done by setting the `isolatedModules` flag to `true`. "isolatedModules": true, // Detect unused imports, variables, and functions - "noUnusedLocals": true, + "noUnusedLocals": true }, "ts-node": { "transpileOnly": true From cfc5790a15392953297a83609ce0f13f499edb62 Mon Sep 17 00:00:00 2001 From: Chen Kai <281165273grape@gmail.com> Date: Thu, 12 Mar 2026 09:44:43 -0400 Subject: [PATCH 33/63] refactor: use BitArray for PTC votes instead of boolean[] (#9021) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Replace `boolean[]` (512 elements, ~4KB per block on mainnet) with `BitArray` from `@chainsafe/ssz` (64 bytes per block on mainnet) for PTC vote storage in fork choice - Use byte-level popcount (Brian Kernighan's algorithm) instead of `Array.filter()` for vote counting in `isPayloadTimely()` - Aligns with how other bitvectors (`aggregationBits`, `syncnets`, etc.) are represented across the codebase ## Details The PTC (Payload Timeliness Committee) tracks 512 validator votes per Gloas block. Previously stored as `boolean[]` (each vote = 8 bytes in V8), now stored as a `BitArray` backed by a 64-byte `Uint8Array` — an 8x memory reduction per tracked block. The vote counting path (`isPayloadTimely()`) previously used `votes.filter(v => v).length` which allocates a new array. Now uses popcount on the underlying bytes directly. No behavioral changes — all 99 existing fork-choice unit tests pass. ## Test plan - [x] All 31 Gloas-specific unit tests pass (PTC vote tests exercise all edge cases) - [x] All 99 fork-choice unit tests pass - [x] Lint passes *AI-assisted: refactoring identified and implemented with Claude* --- .../fork-choice/src/protoArray/protoArray.ts | 18 +++++++++--------- packages/utils/src/bytes/browser.ts | 13 +++++++++++++ packages/utils/src/bytes/nodejs.ts | 11 ++++++++++- 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/packages/fork-choice/src/protoArray/protoArray.ts b/packages/fork-choice/src/protoArray/protoArray.ts index fc55e9e1f388..9c60974357f2 100644 --- a/packages/fork-choice/src/protoArray/protoArray.ts +++ b/packages/fork-choice/src/protoArray/protoArray.ts @@ -1,7 +1,8 @@ +import {BitArray} from "@chainsafe/ssz"; import {GENESIS_EPOCH, PTC_SIZE} from "@lodestar/params"; import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {Epoch, RootHex, Slot} from "@lodestar/types"; -import {toRootHex} from "@lodestar/utils"; +import {bitCount, toRootHex} from "@lodestar/utils"; import {ForkChoiceError, ForkChoiceErrorCode} from "../forkChoice/errors.js"; import {LVHExecError, LVHExecErrorCode, ProtoArrayError, ProtoArrayErrorCode} from "./errors.js"; import { @@ -60,14 +61,14 @@ export class ProtoArray { private previousProposerBoost: ProposerBoost | null = null; /** - * PTC (Payload Timeliness Committee) votes per block - * Maps block root to boolean array of size PTC_SIZE (from params: 512 mainnet, 2 minimal) + * PTC (Payload Timeliness Committee) votes per block as bitvectors + * Maps block root to BitArray of PTC_SIZE bits (512 mainnet, 2 minimal) * Spec: gloas/fork-choice.md#modified-store (line 148) * - * ptcVotes[blockRoot][i] = true if PTC member i voted payload_present=true + * Bit i is set if PTC member i voted payload_present=true * Used by is_payload_timely() to determine if payload is timely */ - private ptcVotes = new Map(); + private ptcVotes = new Map(); constructor({ pruneThreshold, @@ -476,7 +477,7 @@ export class ProtoArray { // Initialize PTC votes for this block (all false initially) // Spec: gloas/fork-choice.md#modified-on_block (line 645) - this.ptcVotes.set(block.blockRoot, new Array(PTC_SIZE).fill(false)); + this.ptcVotes.set(block.blockRoot, BitArray.fromBitLen(PTC_SIZE)); } else { // Pre-Gloas: Only create FULL node (payload embedded in block) const node: ProtoNode = { @@ -605,8 +606,7 @@ export class ProtoArray { throw new Error(`Invalid PTC index: ${ptcIndex}, must be 0..${PTC_SIZE - 1}`); } - // Update the vote - votes[ptcIndex] = payloadPresent; + votes.set(ptcIndex, payloadPresent); } } @@ -636,7 +636,7 @@ export class ProtoArray { } // Count votes for payload_present=true - const yesVotes = votes.filter((v) => v).length; + const yesVotes = bitCount(votes.uint8Array); return yesVotes > PAYLOAD_TIMELY_THRESHOLD; } diff --git a/packages/utils/src/bytes/browser.ts b/packages/utils/src/bytes/browser.ts index 712e99eed25d..12f651c99573 100644 --- a/packages/utils/src/bytes/browser.ts +++ b/packages/utils/src/bytes/browser.ts @@ -187,6 +187,19 @@ export function xor(a: Uint8Array, b: Uint8Array): Uint8Array { return a; } +/** Count set bits in a Uint8Array without allocation (Brian Kernighan's algorithm) */ +export function bitCount(arr: Uint8Array): number { + let count = 0; + for (let i = 0; i < arr.length; i++) { + let byte = arr[i]; + while (byte) { + byte &= byte - 1; + count++; + } + } + return count; +} + /** * Compare two byte arrays for equality. * Note: In Node.js environment, the implementation in nodejs.ts uses Buffer.compare diff --git a/packages/utils/src/bytes/nodejs.ts b/packages/utils/src/bytes/nodejs.ts index 106fbf6563a8..816f925ff003 100644 --- a/packages/utils/src/bytes/nodejs.ts +++ b/packages/utils/src/bytes/nodejs.ts @@ -89,4 +89,13 @@ export function byteArrayEquals(a: Uint8Array, b: Uint8Array): boolean { return Buffer.compare(a, b) === 0; } -export {bigIntToBytes, bytesToBigInt, bytesToInt, fromHexInto, intToBytes, toHexString, xor} from "./browser.js"; +export { + bigIntToBytes, + bitCount, + bytesToBigInt, + bytesToInt, + fromHexInto, + intToBytes, + toHexString, + xor, +} from "./browser.js"; From 8578102bd2aca7ebdc2641000d690f4fcfacac49 Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Thu, 12 Mar 2026 11:45:18 -0500 Subject: [PATCH 34/63] feat: schedule Fulu fork for Chiado testnet (#9027) ## Summary Sets `FULU_FORK_EPOCH` for the Chiado testnet (Gnosis Chain testnet). - **Epoch**: 1353216 - **Slot**: 21651456 - **Timestamp**: 1773653580 - **UTC**: Mon Mar 16 2026 09:33:00 Matches the activation scheduled by other clients: - Lighthouse: sigp/lighthouse#8954 - Nethermind: v1.36.1 - Erigon: erigontech/erigon#19681 --- packages/config/src/chainConfig/networks/chiado.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/config/src/chainConfig/networks/chiado.ts b/packages/config/src/chainConfig/networks/chiado.ts index aa5d573cc8b8..f794b55fc337 100644 --- a/packages/config/src/chainConfig/networks/chiado.ts +++ b/packages/config/src/chainConfig/networks/chiado.ts @@ -42,7 +42,7 @@ export const chiadoChainConfig: ChainConfig = { ELECTRA_FORK_EPOCH: 948224, // Thu Mar 06 2025 09:43:40 GMT+0000 // Fulu FULU_FORK_VERSION: b("0x0600006f"), - FULU_FORK_EPOCH: Infinity, + FULU_FORK_EPOCH: 1353216, // Mon Mar 16 2026 09:33:00 GMT+0000 // Gloas GLOAS_FORK_VERSION: b("0x0700006f"), GLOAS_FORK_EPOCH: Infinity, From 85cdea6cc606897e3ff465cab0f0907d5c8acc33 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 12 Mar 2026 18:00:02 +0000 Subject: [PATCH 35/63] chore: fix seen block input cache metrics (#9020) Fix `seenBlockInputCache` metrics to match actual behavior. - rename the serialized object metric so it reads as a gauge, not a counter - change duplicate / created metrics to counters with `_total` - fix cache creation accounting for block-first and column-first paths - update the Grafana dashboard queries accordingly --- dashboards/lodestar_beacon_chain.json | 27 ++++++++++++----- .../chain/seenCache/seenGossipBlockInput.ts | 5 ++-- .../src/metrics/metrics/lodestar.ts | 30 +++++++++++-------- 3 files changed, 40 insertions(+), 22 deletions(-) diff --git a/dashboards/lodestar_beacon_chain.json b/dashboards/lodestar_beacon_chain.json index 0d738393de53..9c4f2b6ba0a5 100644 --- a/dashboards/lodestar_beacon_chain.json +++ b/dashboards/lodestar_beacon_chain.json @@ -330,7 +330,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(lodestar_seen_block_input_cache_duplicate_block_count[$rate_interval]) * 12", + "expr": "rate(lodestar_seen_block_input_cache_duplicate_block_total[$rate_interval]) * 12", "hide": false, "instant": false, "legendFormat": "duplicate_block_{{source}}", @@ -343,7 +343,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(lodestar_seen_block_input_cache_duplicate_blob_count[$rate_interval]) * 12", + "expr": "rate(lodestar_seen_block_input_cache_duplicate_blob_total[$rate_interval]) * 12", "hide": false, "instant": false, "legendFormat": "duplicate_blob_{{source}}", @@ -356,7 +356,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(lodestar_seen_block_input_cache_duplicate_column_count[$rate_interval]) * 12", + "expr": "rate(lodestar_seen_block_input_cache_duplicate_column_total[$rate_interval]) * 12", "hide": false, "instant": false, "legendFormat": "duplicate_columns_{{source}}", @@ -369,7 +369,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(lodestar_seen_block_input_cache_items_created_by_block[$rate_interval]) * 12", + "expr": "rate(lodestar_seen_block_input_cache_items_created_by_block_total[$rate_interval]) * 12", "hide": false, "instant": false, "legendFormat": "created_by_block", @@ -382,7 +382,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(lodestar_seen_block_input_cache_items_created_by_blob[$rate_interval]) * 12", + "expr": "rate(lodestar_seen_block_input_cache_items_created_by_blob_total[$rate_interval]) * 12", "hide": false, "instant": false, "legendFormat": "created_by_blob", @@ -395,11 +395,24 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "lodestar_seen_block_input_cache_serialized_object_count", + "expr": "rate(lodestar_seen_block_input_cache_items_created_by_column_total[$rate_interval]) * 12", + "hide": false, "instant": false, - "legendFormat": "serialized_objects", + "legendFormat": "created_by_column", "range": true, "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "lodestar_seen_block_input_cache_serialized_object_refs", + "instant": false, + "legendFormat": "serialized_objects", + "range": true, + "refId": "H" } ], "title": "Block Input", diff --git a/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts b/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts index 3d739d265d2b..7570c2a03ffd 100644 --- a/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts +++ b/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts @@ -134,7 +134,7 @@ export class SeenBlockInput { if (metrics) { metrics.seenCache.blockInput.blockInputCount.addCollect(() => { metrics.seenCache.blockInput.blockInputCount.set(this.blockInputs.size); - metrics.seenCache.blockInput.serializedObjectCount.set( + metrics.seenCache.blockInput.serializedObjectRefs.set( Array.from(this.blockInputs.values()).reduce( (count, blockInput) => count + blockInput.getSerializedCacheKeys().length, 0 @@ -250,6 +250,7 @@ export class SeenBlockInput { peerIdStr, }); } + this.metrics?.seenCache.blockInput.createdByBlock.inc(); this.blockInputs.set(blockInput.blockRootHex, blockInput); } @@ -349,7 +350,7 @@ export class SeenBlockInput { custodyColumns: this.custodyConfig.custodyColumns, sampledColumns: this.custodyConfig.sampledColumns, }); - this.metrics?.seenCache.blockInput.createdByBlob.inc(); + this.metrics?.seenCache.blockInput.createdByColumn.inc(); this.blockInputs.set(blockRootHex, blockInput); } diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index b87befc62e22..9ba3c3258225 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -1461,33 +1461,37 @@ export function createLodestarMetrics( name: "lodestar_seen_block_input_cache_size", help: "Number of cached BlockInputs", }), - serializedObjectCount: register.gauge({ - name: "lodestar_seen_block_input_cache_serialized_object_count", - help: "Number of serialized objects retained by cached BlockInputs", + serializedObjectRefs: register.gauge({ + name: "lodestar_seen_block_input_cache_serialized_object_refs", + help: "Number of serialized-cache object refs retained by cached BlockInputs", }), - duplicateBlockCount: register.gauge<{source: BlockInputSource}>({ - name: "lodestar_seen_block_input_cache_duplicate_block_count", + duplicateBlockCount: register.counter<{source: BlockInputSource}>({ + name: "lodestar_seen_block_input_cache_duplicate_block_total", help: "Total number of duplicate blocks that pass validation and attempt to be cached but are known", labelNames: ["source"], }), - duplicateBlobCount: register.gauge<{source: BlockInputSource}>({ - name: "lodestar_seen_block_input_cache_duplicate_blob_count", + duplicateBlobCount: register.counter<{source: BlockInputSource}>({ + name: "lodestar_seen_block_input_cache_duplicate_blob_total", help: "Total number of duplicate blobs that pass validation and attempt to be cached but are known", labelNames: ["source"], }), - duplicateColumnCount: register.gauge<{source: BlockInputSource}>({ - name: "lodestar_seen_block_input_cache_duplicate_column_count", + duplicateColumnCount: register.counter<{source: BlockInputSource}>({ + name: "lodestar_seen_block_input_cache_duplicate_column_total", help: "Total number of duplicate columns that pass validation and attempt to be cached but are known", labelNames: ["source"], }), - createdByBlock: register.gauge({ - name: "lodestar_seen_block_input_cache_items_created_by_block", + createdByBlock: register.counter({ + name: "lodestar_seen_block_input_cache_items_created_by_block_total", help: "Number of BlockInputs created via a block being seen first", }), - createdByBlob: register.gauge({ - name: "lodestar_seen_block_input_cache_items_created_by_blob", + createdByBlob: register.counter({ + name: "lodestar_seen_block_input_cache_items_created_by_blob_total", help: "Number of BlockInputs created via a blob being seen first", }), + createdByColumn: register.counter({ + name: "lodestar_seen_block_input_cache_items_created_by_column_total", + help: "Number of BlockInputs created via a data column being seen first", + }), }, }, From 8cdde189fa2204d5a39cf84f50fc9e04c3a48ad3 Mon Sep 17 00:00:00 2001 From: Chen Kai <281165273grape@gmail.com> Date: Thu, 12 Mar 2026 21:25:27 -0400 Subject: [PATCH 36/63] refactor: simplify default variant lookup to avoid double hash lookup (#9023) ## Summary - Add `getDefaultNodeIndex()` method to `ProtoArray` that combines `getDefaultVariant()` + `getNodeIndexByRootAndStatus()` into a single `this.indices.get()` call - Replace 4 call sites that did redundant double hash lookups: `findHead()`, `hasBlock()`, `shouldExtendPayload()`, and `propagateInvalidExecutionStatusFromExecResponse()` - `getDefaultVariant()` is kept for callers that need the payload status enum value (used in `forkChoice.ts`) ## AI Disclosure AI-assisted ## Test plan - [x] All 31 gloas unit tests pass - [x] All 13 executionStatusUpdates tests pass - [x] Build succeeds - [x] Lint clean --- .../fork-choice/src/protoArray/protoArray.ts | 43 +++++++++++-------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/packages/fork-choice/src/protoArray/protoArray.ts b/packages/fork-choice/src/protoArray/protoArray.ts index 9c60974357f2..683ba6b38145 100644 --- a/packages/fork-choice/src/protoArray/protoArray.ts +++ b/packages/fork-choice/src/protoArray/protoArray.ts @@ -169,6 +169,26 @@ export class ProtoArray { return PayloadStatus.PENDING; } + /** + * Get the node index for the default/canonical variant in a single hash lookup. + * - Pre-Gloas blocks: returns the FULL variant index + * - Gloas blocks: returns the PENDING variant index + */ + getDefaultNodeIndex(blockRoot: RootHex): number | undefined { + const variantOrArr = this.indices.get(blockRoot); + if (variantOrArr == null) { + return undefined; + } + + // Pre-Gloas: value is the index directly + if (!Array.isArray(variantOrArr)) { + return variantOrArr; + } + + // Gloas: PENDING is the canonical variant + return variantOrArr[PayloadStatus.PENDING]; + } + /** * Determine which parent payload status a block extends * Spec: gloas/fork-choice.md#new-get_parent_payload_status @@ -676,8 +696,8 @@ export class ProtoArray { // Get proposer boost block // We don't care about variant here, just need proposer boost block info - const defaultStatus = this.getDefaultVariant(proposerBoostRoot); - const proposerBoostBlock = defaultStatus !== undefined ? this.getNode(proposerBoostRoot, defaultStatus) : undefined; + const proposerBoostIndex = this.getDefaultNodeIndex(proposerBoostRoot); + const proposerBoostBlock = proposerBoostIndex !== undefined ? this.getNodeByIndex(proposerBoostIndex) : undefined; if (!proposerBoostBlock) { // Proposer boost block not found, default to extending payload return true; @@ -745,12 +765,8 @@ export class ProtoArray { // if its in fcU. // const {invalidateFromParentBlockRoot, latestValidExecHash} = execResponse; - // TODO GLOAS: verify if getting default variant is correct here - const defaultStatus = this.getDefaultVariant(invalidateFromParentBlockRoot); - const invalidateFromParentIndex = - defaultStatus !== undefined - ? this.getNodeIndexByRootAndStatus(invalidateFromParentBlockRoot, defaultStatus) - : undefined; + // TODO GLOAS: verify if getting the default/canonical node index is correct here + const invalidateFromParentIndex = this.getDefaultNodeIndex(invalidateFromParentBlockRoot); if (invalidateFromParentIndex === undefined) { throw Error(`Unable to find invalidateFromParentBlockRoot=${invalidateFromParentBlockRoot} in forkChoice`); } @@ -963,9 +979,7 @@ export class ProtoArray { } // Get canonical node: FULL for pre-Gloas, PENDING for Gloas - const defaultStatus = this.getDefaultVariant(justifiedRoot); - const justifiedIndex = - defaultStatus !== undefined ? this.getNodeIndexByRootAndStatus(justifiedRoot, defaultStatus) : undefined; + const justifiedIndex = this.getDefaultNodeIndex(justifiedRoot); if (justifiedIndex === undefined) { throw new ProtoArrayError({ code: ProtoArrayErrorCode.JUSTIFIED_NODE_UNKNOWN, @@ -1654,12 +1668,7 @@ export class ProtoArray { * Uses default variant (PENDING for Gloas, FULL for pre-Gloas) */ hasBlock(blockRoot: RootHex): boolean { - const defaultVariant = this.getDefaultVariant(blockRoot); - if (defaultVariant === undefined) { - return false; - } - const index = this.getNodeIndexByRootAndStatus(blockRoot, defaultVariant); - return index !== undefined; + return this.getDefaultNodeIndex(blockRoot) !== undefined; } /** From 1ddf8e46a9295c45bb747cac103e4fa3cbf3e062 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Fri, 13 Mar 2026 14:22:24 +0000 Subject: [PATCH 37/63] fix: correct pre-Deneb attestation propagation slot range boundary (#9031) `slotWithPastTolerance` floors the disparity-adjusted time to a slot, which is one slot too strict at the exact boundary where `compute_time_at_slot(slot + range + 1) + MAXIMUM_GOSSIP_CLOCK_DISPARITY` equals `current_time_ms`. Use `isCurrentSlotGivenGossipDisparity` to extend the lower bound by one slot only within the disparity window. Caught by consensus-specs gossip validation reference tests https://github.com/ethereum/consensus-specs/pull/4902 --- .../beacon-node/src/chain/validation/attestation.ts | 10 +++++++--- .../beacon-node/test/spec/utils/specTestIterator.ts | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/beacon-node/src/chain/validation/attestation.ts b/packages/beacon-node/src/chain/validation/attestation.ts index 046c482d01ee..d410469ed1c6 100644 --- a/packages/beacon-node/src/chain/validation/attestation.ts +++ b/packages/beacon-node/src/chain/validation/attestation.ts @@ -604,10 +604,14 @@ export function verifyPropagationSlotRange(fork: ForkName, chain: IBeaconChain, // // see: https://github.com/ethereum/consensus-specs/pull/3360 if (ForkSeq[fork] < ForkSeq.deneb) { + const currentSlot = chain.clock.currentSlot; + const withinPastDisparity = currentSlot > 0 && chain.clock.isCurrentSlotGivenGossipDisparity(currentSlot - 1); const earliestPermissibleSlot = Math.max( - // slot with past tolerance of MAXIMUM_GOSSIP_CLOCK_DISPARITY - chain.clock.slotWithPastTolerance(chain.config.MAXIMUM_GOSSIP_CLOCK_DISPARITY / 1000) - - chain.config.ATTESTATION_PROPAGATION_SLOT_RANGE, + // Pre-Deneb propagation is time-bounded: an attestation remains valid at the exact old + // boundary `compute_time_at_slot(slot + range + 1) + MAXIMUM_GOSSIP_CLOCK_DISPARITY`. + // Model that boundary by extending the lower slot bound by one additional slot only while + // the clock still considers the previous slot current given gossip disparity. + currentSlot - chain.config.ATTESTATION_PROPAGATION_SLOT_RANGE - (withinPastDisparity ? 1 : 0), 0 ); diff --git a/packages/beacon-node/test/spec/utils/specTestIterator.ts b/packages/beacon-node/test/spec/utils/specTestIterator.ts index 9e72a5f06ca3..384eadf81b0b 100644 --- a/packages/beacon-node/test/spec/utils/specTestIterator.ts +++ b/packages/beacon-node/test/spec/utils/specTestIterator.ts @@ -59,7 +59,7 @@ const coveredTestRunners = [ // ], // ``` export const defaultSkipOpts: SkipOpts = { - skippedForks: ["eip7805"], + skippedForks: ["eip7805", "heze"], skippedTestSuites: [ // Merge transition tests are skipped because we no longer support performing the merge transition. // All networks have already completed the merge, so this code path is no longer needed. From aeab9f930dae63cd2ced1db65bdbd1504849735a Mon Sep 17 00:00:00 2001 From: Lodekeeper <258435968+lodekeeper@users.noreply.github.com> Date: Fri, 13 Mar 2026 14:57:39 +0000 Subject: [PATCH 38/63] fix(reqresp): clear composed response timeout signals (#9029) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Open the Lodestar side of the memory-leak mitigation by clearing composed response-timeout signals in req/resp request handling. This is the Lodestar-local follow-up for the network-thread leak investigation in #8969. It addresses the req/resp timeout-signal retention path by replacing the `AbortSignal.any(...)` composition with a clearable composed signal and explicitly clearing listeners once the response stream is done. This PR is intentionally scoped to the req/resp fix only. The separate residual gossipsub topic-retention issue is being tracked upstream in `ChainSafe/js-libp2p-gossipsub#543`, and we can pick that up once a new libp2p release lands. ## Change - add `createRespSignal()` helper in `packages/reqresp/src/request/index.ts` - compose request signal + timeout signal via an `AbortController` - remove abort listeners explicitly via `respSignal.clear()` after the response closes ## Validation - `pnpm build` - `pnpm lint` *(warning-only: pre-existing unused biome suppression in `packages/light-client/test/unit/webEsmBundle.browser.test.ts`)* ## AI disclosure This PR was authored with AI assistance (Lodekeeper 🌟). --------- Co-authored-by: lodekeeper --- packages/reqresp/src/request/index.ts | 41 +++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/packages/reqresp/src/request/index.ts b/packages/reqresp/src/request/index.ts index 3745c12424ab..3b101c1ac29b 100644 --- a/packages/reqresp/src/request/index.ts +++ b/packages/reqresp/src/request/index.ts @@ -36,6 +36,42 @@ function scheduleStreamAbortIfNotClosed(stream: Stream, timeoutMs: number): void stream.addEventListener("close", onClose, {once: true}); } +type ClearableSignal = AbortSignal & {clear: () => void}; + +function createRespSignal(signal: AbortSignal | undefined, timeoutMs: number): ClearableSignal { + const timeoutSignal = AbortSignal.timeout(timeoutMs); + const signals = signal ? [signal, timeoutSignal] : [timeoutSignal]; + const controller = new AbortController(); + + const clear = (): void => { + for (const entry of signals) { + entry.removeEventListener("abort", onAbort); + } + }; + + const onAbort = (): void => { + if (controller.signal.aborted) { + return; + } + const reason = signals.find((entry) => entry.aborted)?.reason; + controller.abort(reason); + clear(); + }; + + for (const entry of signals) { + if (entry.aborted) { + onAbort(); + break; + } + entry.addEventListener("abort", onAbort); + } + + const respSignal = controller.signal as ClearableSignal; + respSignal.clear = clear; + + return respSignal; +} + export interface SendRequestOpts { /** The maximum time for complete response transfer. */ respTimeoutMs?: number; @@ -154,9 +190,7 @@ export async function* sendRequest( } // RESP_TIMEOUT: Maximum time for complete response transfer - const respSignal = signal - ? AbortSignal.any([signal, AbortSignal.timeout(RESP_TIMEOUT)]) - : AbortSignal.timeout(RESP_TIMEOUT); + const respSignal = createRespSignal(signal, RESP_TIMEOUT); let responseError: Error | null = null; let responseFullyConsumed = false; @@ -199,6 +233,7 @@ export async function* sendRequest( } } } + respSignal.clear(); metrics?.outgoingClosedStreams?.inc({method}); logger.verbose("Req stream closed", logCtx); } From 8299317d72794a03c09ae88ad163812c91d14c31 Mon Sep 17 00:00:00 2001 From: twoeths <10568965+twoeths@users.noreply.github.com> Date: Fri, 13 Mar 2026 22:10:08 +0700 Subject: [PATCH 39/63] feat(state-transition): add gloas fields/methods to IBeaconStateView (#8963) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Motivation** cherry-pick from #8857 to make it easier to review **Description** Adds two new cross-fork accessors to `IBeaconStateView` / `BeaconStateView`: - **`latestBlockHash: Bytes32`** — returns the execution block hash of the most recently included payload, regardless of fork: - Bellatrix – Fulu: reads `latestExecutionPayloadHeader.blockHash` - Gloas+: reads the dedicated `latestBlockHash` state field (EIP-7732) - Throws before Bellatrix - **`payloadBlockNumber: number`** — returns the execution block number from the payload header: - Only available Bellatrix – Fulu (named `payloadBlockNumber` to mirror `ExecutionPayloadHeader.blockNumber`) - Throws before Bellatrix and from Gloas onward (header no longer exists in EIP-7732) - change **`previousEpochParticipation`** and **`currentEpochParticipation`** to Uint8Array for memory efficiency - add **`getPreviousEpochParticipation`** **`getCurrentEpochParticipation`** when we only check some index, this saves a lot of memory - add **`processExecutionPayloadEnvelope`** for gloas **AI Assistance Disclosure** - [ ] External Contributors: I have read the [contributor guidelines](https://github.com/ChainSafe/lodestar/blob/unstable/CONTRIBUTING.md#ai-assistance-notice) and disclosed my usage of AI below. Claude Sonnet 4.6 assisted with commit message and PR description. --------- Co-authored-by: Tuyen Nguyen Co-authored-by: Claude Sonnet 4.6 --- .../src/stateView/beaconStateView.ts | 79 +++++++++++++++++-- .../src/stateView/interface.ts | 21 ++++- 2 files changed, 91 insertions(+), 9 deletions(-) diff --git a/packages/state-transition/src/stateView/beaconStateView.ts b/packages/state-transition/src/stateView/beaconStateView.ts index 85c3848111d2..47f7f772c111 100644 --- a/packages/state-transition/src/stateView/beaconStateView.ts +++ b/packages/state-transition/src/stateView/beaconStateView.ts @@ -1,7 +1,7 @@ import {CompactMultiProof, ProofType, Tree, createProof} from "@chainsafe/persistent-merkle-tree"; import {ByteViews} from "@chainsafe/ssz"; import {BeaconConfig} from "@lodestar/config"; -import {ForkSeq, SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params"; +import {ForkSeq, SLOTS_PER_HISTORICAL_ROOT, isForkPostGloas} from "@lodestar/params"; import { BeaconBlock, BlindedBeaconBlock, @@ -27,6 +27,7 @@ import { rewards, } from "@lodestar/types"; import {Checkpoint, Fork} from "@lodestar/types/phase0"; +import {processExecutionPayloadEnvelope} from "../block/index.js"; import {VoluntaryExitValidity, getVoluntaryExitValidity} from "../block/processVoluntaryExit.js"; import {getExpectedWithdrawals} from "../block/processWithdrawals.js"; import {EffectiveBalanceIncrements} from "../cache/effectiveBalanceIncrements.js"; @@ -75,10 +76,12 @@ export class BeaconStateView implements IBeaconStateView { // altair private _currentSyncCommittee: SyncCommittee | null = null; private _nextSyncCommittee: SyncCommittee | null = null; - private _previousEpochParticipation: number[] | null = null; - private _currentEpochParticipation: number[] | null = null; + private _previousEpochParticipation: Uint8Array | null = null; + private _currentEpochParticipation: Uint8Array | null = null; // bellatrix private _latestExecutionPayloadHeader: ExecutionPayloadHeader | null = null; + // Caches the cross-fork latestBlockHash value + private _latestBlockHash: Bytes32 | null = null; // capella private _historicalSummaries: capella.HistoricalSummaries | null = null; // electra @@ -159,7 +162,7 @@ export class BeaconStateView implements IBeaconStateView { return getRandaoMix(this.cachedState, epoch); } - get previousEpochParticipation(): number[] { + get previousEpochParticipation(): Uint8Array { if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.altair) { throw new Error("previousEpochParticipation is not available before Altair"); } @@ -167,7 +170,7 @@ export class BeaconStateView implements IBeaconStateView { if (this._previousEpochParticipation === null) { this._previousEpochParticipation = ( this.cachedState as CachedBeaconStateAltair - ).previousEpochParticipation.toValue(); + ).previousEpochParticipation.serialize(); } return this._previousEpochParticipation; @@ -175,7 +178,7 @@ export class BeaconStateView implements IBeaconStateView { // altair - get currentEpochParticipation(): number[] { + get currentEpochParticipation(): Uint8Array { if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.altair) { throw new Error("currentEpochParticipation is not available before Altair"); } @@ -183,12 +186,26 @@ export class BeaconStateView implements IBeaconStateView { if (this._currentEpochParticipation === null) { this._currentEpochParticipation = ( this.cachedState as CachedBeaconStateAltair - ).currentEpochParticipation.toValue(); + ).currentEpochParticipation.serialize(); } return this._currentEpochParticipation; } + getPreviousEpochParticipation(validatorIndex: ValidatorIndex): number { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.altair) { + throw new Error("previousEpochParticipation is not available before Altair"); + } + return (this.cachedState as CachedBeaconStateAltair).previousEpochParticipation.get(validatorIndex); + } + + getCurrentEpochParticipation(validatorIndex: ValidatorIndex): number { + if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.altair) { + throw new Error("currentEpochParticipation is not available before Altair"); + } + return (this.cachedState as CachedBeaconStateAltair).currentEpochParticipation.get(validatorIndex); + } + // bellatrix get latestExecutionPayloadHeader(): ExecutionPayloadHeader { @@ -205,6 +222,46 @@ export class BeaconStateView implements IBeaconStateView { return this._latestExecutionPayloadHeader; } + /** + * Cross-fork accessor for the execution block hash of the most recently included payload. + * Pre-gloas: reads from latestExecutionPayloadHeader.blockHash. + * Gloas+: reads the dedicated latestBlockHash field (EIP-7732). + */ + get latestBlockHash(): Bytes32 { + const forkSeq = this.config.getForkSeq(this.cachedState.slot); + if (forkSeq < ForkSeq.bellatrix) { + throw new Error("latestBlockHash is not available before Bellatrix"); + } + + if (this._latestBlockHash === null) { + if (forkSeq >= ForkSeq.gloas) { + this._latestBlockHash = (this.cachedState as CachedBeaconStateGloas).latestBlockHash; + } else { + this._latestBlockHash = ( + this.cachedState as CachedBeaconStateExecutions + ).latestExecutionPayloadHeader.blockHash; + } + } + + return this._latestBlockHash; + } + + /** + * The execution block number of the most recently included payload. + * Named payloadBlockNumber (not latestBlockNumber) to mirror ExecutionPayloadHeader.blockNumber pre-gloas. + * Only available from bellatrix through fulu — not tracked on BeaconState in gloas+ (EIP-7732). + */ + get payloadBlockNumber(): number { + const forkSeq = this.config.getForkSeq(this.cachedState.slot); + if (forkSeq < ForkSeq.bellatrix) { + throw new Error("payloadBlockNumber is not available before Bellatrix"); + } + if (forkSeq >= ForkSeq.gloas) { + throw new Error("payloadBlockNumber is not available post-gloas"); + } + return (this.cachedState as CachedBeaconStateExecutions).latestExecutionPayloadHeader.blockNumber; + } + // capella get historicalSummaries(): capella.HistoricalSummaries { @@ -703,6 +760,14 @@ export class BeaconStateView implements IBeaconStateView { const newState = processSlots(this.cachedState, slot, epochTransitionCacheOpts, modules); return new BeaconStateView(newState); } + + processExecutionPayloadEnvelope(signedEnvelope: gloas.SignedExecutionPayloadEnvelope, verify: boolean): void { + const fork = this.config.getForkName(this.cachedState.slot); + if (!isForkPostGloas(fork)) { + throw Error(`processExecutionPayloadEnvelope is only available for gloas+ forks, got fork=${fork}`); + } + processExecutionPayloadEnvelope(this.cachedState as CachedBeaconStateGloas, signedEnvelope, verify); + } } /** diff --git a/packages/state-transition/src/stateView/interface.ts b/packages/state-transition/src/stateView/interface.ts index 1059187f3b48..bef62b4bcfdb 100644 --- a/packages/state-transition/src/stateView/interface.ts +++ b/packages/state-transition/src/stateView/interface.ts @@ -55,11 +55,27 @@ export interface IBeaconStateView { getRandaoMix(epoch: Epoch): Bytes32; // altair - previousEpochParticipation: number[]; - currentEpochParticipation: number[]; + previousEpochParticipation: Uint8Array; + currentEpochParticipation: Uint8Array; + getPreviousEpochParticipation(validatorIndex: ValidatorIndex): number; + getCurrentEpochParticipation(validatorIndex: ValidatorIndex): number; // bellatrix latestExecutionPayloadHeader: ExecutionPayloadHeader; + /** + * Cross-fork accessor for the execution block hash of the most recently included payload. + * Pre-gloas: returns latestExecutionPayloadHeader.blockHash (bellatrix–fulu). + * Gloas+: returns the dedicated latestBlockHash state field (EIP-7732). + * Throws before bellatrix. + */ + latestBlockHash: Bytes32; + /** + * The execution block number of the most recently included payload. + * Named payloadBlockNumber (not latestBlockNumber) to mirror ExecutionPayloadHeader.blockNumber pre-gloas. + * Only available from bellatrix through fulu — not tracked on BeaconState in gloas+ (EIP-7732). + * Throws before bellatrix and from gloas onwards. + */ + payloadBlockNumber: number; // capella historicalSummaries: capella.HistoricalSummaries; @@ -193,4 +209,5 @@ export interface IBeaconStateView { epochTransitionCacheOpts?: EpochTransitionCacheOpts & {dontTransferCache?: boolean}, modules?: StateTransitionModules ): IBeaconStateView; + processExecutionPayloadEnvelope(signedEnvelope: gloas.SignedExecutionPayloadEnvelope, verify: boolean): void; } From b90dff673d2e78a1ff1352f7b23f74df2ef7f2eb Mon Sep 17 00:00:00 2001 From: Lodekeeper <258435968+lodekeeper@users.noreply.github.com> Date: Fri, 13 Mar 2026 15:46:42 +0000 Subject: [PATCH 40/63] docs(reqresp): add upstream issue reference to createRespSignal (#9033) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Add a JSDoc comment to `createRespSignal()` in `packages/reqresp/src/request/index.ts` documenting the upstream Node.js issue that motivates the manual `AbortController` composition pattern. Follow-up to #9029 per [review feedback](https://github.com/ChainSafe/lodestar/pull/9029#discussion_r2931776412) — the code should document _why_ we avoid `AbortSignal.any()` and link to the upstream issue for future maintainers. ## References - Upstream: https://github.com/nodejs/node/issues/54614 (`AbortSignal.any()` memory leak) - Investigation: #8969 - Fix PR: #9029 ## AI disclosure This PR was authored with AI assistance (Lodekeeper 🌟). Co-authored-by: lodekeeper --- packages/reqresp/src/request/index.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/packages/reqresp/src/request/index.ts b/packages/reqresp/src/request/index.ts index 3b101c1ac29b..6042a12d738e 100644 --- a/packages/reqresp/src/request/index.ts +++ b/packages/reqresp/src/request/index.ts @@ -38,6 +38,17 @@ function scheduleStreamAbortIfNotClosed(stream: Stream, timeoutMs: number): void type ClearableSignal = AbortSignal & {clear: () => void}; +/** + * Compose an abort signal from an optional parent signal and a timeout, with explicit cleanup. + * + * This replaces a plain `AbortSignal.any([signal, AbortSignal.timeout(timeoutMs)])` to work + * around a memory leak in Node.js where AbortSignal.any() retains references to all source + * signals for the lifetime of the longest-lived signal (the timeout). In a long-running + * req/resp workload this causes unbounded growth of the dependent-signal set. + * + * Upstream issue: https://github.com/nodejs/node/issues/54614 + * Lodestar investigation: https://github.com/ChainSafe/lodestar/issues/8969 + */ function createRespSignal(signal: AbortSignal | undefined, timeoutMs: number): ClearableSignal { const timeoutSignal = AbortSignal.timeout(timeoutMs); const signals = signal ? [signal, timeoutSignal] : [timeoutSignal]; From 92c37f252528d23cd87f6abcd90a3fe7d477020a Mon Sep 17 00:00:00 2001 From: Cayman Date: Sat, 14 Mar 2026 06:11:45 -0400 Subject: [PATCH 41/63] feat: add QUIC support (#8051) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation Add native QUIC transport support to Lodestar via `@chainsafe/libp2p-quic`. QUIC provides lower-latency connections with built-in encryption (TLS 1.3) and multiplexing, eliminating the need for separate stream muxers and reducing handshake overhead compared to TCP+Noise+mplex. Supersedes #7659 ## Description ### Transport configuration - Add QUIC transport alongside TCP using `@chainsafe/libp2p-quic` - QUIC is disabled by default, enabled via `--quic` - Default QUIC port is `port + 1` (9001 when port is 9000), configurable via `--quicPort` / `--quicPort6` - Add `--quic` and `--tcp` CLI flags to selectively enable/disable transports (`--tcp` is hidden, defaults to true) - Fail-early validation: error if both transports disabled, or if `discoveryPort` and `quicPort` collide (both UDP) - QUIC multiaddrs are listed before TCP in `localMultiaddrs` so libp2p prefers QUIC for outbound dials ### ENR (Ethereum Node Record) - Advertise `quic` / `quic6` fields in ENR when QUIC is enabled - Respect `--tcp` / `--quic` flags: only advertise enabled transports - Preserve existing ENR quic port values on restart (fallback chain: CLI arg → derived port → existing ENR value) - Clear all transport fields (`tcp`, `quic`) when `testMultiaddrForLocal` detects a non-local IP ### Peer discovery - `enrRelevance` accepts peers with TCP *or* QUIC (previously required TCP) - `getDiscv5Multiaddrs` prefers QUIC multiaddrs from boot ENRs when QUIC is enabled, falls back to TCP - `handleDiscoveredPeer` selects multiaddrs by protocol (`/tcp/`, `/quic-v1`) instead of assuming array index ordering - `handleDiscoveredPeer` matches peers that share any enabled transport, not just TCP - Normalize transport `Symbol.toStringTag` values for matching (`@libp2p/tcp` → `tcp`, `quic` → `quic`) - `CachedENR.multiaddrTCP` is now optional to support QUIC-only peers ### Documentation - Add QUIC transport section to networking docs covering enabling, port configuration, ENR advertisement - Update firewall ports documentation to include QUIC UDP port ### Testing - Unit tests for `enrRelevance` with tcp-only, quic-only, both, and neither - Unit tests for `getDiscv5Multiaddrs` QUIC preference and fallback behavior - Unit tests for port collision and both-transports-disabled validation - Unit tests for ENR field gating (`--tcp`, `--quic`) and quic port fallback preservation - Unit tests for multiaddr filtering with `--tcp` / `--quic` - Fix sim test port allocation to stride by 2, leaving room for QUIC port ## AI Disclosure AI-assisted (Claude Code) --------- Co-authored-by: matthewkeil Co-authored-by: Claude Opus 4.6 Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Co-authored-by: Nico Flaig --- .wordlist.txt | 2 + dashboards/lodestar_libp2p.json | 123 ++++++++++++++++-- .../pages/run/beacon-management/networking.md | 40 +++++- packages/beacon-node/package.json | 1 + .../beacon-node/src/network/discv5/utils.ts | 9 +- .../src/network/gossip/gossipsub.ts | 19 ++- .../beacon-node/src/network/libp2p/index.ts | 58 ++++++--- packages/beacon-node/src/network/metadata.ts | 1 + packages/beacon-node/src/network/options.ts | 6 +- .../beacon-node/src/network/peers/discover.ts | 57 ++++++-- .../test/unit/network/discv5/utils.test.ts | 65 +++++++++ .../unit/network/gossip/directPeers.test.ts | 42 +++++- .../libp2p/getDiscv5Multiaddrs.test.ts | 56 ++++++++ .../cli/src/cmds/beacon/initPeerIdAndEnr.ts | 16 ++- packages/cli/src/cmds/beacon/options.ts | 14 +- .../src/options/beaconNodeOptions/network.ts | 86 +++++++++++- .../test/unit/cmds/initPeerIdAndEnr.test.ts | 98 +++++++++++++- .../unit/options/beaconNodeOptions.test.ts | 111 ++++++++++++++++ .../cli/test/utils/crucible/utils/ports.ts | 3 +- pnpm-lock.yaml | 98 ++++++++++++++ 20 files changed, 830 insertions(+), 75 deletions(-) create mode 100644 packages/beacon-node/test/unit/network/discv5/utils.test.ts create mode 100644 packages/beacon-node/test/unit/network/libp2p/getDiscv5Multiaddrs.test.ts diff --git a/.wordlist.txt b/.wordlist.txt index c74ea96cdf3d..a49940b25e20 100644 --- a/.wordlist.txt +++ b/.wordlist.txt @@ -84,6 +84,7 @@ PRs Plaintext PoS Prysm +QUIC Quickstart README READMEs @@ -96,6 +97,7 @@ SSD SSZ Somer Stakehouse +TLS TOC TTD Teku diff --git a/dashboards/lodestar_libp2p.json b/dashboards/lodestar_libp2p.json index 3e93c2eb7606..e242d31c7bbe 100644 --- a/dashboards/lodestar_libp2p.json +++ b/dashboards/lodestar_libp2p.json @@ -214,7 +214,7 @@ "x": 12, "y": 1 }, - "id": 28, + "id": 40, "options": { "legend": { "calcs": [], @@ -223,7 +223,7 @@ "showLegend": true }, "tooltip": { - "mode": "single", + "mode": "multi", "sort": "none" } }, @@ -233,14 +233,28 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "exemplar": false, - "expr": "libp2p_connection_manager_protocol_streams_per_connection_90th_percentile", - "interval": "", - "legendFormat": "{{protocol}}", + "editorMode": "code", + "expr": "libp2p_tcp_inbound_connections_total", + "instant": false, + "legendFormat": "tcp", + "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "libp2p_quic_inbound_connections_total", + "hide": false, + "instant": false, + "legendFormat": "quic", + "range": true, + "refId": "B" } ], - "title": "Streams Per Connection (90th percentile)", + "title": "Inbound Connections by Transport", "type": "timeseries" }, { @@ -367,8 +381,7 @@ "mode": "off" } }, - "mappings": [], - "unit": "none" + "mappings": [] }, "overrides": [] }, @@ -378,7 +391,7 @@ "x": 12, "y": 9 }, - "id": 30, + "id": 28, "options": { "legend": { "calcs": [], @@ -397,16 +410,14 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", "exemplar": false, - "expr": "libp2p_protocol_streams_total", + "expr": "libp2p_connection_manager_protocol_streams_per_connection_90th_percentile", "interval": "", "legendFormat": "{{protocol}}", - "range": true, "refId": "A" } ], - "title": "Total Streams", + "title": "Streams Per Connection (90th percentile)", "type": "timeseries" }, { @@ -493,6 +504,90 @@ "title": "Total Bandwidth", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "libp2p_protocol_streams_total", + "interval": "", + "legendFormat": "{{protocol}}", + "range": true, + "refId": "A" + } + ], + "title": "Total Streams", + "type": "timeseries" + }, { "collapsed": false, "gridPos": { diff --git a/docs/pages/run/beacon-management/networking.md b/docs/pages/run/beacon-management/networking.md index 7532fb043a7a..57b0ba9e9007 100644 --- a/docs/pages/run/beacon-management/networking.md +++ b/docs/pages/run/beacon-management/networking.md @@ -10,18 +10,23 @@ Some of the important Lodestar flags related to networking are: - [`--listenAddress`](./beacon-cli#--listenaddress) - [`--port`](./beacon-cli#--port) - [`--discoveryPort`](./beacon-cli#--discoveryport) +- [`--quicPort`](./beacon-cli#--quicport) - [`--listenAddress6`](./beacon-cli#--listenaddress6) - [`--port6`](./beacon-cli#--port6) - [`--discoveryPort6`](./beacon-cli#--discoveryport6) +- [`--quicPort6`](./beacon-cli#--quicport6) +- [`--quic`](./beacon-cli#--quic) - [`--bootnodes`](./beacon-cli#--bootnodes) - [`--subscribeAllSubnets`](./beacon-cli#--subscribeallsubnets) - [`--disablePeerScoring`](./beacon-cli#--disablepeerscoring) - [`--enr.ip`](./beacon-cli#--enrip) - [`--enr.tcp`](./beacon-cli#--enrtcp) - [`--enr.udp`](./beacon-cli#--enrudp) +- [`--enr.quic`](./beacon-cli#--enrquic) - [`--enr.ip6`](./beacon-cli#--enrip6) - [`--enr.tcp6`](./beacon-cli#--enrtcp6) - [`--enr.udp6`](./beacon-cli#--enrudp6) +- [`--enr.quic6`](./beacon-cli#--enrquic6) - [`--nat`](./beacon-cli#--nat) - [`--private`](./beacon-cli#--private) @@ -79,6 +84,38 @@ Libp2p is a modular and extensible network stack that serves as the data transpo Libp2p operates at the lower levels of the OSI model, particularly at the Transport and Network layers. Libp2p supports both TCP and UDP protocols for establishing connections and data transmission. Combined with libp2p's modular design it can integrate with various networking technologies to facilitating both routing and addressing. +### QUIC Transport + +Lodestar supports [QUIC](https://datatracker.ietf.org/doc/html/rfc9000) as a transport alongside TCP. QUIC is a UDP-based transport that provides built-in encryption (TLS 1.3), multiplexed streams, and faster connection establishment compared to TCP. QUIC is disabled by default and can be enabled with the `--quic` flag. When enabled, Lodestar will prefer QUIC when dialing peers that advertise QUIC support. + +#### Enabling QUIC + +To enable QUIC transport: + +```bash +lodestar beacon --quic +``` + +When QUIC is enabled, the node will listen on an additional UDP port and advertise QUIC support in its ENR. + +#### Port Configuration + +With QUIC enabled, Lodestar uses three ports for P2P networking: + +| Port | Protocol | Default | Purpose | +| ----------------- | -------- | ----------------------- | ------------------------- | +| `--port` | TCP | 9000 | TCP transport for libp2p | +| `--discoveryPort` | UDP | 9000 (same as `--port`) | discv5 peer discovery | +| `--quicPort` | UDP | 9001 (`--port` + 1) | QUIC transport for libp2p | + +Note that `--discoveryPort` and `--quicPort` are both UDP but must use different ports. Lodestar will error on startup if they collide. + +For IPv6 dual-stack, equivalent flags are available: `--port6`, `--discoveryPort6`, and `--quicPort6`. + +#### ENR Advertisement + +When QUIC is enabled, the node's ENR automatically includes QUIC port information so that other nodes can discover and connect via QUIC. The ENR fields can be overridden with `--enr.quic` and `--enr.quic6`. + ## Firewall Management If your setup is behind a firewall there are a few ports that will need to be opened to allow for P2P discovery and communication. There are also some ports that need to be protected to prevent unwanted access or DDOS attacks on your node. @@ -86,7 +123,8 @@ If your setup is behind a firewall there are a few ports that will need to be op Ports that must be opened: - 30303/TCP+UDP - Execution layer P2P communication port -- 9000/TCP+UDP - Beacon node IPv4 and IPv6 P2P communication port +- 9000/TCP+UDP - Beacon node P2P communication (TCP transport + discv5 discovery) +- 9001/UDP - Beacon node QUIC transport (only if `--quic` is enabled) Ports that must be protected: diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index e3a34deef0d5..66dc19f1285d 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -113,6 +113,7 @@ "@chainsafe/discv5": "^12.0.0", "@chainsafe/enr": "^6.0.0", "@chainsafe/libp2p-noise": "^17.0.0", + "@chainsafe/libp2p-quic": "^2.0.0", "@chainsafe/persistent-merkle-tree": "^1.2.1", "@chainsafe/prometheus-gc-stats": "^1.0.0", "@chainsafe/pubkey-index-map": "^3.0.0", diff --git a/packages/beacon-node/src/network/discv5/utils.ts b/packages/beacon-node/src/network/discv5/utils.ts index 961607991af0..d0a91a1930cd 100644 --- a/packages/beacon-node/src/network/discv5/utils.ts +++ b/packages/beacon-node/src/network/discv5/utils.ts @@ -4,7 +4,7 @@ import {IClock} from "../../util/clock.js"; import {ENRKey} from "../metadata.js"; export enum ENRRelevance { - no_tcp = "no_tcp", + no_transport = "no_transport", no_eth2 = "no_eth2", // biome-ignore lint/style/useNamingConvention: Need to use the this name for network convention unknown_forkDigest = "unknown_forkDigest", @@ -13,10 +13,11 @@ export enum ENRRelevance { } export function enrRelevance(enr: ENR, config: BeaconConfig, clock: IClock): ENRRelevance { - // We are not interested in peers that don't advertise their tcp addr + // We are not interested in peers that don't advertise at least one transport (tcp or quic) const multiaddrTCP = enr.getLocationMultiaddr(ENRKey.tcp); - if (!multiaddrTCP) { - return ENRRelevance.no_tcp; + const multiaddrQUIC = enr.getLocationMultiaddr(ENRKey.quic); + if (!multiaddrTCP && !multiaddrQUIC) { + return ENRRelevance.no_transport; } // Check if the ENR.eth2 field matches and is of interest diff --git a/packages/beacon-node/src/network/gossip/gossipsub.ts b/packages/beacon-node/src/network/gossip/gossipsub.ts index eb543ba13cc0..f5d5b0a22683 100644 --- a/packages/beacon-node/src/network/gossip/gossipsub.ts +++ b/packages/beacon-node/src/network/gossip/gossipsub.ts @@ -11,7 +11,7 @@ import type {PeerScoreParams, PeerScoreStatsDump} from "@libp2p/gossipsub/score" import type {AddrInfo, PublishOpts, TopicStr} from "@libp2p/gossipsub/types"; import type {PeerId} from "@libp2p/interface"; import {peerIdFromString} from "@libp2p/peer-id"; -import {multiaddr} from "@multiformats/multiaddr"; +import {type Multiaddr, multiaddr} from "@multiformats/multiaddr"; import {ENR} from "@chainsafe/enr"; import {routes} from "@lodestar/api"; import {BeaconConfig, ForkBoundary} from "@lodestar/config"; @@ -537,19 +537,24 @@ export function parseDirectPeers(directPeerStrs: routes.lodestar.DirectPeer[], l const enr = ENR.decodeTxt(peerStr); const peerId = enr.peerId; - // Get TCP multiaddr from ENR - const multiaddrTCP = enr.getLocationMultiaddr("tcp"); - if (!multiaddrTCP) { - logger.warn("ENR does not contain TCP multiaddr", {enr: peerStr}); + // Get all available transport multiaddrs from ENR + const addrs = [enr.getLocationMultiaddr("quic"), enr.getLocationMultiaddr("tcp")].filter( + (a): a is Multiaddr => a != null + ); + if (addrs.length === 0) { + logger.warn("ENR does not contain any transport multiaddr", {enr: peerStr}); continue; } directPeers.push({ id: peerId, - addrs: [multiaddrTCP], + addrs, }); - logger.info("Added direct peer from ENR", {peerId: peerId.toString(), addr: multiaddrTCP.toString()}); + logger.info("Added direct peer from ENR", { + peerId: peerId.toString(), + addrs: addrs.map((a) => a.toString()).join(", "), + }); } catch (e) { logger.warn("Failed to parse direct peer ENR", {enr: peerStr}, e as Error); } diff --git a/packages/beacon-node/src/network/libp2p/index.ts b/packages/beacon-node/src/network/libp2p/index.ts index 9aa53d6fd10b..95fa001bfcbd 100644 --- a/packages/beacon-node/src/network/libp2p/index.ts +++ b/packages/beacon-node/src/network/libp2p/index.ts @@ -5,11 +5,12 @@ import {mdns} from "@libp2p/mdns"; import {mplex} from "@libp2p/mplex"; import {prometheusMetrics} from "@libp2p/prometheus-metrics"; import {tcp} from "@libp2p/tcp"; -import {createLibp2p} from "libp2p"; +import {Libp2pInit, createLibp2p} from "libp2p"; import {Registry} from "prom-client"; import {ENR} from "@chainsafe/enr"; import {noise} from "@chainsafe/libp2p-noise"; import {asCrypto, defaultCrypto} from "@chainsafe/libp2p-noise/crypto"; +import {quic} from "@chainsafe/libp2p-quic"; import {Libp2p, LodestarComponents} from "../interface.js"; import {NetworkOptions, defaultNetworkOptions} from "../options.js"; import {Eth2PeerDataStore} from "../peers/datastore.js"; @@ -21,11 +22,14 @@ export type NodeJsLibp2pOpts = { metricsRegistry?: Registry; }; -export async function getDiscv5Multiaddrs(bootEnrs: string[]): Promise { +export async function getDiscv5Multiaddrs(bootEnrs: string[], quicEnabled?: boolean): Promise { const bootMultiaddrs = []; for (const enrStr of bootEnrs) { const enr = ENR.decodeTxt(enrStr); - const multiaddrWithPeerId = (await enr.getFullMultiaddr("tcp"))?.toString(); + // Prefer QUIC over TCP when available + const quicMultiaddr = quicEnabled ? (await enr.getFullMultiaddr("quic"))?.toString() : undefined; + const tcpMultiaddr = (await enr.getFullMultiaddr("tcp"))?.toString(); + const multiaddrWithPeerId = quicMultiaddr ?? tcpMultiaddr; if (multiaddrWithPeerId) { bootMultiaddrs.push(multiaddrWithPeerId); } @@ -53,7 +57,9 @@ export async function createNodeJsLibp2p( const bootMultiaddrs = [ ...(networkOpts.bootMultiaddrs ?? defaultNetworkOptions.bootMultiaddrs ?? []), // Append discv5.bootEnrs to bootMultiaddrs if requested - ...(networkOpts.connectToDiscv5Bootnodes ? await getDiscv5Multiaddrs(networkOpts.discv5?.bootEnrs ?? []) : []), + ...(networkOpts.connectToDiscv5Bootnodes + ? await getDiscv5Multiaddrs(networkOpts.discv5?.bootEnrs ?? [], networkOpts.quic) + : []), ]; if ((bootMultiaddrs.length ?? 0) > 0) { @@ -64,6 +70,35 @@ export async function createNodeJsLibp2p( peerDiscovery.push(mdns()); } } + const transports: Libp2pInit["transports"] = []; + if (networkOpts.tcp ?? true) { + transports.unshift( + tcp({ + // Reject connections when the server's connection count gets high + maxConnections: networkOpts.maxPeers, + // socket option: the maximum length of the queue of pending connections + // https://nodejs.org/dist/latest-v18.x/docs/api/net.html#serverlisten + // it's not safe if we increase this number + backlog: 5, + closeServerOnMaxConnections: { + closeAbove: networkOpts.maxPeers ?? Infinity, + listenBelow: networkOpts.maxPeers ?? Infinity, + }, + }) + ); + } + if (networkOpts.quic) { + transports.unshift( + quic({ + handshakeTimeout: 5_000, + maxIdleTimeout: 10_000, + keepAliveInterval: 5_000, + maxConcurrentStreamLimit: 256, + maxStreamData: 10_000_000, + maxConnectionData: 15_000_000, + }) + ); + } const noiseCrypto = { ...defaultCrypto, @@ -85,20 +120,7 @@ export async function createNodeJsLibp2p( announce: [], }, connectionEncrypters: [noise({crypto: noiseCrypto})], - // Reject connections when the server's connection count gets high - transports: [ - tcp({ - maxConnections: networkOpts.maxPeers, - // socket option: the maximum length of the queue of pending connections - // https://nodejs.org/dist/latest-v18.x/docs/api/net.html#serverlisten - // it's not safe if we increase this number - backlog: 5, - closeServerOnMaxConnections: { - closeAbove: networkOpts.maxPeers ?? Infinity, - listenBelow: networkOpts.maxPeers ?? Infinity, - }, - }), - ], + transports, streamMuxers: [mplex({disconnectThreshold})], peerDiscovery, metrics: nodeJsLibp2pOpts.metrics diff --git a/packages/beacon-node/src/network/metadata.ts b/packages/beacon-node/src/network/metadata.ts index dfc844ea5fa5..705220cd158e 100644 --- a/packages/beacon-node/src/network/metadata.ts +++ b/packages/beacon-node/src/network/metadata.ts @@ -11,6 +11,7 @@ import {NetworkConfig} from "./networkConfig.js"; export enum ENRKey { tcp = "tcp", + quic = "quic", eth2 = "eth2", attnets = "attnets", syncnets = "syncnets", diff --git a/packages/beacon-node/src/network/options.ts b/packages/beacon-node/src/network/options.ts index f6e7e4374081..6f458ae131ef 100644 --- a/packages/beacon-node/src/network/options.ts +++ b/packages/beacon-node/src/network/options.ts @@ -29,7 +29,6 @@ export interface NetworkOptions useWorker?: boolean; maxYoungGenerationSizeMb?: number; disableLightClientServer?: boolean; - /** * During E2E tests observe a lot of following `missing stream`: * @@ -48,11 +47,14 @@ export interface NetworkOptions * We need to increase this only for the testing purpose */ disconnectThreshold?: number; + quic?: boolean; + tcp?: boolean; } export const defaultNetworkOptions: NetworkOptions = { maxPeers: 210, // Allow some room above targetPeers for new inbound peers targetPeers: 200, + // In CLI usage this is typically overridden; when unset it serves as a fallback default (e.g. programmatic usage/tests) localMultiaddrs: ["/ip4/0.0.0.0/tcp/9000", "/ip6/::/tcp/9000"], bootMultiaddrs: [], /** disabled by default */ @@ -67,6 +69,8 @@ export const defaultNetworkOptions: NetworkOptions = { slotsToSubscribeBeforeAggregatorDuty: 2, // This will enable the light client server by default disableLightClientServer: false, + quic: false, + tcp: true, // specific option for fulu // - this is the same to TARGET_SUBNET_PEERS // - for fusaka-devnets, we have 25-30 peers per subnet diff --git a/packages/beacon-node/src/network/peers/discover.ts b/packages/beacon-node/src/network/peers/discover.ts index e39a61465f56..3da865f8bbc7 100644 --- a/packages/beacon-node/src/network/peers/discover.ts +++ b/packages/beacon-node/src/network/peers/discover.ts @@ -59,6 +59,7 @@ export enum DiscoveredPeerStatus { cached = "cached", dropped = "dropped", no_multiaddrs = "no_multiaddrs", + transport_incompatible = "transport_incompatible", peer_cooling_down = "peer_cooling_down", } @@ -88,7 +89,8 @@ export type SubnetDiscvQueryMs = { type CachedENR = { peerId: PeerId; - multiaddrTCP: Multiaddr; + multiaddrTCP?: Multiaddr; + multiaddrQUIC?: Multiaddr; subnets: Record; addedUnixMs: number; // custodyGroups is null for pre-fulu @@ -114,6 +116,7 @@ export class PeerDiscovery { attnets: new Map(), syncnets: new Map(), }; + private transports: string[]; private custodyGroupQueries: CustodyGroupQueries; @@ -180,6 +183,17 @@ export class PeerDiscovery { } }); } + + // Transport tags vary by library: @libp2p/tcp uses '@libp2p/tcp', @chainsafe/libp2p-quic uses 'quic' + // Normalize to simple 'tcp' / 'quic' strings for matching + this.transports = libp2p.services.components.transportManager + .getTransports() + .map((t) => t[Symbol.toStringTag]) + .map((tag) => { + if (tag?.includes("tcp")) return "tcp"; + if (tag?.includes("quic")) return "quic"; + return tag; + }); } static async init(modules: PeerDiscoveryModules, opts: PeerDiscoveryOpts): Promise { @@ -372,10 +386,15 @@ export class PeerDiscovery { return; } + // Select multiaddrs by protocol rather than index — libp2p discovery events + // don't guarantee ordering or number of addresses + const multiaddrTCP = multiaddrs.find((ma) => ma.toString().includes("/tcp/")); + const multiaddrQUIC = multiaddrs.find((ma) => ma.toString().includes("/quic-v1")); + const attnets = zeroAttnets; const syncnets = zeroSyncnets; - const status = this.handleDiscoveredPeer(id, multiaddrs[0], attnets, syncnets, undefined); + const status = this.handleDiscoveredPeer(id, multiaddrTCP, multiaddrQUIC, attnets, syncnets, undefined); this.logger.debug("Discovered peer via libp2p", {peer: prettyPrintPeerId(id), status}); this.metrics?.discovery.discoveredStatus.inc({status}); }; @@ -388,13 +407,15 @@ export class PeerDiscovery { this.randomNodeQuery.count++; } const peerId = enr.peerId; - // tcp multiaddr is known to be be present, checked inside the worker + // At least one transport is known to be present, checked inside the worker const multiaddrTCP = enr.getLocationMultiaddr(ENRKey.tcp); - if (!multiaddrTCP) { - this.logger.warn("Discv5 worker sent enr without tcp multiaddr", {enr: enr.encodeTxt()}); + const multiaddrQUIC = enr.getLocationMultiaddr(ENRKey.quic); + if (!multiaddrTCP && !multiaddrQUIC) { + this.logger.warn("Discv5 worker sent enr without any transport multiaddr", {enr: enr.encodeTxt()}); this.metrics?.discovery.discoveredStatus.inc({status: DiscoveredPeerStatus.no_multiaddrs}); return; } + // Are this fields mandatory? const attnetsBytes = enr.kvs.get(ENRKey.attnets); // 64 bits const syncnetsBytes = enr.kvs.get(ENRKey.syncnets); // 4 bits @@ -414,7 +435,7 @@ export class PeerDiscovery { const syncnets = syncnetsBytes ? deserializeEnrSubnets(syncnetsBytes, SYNC_COMMITTEE_SUBNET_COUNT) : zeroSyncnets; const custodyGroupCount = custodyGroupCountBytes ? bytesToInt(custodyGroupCountBytes, "be") : undefined; - const status = this.handleDiscoveredPeer(peerId, multiaddrTCP, attnets, syncnets, custodyGroupCount); + const status = this.handleDiscoveredPeer(peerId, multiaddrTCP, multiaddrQUIC, attnets, syncnets, custodyGroupCount); this.logger.debug("Discovered peer via discv5", { peer: prettyPrintPeerId(peerId), status, @@ -428,7 +449,8 @@ export class PeerDiscovery { */ private handleDiscoveredPeer( peerId: PeerId, - multiaddrTCP: Multiaddr, + multiaddrTCP: Multiaddr | undefined, + multiaddrQUIC: Multiaddr | undefined, attnets: boolean[], syncnets: boolean[], custodySubnetCount?: number @@ -454,6 +476,13 @@ export class PeerDiscovery { return DiscoveredPeerStatus.already_connected; } + // ignore peers if they don't share any transport with us + const hasTcpMatch = this.transports.includes("tcp") && multiaddrTCP; + const hasQuicMatch = this.transports.includes("quic") && multiaddrQUIC; + if (!hasTcpMatch && !hasQuicMatch) { + return DiscoveredPeerStatus.transport_incompatible; + } + // Ignore dialing peers if ( this.libp2p.services.components.connectionManager @@ -469,6 +498,7 @@ export class PeerDiscovery { const cachedPeer: CachedENR = { peerId, multiaddrTCP, + multiaddrQUIC, subnets: {attnets, syncnets}, addedUnixMs: Date.now(), // for pre-fulu, custodyGroups is null @@ -566,19 +596,24 @@ export class PeerDiscovery { // are not successful. this.peersToConnect = Math.max(this.peersToConnect - 1, 0); - const {peerId, multiaddrTCP} = cachedPeer; + const {peerId, multiaddrTCP, multiaddrQUIC} = cachedPeer; // Must add the multiaddrs array to the address book before dialing // https://github.com/libp2p/js-libp2p/blob/aec8e3d3bb1b245051b60c2a890550d262d5b062/src/index.js#L638 - const peer = await this.libp2p.peerStore.merge(peerId, {multiaddrs: [multiaddrTCP]}); + const peer = await this.libp2p.peerStore.merge(peerId, { + multiaddrs: [multiaddrQUIC, multiaddrTCP].filter(Boolean) as Multiaddr[], + }); if (peer.addresses.length === 0) { this.metrics?.discovery.notDialReason.inc({reason: NotDialReason.no_multiaddrs}); return; } - // Note: PeerDiscovery adds the multiaddrTCP beforehand + // Note: PeerDiscovery adds the multiaddrs beforehand const peerIdShort = prettyPrintPeerId(peerId); - this.logger.debug("Dialing discovered peer", {peer: peerIdShort}); + this.logger.debug("Dialing discovered peer", { + peer: peerIdShort, + addresses: peer.addresses.map((a) => a.multiaddr.toString()).join(", "), + }); this.metrics?.discovery.dialAttempts.inc(); const timer = this.metrics?.discovery.dialTime.startTimer(); diff --git a/packages/beacon-node/test/unit/network/discv5/utils.test.ts b/packages/beacon-node/test/unit/network/discv5/utils.test.ts new file mode 100644 index 000000000000..55ae813e810e --- /dev/null +++ b/packages/beacon-node/test/unit/network/discv5/utils.test.ts @@ -0,0 +1,65 @@ +import {generateKeyPair} from "@libp2p/crypto/keys"; +import {multiaddr} from "@multiformats/multiaddr"; +import {describe, expect, it} from "vitest"; +import {type ENR, SignableENR} from "@chainsafe/enr"; +import {config} from "@lodestar/config/test-utils"; +import {ssz} from "@lodestar/types"; +import {ENRRelevance, enrRelevance} from "../../../../src/network/discv5/utils.js"; +import {ENRKey, getENRForkID} from "../../../../src/network/metadata.js"; +import {ClockStatic} from "../../../utils/clock.js"; + +describe("network / discv5 / enrRelevance", () => { + const clock = new ClockStatic(0); + + async function createEnr(opts: {tcp?: boolean; quic?: boolean; eth2?: boolean}): Promise { + const privateKey = await generateKeyPair("secp256k1"); + const enr = SignableENR.createFromPrivateKey(privateKey); + + if (opts.tcp) { + enr.setLocationMultiaddr(multiaddr("/ip4/192.168.1.1/tcp/9000")); + } + if (opts.quic) { + enr.setLocationMultiaddr(multiaddr("/ip4/192.168.1.1/udp/9001/quic-v1")); + } + if (opts.eth2) { + const enrForkID = getENRForkID(config, 0); + enr.set(ENRKey.eth2, ssz.phase0.ENRForkID.serialize(enrForkID)); + } + + return enr; + } + + it("should return no_transport for ENR without tcp or quic", async () => { + const enr = await createEnr({eth2: true}); + expect(enrRelevance(enr as unknown as ENR, config, clock)).toBe(ENRRelevance.no_transport); + }); + + it("should return relevant for ENR with tcp only", async () => { + const enr = await createEnr({tcp: true, eth2: true}); + expect(enrRelevance(enr as unknown as ENR, config, clock)).toBe(ENRRelevance.relevant); + }); + + it("should return relevant for ENR with quic only", async () => { + const enr = await createEnr({quic: true, eth2: true}); + expect(enrRelevance(enr as unknown as ENR, config, clock)).toBe(ENRRelevance.relevant); + }); + + it("should return relevant for ENR with both tcp and quic", async () => { + const enr = await createEnr({tcp: true, quic: true, eth2: true}); + expect(enrRelevance(enr as unknown as ENR, config, clock)).toBe(ENRRelevance.relevant); + }); + + it("should return no_eth2 for ENR without eth2 field", async () => { + const enr = await createEnr({tcp: true}); + expect(enrRelevance(enr as unknown as ENR, config, clock)).toBe(ENRRelevance.no_eth2); + }); + + it("should return unknown_forkDigest for ENR with unrecognized fork digest", async () => { + const enr = await createEnr({tcp: true}); + // Set a fake eth2 field with an unknown fork digest + const fakeEth2 = new Uint8Array(16); + fakeEth2.set([0xff, 0xff, 0xff, 0xff], 0); + enr.set(ENRKey.eth2, fakeEth2); + expect(enrRelevance(enr as unknown as ENR, config, clock)).toBe(ENRRelevance.unknown_forkDigest); + }); +}); diff --git a/packages/beacon-node/test/unit/network/gossip/directPeers.test.ts b/packages/beacon-node/test/unit/network/gossip/directPeers.test.ts index 17ef0b151984..5c471530853b 100644 --- a/packages/beacon-node/test/unit/network/gossip/directPeers.test.ts +++ b/packages/beacon-node/test/unit/network/gossip/directPeers.test.ts @@ -115,21 +115,50 @@ describe("network / gossip / directPeers", () => { expect(result[0].addrs[0].toString()).toBe("/ip4/192.168.1.1/tcp/9000"); expect(logger.info).toHaveBeenCalledWith("Added direct peer from ENR", { peerId: enr.peerId.toString(), - addr: "/ip4/192.168.1.1/tcp/9000", + addrs: "/ip4/192.168.1.1/tcp/9000", }); }); - it("should skip ENR without TCP multiaddr and log warning", async () => { + it("should parse ENR with both QUIC and TCP multiaddrs", async () => { + const privateKey = await generateKeyPair("secp256k1"); + const enr = SignableENR.createFromPrivateKey(privateKey); + enr.setLocationMultiaddr(multiaddr("/ip4/192.168.1.1/tcp/9000")); + enr.setLocationMultiaddr(multiaddr("/ip4/192.168.1.1/udp/9001/quic-v1")); + const enrStr = enr.encodeTxt(); + + const result = parseDirectPeers([enrStr], logger); + + expect(result).toHaveLength(1); + expect(result[0].id.toString()).toBe(enr.peerId.toString()); + expect(result[0].addrs).toHaveLength(2); + expect(result[0].addrs[0].toString()).toBe("/ip4/192.168.1.1/udp/9001/quic-v1"); + expect(result[0].addrs[1].toString()).toBe("/ip4/192.168.1.1/tcp/9000"); + }); + + it("should parse ENR with only QUIC multiaddr", async () => { + const privateKey = await generateKeyPair("secp256k1"); + const enr = SignableENR.createFromPrivateKey(privateKey); + enr.setLocationMultiaddr(multiaddr("/ip4/192.168.1.1/udp/9001/quic-v1")); + const enrStr = enr.encodeTxt(); + + const result = parseDirectPeers([enrStr], logger); + + expect(result).toHaveLength(1); + expect(result[0].addrs).toHaveLength(1); + expect(result[0].addrs[0].toString()).toBe("/ip4/192.168.1.1/udp/9001/quic-v1"); + }); + + it("should skip ENR without any transport multiaddr and log warning", async () => { const privateKey = await generateKeyPair("secp256k1"); const enr = SignableENR.createFromPrivateKey(privateKey); - // Only set UDP, not TCP + // Only set UDP discovery, not a dialable transport enr.setLocationMultiaddr(multiaddr("/ip4/192.168.1.1/udp/9000")); const enrStr = enr.encodeTxt(); const result = parseDirectPeers([enrStr], logger); expect(result).toHaveLength(0); - expect(logger.warn).toHaveBeenCalledWith("ENR does not contain TCP multiaddr", {enr: enrStr}); + expect(logger.warn).toHaveBeenCalledWith("ENR does not contain any transport multiaddr", {enr: enrStr}); }); it("should skip invalid ENR and log warning", () => { @@ -146,6 +175,7 @@ describe("network / gossip / directPeers", () => { const privateKey = await generateKeyPair("secp256k1"); const enr = SignableENR.createFromPrivateKey(privateKey); enr.setLocationMultiaddr(multiaddr("/ip4/10.0.0.1/tcp/9001")); + enr.setLocationMultiaddr(multiaddr("/ip4/10.0.0.1/udp/9002/quic-v1")); const enrStr = enr.encodeTxt(); const mixedPeers = [`/ip4/192.168.1.1/tcp/9000/p2p/${peerIdStr}`, enrStr]; @@ -156,7 +186,9 @@ describe("network / gossip / directPeers", () => { expect(result[0].id.toString()).toBe(peerIdStr); expect(result[0].addrs[0].toString()).toBe("/ip4/192.168.1.1/tcp/9000"); expect(result[1].id.toString()).toBe(enr.peerId.toString()); - expect(result[1].addrs[0].toString()).toBe("/ip4/10.0.0.1/tcp/9001"); + expect(result[1].addrs).toHaveLength(2); + expect(result[1].addrs[0].toString()).toBe("/ip4/10.0.0.1/udp/9002/quic-v1"); + expect(result[1].addrs[1].toString()).toBe("/ip4/10.0.0.1/tcp/9001"); }); }); }); diff --git a/packages/beacon-node/test/unit/network/libp2p/getDiscv5Multiaddrs.test.ts b/packages/beacon-node/test/unit/network/libp2p/getDiscv5Multiaddrs.test.ts new file mode 100644 index 000000000000..d042d172d95c --- /dev/null +++ b/packages/beacon-node/test/unit/network/libp2p/getDiscv5Multiaddrs.test.ts @@ -0,0 +1,56 @@ +import {generateKeyPair} from "@libp2p/crypto/keys"; +import {multiaddr} from "@multiformats/multiaddr"; +import {describe, expect, it} from "vitest"; +import {SignableENR} from "@chainsafe/enr"; +import {getDiscv5Multiaddrs} from "../../../../src/network/libp2p/index.js"; + +describe("network / libp2p / getDiscv5Multiaddrs", () => { + async function createEnrTxt(opts: {tcp?: boolean; quic?: boolean}): Promise { + const privateKey = await generateKeyPair("secp256k1"); + const enr = SignableENR.createFromPrivateKey(privateKey); + if (opts.tcp) { + enr.setLocationMultiaddr(multiaddr("/ip4/10.0.0.1/tcp/9000")); + } + if (opts.quic) { + enr.setLocationMultiaddr(multiaddr("/ip4/10.0.0.1/udp/9001/quic-v1")); + } + return enr.encodeTxt(); + } + + it("should prefer quic multiaddr over tcp when quic is enabled", async () => { + const enrTxt = await createEnrTxt({tcp: true, quic: true}); + const result = await getDiscv5Multiaddrs([enrTxt], true); + expect(result).toHaveLength(1); + expect(result[0]).toContain("/quic-v1"); + expect(result[0]).not.toContain("/tcp/"); + }); + + it("should return tcp by default (quic disabled)", async () => { + const enrTxt = await createEnrTxt({tcp: true, quic: true}); + const result = await getDiscv5Multiaddrs([enrTxt]); + expect(result).toHaveLength(1); + expect(result[0]).toContain("/tcp/"); + expect(result[0]).not.toContain("/quic-v1"); + }); + + it("should return tcp when quic is explicitly disabled", async () => { + const enrTxt = await createEnrTxt({tcp: true, quic: true}); + const result = await getDiscv5Multiaddrs([enrTxt], false); + expect(result).toHaveLength(1); + expect(result[0]).toContain("/tcp/"); + expect(result[0]).not.toContain("/quic-v1"); + }); + + it("should skip ENRs with no transport at all", async () => { + const enrTxt = await createEnrTxt({}); + const result = await getDiscv5Multiaddrs([enrTxt]); + expect(result).toHaveLength(0); + }); + + it("should return quic-only ENR when quic is enabled", async () => { + const enrTxt = await createEnrTxt({quic: true}); + const result = await getDiscv5Multiaddrs([enrTxt], true); + expect(result).toHaveLength(1); + expect(result[0]).toContain("/quic-v1"); + }); +}); diff --git a/packages/cli/src/cmds/beacon/initPeerIdAndEnr.ts b/packages/cli/src/cmds/beacon/initPeerIdAndEnr.ts index fa833a194179..a7a12220ae01 100644 --- a/packages/cli/src/cmds/beacon/initPeerIdAndEnr.ts +++ b/packages/cli/src/cmds/beacon/initPeerIdAndEnr.ts @@ -49,7 +49,7 @@ export function isLocalMultiAddr(multiaddr: Multiaddr | undefined): boolean { /** * Only update the enr if the value has changed */ -function maybeUpdateEnr( +function maybeUpdateEnr( enr: SignableENR, key: T, value: SignableENR[T] | undefined @@ -66,14 +66,18 @@ export function overwriteEnrWithCliArgs( opts?: {newEnr?: boolean; bootnode?: boolean} ): void { const preSeq = enr.seq; - const {port, discoveryPort, port6, discoveryPort6} = parseListenArgs(args); + const {port, discoveryPort, quicPort, port6, discoveryPort6, quicPort6} = parseListenArgs(args); + const tcp = args.tcp ?? true; + const quic = args.quic ?? false; maybeUpdateEnr(enr, "ip", args["enr.ip"] ?? enr.ip); maybeUpdateEnr(enr, "ip6", args["enr.ip6"] ?? enr.ip6); maybeUpdateEnr(enr, "udp", args["enr.udp"] ?? discoveryPort ?? enr.udp); maybeUpdateEnr(enr, "udp6", args["enr.udp6"] ?? discoveryPort6 ?? enr.udp6); if (!opts?.bootnode) { - maybeUpdateEnr(enr, "tcp", args["enr.tcp"] ?? port ?? enr.tcp); - maybeUpdateEnr(enr, "tcp6", args["enr.tcp6"] ?? port6 ?? enr.tcp6); + maybeUpdateEnr(enr, "tcp", tcp ? (args["enr.tcp"] ?? port ?? enr.tcp) : undefined); + maybeUpdateEnr(enr, "tcp6", tcp ? (args["enr.tcp6"] ?? port6 ?? enr.tcp6) : undefined); + maybeUpdateEnr(enr, "quic", quic ? (args["enr.quic"] ?? quicPort ?? enr.quic) : undefined); + maybeUpdateEnr(enr, "quic6", quic ? (args["enr.quic6"] ?? quicPort6 ?? enr.quic6) : undefined); } function testMultiaddrForLocal(mu: Multiaddr, ip4: boolean): void { @@ -92,9 +96,13 @@ export function overwriteEnrWithCliArgs( if (ip4) { enr.delete("ip"); enr.delete("udp"); + enr.delete("tcp"); + enr.delete("quic"); } else { enr.delete("ip6"); enr.delete("udp6"); + enr.delete("tcp6"); + enr.delete("quic6"); } } } diff --git a/packages/cli/src/cmds/beacon/options.ts b/packages/cli/src/cmds/beacon/options.ts index 82660e4eabc4..65dff288119a 100644 --- a/packages/cli/src/cmds/beacon/options.ts +++ b/packages/cli/src/cmds/beacon/options.ts @@ -174,10 +174,12 @@ export const beaconExtraOptions: CliCommandOptions = { type ENRArgs = { "enr.ip"?: string; "enr.tcp"?: number; - "enr.ip6"?: string; "enr.udp"?: number; + "enr.quic"?: number; + "enr.ip6"?: string; "enr.tcp6"?: number; "enr.udp6"?: number; + "enr.quic6"?: number; nat?: boolean; }; @@ -197,6 +199,11 @@ const enrOptions: CliCommandOptions = { type: "number", group: "enr", }, + "enr.quic": { + description: "Override ENR QUIC entry", + type: "number", + group: "enr", + }, "enr.ip6": { description: "Override ENR IPv6 entry", type: "string", @@ -212,6 +219,11 @@ const enrOptions: CliCommandOptions = { type: "number", group: "enr", }, + "enr.quic6": { + description: "Override ENR (IPv6-specific) QUIC entry", + type: "number", + group: "enr", + }, nat: { type: "boolean", description: "Allow configuration of non-local addresses", diff --git a/packages/cli/src/options/beaconNodeOptions/network.ts b/packages/cli/src/options/beaconNodeOptions/network.ts index 301a0a0e3bbc..faadadefdc26 100644 --- a/packages/cli/src/options/beaconNodeOptions/network.ts +++ b/packages/cli/src/options/beaconNodeOptions/network.ts @@ -7,20 +7,25 @@ import {YargsError} from "../../util/index.js"; export const defaultListenAddress = "0.0.0.0"; export const defaultListenAddress6 = "::"; export const defaultP2pPort = 9000; +export const defaultQuicPort = 9001; export type NetworkArgs = { discv5?: boolean; listenAddress?: string; port?: number; discoveryPort?: number; + quicPort?: number; listenAddress6?: string; port6?: number; discoveryPort6?: number; + quicPort6?: number; bootnodes?: string[]; targetPeers?: number; subscribeAllSubnets?: boolean; slotsToSubscribeBeforeAggregatorDuty?: number; disablePeerScoring?: boolean; + quic?: boolean; + tcp?: boolean; mdns?: boolean; directPeers?: string[]; "network.maxPeers"?: number; @@ -65,43 +70,79 @@ export function parseListenArgs(args: NetworkArgs) { const listenAddress = args.listenAddress ?? (args.listenAddress6 ? undefined : defaultListenAddress); const port = listenAddress ? (args.port ?? defaultP2pPort) : undefined; const discoveryPort = listenAddress ? (args.discoveryPort ?? port) : undefined; + const quicPort = listenAddress ? (args.quicPort ?? (port !== undefined ? port + 1 : defaultQuicPort)) : undefined; // If listenAddress6 is explicitly set, use it // If listenAddress is not set, use defaultListenAddress6 const listenAddress6 = args.listenAddress6 ?? (args.listenAddress ? undefined : defaultListenAddress6); const port6 = listenAddress6 ? (args.port6 ?? args.port ?? defaultP2pPort) : undefined; const discoveryPort6 = listenAddress6 ? (args.discoveryPort6 ?? port6) : undefined; + const quicPort6 = listenAddress6 + ? (args.quicPort6 ?? (port6 !== undefined ? port6 + 1 : defaultQuicPort)) + : undefined; - return {listenAddress, port, discoveryPort, listenAddress6, port6, discoveryPort6}; + return {listenAddress, port, discoveryPort, quicPort, listenAddress6, port6, discoveryPort6, quicPort6}; } export function parseArgs(args: NetworkArgs): IBeaconNodeOptions["network"] { - const {listenAddress, port, discoveryPort, listenAddress6, port6, discoveryPort6} = parseListenArgs(args); + const {listenAddress, port, discoveryPort, quicPort, listenAddress6, port6, discoveryPort6, quicPort6} = + parseListenArgs(args); + const quic = args.quic ?? false; + const tcp = args.tcp ?? true; + + if (!quic && !tcp) { + throw new YargsError("Cannot disable both TCP and QUIC transports"); + } + // validate ip, ip6, ports const muArgs = { listenAddress: listenAddress ? `/ip4/${listenAddress}` : undefined, port: listenAddress ? `/tcp/${port}` : undefined, discoveryPort: listenAddress ? `/udp/${discoveryPort}` : undefined, + quicPort: listenAddress ? `/udp/${quicPort}/quic-v1` : undefined, listenAddress6: listenAddress6 ? `/ip6/${listenAddress6}` : undefined, port6: listenAddress6 ? `/tcp/${port6}` : undefined, discoveryPort6: listenAddress6 ? `/udp/${discoveryPort6}` : undefined, + quicPort6: listenAddress6 ? `/udp/${quicPort6}/quic-v1` : undefined, }; - for (const key of [ + const keysToValidate: (keyof typeof muArgs)[] = [ "listenAddress", "port", "discoveryPort", "listenAddress6", "port6", "discoveryPort6", - ] as (keyof typeof muArgs)[]) { + ]; + + if (quic) { + keysToValidate.push("quicPort"); + keysToValidate.push("quicPort6"); + } + + for (const key of keysToValidate) { validateMultiaddrArg(muArgs, key); } + if (quic) { + if (discoveryPort !== undefined && quicPort !== undefined && discoveryPort === quicPort) { + throw new YargsError( + `discoveryPort and quicPort must not collide, both are UDP. Got discoveryPort=${discoveryPort} quicPort=${quicPort}` + ); + } + if (discoveryPort6 !== undefined && quicPort6 !== undefined && discoveryPort6 === quicPort6) { + throw new YargsError( + `discoveryPort6 and quicPort6 must not collide, both are UDP. Got discoveryPort6=${discoveryPort6} quicPort6=${quicPort6}` + ); + } + } + const bindMu = listenAddress ? `${muArgs.listenAddress}${muArgs.discoveryPort}` : undefined; - const localMu = listenAddress ? `${muArgs.listenAddress}${muArgs.port}` : undefined; + const localMu = listenAddress && tcp ? `${muArgs.listenAddress}${muArgs.port}` : undefined; + const quicMu = listenAddress && quic ? `${muArgs.listenAddress}${muArgs.quicPort}` : undefined; const bindMu6 = listenAddress6 ? `${muArgs.listenAddress6}${muArgs.discoveryPort6}` : undefined; - const localMu6 = listenAddress6 ? `${muArgs.listenAddress6}${muArgs.port6}` : undefined; + const localMu6 = listenAddress6 && tcp ? `${muArgs.listenAddress6}${muArgs.port6}` : undefined; + const quicMu6 = listenAddress6 && quic ? `${muArgs.listenAddress6}${muArgs.quicPort6}` : undefined; const targetPeers = args.targetPeers; const maxPeers = args["network.maxPeers"] ?? (targetPeers !== undefined ? Math.floor(targetPeers * 1.1) : undefined); @@ -137,11 +178,13 @@ export function parseArgs(args: NetworkArgs): IBeaconNodeOptions["network"] { : null, maxPeers: maxPeers ?? defaultOptions.network.maxPeers, targetPeers: targetPeers ?? defaultOptions.network.targetPeers, - localMultiaddrs: [localMu, localMu6].filter(Boolean) as string[], + localMultiaddrs: [quicMu, quicMu6, localMu, localMu6].filter(Boolean) as string[], subscribeAllSubnets: args.subscribeAllSubnets, slotsToSubscribeBeforeAggregatorDuty: args.slotsToSubscribeBeforeAggregatorDuty ?? defaultOptions.network.slotsToSubscribeBeforeAggregatorDuty, disablePeerScoring: args.disablePeerScoring, + quic, + tcp, connectToDiscv5Bootnodes: args["network.connectToDiscv5Bootnodes"], discv5FirstQueryDelayMs: args["network.discv5FirstQueryDelayMs"], dontSendGossipAttestationsToForkchoice: args["network.dontSendGossipAttestationsToForkchoice"], @@ -192,6 +235,13 @@ export const options: CliCommandOptions = { group: "network", }, + quicPort: { + description: "The UDP port that QUIC will listen on. Defaults to `port` + 1", + type: "number", + defaultDescription: "`port` + 1", + group: "network", + }, + listenAddress6: { type: "string", description: "The IPv6 address to listen for p2p UDP and TCP connections", @@ -214,6 +264,13 @@ export const options: CliCommandOptions = { group: "network", }, + quicPort6: { + description: "The UDP port that QUIC will listen on. Defaults to `port6` + 1", + type: "number", + defaultDescription: "`port6` + 1", + group: "network", + }, + bootnodes: { type: "array", description: "Bootnodes for discv5 discovery", @@ -254,6 +311,21 @@ export const options: CliCommandOptions = { group: "network", }, + quic: { + type: "boolean", + description: "Enable QUIC transport", + default: false, + group: "network", + }, + + tcp: { + hidden: true, + type: "boolean", + description: "Enable TCP transport", + default: true, + group: "network", + }, + mdns: { type: "boolean", description: "Enable mdns local peer discovery", diff --git a/packages/cli/test/unit/cmds/initPeerIdAndEnr.test.ts b/packages/cli/test/unit/cmds/initPeerIdAndEnr.test.ts index b07e9b85873e..a02bf8f8bc69 100644 --- a/packages/cli/test/unit/cmds/initPeerIdAndEnr.test.ts +++ b/packages/cli/test/unit/cmds/initPeerIdAndEnr.test.ts @@ -1,11 +1,107 @@ import fs from "node:fs"; +import {generateKeyPair} from "@libp2p/crypto/keys"; import {peerIdFromPrivateKey} from "@libp2p/peer-id"; import tmp from "tmp"; import {afterEach, beforeEach, describe, expect, it} from "vitest"; -import {initPrivateKeyAndEnr} from "../../../src/cmds/beacon/initPeerIdAndEnr.js"; +import {SignableENR} from "@chainsafe/enr"; +import {initPrivateKeyAndEnr, overwriteEnrWithCliArgs} from "../../../src/cmds/beacon/initPeerIdAndEnr.js"; import {BeaconArgs} from "../../../src/cmds/beacon/options.js"; import {testLogger} from "../../utils.js"; +describe("overwriteEnrWithCliArgs", () => { + it("should set tcp but not quic fields by default", async () => { + const privateKey = await generateKeyPair("secp256k1"); + const enr = SignableENR.createFromPrivateKey(privateKey); + const logger = testLogger(); + + overwriteEnrWithCliArgs(enr, {listenAddress: "0.0.0.0", port: 9000, nat: true} as unknown as BeaconArgs, logger); + + expect(enr.tcp).toBe(9000); + expect(enr.quic).toBeUndefined(); + }); + + it("should set both tcp and quic fields when quic is true", async () => { + const privateKey = await generateKeyPair("secp256k1"); + const enr = SignableENR.createFromPrivateKey(privateKey); + const logger = testLogger(); + + overwriteEnrWithCliArgs( + enr, + {listenAddress: "0.0.0.0", port: 9000, quic: true, nat: true} as unknown as BeaconArgs, + logger + ); + + expect(enr.tcp).toBe(9000); + expect(enr.quic).toBe(9001); + }); + + it("should not set tcp fields when tcp is false", async () => { + const privateKey = await generateKeyPair("secp256k1"); + const enr = SignableENR.createFromPrivateKey(privateKey); + const logger = testLogger(); + + overwriteEnrWithCliArgs( + enr, + {listenAddress: "0.0.0.0", port: 9000, tcp: false, quic: true, nat: true} as unknown as BeaconArgs, + logger + ); + + expect(enr.tcp).toBeUndefined(); + expect(enr.tcp6).toBeUndefined(); + expect(enr.quic).toBe(9001); + }); + + it("should not set quic fields when quic is false", async () => { + const privateKey = await generateKeyPair("secp256k1"); + const enr = SignableENR.createFromPrivateKey(privateKey); + const logger = testLogger(); + + overwriteEnrWithCliArgs( + enr, + {listenAddress: "0.0.0.0", port: 9000, quic: false, nat: true} as unknown as BeaconArgs, + logger + ); + + expect(enr.tcp).toBe(9000); + expect(enr.quic).toBeUndefined(); + expect(enr.quic6).toBeUndefined(); + }); + + it("should clear pre-existing tcp fields when tcp is false", async () => { + const privateKey = await generateKeyPair("secp256k1"); + const enr = SignableENR.createFromPrivateKey(privateKey); + enr.tcp = 9000; + const logger = testLogger(); + + overwriteEnrWithCliArgs( + enr, + {listenAddress: "0.0.0.0", port: 9000, tcp: false, nat: true} as unknown as BeaconArgs, + logger + ); + + expect(enr.tcp).toBeUndefined(); + }); + + it("should preserve existing enr quic port when no explicit quicPort given", async () => { + const privateKey = await generateKeyPair("secp256k1"); + const enr = SignableENR.createFromPrivateKey(privateKey); + // Simulate a persisted ENR that already had quic advertised + enr.quic = 9001; + enr.quic6 = 9002; + const logger = testLogger(); + + // No quicPort arg, no enr.quic arg — should fall back to existing enr value + overwriteEnrWithCliArgs( + enr, + {listenAddress: "0.0.0.0", port: 9000, quic: true, nat: true} as unknown as BeaconArgs, + logger + ); + + expect(enr.quic).toBe(9001); + expect(enr.quic6).toBe(9002); + }); +}); + describe("initPeerIdAndEnr", () => { let tmpDir: tmp.DirResult; diff --git a/packages/cli/test/unit/options/beaconNodeOptions.test.ts b/packages/cli/test/unit/options/beaconNodeOptions.test.ts index cef866b30624..a240ad133eae 100644 --- a/packages/cli/test/unit/options/beaconNodeOptions.test.ts +++ b/packages/cli/test/unit/options/beaconNodeOptions.test.ts @@ -2,6 +2,7 @@ import {describe, expect, it} from "vitest"; import {ArchiveMode, IBeaconNodeOptions} from "@lodestar/beacon-node"; import {RecursivePartial} from "@lodestar/utils"; import {BeaconNodeArgs, parseBeaconNodeArgs} from "../../../src/options/beaconNodeOptions/index.js"; +import {NetworkArgs, parseArgs as parseNetworkArgs} from "../../../src/options/beaconNodeOptions/network.js"; describe("options / beaconNodeOptions", () => { it("Should parse BeaconNodeArgs", () => { @@ -67,6 +68,7 @@ describe("options / beaconNodeOptions", () => { listenAddress: "127.0.0.1", port: 9001, discoveryPort: 9002, + quicPort: 9003, bootnodes: [ "enr:-KG4QOtcP9X1FbIMOe17QNMKqDxCpm14jcX5tiOE4_TyMrFqbmhPZHK_ZPG2Gxb1GE2xdtodOfx9-cgvNtxnRyHEmC0ghGV0aDKQ9aX9QgAAAAD__________4JpZIJ2NIJpcIQDE8KdiXNlY3AyNTZrMaEDhpehBDbZjM_L9ek699Y7vhUJ-eAdMyQW_Fil522Y0fODdGNwgiMog3VkcIIjKA", ], @@ -191,11 +193,13 @@ describe("options / beaconNodeOptions", () => { gossipsubDHigh: 6, gossipsubAwaitHandler: true, mdns: false, + quic: false, rateLimitMultiplier: 1, maxGossipTopicConcurrency: 64, useWorker: true, maxYoungGenerationSizeMb: 152, targetGroupPeers: 12, + tcp: true, directPeers: ["/ip4/192.168.1.1/tcp/9000/p2p/16Uiu2HAkuWPWqF4W3aw9oo5Yw79v5muzBaaGTGKMmuqjPfEyfkwu"], }, sync: { @@ -211,3 +215,110 @@ describe("options / beaconNodeOptions", () => { expect(options).toEqual(expectedOptions); }); }); + +describe("options / network / tcp and quic flags", () => { + it("should include only tcp multiaddrs by default", () => { + const result = parseNetworkArgs({listenAddress: "0.0.0.0", port: 9000} as NetworkArgs); + expect(result.localMultiaddrs).toContain("/ip4/0.0.0.0/tcp/9000"); + expect(result.localMultiaddrs).not.toContain("/ip4/0.0.0.0/udp/9001/quic-v1"); + }); + + it("should include both tcp and quic multiaddrs when quic is true", () => { + const result = parseNetworkArgs({listenAddress: "0.0.0.0", port: 9000, quic: true} as NetworkArgs); + expect(result.localMultiaddrs).toContain("/ip4/0.0.0.0/tcp/9000"); + expect(result.localMultiaddrs).toContain("/ip4/0.0.0.0/udp/9001/quic-v1"); + }); + + it("should exclude tcp multiaddrs when tcp is false", () => { + const result = parseNetworkArgs({listenAddress: "0.0.0.0", port: 9000, tcp: false, quic: true} as NetworkArgs); + const tcpAddrs = result.localMultiaddrs.filter((mu) => mu.includes("/tcp/")); + expect(tcpAddrs).toHaveLength(0); + expect(result.localMultiaddrs).toContain("/ip4/0.0.0.0/udp/9001/quic-v1"); + expect(result.tcp).toBe(false); + }); + + it("should exclude quic multiaddrs when quic is false", () => { + const result = parseNetworkArgs({listenAddress: "0.0.0.0", port: 9000, quic: false} as NetworkArgs); + const quicAddrs = result.localMultiaddrs.filter((mu) => mu.includes("/quic")); + expect(quicAddrs).toHaveLength(0); + expect(result.localMultiaddrs).toContain("/ip4/0.0.0.0/tcp/9000"); + expect(result.quic).toBe(false); + }); + + it("should not validate derived quicPort when quic is false", () => { + const result = parseNetworkArgs({listenAddress: "0.0.0.0", port: 65535, quic: false} as NetworkArgs); + expect(result.localMultiaddrs).toContain("/ip4/0.0.0.0/tcp/65535"); + expect(result.localMultiaddrs).not.toContain("/ip4/0.0.0.0/udp/65536/quic-v1"); + }); + + it("should not validate explicit quicPort when quic is false", () => { + const result = parseNetworkArgs({ + listenAddress: "0.0.0.0", + port: 9000, + quic: false, + quicPort: 65536, + } as NetworkArgs); + expect(result.quic).toBe(false); + }); + + it("should exclude ipv6 tcp multiaddrs when tcp is false", () => { + const result = parseNetworkArgs({ + listenAddress: "0.0.0.0", + listenAddress6: "::", + port: 9000, + tcp: false, + quic: true, + } as NetworkArgs); + const tcpAddrs = result.localMultiaddrs.filter((mu) => mu.includes("/tcp/")); + expect(tcpAddrs).toHaveLength(0); + // quic for both ipv4 and ipv6 should still be present + const quicAddrs = result.localMultiaddrs.filter((mu) => mu.includes("/quic")); + expect(quicAddrs).toHaveLength(2); + }); + + it("should pass tcp through to network options", () => { + const result = parseNetworkArgs({listenAddress: "0.0.0.0", port: 9000, tcp: false, quic: true} as NetworkArgs); + expect(result.tcp).toBe(false); + }); + + it("should throw when both TCP and QUIC are disabled", () => { + expect(() => + parseNetworkArgs({listenAddress: "0.0.0.0", port: 9000, tcp: false, quic: false} as NetworkArgs) + ).toThrow("Cannot disable both TCP and QUIC transports"); + }); + + it("should throw when discoveryPort and quicPort collide", () => { + expect(() => + parseNetworkArgs({ + listenAddress: "0.0.0.0", + port: 9000, + discoveryPort: 9001, + quicPort: 9001, + quic: true, + } as NetworkArgs) + ).toThrow(/discoveryPort and quicPort must not collide/); + }); + + it("should not throw on port collision when quic is false", () => { + const result = parseNetworkArgs({ + listenAddress: "0.0.0.0", + port: 9000, + discoveryPort: 9001, + quicPort: 9001, + quic: false, + } as NetworkArgs); + expect(result.quic).toBe(false); + }); + + it("should throw when discoveryPort6 and quicPort6 collide", () => { + expect(() => + parseNetworkArgs({ + listenAddress6: "::", + port6: 9000, + discoveryPort6: 9001, + quicPort6: 9001, + quic: true, + } as NetworkArgs) + ).toThrow(/discoveryPort6 and quicPort6 must not collide/); + }); +}); diff --git a/packages/cli/test/utils/crucible/utils/ports.ts b/packages/cli/test/utils/crucible/utils/ports.ts index 3b6f51ad1429..e512a77d8133 100644 --- a/packages/cli/test/utils/crucible/utils/ports.ts +++ b/packages/cli/test/utils/crucible/utils/ports.ts @@ -15,7 +15,8 @@ export const getNodePorts = ( execution: {p2pPort: number; enginePort: number; httpPort: number}; } => ({ beacon: { - p2pPort: BN_P2P_BASE_PORT + 1 + nodeIndex, + // Stride by 2 to leave room for the QUIC port (p2pPort + 1) + p2pPort: BN_P2P_BASE_PORT + 1 + nodeIndex * 2, httpPort: BN_REST_BASE_PORT + 1 + nodeIndex, }, validator: { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c41042d0c243..cf033de56844 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -231,6 +231,9 @@ importers: '@chainsafe/libp2p-noise': specifier: ^17.0.0 version: 17.0.0 + '@chainsafe/libp2p-quic': + specifier: ^2.0.0 + version: 2.0.0 '@chainsafe/persistent-merkle-tree': specifier: ^1.2.1 version: 1.2.1 @@ -1443,6 +1446,56 @@ packages: '@chainsafe/libp2p-noise@17.0.0': resolution: {integrity: sha512-vwrmY2Y+L1xYhIDiEpl61KHxwrLCZoXzTpwhyk34u+3+6zCAZPL3GxH3i2cs+u5IYNoyLptORdH17RKFXy7upA==} + '@chainsafe/libp2p-quic-darwin-arm64@2.0.0': + resolution: {integrity: sha512-dY/LEALfn4qRUjY8i3+V5YcmPRzJdeiaE/6eoS92KZr6UoWdVuxUCUz24+3jTl9Ke+EqvqsjZxfIS08CuhWhRA==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + + '@chainsafe/libp2p-quic-darwin-x64@2.0.0': + resolution: {integrity: sha512-XOt+rxMUjJ7W+ainDVUJ76kkmFO/LZYjGI8D9ecVL4P1+SIfxc90awrjikGUDqlozvsk96vjNzvUO6Obw+ftXA==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + + '@chainsafe/libp2p-quic-linux-arm64-gnu@2.0.0': + resolution: {integrity: sha512-TgRv12gyZMWm8G9RMi7j0Uwjmk272ikqXVvYkDICD5yS13vIkvxueVuvAOtFnkt7qO+pPh2hciPUkdfYeqVjuA==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@chainsafe/libp2p-quic-linux-arm64-musl@2.0.0': + resolution: {integrity: sha512-cax7qIL8fGeN41ydH6yRXcYhbYZbSQOYXa9tSr5lLSDJE24JEOjy8BugeP618kXydNFEbbt3VmyjNc+8XrGd4g==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@chainsafe/libp2p-quic-linux-x64-gnu@2.0.0': + resolution: {integrity: sha512-LiiaS3lmebt4USDQAFQCM+t5n1wuMoGeEDmHco6moSmZ9FJhBaM4CryUS3bXey31tpxEyQIx3q6Ub8SOUyLJ/g==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@chainsafe/libp2p-quic-linux-x64-musl@2.0.0': + resolution: {integrity: sha512-G3SS24AoY/1xkAA9IvL7alq2fp4GJwp+YXM8lOaez3cTCZuhwq4CksQFjy11VzyKrNCs/D9H02l0Aat8UeQaBw==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + libc: [musl] + + '@chainsafe/libp2p-quic-win32-x64-msvc@2.0.0': + resolution: {integrity: sha512-IxLdu8uVcus/KmFTiXHX6xqZE51uazKRJSbEn/9r94UY+uA7AEAeu497Zug7LOfzg3j++vFwd/AOJRSbk4U9Lw==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + + '@chainsafe/libp2p-quic@2.0.0': + resolution: {integrity: sha512-RNxHNdo22XhS5DEFbT0fAaZFfrEmuIVRP5InqgMq8md6r0f61AATZdmYqAuNkcj6VSBareLlCAJ/iuhhNZYt7A==} + engines: {node: '>= 22'} + '@chainsafe/netmask@2.0.0': resolution: {integrity: sha512-I3Z+6SWUoaljh3TBzCnCxjlUyN8tA+NAk5L6m9IxvCf1BENQTePzPMis97CoN/iMW1St3WN+AWCCRp+TTBRiDg==} @@ -6122,6 +6175,9 @@ packages: race-event@1.6.1: resolution: {integrity: sha512-vi7WH5g5KoTFpu2mme/HqZiWH14XSOtg5rfp6raBskBHl7wnmy3F/biAIyY5MsK+BHWhoPhxtZ1Y2R7OHHaWyQ==} + race-signal@1.1.3: + resolution: {integrity: sha512-Mt2NznMgepLfORijhQMncE26IhkmjEphig+/1fKC0OtaKwys/gpvpmswSjoN01SS+VO951mj0L4VIDXdXsjnfA==} + race-signal@2.0.0: resolution: {integrity: sha512-P31bLhE4ByBX/70QDXMutxnqgwrF1WUXea1O8DXuviAgkdbQ1iQMQotNgzJIBC9yUSn08u/acZrMUhgw7w6GpA==} @@ -7823,6 +7879,46 @@ snapshots: uint8arrays: 5.1.0 wherearewe: 2.0.1 + '@chainsafe/libp2p-quic-darwin-arm64@2.0.0': + optional: true + + '@chainsafe/libp2p-quic-darwin-x64@2.0.0': + optional: true + + '@chainsafe/libp2p-quic-linux-arm64-gnu@2.0.0': + optional: true + + '@chainsafe/libp2p-quic-linux-arm64-musl@2.0.0': + optional: true + + '@chainsafe/libp2p-quic-linux-x64-gnu@2.0.0': + optional: true + + '@chainsafe/libp2p-quic-linux-x64-musl@2.0.0': + optional: true + + '@chainsafe/libp2p-quic-win32-x64-msvc@2.0.0': + optional: true + + '@chainsafe/libp2p-quic@2.0.0': + dependencies: + '@libp2p/crypto': 5.1.13 + '@libp2p/interface': 3.1.0 + '@libp2p/peer-id': 6.0.4 + '@libp2p/utils': 7.0.12 + '@multiformats/multiaddr': 13.0.1 + '@multiformats/multiaddr-matcher': 3.0.1 + race-signal: 1.1.3 + uint8arraylist: 2.4.8 + optionalDependencies: + '@chainsafe/libp2p-quic-darwin-arm64': 2.0.0 + '@chainsafe/libp2p-quic-darwin-x64': 2.0.0 + '@chainsafe/libp2p-quic-linux-arm64-gnu': 2.0.0 + '@chainsafe/libp2p-quic-linux-arm64-musl': 2.0.0 + '@chainsafe/libp2p-quic-linux-x64-gnu': 2.0.0 + '@chainsafe/libp2p-quic-linux-x64-musl': 2.0.0 + '@chainsafe/libp2p-quic-win32-x64-msvc': 2.0.0 + '@chainsafe/netmask@2.0.0': dependencies: '@chainsafe/is-ip': 2.1.0 @@ -13430,6 +13526,8 @@ snapshots: dependencies: abort-error: 1.0.1 + race-signal@1.1.3: {} + race-signal@2.0.0: {} random-int@3.1.0: {} From 525548199c42c6029909538b42bee689af9d7463 Mon Sep 17 00:00:00 2001 From: Lodekeeper <258435968+lodekeeper@users.noreply.github.com> Date: Sat, 14 Mar 2026 10:22:42 +0000 Subject: [PATCH 42/63] chore: bump libp2p deps to latest published versions (#9035) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Bump `libp2p` and direct `@libp2p/*` dependencies in Lodestar to the latest published versions currently on npm. ### Updated direct deps - `libp2p`: `3.1.5` -> `3.1.6` - `@libp2p/bootstrap`: `^12.0.13` -> `^12.0.14` - `@libp2p/gossipsub`: `^15.0.14` -> `^15.0.15` - `@libp2p/identify`: `^4.0.12` -> `^4.0.13` - `@libp2p/mdns`: `^12.0.13` -> `^12.0.14` - `@libp2p/mplex`: `^12.0.13` -> `^12.0.14` - `@libp2p/prometheus-metrics`: `^5.0.13` -> `^5.0.14` - `@libp2p/tcp`: `^11.0.12` -> `^11.0.13` - `@libp2p/interface-internal`: `^3.0.12` -> `^3.0.13` - `@libp2p/utils`: `^7.0.12` -> `^7.0.13` Lockfile refreshed accordingly. ## Validation - `pnpm build` - `pnpm lint` *(warning-only: pre-existing unused biome suppression in `packages/light-client/test/unit/webEsmBundle.browser.test.ts`)* ## AI disclosure This PR was authored with AI assistance (Lodekeeper 🌟). Co-authored-by: lodekeeper --- package.json | 4 +- packages/beacon-node/package.json | 20 +-- packages/reqresp/package.json | 4 +- pnpm-lock.yaml | 214 +++++++++++++++--------------- 4 files changed, 121 insertions(+), 121 deletions(-) diff --git a/package.json b/package.json index 5323cf833559..6d560f3cd853 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,7 @@ "@lodestar/params": "workspace:", "@types/node": "^24.10.1", "@types/react": "^19.1.12", + "@typescript/native-preview": "7.0.0-dev.20260303.1", "@vitest/browser": "catalog:", "@vitest/browser-playwright": "catalog:", "@vitest/coverage-v8": "catalog:", @@ -64,7 +65,7 @@ "https-browserify": "^1.0.0", "inquirer": "^9.1.5", "jsdom": "^23.0.1", - "libp2p": "3.1.5", + "libp2p": "3.1.6", "node-gyp": "^9.4.0", "npm-run-all": "^4.1.5", "path-browserify": "^1.0.1", @@ -79,7 +80,6 @@ "supertest": "^6.3.3", "ts-node": "^11.0.0-beta.1", "typescript": "^6.0.1-rc", - "@typescript/native-preview": "7.0.0-dev.20260303.1", "vite": "^6.0.11", "vite-plugin-dts": "^4.5.4", "vite-plugin-node-polyfills": "^0.24.0", diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index 66dc19f1285d..11cc8b355d59 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -125,16 +125,16 @@ "@fastify/cors": "^10.0.1", "@fastify/swagger": "^9.0.0", "@fastify/swagger-ui": "^5.0.1", - "@libp2p/bootstrap": "^12.0.13", + "@libp2p/bootstrap": "^12.0.14", "@libp2p/crypto": "^5.1.13", - "@libp2p/gossipsub": "^15.0.14", - "@libp2p/identify": "^4.0.12", + "@libp2p/gossipsub": "^15.0.15", + "@libp2p/identify": "^4.0.13", "@libp2p/interface": "^3.1.0", - "@libp2p/mdns": "^12.0.13", - "@libp2p/mplex": "^12.0.13", + "@libp2p/mdns": "^12.0.14", + "@libp2p/mplex": "^12.0.14", "@libp2p/peer-id": "^6.0.4", - "@libp2p/prometheus-metrics": "^5.0.13", - "@libp2p/tcp": "^11.0.12", + "@libp2p/prometheus-metrics": "^5.0.14", + "@libp2p/tcp": "^11.0.13", "@lodestar/api": "workspace:^", "@lodestar/config": "workspace:^", "@lodestar/db": "workspace:^", @@ -155,7 +155,7 @@ "fastify": "^5.8.1", "interface-datastore": "^9.0.2", "jwt-simple": "0.5.6", - "libp2p": "3.1.5", + "libp2p": "3.1.6", "multiformats": "^13.4.2", "prom-client": "^15.1.0", "qs": "^6.11.1", @@ -166,9 +166,9 @@ }, "devDependencies": { "@chainsafe/swap-or-not-shuffle": "^1.2.1", - "@libp2p/interface-internal": "^3.0.12", + "@libp2p/interface-internal": "^3.0.13", "@libp2p/logger": "^6.2.2", - "@libp2p/utils": "^7.0.12", + "@libp2p/utils": "^7.0.13", "@lodestar/spec-test-util": "workspace:^", "@types/js-yaml": "^4.0.5", "@types/qs": "^6.9.7", diff --git a/packages/reqresp/package.json b/packages/reqresp/package.json index a17eedcd812b..f34eef7498f9 100644 --- a/packages/reqresp/package.json +++ b/packages/reqresp/package.json @@ -46,7 +46,7 @@ "dependencies": { "@chainsafe/fast-crc32c": "^4.2.0", "@libp2p/interface": "^3.1.0", - "@libp2p/utils": "^7.0.12", + "@libp2p/utils": "^7.0.13", "@lodestar/config": "workspace:^", "@lodestar/params": "workspace:^", "@lodestar/utils": "workspace:^", @@ -62,7 +62,7 @@ "@libp2p/peer-id": "^6.0.4", "@lodestar/logger": "workspace:^", "@lodestar/types": "workspace:^", - "libp2p": "3.1.5" + "libp2p": "3.1.6" }, "peerDependencies": { "libp2p": "^3.1.4" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index cf033de56844..cab730877593 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -108,8 +108,8 @@ importers: specifier: ^23.0.1 version: 23.0.1 libp2p: - specifier: 3.1.5 - version: 3.1.5 + specifier: 3.1.6 + version: 3.1.6 node-gyp: specifier: ^9.4.0 version: 9.4.1 @@ -268,35 +268,35 @@ importers: specifier: ^5.0.1 version: 5.0.1 '@libp2p/bootstrap': - specifier: ^12.0.13 - version: 12.0.13 + specifier: ^12.0.14 + version: 12.0.14 '@libp2p/crypto': specifier: ^5.1.13 version: 5.1.13 '@libp2p/gossipsub': - specifier: ^15.0.14 - version: 15.0.14 + specifier: ^15.0.15 + version: 15.0.15 '@libp2p/identify': - specifier: ^4.0.12 - version: 4.0.12 + specifier: ^4.0.13 + version: 4.0.13 '@libp2p/interface': specifier: ^3.1.0 version: 3.1.0 '@libp2p/mdns': - specifier: ^12.0.13 - version: 12.0.13 + specifier: ^12.0.14 + version: 12.0.14 '@libp2p/mplex': - specifier: ^12.0.13 - version: 12.0.13 + specifier: ^12.0.14 + version: 12.0.14 '@libp2p/peer-id': specifier: ^6.0.4 version: 6.0.4 '@libp2p/prometheus-metrics': - specifier: ^5.0.13 - version: 5.0.13 + specifier: ^5.0.14 + version: 5.0.14 '@libp2p/tcp': - specifier: ^11.0.12 - version: 11.0.12 + specifier: ^11.0.13 + version: 11.0.13 '@lodestar/api': specifier: workspace:^ version: link:../api @@ -358,8 +358,8 @@ importers: specifier: 0.5.6 version: 0.5.6 libp2p: - specifier: 3.1.5 - version: 3.1.5 + specifier: 3.1.6 + version: 3.1.6 multiformats: specifier: ^13.4.2 version: 13.4.2 @@ -386,14 +386,14 @@ importers: specifier: ^1.2.1 version: 1.2.1 '@libp2p/interface-internal': - specifier: ^3.0.12 - version: 3.0.12 + specifier: ^3.0.13 + version: 3.0.13 '@libp2p/logger': specifier: ^6.2.2 version: 6.2.2 '@libp2p/utils': - specifier: ^7.0.12 - version: 7.0.12 + specifier: ^7.0.13 + version: 7.0.13 '@lodestar/spec-test-util': specifier: workspace:^ version: link:../spec-test-util @@ -899,8 +899,8 @@ importers: specifier: ^3.1.0 version: 3.1.0 '@libp2p/utils': - specifier: ^7.0.12 - version: 7.0.12 + specifier: ^7.0.13 + version: 7.0.13 '@lodestar/config': specifier: workspace:^ version: link:../config @@ -942,8 +942,8 @@ importers: specifier: workspace:^ version: link:../types libp2p: - specifier: 3.1.5 - version: 3.1.5 + specifier: 3.1.6 + version: 3.1.6 packages/spec-test-util: dependencies: @@ -964,7 +964,7 @@ importers: version: 0.7.0 vitest: specifier: 'catalog:' - version: 4.0.7(@types/node@25.3.3)(@vitest/browser-playwright@4.0.7)(jsdom@23.0.1)(yaml@2.8.2) + version: 4.0.7(@types/node@25.4.0)(@vitest/browser-playwright@4.0.7)(jsdom@23.0.1)(yaml@2.8.2) packages/state-transition: dependencies: @@ -1031,7 +1031,7 @@ importers: version: 0.2.4 vitest: specifier: 'catalog:' - version: 4.0.7(@types/node@25.3.3)(@vitest/browser-playwright@4.0.7)(jsdom@23.0.1)(yaml@2.8.2) + version: 4.0.7(@types/node@25.4.0)(@vitest/browser-playwright@4.0.7)(jsdom@23.0.1)(yaml@2.8.2) devDependencies: '@types/tmp': specifier: ^0.2.3 @@ -2256,21 +2256,21 @@ packages: resolution: {integrity: sha512-hRFGB47RWHNKZkh9t+ka34kNlv5DQwrIDBvjI+mmzZqZK7UAtPRsJoE+MlQHVlQApDFwUPiSTdxY9UXzJrYFdQ==} engines: {node: ^20.17.0 || >=22.9.0} - '@libp2p/bootstrap@12.0.13': - resolution: {integrity: sha512-txdW/5IK/2gm76NpnRYcTpZgaT7FIH/t59171ENc1t2lnvwXeaP6l/6bTXX23TSDCINuZUHOv0Hm8f+CpGNB8Q==} + '@libp2p/bootstrap@12.0.14': + resolution: {integrity: sha512-kVg/t303ac6l7eSo5M1oZs72cs54FRRjYhmXwvHbPEz5v7NgglDpnNgz7ScBJGaS/z4Xn5qeyghVZFc8Qz7BaA==} '@libp2p/crypto@5.1.13': resolution: {integrity: sha512-8NN9cQP3jDn+p9+QE9ByiEoZ2lemDFf/unTgiKmS3JF93ph240EUVdbCyyEgOMfykzb0okTM4gzvwfx9osJebQ==} - '@libp2p/gossipsub@15.0.14': - resolution: {integrity: sha512-F+Rjyz6sPvKvZr588D1l7rRZPefPTOxvVYtH3/WXUyu4h6EA+trwMax/WOrbrAYopfwJd++/1RXHMEuZpGzirg==} + '@libp2p/gossipsub@15.0.15': + resolution: {integrity: sha512-uZLQj2EH0cDPX8rLgXDjJNL8sY1wYupGfvrY2oPUV/QsyZZQLqJ/a1BrYf1SjWCIqfqqGvyTtpBO8XvK4Ln8OA==} engines: {npm: '>=8.7.0'} - '@libp2p/identify@4.0.12': - resolution: {integrity: sha512-jolOVcJ9zj4306btDuDQw5mZId4tE20UndS4w5+v+KhXcNZWsntZ6p/F0l3AgOjPRxHUw1d0VabQQEKjHDbOjw==} + '@libp2p/identify@4.0.13': + resolution: {integrity: sha512-/zAhl2yMuQeHMyghJZDBRvQ1l3fRwxbsq3zPhT5nDscu9qVDa/CB4xDwquV4jV2Y/pnvefZtngJgj5c+bBIxug==} - '@libp2p/interface-internal@3.0.12': - resolution: {integrity: sha512-WfUwypkIEwy0YwS05a1E8o7VicbUa259rSjzSpbGFFw8dlVqGCaKKU+6hj4mSbkXhRWq56xXkvt9MjzdEZ/fcA==} + '@libp2p/interface-internal@3.0.13': + resolution: {integrity: sha512-qZTn1CKOro/1m8Eizb/B1pUvW/eJe5KhP/dvqKETqka26qH89eX5SlTS1OPTINXzJvfbnDFptVJOPxmpa3BfgA==} '@libp2p/interface@3.1.0': resolution: {integrity: sha512-RE7/XyvC47fQBe1cHxhMvepYKa5bFCUyFrrpj8PuM0E7JtzxU7F+Du5j4VXbg2yLDcToe0+j8mB7jvwE2AThYw==} @@ -2278,17 +2278,17 @@ packages: '@libp2p/logger@6.2.2': resolution: {integrity: sha512-XtanXDT+TuMuZoCK760HGV1AmJsZbwAw5AiRUxWDbsZPwAroYq64nb41AHRu9Gyc0TK9YD+p72+5+FIxbw0hzw==} - '@libp2p/mdns@12.0.13': - resolution: {integrity: sha512-XDQbX8PBvGWHeL+pvGGTGB2NYY2ib1lTZg7e4CV3pqnK9TxizIVQNtp7sdRx9YpxvvJdxuBtHezUZT714VtLJw==} + '@libp2p/mdns@12.0.14': + resolution: {integrity: sha512-tGbpZO7veBUQTOsIAHtXhFRLqGP3ek4BN+aE2f9yCVvuX//AmDX5HZLhjuAyd+lLabhaNv4+2MFeNqexC1P6/w==} - '@libp2p/mplex@12.0.13': - resolution: {integrity: sha512-fDscM89OmkQngE/CUMDvdQDAP3GLp7lqC/95OXvl/fmNxQBDd1mV3UoHsmmrbcVI0auinC/keIqBGZG9NJGDqQ==} + '@libp2p/mplex@12.0.14': + resolution: {integrity: sha512-aPa1nEvxkWYAYmIQuE2FdMHrgHWgF3j7E/X82M87cT8irBFQRwWFCjiE/gsxw+Pec09uMubTbSROHHbPxjd6Wg==} - '@libp2p/multistream-select@7.0.12': - resolution: {integrity: sha512-fYvGllfzb3J5CJqM8E/uKmcsPqQc0mNR5zdYDFhYpqz7YjPrx2r9tYXisDrH8J2nW71a68RMPfRavSwi7Bs1DQ==} + '@libp2p/multistream-select@7.0.13': + resolution: {integrity: sha512-nX13GinXiuBFgN+zA/CvIyXZyR/DaftT26agsw6dDfhRvH2RWsoPvf0IGqxk90DsLhpmVxZnTE31rITjmLIKww==} - '@libp2p/peer-collections@7.0.12': - resolution: {integrity: sha512-1QtoSWo/vJv2pXDjUW1/QYMtYS4WzmJT9LPjBqoosKC6fJUiMoumcF9uh0bZKKDH74Biy5ZEKx6kaW18Azwsnw==} + '@libp2p/peer-collections@7.0.13': + resolution: {integrity: sha512-SwNQFT0tfSyfbdUUKZFzHv9DXxsabuT99ch/40as8qC7xgoJJfUmhoa9FSuAuABdpTVHDJmxCI2pIbcb1kBqfg==} '@libp2p/peer-id@6.0.4': resolution: {integrity: sha512-Z3xK0lwwKn4bPg3ozEpPr1HxsRi2CxZdghOL+MXoFah/8uhJJHxHFA8A/jxtKn4BB8xkk6F8R5vKNIS05yaCYw==} @@ -2296,17 +2296,17 @@ packages: '@libp2p/peer-record@9.0.5': resolution: {integrity: sha512-disk23OO00yD52O4VmItbDkjJZ/YZJsKbMsqNgVhr+D3PcM+KRpu9VVbiCnN5Tzn9XvFEHhrMJY7BPE+rvT5MQ==} - '@libp2p/peer-store@12.0.12': - resolution: {integrity: sha512-Ra4yqaE4RSSmYSbfWLw+Gfi6Zkb/Khf4r5tAZiafO5UyX8mBnaDqTnL1iInUwTrHVJX/neB/lfLYLD9m6Yb89g==} + '@libp2p/peer-store@12.0.13': + resolution: {integrity: sha512-hXiIrXEUlXNDJe7i0O32qXRGmLA3ckoRHDjGZcNKMzPnkRDPkGEUQ42v1keA+1QoysMkm95xYyyhF6S3dA6nxg==} - '@libp2p/prometheus-metrics@5.0.13': - resolution: {integrity: sha512-VovjUEMzolaKtxGUn+AY7YCNjMgh7gX2rjjoUyWBBZ/tv+KJjX0kJZ5tHVpwqrOsXsuIzRl9oRiaPLQ+m0ZtRA==} + '@libp2p/prometheus-metrics@5.0.14': + resolution: {integrity: sha512-v9IkdtvctmY+LVI6tzfjzGpLCEbLyncd4YH54l2ki+5w/U1oXPMTUbO01A94Zlywtt5lTKakQxxOnuliuRBfFA==} - '@libp2p/tcp@11.0.12': - resolution: {integrity: sha512-oz77ksfE/eQbpz2Vkx8Kr6rklOLkhDq0NC0Plc+ScIqTyYbnWJ6L3tMd/+3vaEArF+uKeeeBA7VfGgI1+TOzaQ==} + '@libp2p/tcp@11.0.13': + resolution: {integrity: sha512-YTV6rX1NpQVbixYDlsWtIAHALBkW4vYdk4DPmHbFyvvQJdtBofbeHIQakyjPexKrbUtDGaoIXgT5KC2osLRp0g==} - '@libp2p/utils@7.0.12': - resolution: {integrity: sha512-+qHXhPubw1KJtuzNH2IR02frGJxebnD/wrDIP+0GEwfUS3h76dN0TzPWLjSghzDQaUybbzlaCHTsN6c3OVcj4Q==} + '@libp2p/utils@7.0.13': + resolution: {integrity: sha512-BZAn+60HN8EDNwevWhiblopDXFpQ5Z5FnbML73btZKsJ9GGa4yQ2R18CTpWfsbHL8LUo21gTRC0XxUpYWq+UZg==} '@lukeed/ms@2.0.2': resolution: {integrity: sha512-9I2Zn6+NJLfaGoz9jN3lpwDgAYvfGeNYdbAIjJOqzs4Tpc+VU3Jqq4IofSUBKajiDS8k9fZIg18/z13mpk1bsA==} @@ -3184,8 +3184,8 @@ packages: '@types/node@24.10.1': resolution: {integrity: sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ==} - '@types/node@25.3.3': - resolution: {integrity: sha512-DpzbrH7wIcBaJibpKo9nnSQL0MTRdnWttGyE5haGwK86xgMOkFLp7vEyfQPGLOJh5wNYiJ3V9PmUMDhV9u8kkQ==} + '@types/node@25.4.0': + resolution: {integrity: sha512-9wLpoeWuBlcbBpOY3XmzSTG3oscB6xjBEEtn+pYXTfhyXhIxC5FsBer2KTopBlvKEiW9l13po9fq+SJY/5lkhw==} '@types/normalize-package-data@2.4.4': resolution: {integrity: sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA==} @@ -5242,8 +5242,8 @@ packages: resolution: {integrity: sha512-NVPTth/71cfbdYHqypcO9Lt5WFGTzFEcx81lWd7GDJIgZ95ERdYHGUfCtFejHCyqodKsQkNEx2JCkMpreDty/A==} engines: {node: ^20.17.0 || >=22.9.0} - libp2p@3.1.5: - resolution: {integrity: sha512-C+ZL03De0WFr9e09A27hvoCmlvWciZ9FyXVonuXUgPEoob5XyTxjJedfmQ2gXxkh2Pqz1W2vZSUqi9f0BYIT+A==} + libp2p@3.1.6: + resolution: {integrity: sha512-p1Tg8htMjQbbyNOQd5GtSsZJXKkJQYQBvRrPGMCa3PZBjGs2pNV4Utr7z0na+WgfJJn+mIbcNvP7NzzcrSD1nw==} light-my-request@6.6.0: resolution: {integrity: sha512-CHYbu8RtboSIoVsHZ6Ye4cj4Aw/yg2oAFimlF7mNvfDV192LR7nDiKtSIfCuLT7KokPSTn/9kfVLm5OGN0A28A==} @@ -7870,7 +7870,7 @@ snapshots: '@libp2p/crypto': 5.1.13 '@libp2p/interface': 3.1.0 '@libp2p/peer-id': 6.0.4 - '@libp2p/utils': 7.0.12 + '@libp2p/utils': 7.0.13 '@noble/ciphers': 2.1.1 '@noble/curves': 2.0.1 '@noble/hashes': 2.0.1 @@ -9022,10 +9022,10 @@ snapshots: - conventional-commits-filter - supports-color - '@libp2p/bootstrap@12.0.13': + '@libp2p/bootstrap@12.0.14': dependencies: '@libp2p/interface': 3.1.0 - '@libp2p/interface-internal': 3.0.12 + '@libp2p/interface-internal': 3.0.13 '@libp2p/peer-id': 6.0.4 '@multiformats/multiaddr': 13.0.1 '@multiformats/multiaddr-matcher': 3.0.1 @@ -9041,13 +9041,13 @@ snapshots: uint8arraylist: 2.4.8 uint8arrays: 5.1.0 - '@libp2p/gossipsub@15.0.14': + '@libp2p/gossipsub@15.0.15': dependencies: '@libp2p/crypto': 5.1.13 '@libp2p/interface': 3.1.0 - '@libp2p/interface-internal': 3.0.12 + '@libp2p/interface-internal': 3.0.13 '@libp2p/peer-id': 6.0.4 - '@libp2p/utils': 7.0.12 + '@libp2p/utils': 7.0.13 '@multiformats/multiaddr': 13.0.1 denque: 2.1.0 it-length-prefixed: 10.0.1 @@ -9058,14 +9058,14 @@ snapshots: uint8arraylist: 2.4.8 uint8arrays: 5.1.0 - '@libp2p/identify@4.0.12': + '@libp2p/identify@4.0.13': dependencies: '@libp2p/crypto': 5.1.13 '@libp2p/interface': 3.1.0 - '@libp2p/interface-internal': 3.0.12 + '@libp2p/interface-internal': 3.0.13 '@libp2p/peer-id': 6.0.4 '@libp2p/peer-record': 9.0.5 - '@libp2p/utils': 7.0.12 + '@libp2p/utils': 7.0.13 '@multiformats/multiaddr': 13.0.1 '@multiformats/multiaddr-matcher': 3.0.1 it-drain: 3.0.10 @@ -9075,10 +9075,10 @@ snapshots: uint8arraylist: 2.4.8 uint8arrays: 5.1.0 - '@libp2p/interface-internal@3.0.12': + '@libp2p/interface-internal@3.0.13': dependencies: '@libp2p/interface': 3.1.0 - '@libp2p/peer-collections': 7.0.12 + '@libp2p/peer-collections': 7.0.13 '@multiformats/multiaddr': 13.0.1 progress-events: 1.0.1 @@ -9099,40 +9099,40 @@ snapshots: multiformats: 13.4.2 weald: 1.1.1 - '@libp2p/mdns@12.0.13': + '@libp2p/mdns@12.0.14': dependencies: '@libp2p/interface': 3.1.0 - '@libp2p/interface-internal': 3.0.12 + '@libp2p/interface-internal': 3.0.13 '@libp2p/peer-id': 6.0.4 - '@libp2p/utils': 7.0.12 + '@libp2p/utils': 7.0.13 '@multiformats/multiaddr': 13.0.1 '@types/multicast-dns': 7.2.4 dns-packet: 5.6.1 main-event: 1.0.1 multicast-dns: 7.2.5 - '@libp2p/mplex@12.0.13': + '@libp2p/mplex@12.0.14': dependencies: '@libp2p/interface': 3.1.0 - '@libp2p/utils': 7.0.12 + '@libp2p/utils': 7.0.13 it-pushable: 3.2.3 uint8-varint: 2.0.4 uint8arraylist: 2.4.8 uint8arrays: 5.1.0 - '@libp2p/multistream-select@7.0.12': + '@libp2p/multistream-select@7.0.13': dependencies: '@libp2p/interface': 3.1.0 - '@libp2p/utils': 7.0.12 + '@libp2p/utils': 7.0.13 it-length-prefixed: 10.0.1 uint8arraylist: 2.4.8 uint8arrays: 5.1.0 - '@libp2p/peer-collections@7.0.12': + '@libp2p/peer-collections@7.0.13': dependencies: '@libp2p/interface': 3.1.0 '@libp2p/peer-id': 6.0.4 - '@libp2p/utils': 7.0.12 + '@libp2p/utils': 7.0.13 multiformats: 13.4.2 '@libp2p/peer-id@6.0.4': @@ -9154,11 +9154,11 @@ snapshots: uint8arraylist: 2.4.8 uint8arrays: 5.1.0 - '@libp2p/peer-store@12.0.12': + '@libp2p/peer-store@12.0.13': dependencies: '@libp2p/crypto': 5.1.13 '@libp2p/interface': 3.1.0 - '@libp2p/peer-collections': 7.0.12 + '@libp2p/peer-collections': 7.0.13 '@libp2p/peer-id': 6.0.4 '@libp2p/peer-record': 9.0.5 '@multiformats/multiaddr': 13.0.1 @@ -9171,15 +9171,15 @@ snapshots: uint8arraylist: 2.4.8 uint8arrays: 5.1.0 - '@libp2p/prometheus-metrics@5.0.13': + '@libp2p/prometheus-metrics@5.0.14': dependencies: '@libp2p/interface': 3.1.0 prom-client: 15.1.3 - '@libp2p/tcp@11.0.12': + '@libp2p/tcp@11.0.13': dependencies: '@libp2p/interface': 3.1.0 - '@libp2p/utils': 7.0.12 + '@libp2p/utils': 7.0.13 '@multiformats/multiaddr': 13.0.1 '@multiformats/multiaddr-matcher': 3.0.1 '@types/sinon': 20.0.0 @@ -9188,7 +9188,7 @@ snapshots: progress-events: 1.0.1 uint8arraylist: 2.4.8 - '@libp2p/utils@7.0.12': + '@libp2p/utils@7.0.13': dependencies: '@chainsafe/is-ip': 2.1.0 '@chainsafe/netmask': 2.0.0 @@ -10011,7 +10011,7 @@ snapshots: '@types/dns-packet@5.6.5': dependencies: - '@types/node': 25.3.3 + '@types/node': 25.4.0 '@types/estree@1.0.8': {} @@ -10039,7 +10039,7 @@ snapshots: '@types/multicast-dns@7.2.4': dependencies: '@types/dns-packet': 5.6.5 - '@types/node': 25.3.3 + '@types/node': 25.4.0 '@types/node@18.15.13': {} @@ -10053,7 +10053,7 @@ snapshots: dependencies: undici-types: 7.16.0 - '@types/node@25.3.3': + '@types/node@25.4.0': dependencies: undici-types: 7.18.2 @@ -10159,13 +10159,13 @@ snapshots: - utf-8-validate - vite - '@vitest/browser-playwright@4.0.7(playwright@1.56.1)(vite@6.1.6(@types/node@25.3.3)(yaml@2.8.2))(vitest@4.0.7)': + '@vitest/browser-playwright@4.0.7(playwright@1.56.1)(vite@6.1.6(@types/node@25.4.0)(yaml@2.8.2))(vitest@4.0.7)': dependencies: - '@vitest/browser': 4.0.7(vite@6.1.6(@types/node@25.3.3)(yaml@2.8.2))(vitest@4.0.7) - '@vitest/mocker': 4.0.7(vite@6.1.6(@types/node@25.3.3)(yaml@2.8.2)) + '@vitest/browser': 4.0.7(vite@6.1.6(@types/node@25.4.0)(yaml@2.8.2))(vitest@4.0.7) + '@vitest/mocker': 4.0.7(vite@6.1.6(@types/node@25.4.0)(yaml@2.8.2)) playwright: 1.56.1 tinyrainbow: 3.0.3 - vitest: 4.0.7(@types/node@25.3.3)(@vitest/browser-playwright@4.0.7)(jsdom@23.0.1)(yaml@2.8.2) + vitest: 4.0.7(@types/node@25.4.0)(@vitest/browser-playwright@4.0.7)(jsdom@23.0.1)(yaml@2.8.2) transitivePeerDependencies: - bufferutil - msw @@ -10190,16 +10190,16 @@ snapshots: - utf-8-validate - vite - '@vitest/browser@4.0.7(vite@6.1.6(@types/node@25.3.3)(yaml@2.8.2))(vitest@4.0.7)': + '@vitest/browser@4.0.7(vite@6.1.6(@types/node@25.4.0)(yaml@2.8.2))(vitest@4.0.7)': dependencies: - '@vitest/mocker': 4.0.7(vite@6.1.6(@types/node@25.3.3)(yaml@2.8.2)) + '@vitest/mocker': 4.0.7(vite@6.1.6(@types/node@25.4.0)(yaml@2.8.2)) '@vitest/utils': 4.0.7 magic-string: 0.30.21 pixelmatch: 7.1.0 pngjs: 7.0.0 sirv: 3.0.2 tinyrainbow: 3.0.3 - vitest: 4.0.7(@types/node@25.3.3)(@vitest/browser-playwright@4.0.7)(jsdom@23.0.1)(yaml@2.8.2) + vitest: 4.0.7(@types/node@25.4.0)(@vitest/browser-playwright@4.0.7)(jsdom@23.0.1)(yaml@2.8.2) ws: 8.18.3 transitivePeerDependencies: - bufferutil @@ -10244,13 +10244,13 @@ snapshots: optionalDependencies: vite: 6.1.6(@types/node@24.10.1)(yaml@2.8.2) - '@vitest/mocker@4.0.7(vite@6.1.6(@types/node@25.3.3)(yaml@2.8.2))': + '@vitest/mocker@4.0.7(vite@6.1.6(@types/node@25.4.0)(yaml@2.8.2))': dependencies: '@vitest/spy': 4.0.7 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 6.1.6(@types/node@25.3.3)(yaml@2.8.2) + vite: 6.1.6(@types/node@25.4.0)(yaml@2.8.2) '@vitest/pretty-format@2.1.8': dependencies: @@ -12499,19 +12499,19 @@ snapshots: transitivePeerDependencies: - supports-color - libp2p@3.1.5: + libp2p@3.1.6: dependencies: '@chainsafe/is-ip': 2.1.0 '@chainsafe/netmask': 2.0.0 '@libp2p/crypto': 5.1.13 '@libp2p/interface': 3.1.0 - '@libp2p/interface-internal': 3.0.12 + '@libp2p/interface-internal': 3.0.13 '@libp2p/logger': 6.2.2 - '@libp2p/multistream-select': 7.0.12 - '@libp2p/peer-collections': 7.0.12 + '@libp2p/multistream-select': 7.0.13 + '@libp2p/peer-collections': 7.0.13 '@libp2p/peer-id': 6.0.4 - '@libp2p/peer-store': 12.0.12 - '@libp2p/utils': 7.0.12 + '@libp2p/peer-store': 12.0.13 + '@libp2p/utils': 7.0.13 '@multiformats/dns': 1.0.13 '@multiformats/multiaddr': 13.0.1 '@multiformats/multiaddr-matcher': 3.0.1 @@ -14485,13 +14485,13 @@ snapshots: fsevents: 2.3.3 yaml: 2.8.2 - vite@6.1.6(@types/node@25.3.3)(yaml@2.8.2): + vite@6.1.6(@types/node@25.4.0)(yaml@2.8.2): dependencies: esbuild: 0.24.2 postcss: 8.5.6 rollup: 4.52.5 optionalDependencies: - '@types/node': 25.3.3 + '@types/node': 25.4.0 fsevents: 2.3.3 yaml: 2.8.2 @@ -14542,10 +14542,10 @@ snapshots: - tsx - yaml - vitest@4.0.7(@types/node@25.3.3)(@vitest/browser-playwright@4.0.7)(jsdom@23.0.1)(yaml@2.8.2): + vitest@4.0.7(@types/node@25.4.0)(@vitest/browser-playwright@4.0.7)(jsdom@23.0.1)(yaml@2.8.2): dependencies: '@vitest/expect': 4.0.7 - '@vitest/mocker': 4.0.7(vite@6.1.6(@types/node@25.3.3)(yaml@2.8.2)) + '@vitest/mocker': 4.0.7(vite@6.1.6(@types/node@25.4.0)(yaml@2.8.2)) '@vitest/pretty-format': 4.0.7 '@vitest/runner': 4.0.7 '@vitest/snapshot': 4.0.7 @@ -14562,11 +14562,11 @@ snapshots: tinyexec: 0.3.2 tinyglobby: 0.2.15 tinyrainbow: 3.0.3 - vite: 6.1.6(@types/node@25.3.3)(yaml@2.8.2) + vite: 6.1.6(@types/node@25.4.0)(yaml@2.8.2) why-is-node-running: 2.3.0 optionalDependencies: - '@types/node': 25.3.3 - '@vitest/browser-playwright': 4.0.7(playwright@1.56.1)(vite@6.1.6(@types/node@25.3.3)(yaml@2.8.2))(vitest@4.0.7) + '@types/node': 25.4.0 + '@vitest/browser-playwright': 4.0.7(playwright@1.56.1)(vite@6.1.6(@types/node@25.4.0)(yaml@2.8.2))(vitest@4.0.7) jsdom: 23.0.1 transitivePeerDependencies: - jiti From c11b797bd61763a3bc7dbcabba0119da12dbafbf Mon Sep 17 00:00:00 2001 From: Lodekeeper <258435968+lodekeeper@users.noreply.github.com> Date: Sat, 14 Mar 2026 10:46:19 +0000 Subject: [PATCH 43/63] chore: update lockfile for @libp2p/utils 7.0.13 (#9037) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Follow-up to #9035 — `@libp2p/utils` 7.0.13 was published after the lockfile was generated, causing `pnpm install` to produce a dirty lockfile on CI. ### Changes - Regenerated `pnpm-lock.yaml` to pick up `@libp2p/utils` 7.0.12 → 7.0.13 (transitive dep of `@libp2p/peer-store`) > [!NOTE] > This PR was authored with AI assistance (Lodekeeper 🌟) Co-authored-by: lodekeeper --- pnpm-lock.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index cab730877593..cb36fab99030 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -7905,7 +7905,7 @@ snapshots: '@libp2p/crypto': 5.1.13 '@libp2p/interface': 3.1.0 '@libp2p/peer-id': 6.0.4 - '@libp2p/utils': 7.0.12 + '@libp2p/utils': 7.0.13 '@multiformats/multiaddr': 13.0.1 '@multiformats/multiaddr-matcher': 3.0.1 race-signal: 1.1.3 From 6a85a50c16636341b55a434657a997a96bf38da6 Mon Sep 17 00:00:00 2001 From: Chen Kai <281165273grape@gmail.com> Date: Sun, 15 Mar 2026 21:32:03 -0400 Subject: [PATCH 44/63] perf(fork-choice): hoist proposerBoostRoot out of applyScoreChanges loop (#9040) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description In `applyScoreChanges`, the expression `proposerBoost?.root ?? null` was being evaluated on every iteration of the second loop (which iterates over all proto-array nodes in reverse). Since `proposerBoost` does not change during the loop, this is a loop-invariant computation that can be hoisted out. This is a micro-optimization but it applies to a hot path — `applyScoreChanges` runs on every attestation/block import and iterates over all fork-choice nodes. ### Changes - Hoist `proposerBoost?.root ?? null` into a `const proposerBoostRoot` before the loop - Pass the pre-computed value to `maybeUpdateBestChildAndDescendant` > This PR was authored with AI assistance (GitHub Copilot). --- packages/fork-choice/src/protoArray/protoArray.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/fork-choice/src/protoArray/protoArray.ts b/packages/fork-choice/src/protoArray/protoArray.ts index 683ba6b38145..57afd0d2a1e9 100644 --- a/packages/fork-choice/src/protoArray/protoArray.ts +++ b/packages/fork-choice/src/protoArray/protoArray.ts @@ -391,6 +391,7 @@ export class ProtoArray { // We _must_ perform these functions separate from the weight-updating loop above to ensure // that we have a fully coherent set of weights before updating parent // best-child/descendant. + const proposerBoostRoot = proposerBoost?.root ?? null; for (let nodeIndex = this.nodes.length - 1; nodeIndex >= 0; nodeIndex--) { const node = this.nodes[nodeIndex]; if (node === undefined) { @@ -403,7 +404,7 @@ export class ProtoArray { // If the node has a parent, try to update its best-child and best-descendant. const parentIndex = node.parent; if (parentIndex !== undefined) { - this.maybeUpdateBestChildAndDescendant(parentIndex, nodeIndex, currentSlot, proposerBoost?.root ?? null); + this.maybeUpdateBestChildAndDescendant(parentIndex, nodeIndex, currentSlot, proposerBoostRoot); } } // Update the previous proposer boost From 241ad7952a07cfa460900b9362f7e7b130c2fa17 Mon Sep 17 00:00:00 2001 From: Lodekeeper <258435968+lodekeeper@users.noreply.github.com> Date: Mon, 16 Mar 2026 17:22:02 +0000 Subject: [PATCH 45/63] chore: bump @chainsafe/discv5 to v12.0.1 and @chainsafe/enr to v6.0.1 (#9049) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation Fixes #8808 — Lodestar nodes running dual-stack (IPv4 + IPv6) fail to auto-discover their external IPv6 address in the ENR. ## Description Bumps `@chainsafe/discv5` to v12.0.1 ([ChainSafe/discv5#334](https://github.com/ChainSafe/discv5/pull/334)) and `@chainsafe/enr` to v6.0.1. ### Root causes fixed in discv5 v12.0.1 1. **Single vote pool** — IPv4 votes reach threshold first, `addVote()` calls `clear()` which wipes accumulated IPv6 votes 2. **No immediate ping on session establishment** — first PONG votes arrive only after `pingInterval` (5 min default) 3. **IPv4-mapped IPv6 addresses pollute IPv6 pool** — some peers report IPv4 as `::ffff:x.x.x.x`, routing to the wrong pool ### Changes in this PR - Bump `@chainsafe/discv5` from `^12.0.0` to `^12.0.1` - Bump `@chainsafe/enr` from `^6.0.0` to `^6.0.1` (required by discv5 v12.0.1) - Add `@chainsafe/discv5` and `@chainsafe/enr` to `minimumReleaseAgeExclude` in `pnpm-workspace.yaml` ### Testing Ran a local Lodestar beacon node (checkpoint sync + engineMock) on mainnet with the bumped discv5: - IPv4 discovered in ~1 min ✅ - IPv6 discovered in ~2 min ✅ - Both addresses stable for 8+ minutes across all samples - 75-92 connected peers *This PR was authored by an AI contributor — @lodekeeper, with human review by @nflaig.* --------- Co-authored-by: lodekeeper --- packages/beacon-node/package.json | 4 +-- packages/cli/package.json | 4 +-- pnpm-lock.yaml | 46 ++++++++++++++----------------- pnpm-workspace.yaml | 1 + 4 files changed, 25 insertions(+), 30 deletions(-) diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index 11cc8b355d59..098b2910e7c0 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -110,8 +110,8 @@ "dependencies": { "@chainsafe/as-sha256": "^1.2.0", "@chainsafe/blst": "^2.2.0", - "@chainsafe/discv5": "^12.0.0", - "@chainsafe/enr": "^6.0.0", + "@chainsafe/discv5": "^12.0.1", + "@chainsafe/enr": "^6.0.1", "@chainsafe/libp2p-noise": "^17.0.0", "@chainsafe/libp2p-quic": "^2.0.0", "@chainsafe/persistent-merkle-tree": "^1.2.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index a41ae747d102..2585e144803b 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -60,8 +60,8 @@ "@chainsafe/bls-keygen": "^0.4.0", "@chainsafe/bls-keystore": "^3.1.0", "@chainsafe/blst": "^2.2.0", - "@chainsafe/discv5": "^12.0.0", - "@chainsafe/enr": "^6.0.0", + "@chainsafe/discv5": "^12.0.1", + "@chainsafe/enr": "^6.0.1", "@chainsafe/persistent-merkle-tree": "^1.2.1", "@chainsafe/pubkey-index-map": "^3.0.0", "@chainsafe/ssz": "^1.2.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index cb36fab99030..4cad8ddeeaa0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -223,11 +223,11 @@ importers: specifier: ^2.2.0 version: 2.2.0 '@chainsafe/discv5': - specifier: ^12.0.0 - version: 12.0.0 + specifier: ^12.0.1 + version: 12.0.1 '@chainsafe/enr': - specifier: ^6.0.0 - version: 6.0.0 + specifier: ^6.0.1 + version: 6.0.1 '@chainsafe/libp2p-noise': specifier: ^17.0.0 version: 17.0.0 @@ -437,11 +437,11 @@ importers: specifier: ^2.2.0 version: 2.2.0 '@chainsafe/discv5': - specifier: ^12.0.0 - version: 12.0.0 + specifier: ^12.0.1 + version: 12.0.1 '@chainsafe/enr': - specifier: ^6.0.0 - version: 6.0.0 + specifier: ^6.0.1 + version: 6.0.1 '@chainsafe/persistent-merkle-tree': specifier: ^1.2.1 version: 1.2.1 @@ -1387,11 +1387,11 @@ packages: resolution: {integrity: sha512-VBaQoNE2a9d9+skAjQKv3Suk0yGKqp3mZM0YWYJNPj/Ae/f6lAyeVSgKqo2LrsNQBzD/LqrJLKUY8rJT3vDKLA==} engines: {node: '>= 16'} - '@chainsafe/discv5@12.0.0': - resolution: {integrity: sha512-ESP0UU1K9WySq35/9d6usB1B3jNP2QYtlbknriddQFwWMtp7bBY9aIK4X2/g6ccutD1ILt1RYxdBrccj555nXg==} + '@chainsafe/discv5@12.0.1': + resolution: {integrity: sha512-isFDtl+DiBOD6R0cV4+4Zch25zlj5mv5nytw2x6c+eR6xDte7xlB6+/UutjfnSScMH/CwysytQCiOi8/pMxrTg==} - '@chainsafe/enr@6.0.0': - resolution: {integrity: sha512-uRnTpjJlafTQI6RinhN1EgEGZL3VMOcHUcnpDCL9LclTglKUmDB7RdTi+Qqj8tJbYEehYXhCwZiWq2dgbnpKww==} + '@chainsafe/enr@6.0.1': + resolution: {integrity: sha512-AoPgLrFAtsnIgyGx6ATzb7dRZkUmHBxA2wXAqf3sLlBeojeoaNprePcYXnU0UxB1E9Y8ZWgF7PI567sF+VrDQw==} '@chainsafe/fast-crc32c@4.2.0': resolution: {integrity: sha512-wbVJ3q1DBuYqs1VGwLe9o5VIDQtxGpSVzr7UY9Hq2SQ283fU6AouII/pwioS9MbiiIK42jDxtY0uLFmTuenb0g==} @@ -5316,10 +5316,6 @@ packages: lru-cache@10.4.3: resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} - lru-cache@11.0.1: - resolution: {integrity: sha512-CgeuL5uom6j/ZVrg7G/+1IXqRY8JXX4Hghfy5YE0EhoYQWvndP1kufu58cmZLNIDKnRhZrXfdS9urVWx98AipQ==} - engines: {node: 20 || >=22} - lru-cache@11.2.4: resolution: {integrity: sha512-B5Y16Jr9LB9dHVkh6ZevG+vAbOsNOYCX+sXvFWFu7B3Iz5mijW3zdbMyhsh8ANd2mSWBYdJgnqi+mL7/LrOPYg==} engines: {node: 20 || >=22} @@ -7805,9 +7801,9 @@ snapshots: '@chainsafe/blst-linux-x64-musl': 2.2.0 '@chainsafe/blst-win32-x64-msvc': 2.2.0 - '@chainsafe/discv5@12.0.0': + '@chainsafe/discv5@12.0.1': dependencies: - '@chainsafe/enr': 6.0.0 + '@chainsafe/enr': 6.0.1 '@ethereumjs/rlp': 10.1.1 '@libp2p/crypto': 5.1.13 '@libp2p/interface': 3.1.0 @@ -7820,7 +7816,7 @@ snapshots: strict-event-emitter-types: 2.0.0 weald: 1.1.1 - '@chainsafe/enr@6.0.0': + '@chainsafe/enr@6.0.1': dependencies: '@ethereumjs/rlp': 10.1.1 '@libp2p/crypto': 5.1.13 @@ -9424,7 +9420,7 @@ snapshots: agent-base: 7.1.3 http-proxy-agent: 7.0.0 https-proxy-agent: 7.0.6 - lru-cache: 11.2.4 + lru-cache: 11.2.5 socks-proxy-agent: 8.0.5 transitivePeerDependencies: - supports-color @@ -9480,7 +9476,7 @@ snapshots: dependencies: '@npmcli/promise-spawn': 9.0.1 ini: 6.0.0 - lru-cache: 11.2.4 + lru-cache: 11.2.5 npm-pick-manifest: 11.0.3 proc-log: 6.1.0 promise-retry: 2.0.1 @@ -10723,7 +10719,7 @@ snapshots: '@npmcli/fs': 5.0.0 fs-minipass: 3.0.1 glob: 13.0.0 - lru-cache: 11.2.4 + lru-cache: 11.2.5 minipass: 7.1.2 minipass-collect: 2.0.1 minipass-flush: 1.0.5 @@ -11953,7 +11949,7 @@ snapshots: hosted-git-info@9.0.2: dependencies: - lru-cache: 11.2.4 + lru-cache: 11.2.5 html-encoding-sniffer@4.0.0: dependencies: @@ -12610,8 +12606,6 @@ snapshots: lru-cache@10.4.3: {} - lru-cache@11.0.1: {} - lru-cache@11.2.4: {} lru-cache@11.2.5: {} @@ -13310,7 +13304,7 @@ snapshots: path-scurry@2.0.0: dependencies: - lru-cache: 11.0.1 + lru-cache: 11.2.5 minipass: 7.1.2 path-to-regexp@6.3.0: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 1bf49212904a..aeba7a2b0dcb 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -6,6 +6,7 @@ linkWorkspacePackages: true minimumReleaseAge: 2880 minimumReleaseAgeExclude: + - "@chainsafe/*" - "@libp2p/*" - "libp2p" From 21d4a81d4e6886d78ce7cb9f5780f3002b70a93d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Mar 2026 17:42:32 +0000 Subject: [PATCH 46/63] chore(deps): bump the actions group with 7 updates (#9048) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps the actions group with 7 updates: | Package | From | To | | --- | --- | --- | | [actions/cache](https://github.com/actions/cache) | `4.3.0` | `5.0.3` | | [github/codeql-action](https://github.com/github/codeql-action) | `2.28.1` | `4.33.0` | | [pnpm/action-setup](https://github.com/pnpm/action-setup) | `4.2.0` | `4.4.0` | | [rojopolis/spellcheck-github-actions](https://github.com/rojopolis/spellcheck-github-actions) | `0.32.0` | `0.60.0` | | [peaceiris/actions-gh-pages](https://github.com/peaceiris/actions-gh-pages) | `3.9.3` | `4.0.0` | | [actions/download-artifact](https://github.com/actions/download-artifact) | `8.0.0` | `8.0.1` | | [softprops/action-gh-release](https://github.com/softprops/action-gh-release) | `2.5.0` | `2.6.1` | Updates `actions/cache` from 4.3.0 to 5.0.3
Release notes

Sourced from actions/cache's releases.

v5.0.3

What's Changed

Full Changelog: https://github.com/actions/cache/compare/v5...v5.0.3

v.5.0.2

v5.0.2

What's Changed

When creating cache entries, 429s returned from the cache service will not be retried.

v5.0.1

[!IMPORTANT] actions/cache@v5 runs on the Node.js 24 runtime and requires a minimum Actions Runner version of 2.327.1.

If you are using self-hosted runners, ensure they are updated before upgrading.

v5.0.1

What's Changed

v5.0.0

What's Changed

Full Changelog: https://github.com/actions/cache/compare/v5...v5.0.1

v5.0.0

[!IMPORTANT] actions/cache@v5 runs on the Node.js 24 runtime and requires a minimum Actions Runner version of 2.327.1.

If you are using self-hosted runners, ensure they are updated before upgrading.

What's Changed

... (truncated)

Changelog

Sourced from actions/cache's changelog.

Releases

How to prepare a release

[!NOTE]
Relevant for maintainers with write access only.

  1. Switch to a new branch from main.
  2. Run npm test to ensure all tests are passing.
  3. Update the version in https://github.com/actions/cache/blob/main/package.json.
  4. Run npm run build to update the compiled files.
  5. Update this https://github.com/actions/cache/blob/main/RELEASES.md with the new version and changes in the ## Changelog section.
  6. Run licensed cache to update the license report.
  7. Run licensed status and resolve any warnings by updating the https://github.com/actions/cache/blob/main/.licensed.yml file with the exceptions.
  8. Commit your changes and push your branch upstream.
  9. Open a pull request against main and get it reviewed and merged.
  10. Draft a new release https://github.com/actions/cache/releases use the same version number used in package.json
    1. Create a new tag with the version number.
    2. Auto generate release notes and update them to match the changes you made in RELEASES.md.
    3. Toggle the set as the latest release option.
    4. Publish the release.
  11. Navigate to https://github.com/actions/cache/actions/workflows/release-new-action-version.yml
    1. There should be a workflow run queued with the same version number.
    2. Approve the run to publish the new version and update the major tags for this action.

Changelog

5.0.3

5.0.2

  • Bump @actions/cache to v5.0.3 #1692

5.0.1

  • Update @azure/storage-blob to ^12.29.1 via @actions/cache@5.0.1 #1685

5.0.0

[!IMPORTANT] actions/cache@v5 runs on the Node.js 24 runtime and requires a minimum Actions Runner version of 2.327.1. If you are using self-hosted runners, ensure they are updated before upgrading.

4.3.0

  • Bump @actions/cache to v4.1.0

... (truncated)

Commits

Updates `github/codeql-action` from 2.28.1 to 4.33.0
Release notes

Sourced from github/codeql-action's releases.

v4.33.0

  • Upcoming change: Starting April 2026, the CodeQL Action will skip collecting file coverage information on pull requests to improve analysis performance. File coverage information will still be computed on non-PR analyses. Pull request analyses will log a warning about this upcoming change. #3562

    To opt out of this change:

    • Repositories owned by an organization: Create a custom repository property with the name github-codeql-file-coverage-on-prs and the type "True/false", then set this property to true in the repository's settings. For more information, see Managing custom properties for repositories in your organization. Alternatively, if you are using an advanced setup workflow, you can set the CODEQL_ACTION_FILE_COVERAGE_ON_PRS environment variable to true in your workflow.
    • User-owned repositories using default setup: Switch to an advanced setup workflow and set the CODEQL_ACTION_FILE_COVERAGE_ON_PRS environment variable to true in your workflow.
    • User-owned repositories using advanced setup: Set the CODEQL_ACTION_FILE_COVERAGE_ON_PRS environment variable to true in your workflow.

  • Fixed a bug which caused the CodeQL Action to fail loading repository properties if a "Multi select" repository property was configured for the repository. #3557

  • The CodeQL Action now loads custom repository properties on GitHub Enterprise Server, enabling the customization of features such as github-codeql-disable-overlay that was previously only available on GitHub.com. #3559

  • Once private package registries can be configured with OIDC-based authentication for organizations, the CodeQL Action will now be able to accept such configurations. #3563

  • Fixed the retry mechanism for database uploads. Previously this would fail with the error "Response body object should not be disturbed or locked". #3564

  • A warning is now emitted if the CodeQL Action detects a repository property whose name suggests that it relates to the CodeQL Action, but which is not one of the properties recognised by the current version of the CodeQL Action. #3570

v4.32.6

  • Update default CodeQL bundle version to 2.24.3. #3548

v4.32.5

  • Repositories owned by an organization can now set up the github-codeql-disable-overlay custom repository property to disable improved incremental analysis for CodeQL. First, create a custom repository property with the name github-codeql-disable-overlay and the type "True/false" in the organization's settings. Then in the repository's settings, set this property to true to disable improved incremental analysis. For more information, see Managing custom properties for repositories in your organization. This feature is not yet available on GitHub Enterprise Server. #3507
  • Added an experimental change so that when improved incremental analysis fails on a runner — potentially due to insufficient disk space — the failure is recorded in the Actions cache so that subsequent runs will automatically skip improved incremental analysis until something changes (e.g. a larger runner is provisioned or a new CodeQL version is released). We expect to roll this change out to everyone in March. #3487
  • The minimum memory check for improved incremental analysis is now skipped for CodeQL 2.24.3 and later, which has reduced peak RAM usage. #3515
  • Reduced log levels for best-effort private package registry connection check failures to reduce noise from workflow annotations. #3516
  • Added an experimental change which lowers the minimum disk space requirement for improved incremental analysis, enabling it to run on standard GitHub Actions runners. We expect to roll this change out to everyone in March. #3498
  • Added an experimental change which allows the start-proxy action to resolve the CodeQL CLI version from feature flags instead of using the linked CLI bundle version. We expect to roll this change out to everyone in March. #3512
  • The previously experimental changes from versions 4.32.3, 4.32.4, 3.32.3 and 3.32.4 are now enabled by default. #3503, #3504

v4.32.4

  • Update default CodeQL bundle version to 2.24.2. #3493
  • Added an experimental change which improves how certificates are generated for the authentication proxy that is used by the CodeQL Action in Default Setup when private package registries are configured. This is expected to generate more widely compatible certificates and should have no impact on analyses which are working correctly already. We expect to roll this change out to everyone in February. #3473
  • When the CodeQL Action is run with debugging enabled in Default Setup and private package registries are configured, the "Setup proxy for registries" step will output additional diagnostic information that can be used for troubleshooting. #3486
  • Added a setting which allows the CodeQL Action to enable network debugging for Java programs. This will help GitHub staff support customers with troubleshooting issues in GitHub-managed CodeQL workflows, such as Default Setup. This setting can only be enabled by GitHub staff. #3485
  • Added a setting which enables GitHub-managed workflows, such as Default Setup, to use a nightly CodeQL CLI release instead of the latest, stable release that is used by default. This will help GitHub staff support customers whose analyses for a given repository or organization require early access to a change in an upcoming CodeQL CLI release. This setting can only be enabled by GitHub staff. #3484

v4.32.3

  • Added experimental support for testing connections to private package registries. This feature is not currently enabled for any analysis. In the future, it may be enabled by default for Default Setup. #3466

v4.32.2

  • Update default CodeQL bundle version to 2.24.1. #3460

v4.32.1

  • A warning is now shown in Default Setup workflow logs if a private package registry is configured using a GitHub Personal Access Token (PAT), but no username is configured. #3422
  • Fixed a bug which caused the CodeQL Action to fail when repository properties cannot successfully be retrieved. #3421

v4.32.0

  • Update default CodeQL bundle version to 2.24.0. #3425

v4.31.11

  • When running a Default Setup workflow with Actions debugging enabled, the CodeQL Action will now use more unique names when uploading logs from the Dependabot authentication proxy as workflow artifacts. This ensures that the artifact names do not clash between multiple jobs in a build matrix. #3409
  • Improved error handling throughout the CodeQL Action. #3415
  • Added experimental support for automatically excluding generated files from the analysis. This feature is not currently enabled for any analysis. In the future, it may be enabled by default for some GitHub-managed analyses. #3318
  • The changelog extracts that are included with releases of the CodeQL Action are now shorter to avoid duplicated information from appearing in Dependabot PRs. #3403

... (truncated)

Changelog

Sourced from github/codeql-action's changelog.

CodeQL Action Changelog

See the releases page for the relevant changes to the CodeQL CLI and language packs.

[UNRELEASED]

No user facing changes.

4.33.0 - 16 Mar 2026

  • Upcoming change: Starting April 2026, the CodeQL Action will skip collecting file coverage information on pull requests to improve analysis performance. File coverage information will still be computed on non-PR analyses. Pull request analyses will log a warning about this upcoming change. #3562

    To opt out of this change:

    • Repositories owned by an organization: Create a custom repository property with the name github-codeql-file-coverage-on-prs and the type "True/false", then set this property to true in the repository's settings. For more information, see Managing custom properties for repositories in your organization. Alternatively, if you are using an advanced setup workflow, you can set the CODEQL_ACTION_FILE_COVERAGE_ON_PRS environment variable to true in your workflow.
    • User-owned repositories using default setup: Switch to an advanced setup workflow and set the CODEQL_ACTION_FILE_COVERAGE_ON_PRS environment variable to true in your workflow.
    • User-owned repositories using advanced setup: Set the CODEQL_ACTION_FILE_COVERAGE_ON_PRS environment variable to true in your workflow.

  • Fixed a bug which caused the CodeQL Action to fail loading repository properties if a "Multi select" repository property was configured for the repository. #3557

  • The CodeQL Action now loads custom repository properties on GitHub Enterprise Server, enabling the customization of features such as github-codeql-disable-overlay that was previously only available on GitHub.com. #3559

  • Once private package registries can be configured with OIDC-based authentication for organizations, the CodeQL Action will now be able to accept such configurations. #3563

  • Fixed the retry mechanism for database uploads. Previously this would fail with the error "Response body object should not be disturbed or locked". #3564

  • A warning is now emitted if the CodeQL Action detects a repository property whose name suggests that it relates to the CodeQL Action, but which is not one of the properties recognised by the current version of the CodeQL Action. #3570

4.32.6 - 05 Mar 2026

  • Update default CodeQL bundle version to 2.24.3. #3548

4.32.5 - 02 Mar 2026

  • Repositories owned by an organization can now set up the github-codeql-disable-overlay custom repository property to disable improved incremental analysis for CodeQL. First, create a custom repository property with the name github-codeql-disable-overlay and the type "True/false" in the organization's settings. Then in the repository's settings, set this property to true to disable improved incremental analysis. For more information, see Managing custom properties for repositories in your organization. This feature is not yet available on GitHub Enterprise Server. #3507
  • Added an experimental change so that when improved incremental analysis fails on a runner — potentially due to insufficient disk space — the failure is recorded in the Actions cache so that subsequent runs will automatically skip improved incremental analysis until something changes (e.g. a larger runner is provisioned or a new CodeQL version is released). We expect to roll this change out to everyone in March. #3487
  • The minimum memory check for improved incremental analysis is now skipped for CodeQL 2.24.3 and later, which has reduced peak RAM usage. #3515
  • Reduced log levels for best-effort private package registry connection check failures to reduce noise from workflow annotations. #3516
  • Added an experimental change which lowers the minimum disk space requirement for improved incremental analysis, enabling it to run on standard GitHub Actions runners. We expect to roll this change out to everyone in March. #3498
  • Added an experimental change which allows the start-proxy action to resolve the CodeQL CLI version from feature flags instead of using the linked CLI bundle version. We expect to roll this change out to everyone in March. #3512
  • The previously experimental changes from versions 4.32.3, 4.32.4, 3.32.3 and 3.32.4 are now enabled by default. #3503, #3504

4.32.4 - 20 Feb 2026

  • Update default CodeQL bundle version to 2.24.2. #3493
  • Added an experimental change which improves how certificates are generated for the authentication proxy that is used by the CodeQL Action in Default Setup when private package registries are configured. This is expected to generate more widely compatible certificates and should have no impact on analyses which are working correctly already. We expect to roll this change out to everyone in February. #3473
  • When the CodeQL Action is run with debugging enabled in Default Setup and private package registries are configured, the "Setup proxy for registries" step will output additional diagnostic information that can be used for troubleshooting. #3486
  • Added a setting which allows the CodeQL Action to enable network debugging for Java programs. This will help GitHub staff support customers with troubleshooting issues in GitHub-managed CodeQL workflows, such as Default Setup. This setting can only be enabled by GitHub staff. #3485
  • Added a setting which enables GitHub-managed workflows, such as Default Setup, to use a nightly CodeQL CLI release instead of the latest, stable release that is used by default. This will help GitHub staff support customers whose analyses for a given repository or organization require early access to a change in an upcoming CodeQL CLI release. This setting can only be enabled by GitHub staff. #3484

4.32.3 - 13 Feb 2026

  • Added experimental support for testing connections to private package registries. This feature is not currently enabled for any analysis. In the future, it may be enabled by default for Default Setup. #3466

4.32.2 - 05 Feb 2026

... (truncated)

Commits
  • b1bff81 Merge pull request #3574 from github/update-v4.32.7-7dd76e6bf
  • e682234 Add changelog entry for #3570
  • 95be291 Bump minor version
  • 59bcb60 Update changelog for v4.32.7
  • 7dd76e6 Merge pull request #3572 from github/mbg/pr-checks/eslint
  • e3200e3 Merge pull request #3563 from github/mbg/private-registry/oidc
  • 4c356c7 Merge pull request #3570 from github/mbg/repo-props/warn-on-unexpected-props
  • b4937c1 Only emit one message with accumulated property names
  • 136b8ab Remove cache-dependency-path options as well
  • a5aba59 Remove package-lock.json that's no longer needed
  • Additional commits viewable in compare view

Updates `pnpm/action-setup` from 4.2.0 to 4.4.0
Release notes

Sourced from pnpm/action-setup's releases.

v4.4.0

Updated the action to use Node.js 24.

v4.3.0

What's Changed

New Contributors

Full Changelog: https://github.com/pnpm/action-setup/compare/v4.2.0...v4.3.0

Commits

Updates `rojopolis/spellcheck-github-actions` from 0.32.0 to 0.60.0
Release notes

Sourced from rojopolis/spellcheck-github-actions's releases.

0.60.0

What's Changed

Full Changelog: https://github.com/rojopolis/spellcheck-github-actions/compare/0.59.0...0.60.0

0.59.0

What's Changed

New Contributors

Full Changelog: https://github.com/rojopolis/spellcheck-github-actions/compare/0.58.0...0.59.0

0.58.0

What's Changed

Full Changelog: https://github.com/rojopolis/spellcheck-github-actions/compare/0.57.0...0.58.0

0.57.0

What's Changed

Full Changelog: https://github.com/rojopolis/spellcheck-github-actions/compare/0.56.0...0.57.0

0.56.0

What's Changed

... (truncated)

Changelog

Sourced from rojopolis/spellcheck-github-actions's changelog.

Change Log for spellcheck-github-actions

0.60, 2026-03-14, minor feature release, update not required

  • Docker based image updated for Python 3.14.3 slim trixie via PR #325 from Dependabot.

  • Cleaned up the error messaging, to address issue #328 from @​akohout-hai, the error message is now more correct, but not improved in general

0.59.0, 2026-03-02, feature release, update recommended

  • Improvements have been added to the docker entrypoint, based on a PR from @​akohout-hai which fixes an issue with handling of spaces in files names and directories, see PR #322 for details. This is his first contribution to the project and I want to thank him for his contribution, which is highly appreciated.

  • Docker based image updated to Python 3.14.3 slim trixie via PR #320 from Dependabot.

0.58.0, 2026-01-20, security release, update not required

0.57.0, 2026-01-14, maintenance release, update not required

  • Docker based image updated to Python 3.14.2 slim trixie via PR #310 from Dependabot. The version is the same, the image has had updates.

0.56.0, 2025-12-27, feature and maintenance release, update not required

0.55.0, 2025-11-27, maintenance release, update not required

  • Via an issue #293 from @​shoverbj, an update to the core component PySpelling from version 2.12.0 to version 2.12.1 was made, this allows for use of large dictionaries with Aspell

0.54.0, 2025-11-05, feature release, update not required

0.53.0, 2025-10-25, maintenance release, update not required

  • Docker image updated to Python 3.14.0 trixie slim Release notes for Python 3.14.0, this originated from the PR mentioned below, however updated to Trixie from Bookworm and as always the slim variant is used

  • Bumped the requirement for cython to 3.0.11 or above, addressing a build issue with lxml, located when testing the PR : #274 from @​dependabot, the above update of Python

  • In general the Docker build file had a few updates since the above changes required some tweaking of the Dockerfile

    • Order of installation of dependencies adjusted to ensure that lxml can build correctly
    • Installation of:
      • build-essential
      • pkg-config

... (truncated)

Commits
  • e3cd8e9 Merge pull request #335 from rojopolis/release_candidate_0.60.0
  • 1d3820d Preparing release 0.60.0
  • bafc7d3 More correct error message, issue #328 (#334)
  • df486cb Added update of Docker base image
  • 6d6eb52 Bump python from 486b809 to 6a27522 (#325)
  • 631e4a7 Bump docker/setup-qemu-action from 3.7.0 to 4.0.0 (#331)
  • ebefb9d Bump docker/build-push-action from 6.19.2 to 7.0.0 (#330)
  • b3b2580 Bump docker/setup-buildx-action from 3.12.0 to 4.0.0 (#329)
  • dc29b58 Bump docker/login-action from 3.7.0 to 4.0.0 (#333)
  • 5d55413 Bump docker/metadata-action from 5.10.0 to 6.0.0 (#332)
  • Additional commits viewable in compare view

Updates `peaceiris/actions-gh-pages` from 3.9.3 to 4.0.0
Release notes

Sourced from peaceiris/actions-gh-pages's releases.

actions-github-pages v4.0.0

See CHANGELOG.md for more details.

Changelog

Sourced from peaceiris/actions-gh-pages's changelog.

Changelog

All notable changes to this project will be documented in this file. See standard-version for commit guidelines.

4.0.0 (2024-04-08)

build

chore

ci

docs

3.9.3 (2023-03-30)

docs

fix

3.9.2 (2023-01-17)

chore

... (truncated)

Commits
  • 4f9cc66 chore(release): 4.0.0
  • 9c75028 chore(release): Add build assets
  • 5049354 build: node 20.11.1
  • 4eb285e chore: bump node16 to node20 (#1067)
  • cdc09a3 chore(deps): update dependency @​types/node to v16.18.77 (#1065)
  • d830378 chore(deps): update dependency @​types/node to v16.18.76 (#1063)
  • 80daa1d chore(deps): update dependency @​types/node to v16.18.75 (#1061)
  • 108285e chore(deps): update dependency ts-jest to v29.1.2 (#1060)
  • 99c95ff chore(deps): update dependency @​types/node to v16.18.74 (#1058)
  • 1f46537 chore(deps): update dependency @​types/node to v16.18.73 (#1057)
  • Additional commits viewable in compare view

Updates `actions/download-artifact` from 8.0.0 to 8.0.1
Release notes

Sourced from actions/download-artifact's releases.

v8.0.1

What's Changed

Full Changelog: https://github.com/actions/download-artifact/compare/v8...v8.0.1

Commits

Updates `softprops/action-gh-release` from 2.5.0 to 2.6.1
Release notes

Sourced from softprops/action-gh-release's releases.

v2.6.1

2.6.1 is a patch release focused on restoring linked discussion thread creation when discussion_category_name is set. It fixes [#764](https://github.com/softprops/action-gh-release/issues/764), where the draft-first publish flow stopped carrying the discussion category through the final publish step.

If you still hit an issue after upgrading, please open a report with the bug template and include a minimal repro or sanitized workflow snippet where possible.

What's Changed

Bug fixes 🐛

  • fix: preserve discussion c... _Description has been truncated_ Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql-analysis.yml | 6 +++--- .github/workflows/docs-backfill.yml | 2 +- .github/workflows/docs-check.yml | 2 +- .github/workflows/docs-version.yml | 2 +- .github/workflows/docs.yml | 4 ++-- .github/workflows/publish-dev.yml | 2 +- .github/workflows/publish-nextfork.yml | 4 ++-- .github/workflows/publish-rc.yml | 4 ++-- .github/workflows/publish-stable.yml | 4 ++-- 9 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index b3b31e370a4a..b1603921d1b1 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -51,7 +51,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@b8d3b6e8af63cde30bdc382c0bc28114f4346c88 # v2 + uses: github/codeql-action/init@b1bff81932f5cdfc8695c7752dcee935dcd061c8 # v4.33.0 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -64,7 +64,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@b8d3b6e8af63cde30bdc382c0bc28114f4346c88 # v2 + uses: github/codeql-action/autobuild@b1bff81932f5cdfc8695c7752dcee935dcd061c8 # v4.33.0 # ℹ️ Command-line programs to run using the OS shell. # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun @@ -77,4 +77,4 @@ jobs: # ./location_of_script_within_repo/buildscript.sh - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@b8d3b6e8af63cde30bdc382c0bc28114f4346c88 # v2 + uses: github/codeql-action/analyze@b1bff81932f5cdfc8695c7752dcee935dcd061c8 # v4.33.0 diff --git a/.github/workflows/docs-backfill.yml b/.github/workflows/docs-backfill.yml index bc8bd71fe427..19c69af10a65 100644 --- a/.github/workflows/docs-backfill.yml +++ b/.github/workflows/docs-backfill.yml @@ -73,7 +73,7 @@ jobs: - name: Install pnpm if: steps.versions.outputs.missing != '' && github.event.inputs.dry_run != 'true' - uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4 + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4 - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 if: steps.versions.outputs.missing != '' && github.event.inputs.dry_run != 'true' diff --git a/.github/workflows/docs-check.yml b/.github/workflows/docs-check.yml index b69f45ee3d16..105faa1744d7 100644 --- a/.github/workflows/docs-check.yml +++ b/.github/workflows/docs-check.yml @@ -29,4 +29,4 @@ jobs: # Run spellcheck AFTER building docs, in case the CLI reference has issues - name: Spellcheck - uses: rojopolis/spellcheck-github-actions@ac9fbc2d852c70d7e9d305a62bdd7b1517c50e3d # 0.32.0 + uses: rojopolis/spellcheck-github-actions@e3cd8e9aec4587ec73bc0e60745aafd45c37aa2e # 0.60.0 diff --git a/.github/workflows/docs-version.yml b/.github/workflows/docs-version.yml index 0d5f36940cf7..2738e73c27a1 100644 --- a/.github/workflows/docs-version.yml +++ b/.github/workflows/docs-version.yml @@ -38,7 +38,7 @@ jobs: ref: ${{ github.ref }} - name: Install pnpm - uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4 + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4 - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index c0dd4c7ad9a8..e23ca9be580f 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -35,7 +35,7 @@ jobs: ref: ${{ env.DEPLOY_REF }} - name: Install pnpm before setup node - uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4 + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4 - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: @@ -91,7 +91,7 @@ jobs: run: pnpm install && pnpm build - name: Deploy - uses: peaceiris/actions-gh-pages@373f7f263a76c20808c831209c920827a82a2847 # v3 + uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4.0.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: ./docs/build diff --git a/.github/workflows/publish-dev.yml b/.github/workflows/publish-dev.yml index e11b96ba08a9..589620b5e2f1 100644 --- a/.github/workflows/publish-dev.yml +++ b/.github/workflows/publish-dev.yml @@ -24,7 +24,7 @@ jobs: fetch-depth: 0 - name: Install pnpm before setup node - uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4 + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4 - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: diff --git a/.github/workflows/publish-nextfork.yml b/.github/workflows/publish-nextfork.yml index 8ccac06fcc93..45f74ff069be 100644 --- a/.github/workflows/publish-nextfork.yml +++ b/.github/workflows/publish-nextfork.yml @@ -26,7 +26,7 @@ jobs: with: fetch-depth: 0 - name: Install pnpm before setup node - uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4 + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4 - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version: 24 @@ -37,7 +37,7 @@ jobs: id: node run: echo "v8CppApiVersion=$(node --print "process.versions.modules")" >> $GITHUB_OUTPUT - name: Restore dependencies - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # master + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # master id: cache-deps with: path: | diff --git a/.github/workflows/publish-rc.yml b/.github/workflows/publish-rc.yml index c2dc5537b954..dcd3d8b59c39 100644 --- a/.github/workflows/publish-rc.yml +++ b/.github/workflows/publish-rc.yml @@ -71,14 +71,14 @@ jobs: run: node scripts/generate_changelog.mjs ${{ needs.tag.outputs.prev_tag }} ${{ needs.tag.outputs.tag }} CHANGELOG.md - name: Get binaries - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: path: dist/ merge-multiple: true - name: Create Release id: create_release - uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2.5.0 + uses: softprops/action-gh-release@153bb8e04406b158c6c84fc1615b65b24149a1fe # v2.6.1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: diff --git a/.github/workflows/publish-stable.yml b/.github/workflows/publish-stable.yml index 02fa0e627dbb..6e292260f505 100644 --- a/.github/workflows/publish-stable.yml +++ b/.github/workflows/publish-stable.yml @@ -77,14 +77,14 @@ jobs: run: node scripts/generate_changelog.mjs ${{ needs.tag.outputs.prev_tag }} ${{ needs.tag.outputs.tag }} CHANGELOG.md - name: Get binaries - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: path: dist/ merge-multiple: true - name: Create Release id: create_release - uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2.5.0 + uses: softprops/action-gh-release@153bb8e04406b158c6c84fc1615b65b24149a1fe # v2.6.1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: From 8e34c7560f1239897e257d851950bb55547fae18 Mon Sep 17 00:00:00 2001 From: rnfairchild Date: Mon, 16 Mar 2026 19:17:09 -0400 Subject: [PATCH 47/63] chore: fix peerdas metrics (#8920) **Motivation** - https://github.com/ChainSafe/lodestar/issues/8850 **Description** Resolves issues with peerDAS metrics. Credit: [Vedant](https://github.com/vedant-asati/lodestar/tree/fix-peerdas-metrics-clean) Issue_number #8850 --------- Co-authored-by: Vedant Asati Co-authored-by: Vedant Asati <114929867+vedant-asati@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Lodekeeper <258435968+lodekeeper@users.noreply.github.com> Co-authored-by: lodekeeper Co-authored-by: Nico Flaig Co-authored-by: twoeths <10568965+twoeths@users.noreply.github.com> Co-authored-by: Tuyen Nguyen Co-authored-by: Claude Sonnet 4.5 Co-authored-by: Cayman Co-authored-by: NC <17676176+ensi321@users.noreply.github.com> --- dashboards/lodestar_peerdas.json | 973 +++++++++--------- .../src/api/impl/beacon/blocks/index.ts | 11 + .../src/chain/blocks/importBlock.ts | 2 +- packages/beacon-node/src/chain/chain.ts | 6 +- .../chain/produceBlock/produceBlockBody.ts | 9 +- .../src/chain/validation/dataColumnSidecar.ts | 272 ++--- .../beacon-node/src/metrics/metrics/beacon.ts | 12 +- .../src/metrics/metrics/lodestar.ts | 30 +- packages/beacon-node/src/sync/range/range.ts | 1 + .../src/sync/utils/downloadByRange.ts | 15 +- .../src/sync/utils/downloadByRoot.ts | 2 +- packages/beacon-node/src/util/dataColumns.ts | 8 +- packages/beacon-node/src/util/execution.ts | 35 +- .../test/spec/presets/fork_choice.test.ts | 3 +- 14 files changed, 743 insertions(+), 636 deletions(-) diff --git a/dashboards/lodestar_peerdas.json b/dashboards/lodestar_peerdas.json index 32cc741baccd..92f86bc8cfab 100644 --- a/dashboards/lodestar_peerdas.json +++ b/dashboards/lodestar_peerdas.json @@ -21,6 +21,12 @@ "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, "type": "dashboard" } ] @@ -46,6 +52,7 @@ "url": "" } ], + "liveNow": false, "panels": [ { "collapsed": false, @@ -55,9 +62,9 @@ "x": 0, "y": 0 }, - "id": 61, + "id": 100, "panels": [], - "title": "Network", + "title": "Health Overview", "type": "row" }, { @@ -65,14 +72,175 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "Time elapsed between data column sidecar slot time and the time data column received", + "description": "Total number of custody groups", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "thresholds" + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 5, + "x": 0, + "y": 1 + }, + "id": 36, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "8.5.16", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "beacon_custody_groups", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Custody groups", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of custody data columns expected to have but are missing currently.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 5, + "x": 5, + "y": 1 + }, + "id": 101, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.16", + "targets": [ + { + "expr": "lodestar_data_columns_missing_custody_columns_count", + "legendFormat": "missing", + "refId": "A" + } + ], + "title": "Missing Custody Columns", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total number of backfilled custody groups", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "thresholds" + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 5, + "x": 10, + "y": 1 + }, + "id": 53, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "8.5.16", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "beacon_custody_groups_backfilled", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Backfilled custody groups", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Average number of data column sidecars per slot that were already received via another source per slot (ie scaled x12).", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -86,7 +254,6 @@ "tooltip": false, "viz": false }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -108,10 +275,85 @@ "overrides": [] }, "gridPos": { - "h": 6, + "h": 7, + "w": 9, + "x": 15, + "y": 1 + }, + "id": 103, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "expr": "rate(lodestar_data_column_sidecar_already_added[$rate_interval]) * 12", + "legendFormat": "duplicates", + "refId": "A" + } + ], + "title": "Duplicate Columns per slot", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Average time elapsed between data column's slot time and when it is received", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, "w": 12, "x": 0, - "y": 1 + "y": 8 }, "id": 71, "options": { @@ -133,15 +375,15 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(lodestar_data_column_elapsed_time_till_received_seconds_sum{source=\"gossip\"}[$rate_interval])\n/\nrate(lodestar_data_column_elapsed_time_till_received_seconds_count{source=\"gossip\"}[$rate_interval])", + "expr": "sum by (receivedOrder) (rate(lodestar_data_column_elapsed_time_till_received_seconds_sum[$rate_interval]))\n/\nsum by (receivedOrder) (rate(lodestar_data_column_elapsed_time_till_received_seconds_count[$rate_interval]))", "instant": false, "interval": "", - "legendFormat": "{{source}}", + "legendFormat": "{{receivedOrder}}", "range": true, "refId": "A" } ], - "title": "Data columns receiving delay", + "title": "Receiving delay (avg)", "type": "timeseries" }, { @@ -149,7 +391,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "Time elapsed between data column sidecar slot time and the time data column received", + "description": "Heatmap of data column receiving latency, (columns/second) received within each latency bucket.", "fieldConfig": { "defaults": { "custom": { @@ -166,17 +408,17 @@ "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 12, - "y": 1 + "y": 8 }, "id": 62, "options": { "calculate": false, - "cellGap": 2, + "cellGap": 1, "color": { - "exponent": 0.5, + "exponent": 0.15, "fill": "dark-orange", "mode": "scheme", "reverse": false, @@ -198,16 +440,17 @@ }, "tooltip": { "mode": "single", - "showColorScale": false, - "yHistogram": false + "show": true, + "showColorScale": true, + "yHistogram": true }, "yAxis": { "axisPlacement": "left", "reverse": false, - "unit": "short" + "unit": "s" } }, - "pluginVersion": "10.4.1", + "pluginVersion": "8.5.16", "targets": [ { "datasource": { @@ -215,7 +458,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "12*rate(lodestar_data_column_elapsed_time_till_received_seconds_bucket{source=\"gossip\"}[$rate_interval])", + "expr": "sum by (le) (rate(lodestar_data_column_elapsed_time_till_received_seconds_bucket[$rate_interval]))", "format": "heatmap", "instant": false, "interval": "", @@ -224,25 +467,37 @@ "refId": "A" } ], - "title": "Data columns receiving delay", + "title": "Receiving delay (heatmap)", "type": "heatmap" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 61, + "panels": [], + "title": "Gossip", + "type": "row" + }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "Number of received data columns by source", + "description": "Average number of data columns received per slot, by source, ( ie scaled x12).", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "", + "axisLabel": "columns / slot (scaled)", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", @@ -253,7 +508,6 @@ "tooltip": false, "viz": false }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -275,10 +529,10 @@ "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 0, - "y": 7 + "y": 17 }, "id": 63, "options": { @@ -308,37 +562,24 @@ "refId": "A" } ], - "title": "Data columns source", + "title": "Data columns received per slot (by source)", "type": "timeseries" }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 13 - }, - "id": 16, - "panels": [], - "title": "Data columns gossip verification", - "type": "row" - }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "description": "Number of data column sidecars processed per slot, by outcome (ie scaled x12).", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "", + "axisLabel": "columns / slot (scaled)", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", @@ -349,7 +590,6 @@ "tooltip": false, "viz": false }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -360,21 +600,23 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, - "mappings": [] + "decimals": 2, + "mappings": [], + "unit": "none" }, "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 12, - "x": 0, - "y": 14 + "x": 12, + "y": 17 }, "id": 13, "interval": "1m", @@ -401,11 +643,11 @@ }, "editorMode": "code", "exemplar": false, - "expr": "avg_over_time(\n (rate(beacon_data_column_sidecar_processing_requests_total[1m]) * 12 > 0)[$rate_interval:]\n)", + "expr": "rate(beacon_data_column_sidecar_processing_requests_total[$rate_interval]) * 12", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "befor verification", + "legendFormat": "total", "range": true, "refId": "A" }, @@ -415,11 +657,11 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "avg_over_time(\n (rate(beacon_data_column_sidecar_processing_successes_total[1m]) * 12 > 0)[$rate_interval:]\n)", + "expr": "rate(beacon_data_column_sidecar_processing_successes_total[$rate_interval]) * 12", "hide": false, "instant": false, "interval": "", - "legendFormat": "after verification", + "legendFormat": "verified", "range": true, "refId": "B" }, @@ -429,100 +671,30 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "avg_over_time(\n (rate(beacon_data_column_sidecar_processing_requests_total[1m]) * 12 > 0)[$rate_interval:]\n)\n-\navg_over_time(\n (rate(beacon_data_column_sidecar_processing_successes_total[1m]) * 12 > 0)[$rate_interval:]\n) != 0", + "expr": "rate(beacon_data_column_sidecar_processing_skip_total[$rate_interval]) * 12", "hide": false, "instant": false, "interval": "", - "legendFormat": "failures", + "legendFormat": "skipped", "range": true, "refId": "C" - } - ], - "title": "Data column sidecars gossip verification", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [] - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 14 - }, - "id": 70, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(lodestar_gossip_validation_error_total{topic=\"data_column_sidecar\"}[$rate_interval])", + "expr": "(\n rate(beacon_data_column_sidecar_processing_requests_total[$rate_interval])\n -\n rate(beacon_data_column_sidecar_processing_successes_total[$rate_interval])\n -\n rate(beacon_data_column_sidecar_processing_skip_total[$rate_interval])\n) * 12", + "hide": false, "instant": false, "interval": "", - "legendFormat": "{{error}}", + "legendFormat": "failed", "range": true, - "refId": "A" + "refId": "D" } ], - "title": "Gossip DataColumnSidecar Error", + "title": "Data column sidecars processing(verification) (per slot)", "type": "timeseries" }, { @@ -530,14 +702,13 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "Time elapsed between data column received and data column validated", + "description": "p99 time to verify a gossip data column sidecar", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -551,7 +722,6 @@ "tooltip": false, "viz": false }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -568,17 +738,19 @@ "mode": "off" } }, - "mappings": [] + "mappings": [], + "unit": "s" }, "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 0, - "y": 20 + "y": 25 }, - "id": 69, + "id": 35, + "interval": "30s", "options": { "legend": { "calcs": [], @@ -591,6 +763,7 @@ "sort": "none" } }, + "pluginVersion": "12.0.1", "targets": [ { "datasource": { @@ -598,14 +771,15 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(lodestar_gossip_data_column_received_to_gossip_validate_seconds_bucket[$rate_interval])", + "expr": "histogram_quantile(0.99, rate(beacon_data_column_sidecar_gossip_verification_seconds_bucket [$rate_interval]))\n", "instant": false, - "legendFormat": "__auto", + "interval": "", + "legendFormat": "p99", "range": true, "refId": "A" } ], - "title": "Data column recv to gossip validation delay", + "title": "Gossip verification runtime (p99)", "type": "timeseries" }, { @@ -613,16 +787,16 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "description": "Validation errors while processing gossip sidecars (errors / slot) (ie scaled x12)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "", + "axisLabel": "errors / slot (scaled)", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", @@ -633,7 +807,6 @@ "tooltip": false, "viz": false }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -651,32 +824,29 @@ } }, "mappings": [], - "unit": "s" + "unit": "none" }, "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 12, - "y": 20 + "y": 25 }, - "id": 35, - "interval": "30s", + "id": 70, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": false + "showLegend": true }, "tooltip": { - "hideZeros": false, "mode": "single", "sort": "none" } }, - "pluginVersion": "12.0.1", "targets": [ { "datasource": { @@ -684,15 +854,15 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, rate(beacon_data_column_sidecar_gossip_verification_seconds_bucket [$rate_interval]))\n", + "expr": "rate(lodestar_gossip_validation_error_total{topic=\"data_column_sidecar\"}[$rate_interval]) * 12", "instant": false, "interval": "", - "legendFormat": "{{instance}}", + "legendFormat": "{{error}}", "range": true, "refId": "A" } ], - "title": "Runtime of data column sidecars gossip verification", + "title": "Gossip DataColumnSidecar validation errors (per slot)", "type": "timeseries" }, { @@ -700,6 +870,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "description": "Heatmap of gossip data column sidecar verification latency. Shows (sidecars/sec) verified within each latency bucket over time.", "fieldConfig": { "defaults": { "custom": { @@ -716,10 +887,10 @@ "overrides": [] }, "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 26 + "h": 9, + "w": 24, + "x": 0, + "y": 33 }, "id": 72, "interval": "30s", @@ -727,7 +898,7 @@ "calculate": false, "cellGap": 1, "color": { - "exponent": 0.5, + "exponent": 0.7, "fill": "dark-orange", "mode": "scheme", "reverse": false, @@ -742,22 +913,24 @@ "le": 1e-9 }, "legend": { - "show": true + "show": false }, "rowsFrame": { "layout": "auto" }, "tooltip": { "mode": "single", - "showColorScale": false, - "yHistogram": false + "show": true, + "showColorScale": true, + "yHistogram": true }, "yAxis": { "axisPlacement": "left", - "reverse": false + "reverse": false, + "unit": "s" } }, - "pluginVersion": "10.4.1", + "pluginVersion": "8.5.16", "targets": [ { "datasource": { @@ -769,12 +942,12 @@ "format": "heatmap", "instant": false, "interval": "", - "legendFormat": "{{instance}}", + "legendFormat": "{{le}}", "range": true, "refId": "A" } ], - "title": "Runtime of data column sidecars gossip verification", + "title": "Gossip verification runtime (heatmap)", "type": "heatmap" }, { @@ -783,11 +956,11 @@ "h": 1, "w": 24, "x": 0, - "y": 32 + "y": 42 }, "id": 58, "panels": [], - "title": "engine_getBlobsV2", + "title": "Engine", "type": "row" }, { @@ -795,14 +968,13 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "Total number of engine_getBlobsV2 requests sent and responses received", + "description": "Rate of engine_getBlobsV2 requests sent and responses received (req/s)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -816,7 +988,6 @@ "tooltip": false, "viz": false }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -835,15 +1006,15 @@ }, "decimals": 0, "mappings": [], - "unit": "none" + "unit": "ops" }, "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 0, - "y": 33 + "y": 43 }, "id": 50, "interval": "30s", @@ -883,16 +1054,100 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(beacon_engine_getBlobsV2_responses_total[$rate_interval])", - "hide": false, - "instant": false, + "expr": "rate(beacon_engine_getBlobsV2_responses_total[$rate_interval])", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "responses", + "range": true, + "refId": "B" + } + ], + "title": "engine_getBlobsV2 requests & responses", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Duration of engine_getBlobsV2 requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 43 + }, + "id": 49, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, rate(beacon_engine_getBlobsV2_request_duration_seconds_bucket[$rate_interval]))", "interval": "", - "legendFormat": "responses", + "legendFormat": "p99", "range": true, - "refId": "B" + "refId": "A" } ], - "title": "engine_getBlobsV2 requests & responses", + "title": "Duration of engine_getBlobsV2 requests (p99)", "type": "timeseries" }, { @@ -900,14 +1155,13 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "Duration of engine_getBlobsV2 requests", + "description": "Percentage of engine_getBlobsV2 requests that did not receive a successful (not null) response", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -921,7 +1175,6 @@ "tooltip": false, "viz": false }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -938,18 +1191,20 @@ "mode": "off" } }, + "decimals": 1, "mappings": [], - "unit": "s" + "unit": "percent" }, "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 12, - "x": 12, - "y": 33 + "x": 0, + "y": 51 }, - "id": 49, + "id": 57, + "interval": "30s", "options": { "legend": { "calcs": [], @@ -971,14 +1226,17 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, rate(beacon_engine_getBlobsV2_request_duration_seconds_bucket[$rate_interval]))", + "exemplar": false, + "expr": "(rate(beacon_engine_getBlobsV2_requests_total[$rate_interval])-rate(beacon_engine_getBlobsV2_responses_total[$rate_interval])) / clamp_min(rate(beacon_engine_getBlobsV2_requests_total[$rate_interval]), 1)* 100", + "format": "time_series", "interval": "", - "legendFormat": "{{instance}}", + "intervalFactor": 1, "range": true, - "refId": "A" + "refId": "A", + "legendFormat": "unsuccessful %" } ], - "title": "Duration of engine_getBlobsV2 requests", + "title": "Unsuccessful engine_getBlobsV2 requests rate (%)", "type": "timeseries" }, { @@ -986,28 +1244,26 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "Percentage of engine_getBlobsV2 requests that did not receive a successful (not null) response", + "description": "Rate of engine API results per second, broken down by outcome", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "opacity", + "fillOpacity": 0, + "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1018,58 +1274,44 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, - "decimals": 0, "mappings": [], - "unit": "none" + "unit": "short" }, "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 12, - "x": 0, - "y": 39 + "x": 12, + "y": 51 }, - "id": 57, - "interval": "30s", + "id": 102, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": false + "showLegend": true }, "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" + "mode": "single", + "sort": "none" } }, - "pluginVersion": "12.0.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "(rate(beacon_engine_getBlobsV2_requests_total[$rate_interval])\n-\nrate(beacon_engine_getBlobsV2_responses_total[$rate_interval])) / rate(beacon_engine_getBlobsV2_requests_total[$rate_interval]) * 100", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}", - "range": true, + "expr": "rate(lodestar_data_column_engine_result_total[$rate_interval])", + "legendFormat": "{{result}}", "refId": "A" } ], - "title": "Unsuccessful engine_getBlobsV2 requests rate (%)", + "title": "Engine API Results (per second)", "type": "timeseries" }, { @@ -1078,7 +1320,7 @@ "h": 1, "w": 24, "x": 0, - "y": 45 + "y": 59 }, "id": 60, "panels": [], @@ -1097,7 +1339,6 @@ "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -1111,7 +1352,6 @@ "tooltip": false, "viz": false }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1134,10 +1374,10 @@ "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 0, - "y": 46 + "y": 60 }, "id": 21, "options": { @@ -1161,14 +1401,14 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, rate(beacon_data_column_sidecar_computation_seconds_bucket[$rate_interval]))\n", + "expr": "histogram_quantile(0.99, sum by (le) (rate(beacon_data_column_sidecar_computation_seconds_bucket[$rate_interval])))", "interval": "", - "legendFormat": "{{instance}}", + "legendFormat": "p99", "range": true, "refId": "A" } ], - "title": "Time taken to compute data column sidecar", + "title": "Data column sidecar computation time (p99)", "type": "timeseries" }, { @@ -1182,7 +1422,6 @@ "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -1196,7 +1435,6 @@ "tooltip": false, "viz": false }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1219,10 +1457,10 @@ "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 12, - "y": 46 + "y": 60 }, "id": 22, "options": { @@ -1247,12 +1485,12 @@ }, "editorMode": "code", "expr": "histogram_quantile(0.99, rate(beacon_data_column_sidecar_inclusion_proof_verification_seconds_bucket[$rate_interval]))", - "legendFormat": "{{instance}}", + "legendFormat": "p99", "range": true, "refId": "A" } ], - "title": "Time taken to verify data column sidecar inclusion proof", + "title": "Time taken to verify data column sidecar inclusion proof (p99)", "type": "timeseries" }, { @@ -1266,7 +1504,6 @@ "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -1280,7 +1517,6 @@ "tooltip": false, "viz": false }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1303,10 +1539,10 @@ "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 0, - "y": 52 + "y": 68 }, "id": 24, "options": { @@ -1332,135 +1568,21 @@ "editorMode": "code", "expr": "histogram_quantile(0.99, rate(beacon_kzg_verification_data_column_batch_seconds_bucket[$rate_interval]))", "interval": "", - "legendFormat": "{{instance}}", + "legendFormat": "p99", "range": true, "refId": "A" } ], - "title": "Runtime of batched data column kzg verification", + "title": "Runtime of batched data column kzg verification (p99)", "type": "timeseries" }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Total number of custody groups", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "green", - "mode": "thresholds" - }, - "mappings": [] - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 52 - }, - "id": 36, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "beacon_target_custody_group_count", - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Custody groups", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Total number of backfilled custody groups", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "green", - "mode": "thresholds" - }, - "mappings": [] - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 52 - }, - "id": 53, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "beacon_custody_groups_backfilled", - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Backfilled custody groups", - "type": "stat" - }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 58 + "y": 76 }, "id": 59, "panels": [], @@ -1478,10 +1600,9 @@ "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "", + "axisLabel": "columns/s", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", @@ -1492,7 +1613,6 @@ "tooltip": false, "viz": false }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1510,15 +1630,16 @@ } }, "decimals": 0, - "mappings": [] + "mappings": [], + "unit": "none" }, "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 0, - "y": 59 + "y": 77 }, "id": 14, "interval": "30s", @@ -1543,7 +1664,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "avg_over_time(\n (rate(beacon_data_availability_reconstructed_columns_total[1m]) * 12 > 0)[$rate_interval:]\n)", + "expr": "rate(beacon_data_availability_reconstructed_columns_total[$rate_interval])", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -1552,7 +1673,7 @@ "refId": "A" } ], - "title": "Total count of reconstructed columns", + "title": "Rate of column reconstruction", "type": "timeseries" }, { @@ -1566,7 +1687,6 @@ "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -1580,7 +1700,6 @@ "tooltip": false, "viz": false }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1603,10 +1722,10 @@ "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 12, - "y": 59 + "y": 77 }, "id": 25, "interval": "30s", @@ -1631,13 +1750,13 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, rate(beacon_data_availability_reconstruction_time_seconds_bucket[$rate_interval]))", - "legendFormat": "{{job}}", + "expr": "histogram_quantile(0.99,sum by (le) (rate(beacon_data_availability_reconstruction_time_seconds_bucket[$rate_interval])))", + "legendFormat": "p99", "range": true, "refId": "A" } ], - "title": "Time taken to reconstruct columns", + "title": "Time taken to reconstruct columns (p99)", "type": "timeseries" }, { @@ -1652,7 +1771,6 @@ "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -1666,7 +1784,6 @@ "tooltip": false, "viz": false }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1683,15 +1800,16 @@ "mode": "off" } }, - "mappings": [] + "mappings": [], + "unit": "short" }, "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 0, - "y": 65 + "y": 85 }, "id": 64, "options": { @@ -1713,9 +1831,9 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(lodestar_data_columns_in_custody_before_reconstruction[$rate_interval]) * 12", + "expr": "lodestar_data_columns_in_custody_before_reconstruction", "instant": false, - "legendFormat": "__auto", + "legendFormat": "{{job}}", "range": true, "refId": "A" } @@ -1728,14 +1846,13 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "Data column sidecars reconstruction result ", + "description": "Reconstruction outcomes per slot (ie rate scaled x12), grouped by result code.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -1749,7 +1866,6 @@ "tooltip": false, "viz": false }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1760,7 +1876,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" @@ -1771,10 +1887,10 @@ "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 12, - "y": 65 + "y": 85 }, "id": 67, "options": { @@ -1804,90 +1920,7 @@ "refId": "A" } ], - "title": "Reconstruction result", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Time elapsed between block slot time and the time data column sidecar reconstructed", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [] - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 71 - }, - "id": 68, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "rate(lodestar_data_column_sidecar_elapsed_time_till_reconstructed_seconds_sum[$rate_interval])\n/\nrate(lodestar_data_column_sidecar_elapsed_time_till_reconstructed_seconds_count[$rate_interval])", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Data columns reconstruction delay", + "title": "Reconstruction result (per slot)", "type": "timeseries" } ], diff --git a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts index af41e4d33fbc..1b45b9457033 100644 --- a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts @@ -815,11 +815,18 @@ export function getBeaconBlockApi({ } const indicesToReconstruct = indices ?? Array.from({length: blobCount}, (_, i) => i); + + const timer = metrics?.recoverBlobSidecars.reconstructionTime.startTimer(); const blobs = await reconstructBlobs(dataColumnSidecars, indicesToReconstruct); + timer?.(); + metrics?.recoverBlobSidecars.blobsReconstructed.inc(indicesToReconstruct.length); + const signedBlockHeader = signedBlockToSignedHeader(config, block); data = await Promise.all( indicesToReconstruct.map(async (index, i) => { + // record per column computation time + const compTimer = metrics?.peerDas.dataColumnSidecarComputationTime.startTimer(); // Reconstruct blob sidecar from blob const kzgCommitment = blobKzgCommitments[index]; const blob = blobs[i]; // Use i since blobs only contains requested indices @@ -829,6 +836,7 @@ export function getBeaconBlockApi({ block.message.body, index ); + compTimer?.(); return {index, blob, kzgCommitment, kzgProof, signedBlockHeader, kzgCommitmentInclusionProof}; }) ); @@ -910,7 +918,10 @@ export function getBeaconBlockApi({ indicesToReconstruct = Array.from({length: blobCount}, (_, i) => i); } + const timer = metrics?.peerDas.dataColumnsReconstructionTime.startTimer(); blobs = await reconstructBlobs(dataColumnSidecars, indicesToReconstruct); + timer?.(); + metrics?.peerDas.reconstructedColumns.inc(indicesToReconstruct.length); } else { blobs = []; } diff --git a/packages/beacon-node/src/chain/blocks/importBlock.ts b/packages/beacon-node/src/chain/blocks/importBlock.ts index 7e61ce6671d9..0d5af688580c 100644 --- a/packages/beacon-node/src/chain/blocks/importBlock.ts +++ b/packages/beacon-node/src/chain/blocks/importBlock.ts @@ -554,7 +554,7 @@ export async function importBlock( if (isBlockInputColumns(blockInput)) { for (const {source} of blockInput.getSampledColumnsWithSource()) { - this.metrics?.importBlock.columnsBySource.inc({source}); + this.metrics?.dataColumns.bySource.inc({source}); } } else if (isBlockInputBlobs(blockInput)) { for (const {source} of blockInput.getAllBlobsWithSource()) { diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 5338a6bee134..f0a9aeace741 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -311,7 +311,9 @@ export class BeaconChain implements IBeaconChain { const nodeId = computeNodeIdFromPrivateKey(privateKey); const initialCustodyGroupCount = opts.initialCustodyGroupCount ?? config.CUSTODY_REQUIREMENT; - this.metrics?.peerDas.targetCustodyGroupCount.set(initialCustodyGroupCount); + this.metrics?.peerDas.custodyGroupCount.set(initialCustodyGroupCount); + // TODO: backfill not implemented yet + this.metrics?.peerDas.custodyGroupsBackfilled.set(0); this.custodyConfig = new CustodyConfig({ nodeId, config, @@ -1473,7 +1475,7 @@ export class BeaconChain implements IBeaconChain { // Only update if target is increased if (targetCustodyGroupCount > this.custodyConfig.targetCustodyGroupCount) { this.custodyConfig.updateTargetCustodyGroupCount(targetCustodyGroupCount); - this.metrics?.peerDas.targetCustodyGroupCount.set(targetCustodyGroupCount); + this.metrics?.peerDas.custodyGroupCount.set(targetCustodyGroupCount); this.logger.verbose("Updated target custody group count", { finalizedEpoch: finalizedCheckpoint.epoch, validatorCount: validatorIndices.length, diff --git a/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts b/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts index 9642c483b63b..8eaf1bf36791 100644 --- a/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts +++ b/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts @@ -511,9 +511,16 @@ export async function produceBlockBody( // NOTE: Even though the fulu.BlobsBundle type is superficially the same as deneb.BlobsBundle, it is NOT. // In fulu, proofs are _cell_ proofs, vs in deneb they are _blob_ proofs. + const timer = this?.metrics?.peerDas.dataColumnSidecarComputationTime.startTimer(); const cells = blobsBundle.blobs.map((blob) => kzg.computeCells(blob)); + timer?.(); if (this.opts.sanityCheckExecutionEngineBlobs) { - await validateCellsAndKzgCommitments(blobsBundle.commitments, blobsBundle.proofs, cells); + const validationTimer = this.metrics?.peerDas.kzgVerificationDataColumnBatchTime.startTimer(); + try { + await validateCellsAndKzgCommitments(blobsBundle.commitments, blobsBundle.proofs, cells); + } finally { + validationTimer?.(); + } } (blockBody as deneb.BeaconBlockBody).blobKzgCommitments = blobsBundle.commitments; diff --git a/packages/beacon-node/src/chain/validation/dataColumnSidecar.ts b/packages/beacon-node/src/chain/validation/dataColumnSidecar.ts index 0aa5fde20cae..d2ae63072f3a 100644 --- a/packages/beacon-node/src/chain/validation/dataColumnSidecar.ts +++ b/packages/beacon-node/src/chain/validation/dataColumnSidecar.ts @@ -12,6 +12,7 @@ import { } from "@lodestar/state-transition"; import {DataColumnSidecar, Root, Slot, SubnetID, fulu, ssz} from "@lodestar/types"; import {byteArrayEquals, toRootHex, verifyMerkleBranch} from "@lodestar/utils"; +import {BeaconMetrics} from "../../metrics/metrics/beacon.js"; import {Metrics} from "../../metrics/metrics.js"; import {kzg} from "../../util/kzg.js"; import { @@ -177,6 +178,7 @@ export async function validateGossipDataColumnSidecar( }); } + // single data column is being verified here const kzgProofTimer = metrics?.peerDas.dataColumnSidecarKzgProofsVerificationTime.startTimer(); // 11) [REJECT] The sidecar's column data is valid as verified by verify_data_column_sidecar_kzg_proofs try { @@ -297,159 +299,177 @@ export async function validateBlockDataColumnSidecars( blockSlot: Slot, blockRoot: Root, blockBlobCount: number, - dataColumnSidecars: fulu.DataColumnSidecars + dataColumnSidecars: fulu.DataColumnSidecars, + metrics?: BeaconMetrics["peerDas"] | null ): Promise { - if (dataColumnSidecars.length === 0) { - return; - } + metrics?.dataColumnSidecarProcessingRequests.inc(dataColumnSidecars.length); + const verificationTimer = metrics?.dataColumnSidecarGossipVerificationTime.startTimer(); + try { + if (dataColumnSidecars.length === 0) { + return; + } - if (blockBlobCount === 0) { - throw new DataColumnSidecarValidationError( - { - code: DataColumnSidecarErrorCode.INCORRECT_SIDECAR_COUNT, - slot: blockSlot, - expected: 0, - actual: dataColumnSidecars.length, - }, - "Block has no blob commitments but data column sidecars were provided" - ); - } - // Hash the first sidecar block header and compare the rest via (cheaper) equality - const firstSidecarSignedBlockHeader = dataColumnSidecars[0].signedBlockHeader; - const firstSidecarBlockHeader = firstSidecarSignedBlockHeader.message; - const firstBlockRoot = ssz.phase0.BeaconBlockHeader.hashTreeRoot(firstSidecarBlockHeader); - if (!byteArrayEquals(blockRoot, firstBlockRoot)) { - throw new DataColumnSidecarValidationError( - { - code: DataColumnSidecarErrorCode.INCORRECT_BLOCK, - slot: blockSlot, - columnIndex: 0, - expected: toRootHex(blockRoot), - actual: toRootHex(firstBlockRoot), - }, - "DataColumnSidecar doesn't match corresponding block" - ); - } + if (blockBlobCount === 0) { + throw new DataColumnSidecarValidationError( + { + code: DataColumnSidecarErrorCode.INCORRECT_SIDECAR_COUNT, + slot: blockSlot, + expected: 0, + actual: dataColumnSidecars.length, + }, + "Block has no blob commitments but data column sidecars were provided" + ); + } + // Hash the first sidecar block header and compare the rest via (cheaper) equality + const firstSidecarSignedBlockHeader = dataColumnSidecars[0].signedBlockHeader; + const firstSidecarBlockHeader = firstSidecarSignedBlockHeader.message; + const firstBlockRoot = ssz.phase0.BeaconBlockHeader.hashTreeRoot(firstSidecarBlockHeader); + if (!byteArrayEquals(blockRoot, firstBlockRoot)) { + throw new DataColumnSidecarValidationError( + { + code: DataColumnSidecarErrorCode.INCORRECT_BLOCK, + slot: blockSlot, + columnIndex: 0, + expected: toRootHex(blockRoot), + actual: toRootHex(firstBlockRoot), + }, + "DataColumnSidecar doesn't match corresponding block" + ); + } + + if (chain !== null) { + const rootHex = toRootHex(blockRoot); + const slot = firstSidecarSignedBlockHeader.message.slot; + const signature = firstSidecarSignedBlockHeader.signature; + if (!chain.seenBlockInputCache.isVerifiedProposerSignature(slot, rootHex, signature)) { + const signatureSet = getBlockHeaderProposerSignatureSetByHeaderSlot( + chain.config, + firstSidecarSignedBlockHeader + ); + + if ( + !(await chain.bls.verifySignatureSets([signatureSet], { + verifyOnMainThread: true, + })) + ) { + throw new DataColumnSidecarValidationError({ + code: DataColumnSidecarErrorCode.PROPOSAL_SIGNATURE_INVALID, + blockRoot: rootHex, + slot: blockSlot, + index: dataColumnSidecars[0].index, + }); + } + + chain.seenBlockInputCache.markVerifiedProposerSignature(slot, rootHex, signature); + } + } - if (chain !== null) { - const rootHex = toRootHex(blockRoot); - const slot = firstSidecarSignedBlockHeader.message.slot; - const signature = firstSidecarSignedBlockHeader.signature; - if (!chain.seenBlockInputCache.isVerifiedProposerSignature(slot, rootHex, signature)) { - const signatureSet = getBlockHeaderProposerSignatureSetByHeaderSlot(chain.config, firstSidecarSignedBlockHeader); + const commitments: Uint8Array[] = []; + const cellIndices: number[] = []; + const cells: Uint8Array[] = []; + const proofs: Uint8Array[] = []; + for (let i = 0; i < dataColumnSidecars.length; i++) { + const columnSidecar = dataColumnSidecars[i]; if ( - !(await chain.bls.verifySignatureSets([signatureSet], { - verifyOnMainThread: true, - })) + i !== 0 && + !ssz.phase0.SignedBeaconBlockHeader.equals(firstSidecarSignedBlockHeader, columnSidecar.signedBlockHeader) ) { throw new DataColumnSidecarValidationError({ - code: DataColumnSidecarErrorCode.PROPOSAL_SIGNATURE_INVALID, - blockRoot: rootHex, + code: DataColumnSidecarErrorCode.INCORRECT_HEADER_ROOT, slot: blockSlot, - index: dataColumnSidecars[0].index, + expected: toRootHex(blockRoot), + actual: toRootHex(ssz.phase0.BeaconBlockHeader.hashTreeRoot(columnSidecar.signedBlockHeader.message)), }); } - chain.seenBlockInputCache.markVerifiedProposerSignature(slot, rootHex, signature); - } - } + if (columnSidecar.index >= NUMBER_OF_COLUMNS) { + throw new DataColumnSidecarValidationError( + { + code: DataColumnSidecarErrorCode.INVALID_INDEX, + slot: blockSlot, + columnIndex: columnSidecar.index, + }, + "DataColumnSidecar has invalid index" + ); + } - const commitments: Uint8Array[] = []; - const cellIndices: number[] = []; - const cells: Uint8Array[] = []; - const proofs: Uint8Array[] = []; - for (let i = 0; i < dataColumnSidecars.length; i++) { - const columnSidecar = dataColumnSidecars[i]; + if (columnSidecar.column.length !== blockBlobCount) { + throw new DataColumnSidecarValidationError({ + code: DataColumnSidecarErrorCode.INCORRECT_CELL_COUNT, + slot: blockSlot, + columnIndex: columnSidecar.index, + expected: blockBlobCount, + actual: columnSidecar.column.length, + }); + } - if ( - i !== 0 && - !ssz.phase0.SignedBeaconBlockHeader.equals(firstSidecarSignedBlockHeader, columnSidecar.signedBlockHeader) - ) { - throw new DataColumnSidecarValidationError({ - code: DataColumnSidecarErrorCode.INCORRECT_HEADER_ROOT, - slot: blockSlot, - expected: toRootHex(blockRoot), - actual: toRootHex(ssz.phase0.BeaconBlockHeader.hashTreeRoot(columnSidecar.signedBlockHeader.message)), - }); - } + if (columnSidecar.column.length !== columnSidecar.kzgCommitments.length) { + throw new DataColumnSidecarValidationError({ + code: DataColumnSidecarErrorCode.INCORRECT_KZG_COMMITMENTS_COUNT, + slot: blockSlot, + columnIndex: columnSidecar.index, + expected: columnSidecar.column.length, + actual: columnSidecar.kzgCommitments.length, + }); + } - if (columnSidecar.index >= NUMBER_OF_COLUMNS) { - throw new DataColumnSidecarValidationError( - { - code: DataColumnSidecarErrorCode.INVALID_INDEX, + if (columnSidecar.column.length !== columnSidecar.kzgProofs.length) { + throw new DataColumnSidecarValidationError({ + code: DataColumnSidecarErrorCode.INCORRECT_KZG_PROOF_COUNT, slot: blockSlot, columnIndex: columnSidecar.index, - }, - "DataColumnSidecar has invalid index" - ); - } + expected: columnSidecar.column.length, + actual: columnSidecar.kzgProofs.length, + }); + } - if (columnSidecar.column.length !== blockBlobCount) { - throw new DataColumnSidecarValidationError({ - code: DataColumnSidecarErrorCode.INCORRECT_CELL_COUNT, - slot: blockSlot, - columnIndex: columnSidecar.index, - expected: blockBlobCount, - actual: columnSidecar.column.length, - }); - } + const inclusionProofTimer = metrics?.dataColumnSidecarInclusionProofVerificationTime.startTimer(); + const validInclusionProof = verifyDataColumnSidecarInclusionProof(columnSidecar); + inclusionProofTimer?.(); + if (!validInclusionProof) { + throw new DataColumnSidecarValidationError( + { + code: DataColumnSidecarErrorCode.INCLUSION_PROOF_INVALID, + slot: blockSlot, + columnIndex: columnSidecar.index, + }, + "DataColumnSidecar has invalid inclusion proof" + ); + } - if (columnSidecar.column.length !== columnSidecar.kzgCommitments.length) { - throw new DataColumnSidecarValidationError({ - code: DataColumnSidecarErrorCode.INCORRECT_KZG_COMMITMENTS_COUNT, - slot: blockSlot, - columnIndex: columnSidecar.index, - expected: columnSidecar.column.length, - actual: columnSidecar.kzgCommitments.length, - }); + commitments.push(...columnSidecar.kzgCommitments); + cellIndices.push(...Array.from({length: columnSidecar.column.length}, () => columnSidecar.index)); + cells.push(...columnSidecar.column); + proofs.push(...columnSidecar.kzgProofs); } - if (columnSidecar.column.length !== columnSidecar.kzgProofs.length) { - throw new DataColumnSidecarValidationError({ - code: DataColumnSidecarErrorCode.INCORRECT_KZG_PROOF_COUNT, - slot: blockSlot, - columnIndex: columnSidecar.index, - expected: columnSidecar.column.length, - actual: columnSidecar.kzgProofs.length, - }); + let reason: string | undefined; + // batch verification for the cases: downloadByRange and downloadByRoot + const kzgVerificationTimer = metrics?.kzgVerificationDataColumnBatchTime.startTimer(); + try { + const valid = await kzg.asyncVerifyCellKzgProofBatch(commitments, cellIndices, cells, proofs); + if (!valid) { + reason = "Invalid KZG proof batch"; + } + } catch (e) { + reason = (e as Error).message; + } finally { + kzgVerificationTimer?.(); } - - if (!verifyDataColumnSidecarInclusionProof(columnSidecar)) { + if (reason !== undefined) { throw new DataColumnSidecarValidationError( { - code: DataColumnSidecarErrorCode.INCLUSION_PROOF_INVALID, + code: DataColumnSidecarErrorCode.INVALID_KZG_PROOF_BATCH, slot: blockSlot, - columnIndex: columnSidecar.index, + reason, }, - "DataColumnSidecar has invalid inclusion proof" + "DataColumnSidecar has invalid KZG proof batch" ); } - - commitments.push(...columnSidecar.kzgCommitments); - cellIndices.push(...Array.from({length: columnSidecar.column.length}, () => columnSidecar.index)); - cells.push(...columnSidecar.column); - proofs.push(...columnSidecar.kzgProofs); - } - - let reason: string | undefined; - try { - const valid = await kzg.asyncVerifyCellKzgProofBatch(commitments, cellIndices, cells, proofs); - if (!valid) { - reason = "Invalid KZG proof batch"; - } - } catch (e) { - reason = (e as Error).message; - } - if (reason !== undefined) { - throw new DataColumnSidecarValidationError( - { - code: DataColumnSidecarErrorCode.INVALID_KZG_PROOF_BATCH, - slot: blockSlot, - reason, - }, - "DataColumnSidecar has invalid KZG proof batch" - ); + metrics?.dataColumnSidecarProcessingSuccesses.inc(); + } finally { + verificationTimer?.(); } } diff --git a/packages/beacon-node/src/metrics/metrics/beacon.ts b/packages/beacon-node/src/metrics/metrics/beacon.ts index 09b5c89b7ee4..4698600dd65c 100644 --- a/packages/beacon-node/src/metrics/metrics/beacon.ts +++ b/packages/beacon-node/src/metrics/metrics/beacon.ts @@ -333,11 +333,13 @@ export function createBeaconMetrics(register: RegistryMetricCreator) { help: "Time taken to verify data_column sidecar inclusion proof", buckets: [0.002, 0.004, 0.006, 0.008, 0.01, 0.05, 1, 2], }), + // single verification dataColumnSidecarKzgProofsVerificationTime: register.histogram({ name: "beacon_data_column_sidecar_kzg_proofs_verification_seconds", - help: "Time taken to verify data_column sidecar kzg proofs", + help: "Time taken to verify single data_column sidecar kzg proofs", buckets: [0.01, 0.02, 0.03, 0.04, 0.05, 0.1, 0.2, 0.5, 1], }), + // batch verification kzgVerificationDataColumnBatchTime: register.histogram({ name: "beacon_kzg_verification_data_column_batch_seconds", help: "Runtime of batched data column kzg verification", @@ -361,10 +363,14 @@ export function createBeaconMetrics(register: RegistryMetricCreator) { help: "Duration of engine_getBlobsV2 requests", buckets: [0.01, 0.05, 0.1, 0.5, 1, 2.5, 5, 7.5], }), - targetCustodyGroupCount: register.gauge({ - name: "beacon_target_custody_group_count", + custodyGroupCount: register.gauge({ + name: "beacon_custody_groups", help: "Total number of custody groups within a node", }), + custodyGroupsBackfilled: register.gauge({ + name: "beacon_custody_groups_backfilled", + help: "Total number of custody groups backfilled by a node", + }), reconstructedColumns: register.counter({ name: "beacon_data_availability_reconstructed_columns_total", help: "Total count of reconstructed columns", diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index 9ba3c3258225..32955d5a9e4d 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -836,20 +836,23 @@ export function createLodestarMetrics( buckets: [0.5, 1, 2, 4, 6, 12], }), }, - recoverDataColumnSidecars: { - recoverTime: register.histogram({ - name: "lodestar_recover_data_column_sidecar_recover_time_seconds", - help: "Time elapsed to recover data column sidecar", - buckets: [0.5, 1.0, 1.5, 2], + // recovery in the case of specific blob rows required + recoverBlobSidecars: { + blobsReconstructed: register.counter({ + name: "lodestar_blobs_reconstructed_total", + help: "Total count of reconstructed blobs", + }), + reconstructionTime: register.histogram({ + name: "lodestar_blob_reconstruction_seconds", + help: "Time taken to reconstruct blobs", + buckets: [0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 2, 5], }), + }, + recoverDataColumnSidecars: { custodyBeforeReconstruction: register.gauge({ name: "lodestar_data_columns_in_custody_before_reconstruction", help: "Number of data columns in custody before reconstruction", }), - numberOfColumnsRecovered: register.gauge({ - name: "lodestar_recover_data_column_sidecar_recovered_columns_total", - help: "Total number of columns that were recovered", - }), reconstructionResult: register.counter<{result: DataColumnReconstructionCode}>({ name: "lodestar_data_column_sidecars_reconstruction_result", help: "Data column sidecars reconstruction result", @@ -857,6 +860,10 @@ export function createLodestarMetrics( }), }, dataColumns: { + alreadyAdded: register.counter({ + name: "lodestar_data_column_sidecar_already_added", + help: "Total number of columns that were already added by other sources while waiting", + }), bySource: register.gauge<{source: BlockInputSource}>({ name: "lodestar_data_columns_by_source", help: "Number of received data columns by source", @@ -912,11 +919,6 @@ export function createLodestarMetrics( help: "Total number of imported blobs by source", labelNames: ["blobsSource"], }), - columnsBySource: register.gauge<{source: BlockInputSource}>({ - name: "lodestar_import_columns_by_source_total", - help: "Total number of imported columns (sampled columns) by source", - labelNames: ["source"], - }), notOverrideFcuReason: register.counter<{reason: NotReorgedReason}>({ name: "lodestar_import_block_not_override_fcu_reason_total", help: "Reason why the fcu call is not suppressed during block import", diff --git a/packages/beacon-node/src/sync/range/range.ts b/packages/beacon-node/src/sync/range/range.ts index dbd5213ae4ef..64c284c65936 100644 --- a/packages/beacon-node/src/sync/range/range.ts +++ b/packages/beacon-node/src/sync/range/range.ts @@ -206,6 +206,7 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) { logger: this.logger, peerIdStr: peer.peerId, batchBlocks, + peerDasMetrics: this.chain.metrics?.peerDas, ...batch.getRequestsForPeer(peer), }); const cached = cacheByRangeResponses({ diff --git a/packages/beacon-node/src/sync/utils/downloadByRange.ts b/packages/beacon-node/src/sync/utils/downloadByRange.ts index ba7b4df98dfc..849fd6448ed7 100644 --- a/packages/beacon-node/src/sync/utils/downloadByRange.ts +++ b/packages/beacon-node/src/sync/utils/downloadByRange.ts @@ -12,6 +12,7 @@ import { import {SeenBlockInput} from "../../chain/seenCache/seenGossipBlockInput.js"; import {validateBlockBlobSidecars} from "../../chain/validation/blobSidecar.js"; import {validateBlockDataColumnSidecars} from "../../chain/validation/dataColumnSidecar.js"; +import {BeaconMetrics} from "../../metrics/metrics/beacon.js"; import {INetwork} from "../../network/index.js"; import {getBlobKzgCommitments} from "../../util/dataColumns.js"; import {PeerIdStr} from "../../util/peerId.js"; @@ -35,6 +36,7 @@ export type DownloadAndCacheByRangeProps = DownloadByRangeRequests & { logger: Logger; peerIdStr: string; batchBlocks?: IBlockInput[]; + peerDasMetrics?: BeaconMetrics["peerDas"] | null; }; export type CacheByRangeResponsesProps = { @@ -196,6 +198,7 @@ export async function downloadByRange({ blocksRequest, blobsRequest, columnsRequest, + peerDasMetrics, }: DownloadAndCacheByRangeProps): Promise> { let response: DownloadByRangeResponses; try { @@ -220,6 +223,7 @@ export async function downloadByRange({ blocksRequest, blobsRequest, columnsRequest, + peerDasMetrics, ...response, }); @@ -290,10 +294,12 @@ export async function validateResponses({ blocks, blobSidecars, columnSidecars, + peerDasMetrics, }: DownloadByRangeRequests & DownloadByRangeResponses & { config: ChainForkConfig; batchBlocks?: IBlockInput[]; + peerDasMetrics?: BeaconMetrics["peerDas"] | null; }): Promise> { // Blocks are always required for blob/column validation // If a blocksRequest is provided, blocks have just been downloaded @@ -372,7 +378,8 @@ export async function validateResponses({ config, columnsRequest, blocksForDataValidation, - columnSidecars + columnSidecars, + peerDasMetrics ); validatedResponses.validatedColumnSidecars = validatedColumnSidecarsResult.result; warnings = validatedColumnSidecarsResult.warnings; @@ -608,7 +615,8 @@ export async function validateColumnsByRangeResponse( config: ChainForkConfig, request: fulu.DataColumnSidecarsByRangeRequest, blocks: ValidatedBlock[], - columnSidecars: fulu.DataColumnSidecars + columnSidecars: fulu.DataColumnSidecars, + peerDasMetrics?: BeaconMetrics["peerDas"] | null ): Promise> { const warnings: DownloadByRangeError[] = []; @@ -764,7 +772,8 @@ export async function validateColumnsByRangeResponse( slot, blockRoot, blobCount, - columnSidecars + columnSidecars, + peerDasMetrics ).then(() => ({ blockRoot, columnSidecars, diff --git a/packages/beacon-node/src/sync/utils/downloadByRoot.ts b/packages/beacon-node/src/sync/utils/downloadByRoot.ts index 40cd33a56fba..fcfa8053bd8b 100644 --- a/packages/beacon-node/src/sync/utils/downloadByRoot.ts +++ b/packages/beacon-node/src/sync/utils/downloadByRoot.ts @@ -440,7 +440,7 @@ export async function fetchAndValidateColumns({ ); } - await validateBlockDataColumnSidecars(chain, slot, blockRoot, blobCount, columnSidecars); + await validateBlockDataColumnSidecars(chain, slot, blockRoot, blobCount, columnSidecars, chain?.metrics?.peerDas); return {result: columnSidecars, warnings: warnings.length > 0 ? warnings : null}; } diff --git a/packages/beacon-node/src/util/dataColumns.ts b/packages/beacon-node/src/util/dataColumns.ts index 69bca00577e4..ac16f9683d3e 100644 --- a/packages/beacon-node/src/util/dataColumns.ts +++ b/packages/beacon-node/src/util/dataColumns.ts @@ -425,7 +425,8 @@ export async function recoverDataColumnSidecars( partialSidecars.set(columnSidecar.index, columnSidecar); } - const timer = metrics?.recoverDataColumnSidecars.recoverTime.startTimer(); + const timer = metrics?.peerDas.dataColumnsReconstructionTime.startTimer(); + // if this function throws, we catch at the consumer side const fullSidecars = await dataColumnMatrixRecovery(partialSidecars).catch(() => null); timer?.(); @@ -435,6 +436,7 @@ export async function recoverDataColumnSidecars( if (blockInput.getAllColumns().length === NUMBER_OF_COLUMNS) { // either gossip or getBlobsV2 resolved availability while we were recovering + metrics?.dataColumns.alreadyAdded.inc(fullSidecars.length); return DataColumnReconstructionCode.SuccessLate; } @@ -458,8 +460,10 @@ export async function recoverDataColumnSidecars( sidecarsToPublish.push(columnSidecar); } } + metrics?.peerDas.reconstructedColumns.inc(sidecarsToPublish.length); + metrics?.dataColumns.bySource.inc({source: BlockInputSource.recovery}, sidecarsToPublish.length); emitter.emit(ChainEvent.publishDataColumns, sidecarsToPublish); - + // TODO: Can we record dataColumns.sentPeersPerSubnet metric somehow return DataColumnReconstructionCode.SuccessResolved; } diff --git a/packages/beacon-node/src/util/execution.ts b/packages/beacon-node/src/util/execution.ts index c05393769829..18fcf070c0d5 100644 --- a/packages/beacon-node/src/util/execution.ts +++ b/packages/beacon-node/src/util/execution.ts @@ -173,16 +173,21 @@ export async function getDataColumnSidecarsFromExecution( } let dataColumnSidecars: fulu.DataColumnSidecars; - const cellsAndProofs = await getCellsAndProofs(blobs); - if (blockInput.hasBlock()) { - dataColumnSidecars = getDataColumnSidecarsFromBlock( - config, - blockInput.getBlock() as fulu.SignedBeaconBlock, - cellsAndProofs - ); - } else { - const firstSidecar = blockInput.getAllColumns()[0]; - dataColumnSidecars = getDataColumnSidecarsFromColumnSidecar(firstSidecar, cellsAndProofs); + const compTimer = metrics?.peerDas.dataColumnSidecarComputationTime.startTimer(); + try { + const cellsAndProofs = await getCellsAndProofs(blobs); + if (blockInput.hasBlock()) { + dataColumnSidecars = getDataColumnSidecarsFromBlock( + config, + blockInput.getBlock() as fulu.SignedBeaconBlock, + cellsAndProofs + ); + } else { + const firstSidecar = blockInput.getAllColumns()[0]; + dataColumnSidecars = getDataColumnSidecarsFromColumnSidecar(firstSidecar, cellsAndProofs); + } + } finally { + compTimer?.(); } // Publish columns if and only if subscribed to them @@ -191,13 +196,15 @@ export async function getDataColumnSidecarsFromExecution( // for columns that we already seen, it will be ignored through `ignoreDuplicatePublishError` gossip option emitter.emit(ChainEvent.publishDataColumns, sampledColumns); + // TODO: Can we record dataColumns.sentPeersPerSubnet metric here somehow // add all sampled columns to the block input, even if we didn't sample them const seenTimestampSec = Date.now() / 1000; + let alreadyAddedColumnsCount = 0; for (const columnSidecar of sampledColumns) { if (blockInput.hasColumn(columnSidecar.index)) { // columns may have been added while waiting - // TODO(fulu): add metrics for this condition + alreadyAddedColumnsCount++; continue; } @@ -217,7 +224,11 @@ export async function getDataColumnSidecarsFromExecution( }); } } + metrics?.dataColumns.alreadyAdded.inc(alreadyAddedColumnsCount); - metrics?.dataColumns.bySource.inc({source: BlockInputSource.engine}, previouslyMissingColumns.length); + metrics?.dataColumns.bySource.inc( + {source: BlockInputSource.engine}, + previouslyMissingColumns.length - alreadyAddedColumnsCount + ); return DataColumnEngineResult.SuccessResolved; } diff --git a/packages/beacon-node/test/spec/presets/fork_choice.test.ts b/packages/beacon-node/test/spec/presets/fork_choice.test.ts index dc3e9dcf5a33..8a3f11940e86 100644 --- a/packages/beacon-node/test/spec/presets/fork_choice.test.ts +++ b/packages/beacon-node/test/spec/presets/fork_choice.test.ts @@ -245,7 +245,8 @@ const forkChoiceTest = blockRoot, (signedBlock as SignedBeaconBlock).message.body.blobKzgCommitments .length, - columns + columns, + chain.metrics?.peerDas ); blockImport = BlockInputColumns.createFromBlock({ From 26ed5ad3530085916a30e304076feb26bf0a7f62 Mon Sep 17 00:00:00 2001 From: Lodekeeper <258435968+lodekeeper@users.noreply.github.com> Date: Tue, 17 Mar 2026 13:40:26 +0000 Subject: [PATCH 48/63] ci: add native module CPU portability check (#9044) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Adds a CI workflow that scans prebuilt native (`.node`) binaries for unconditional AVX/AVX2 instructions. Native modules using AVX **must** have CPUID-based runtime dispatch to avoid SIGILL crashes on CPUs without AVX support. ## Why Closes #9042. A user running Lodestar v1.40.0 on a Celeron N5105 (no AVX) hit exit code 132 (SIGILL). Root cause: [`@vekexasia/bigint-buffer2@1.1.0`](https://github.com/vekexasia/bigint-swissknife/issues/6) is compiled with hard-coded `-C target-feature=+avx2,+bmi2`, producing 1292 AVX instructions with zero CPUID dispatch. All other native modules in our tree (`blst`, `node-eth-kzg`, `hashtree`, `libp2p-quic`) have proper CPUID-based runtime detection. ## How it works After `pnpm install`, the script: 1. Finds all `linux-x64` native `.node` files 2. Counts AVX instructions (YMM register references) via `objdump` 3. Counts CPUID calls (runtime CPU feature detection) 4. Fails if any module has AVX instructions but zero CPUID calls Triggers on dependency changes (`package.json`, `pnpm-lock.yaml`) and on `workflow_dispatch`. ## Example output ``` OK: @chainsafe/blst-linux-x64-gnu (104 AVX insns, 9 CPUID calls) OK: @chainsafe/hashtree-linux-x64-gnu (6067 AVX insns, 10 CPUID calls) OK: @chainsafe/libp2p-quic-linux-x64-gnu (6755 AVX insns, 21 CPUID calls) OK: @crate-crypto/node-eth-kzg-linux-x64-gnu (100 AVX insns, 11 CPUID calls) FAIL: @vekexasia/bigint-buffer2 1292 AVX instructions, 0 CPUID dispatch calls Binary will crash with SIGILL on CPUs without AVX (e.g. Celeron N5105) ``` Upstream issue filed: https://github.com/vekexasia/bigint-swissknife/issues/6 --- > This PR was authored with AI assistance (Lodekeeper 🌟) --------- Co-authored-by: lodekeeper Co-authored-by: Nico Flaig --- .github/workflows/native-portability.yml | 32 +++++++++++ packages/state-transition/package.json | 2 +- packages/utils/package.json | 2 +- packages/validator/package.json | 2 +- pnpm-lock.yaml | 18 +++---- pnpm-workspace.yaml | 1 + scripts/ci/check_native_portability.sh | 68 ++++++++++++++++++++++++ 7 files changed, 113 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/native-portability.yml create mode 100755 scripts/ci/check_native_portability.sh diff --git a/.github/workflows/native-portability.yml b/.github/workflows/native-portability.yml new file mode 100644 index 000000000000..0cfb51047e07 --- /dev/null +++ b/.github/workflows/native-portability.yml @@ -0,0 +1,32 @@ +name: Native portability +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +on: + push: + branches: [unstable, stable] + pull_request: + workflow_dispatch: + +jobs: + check-native-portability: + name: Check native module portability + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4 + + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + with: + node-version: 24 + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Install binutils + run: sudo apt-get install -y binutils + + - name: Check native module CPU portability + run: bash scripts/ci/check_native_portability.sh diff --git a/packages/state-transition/package.json b/packages/state-transition/package.json index e2c44d09a8e6..937ba9bc2b8c 100644 --- a/packages/state-transition/package.json +++ b/packages/state-transition/package.json @@ -71,7 +71,7 @@ "@lodestar/params": "workspace:^", "@lodestar/types": "workspace:^", "@lodestar/utils": "workspace:^", - "@vekexasia/bigint-buffer2": "^1.1.0" + "@vekexasia/bigint-buffer2": "^1.1.1" }, "devDependencies": { "@lodestar/api": "workspace:^" diff --git a/packages/utils/package.json b/packages/utils/package.json index 0e3d1e24a6eb..5c91e5d3aa00 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -47,7 +47,7 @@ "types": "lib/index.d.ts", "dependencies": { "@chainsafe/as-sha256": "^1.2.0", - "@vekexasia/bigint-buffer2": "^1.1.0", + "@vekexasia/bigint-buffer2": "^1.1.1", "any-signal": "^4.1.1", "case": "^1.6.3", "js-yaml": "^4.1.0" diff --git a/packages/validator/package.json b/packages/validator/package.json index 3364107ab993..3a3cf095e926 100644 --- a/packages/validator/package.json +++ b/packages/validator/package.json @@ -63,7 +63,7 @@ "@lodestar/logger": "workspace:^", "@lodestar/spec-test-util": "workspace:^", "@lodestar/test-utils": "workspace:^", - "@vekexasia/bigint-buffer2": "^1.1.0", + "@vekexasia/bigint-buffer2": "^1.1.1", "rimraf": "^4.4.1" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4cad8ddeeaa0..0f8017b11771 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1002,8 +1002,8 @@ importers: specifier: workspace:^ version: link:../utils '@vekexasia/bigint-buffer2': - specifier: ^1.1.0 - version: 1.1.0 + specifier: ^1.1.1 + version: 1.1.1 devDependencies: '@lodestar/api': specifier: workspace:^ @@ -1058,8 +1058,8 @@ importers: specifier: ^1.2.0 version: 1.2.0 '@vekexasia/bigint-buffer2': - specifier: ^1.1.0 - version: 1.1.0 + specifier: ^1.1.1 + version: 1.1.1 any-signal: specifier: ^4.1.1 version: 4.1.1 @@ -1126,8 +1126,8 @@ importers: specifier: workspace:^ version: link:../test-utils '@vekexasia/bigint-buffer2': - specifier: ^1.1.0 - version: 1.1.0 + specifier: ^1.1.1 + version: 1.1.1 rimraf: specifier: ^4.4.1 version: 4.4.1 @@ -3278,8 +3278,8 @@ packages: resolution: {integrity: sha512-BDHJjXlPldInEogbzAc7OCLvT75p3rdkmb5YIA6Je0vjg+5z1UQp3moAvcBGvZQflO/gusOd9a74EfrMVUU/4g==} hasBin: true - '@vekexasia/bigint-buffer2@1.1.0': - resolution: {integrity: sha512-CB19/UHoop2Q+HaRU1lI3fWKUkwju7XtwGTvdDfJgZyPHI+lLXKDYOOkz+NOCLfcBSXpJXpue/vLN0PDZtBT/Q==} + '@vekexasia/bigint-buffer2@1.1.1': + resolution: {integrity: sha512-uL63NGANn85oKrsrvD0CzgmkuMlWQnGv/J7sz0QQ2CnRaF1nMF0fAoRFH9PFtGRr62ZCA14qkHZATyBZbKYlRQ==} engines: {node: '>= 14.0.0'} peerDependencies: '@vekexasia/bigint-uint8array': '*' @@ -10140,7 +10140,7 @@ snapshots: '@typescript/native-preview-win32-arm64': 7.0.0-dev.20260303.1 '@typescript/native-preview-win32-x64': 7.0.0-dev.20260303.1 - '@vekexasia/bigint-buffer2@1.1.0': {} + '@vekexasia/bigint-buffer2@1.1.1': {} '@vitest/browser-playwright@4.0.7(playwright@1.56.1)(vite@6.1.6(@types/node@24.10.1)(yaml@2.8.2))(vitest@4.0.7)': dependencies: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index aeba7a2b0dcb..74aa36e450d8 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -6,6 +6,7 @@ linkWorkspacePackages: true minimumReleaseAge: 2880 minimumReleaseAgeExclude: + - "@vekexasia/bigint-buffer2" - "@chainsafe/*" - "@libp2p/*" - "libp2p" diff --git a/scripts/ci/check_native_portability.sh b/scripts/ci/check_native_portability.sh new file mode 100755 index 000000000000..4061b1021f7f --- /dev/null +++ b/scripts/ci/check_native_portability.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# check_native_portability.sh +# +# Scans prebuilt native (.node) binaries for unconditional AVX/AVX2 usage. +# Native modules that use AVX instructions MUST have CPUID-based runtime +# dispatch to fall back on CPUs without AVX support (e.g. Intel Atom/Celeron). +# +# Catches issues like https://github.com/ChainSafe/lodestar/issues/9042 +# where a dependency was compiled with hard-coded -C target-feature=+avx2. + +set -euo pipefail + +if ! command -v objdump > /dev/null 2>&1; then + echo "warning: objdump not found, skipping CPU portability check." + exit 0 +fi + +EXIT_CODE=0 + +echo "Checking native modules for CPU portability..." +echo "" + +found=0 +while IFS= read -r node_file; do + found=1 + # Relative path from node_modules for readable output + name=$(echo "$node_file" | sed 's|.*node_modules/||;s|/[^/]*\.node$||') + + # Disassemble once, then grep for AVX indicators and CPUID calls + dump=$(objdump -d "$node_file" 2>/dev/null || true) + + # Count 256-bit AVX (YMM registers) + ymm_count=$(echo "$dump" | grep -c "ymm" || true) + # Count VEX-encoded instructions (v-prefixed SIMD mnemonics like vmovaps, vzeroupper) + # These also require AVX support and will SIGILL on non-AVX CPUs + vex_count=$(echo "$dump" | grep -cE "\b(vmov|vadd|vsub|vmul|vdiv|vxor|vpxor|vpand|vpor|vpcmp|vshuf|vperm|vbroadcast|vinsert|vextract|vzero|vfmadd|vfmsub|vfnmadd|vfnmsub|vcvt|vpack|vpunpck|vpalign|vblend|vround|vtest|vptest)[a-z]*\b" || true) + avx_count=$((ymm_count + vex_count)) + # Count CPUID calls (runtime CPU feature detection) + cpuid_count=$(echo "$dump" | grep -c "cpuid" || true) + + if [ "$avx_count" -gt 0 ] && [ "$cpuid_count" -eq 0 ]; then + echo "FAIL: $name" + echo " $avx_count AVX instructions ($ymm_count ymm, $vex_count vex-encoded), 0 CPUID dispatch calls" + echo " Binary will crash with SIGILL on CPUs without AVX (e.g. Celeron N5105)" + EXIT_CODE=1 + elif [ "$avx_count" -gt 0 ]; then + echo "OK: $name ($avx_count AVX insns, $cpuid_count CPUID calls)" + else + echo "OK: $name (no AVX)" + fi +done < <(find node_modules -name "*.node" -path "*linux-x64*" ! -path "*/rollup/*" ! -path "*/swc/*" ! -path "*musl*" 2>/dev/null | sort) + +if [ "$found" -eq 0 ]; then + echo "No linux-x64 native modules found (skipped)." + exit 0 +fi + +echo "" +if [ "$EXIT_CODE" -eq 0 ]; then + echo "All native modules have proper CPU feature detection" +else + echo "" + echo "Some native modules use AVX without runtime CPU detection." + echo "These will crash with SIGILL (exit code 132) on CPUs without AVX support." + echo "See: https://github.com/ChainSafe/lodestar/issues/9042" +fi + +exit $EXIT_CODE From 1ff95eaa7b131024d3327c29d330de7066b5e253 Mon Sep 17 00:00:00 2001 From: Lodekeeper <258435968+lodekeeper@users.noreply.github.com> Date: Tue, 17 Mar 2026 17:57:46 +0000 Subject: [PATCH 49/63] chore: create GitHub releases as draft to support immutable releases (#9056) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Creates GitHub releases as draft first, uploads assets, then publishes (undrafts). Fixes the `Create Release` step failing on repos with immutable releases enabled. ## Why Closes the RC publish failure: https://github.com/ChainSafe/lodestar/actions/runs/23205012815/job/67438769695 ``` Error: Cannot upload asset lodestar-v1.41.0-rc.1-linux-arm64.tar.gz to an immutable release. GitHub only allows asset uploads before a release is published... ``` GitHub's immutable releases feature fires `release.published` immediately for prereleases, making the release immutable before assets finish uploading. The fix: create as `draft: true`, upload assets, then `gh release edit --draft=false`. ## Changes - **`publish-rc.yml`**: Add `draft: true` to release creation + publish step - **`publish-stable.yml`**: Same fix for stable releases The rollback step still works — it references `steps.create_release.outputs.id` which is set during draft creation. > This PR was authored by Lodekeeper 🌟 with AI assistance. --------- Co-authored-by: lodekeeper Co-authored-by: Nico Flaig --- .github/workflows/publish-rc.yml | 6 ++++++ .github/workflows/publish-stable.yml | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/.github/workflows/publish-rc.yml b/.github/workflows/publish-rc.yml index dcd3d8b59c39..2ea480971a69 100644 --- a/.github/workflows/publish-rc.yml +++ b/.github/workflows/publish-rc.yml @@ -88,6 +88,7 @@ jobs: body_path: "CHANGELOG.md" name: Release ${{ needs.tag.outputs.tag }} prerelease: true + draft: true - name: Change and commit version # Write version before publishing so it's picked up by `lerna publish from-package`. @@ -108,6 +109,11 @@ jobs: - name: Publish to npm registry run: pnpm run release:publish --dist-tag rc + - name: Publish release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: gh release edit ${{ needs.tag.outputs.tag }} --draft=false + # In case of failure - name: Rollback on failure if: failure() diff --git a/.github/workflows/publish-stable.yml b/.github/workflows/publish-stable.yml index 6e292260f505..2d13ce1ea3b3 100644 --- a/.github/workflows/publish-stable.yml +++ b/.github/workflows/publish-stable.yml @@ -94,10 +94,16 @@ jobs: body_path: "CHANGELOG.md" name: Release ${{ needs.tag.outputs.tag }} prerelease: false + draft: true - name: Publish to npm registry (release) run: pnpm run release:publish + - name: Publish release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: gh release edit ${{ needs.tag.outputs.tag }} --draft=false + # In case of failure - name: Rollback on failure if: failure() From e8653ab66cb70a3c42c67dcc6fab9ea9d959a988 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Sat, 14 Mar 2026 10:48:22 +0000 Subject: [PATCH 50/63] chore: bump package versions to 1.41.0 --- lerna.json | 2 +- packages/api/package.json | 2 +- packages/beacon-node/package.json | 2 +- packages/cli/package.json | 2 +- packages/config/package.json | 6 +++--- packages/db/package.json | 2 +- packages/era/package.json | 2 +- packages/flare/package.json | 2 +- packages/fork-choice/package.json | 2 +- packages/light-client/package.json | 2 +- packages/logger/package.json | 2 +- packages/params/package.json | 2 +- packages/prover/package.json | 2 +- packages/reqresp/package.json | 2 +- packages/spec-test-util/package.json | 2 +- packages/state-transition/package.json | 2 +- packages/test-utils/package.json | 2 +- packages/types/package.json | 2 +- packages/utils/package.json | 2 +- packages/validator/package.json | 2 +- 20 files changed, 22 insertions(+), 22 deletions(-) diff --git a/lerna.json b/lerna.json index e6666ade7b69..55881c183c53 100644 --- a/lerna.json +++ b/lerna.json @@ -2,7 +2,7 @@ "packages": [ "packages/*" ], - "version": "1.40.0", + "version": "1.41.0", "stream": true, "command": { "version": { diff --git a/packages/api/package.json b/packages/api/package.json index 8272a90a3ab2..465ddb8c6b5d 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.40.0", + "version": "1.41.0", "type": "module", "exports": { ".": { diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index 098b2910e7c0..c9b37e3378ee 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.40.0", + "version": "1.41.0", "type": "module", "exports": { ".": { diff --git a/packages/cli/package.json b/packages/cli/package.json index 2585e144803b..80265a3e7b37 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@chainsafe/lodestar", - "version": "1.40.0", + "version": "1.41.0", "description": "Command line interface for lodestar", "author": "ChainSafe Systems", "license": "Apache-2.0", diff --git a/packages/config/package.json b/packages/config/package.json index 784e6e2d1add..552ea9d5a336 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/config", - "version": "1.40.0", + "version": "1.41.0", "description": "Chain configuration required for lodestar", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -68,8 +68,8 @@ "@chainsafe/as-sha256": "^1.2.0", "@chainsafe/ssz": "^1.2.2", "@lodestar/params": "workspace:^", + "@lodestar/spec-test-util": "workspace:^", "@lodestar/types": "workspace:^", - "@lodestar/utils": "workspace:^", - "@lodestar/spec-test-util": "workspace:^" + "@lodestar/utils": "workspace:^" } } diff --git a/packages/db/package.json b/packages/db/package.json index 6feaf4b9fb3d..0d74a4ec2135 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/db", - "version": "1.40.0", + "version": "1.41.0", "description": "DB modules of Lodestar", "author": "ChainSafe Systems", "homepage": "https://github.com/ChainSafe/lodestar#readme", diff --git a/packages/era/package.json b/packages/era/package.json index d60f46033e04..ca0add58b0aa 100644 --- a/packages/era/package.json +++ b/packages/era/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.40.0", + "version": "1.41.0", "type": "module", "exports": { ".": { diff --git a/packages/flare/package.json b/packages/flare/package.json index ac4328cd4c76..a65332cfbdb2 100644 --- a/packages/flare/package.json +++ b/packages/flare/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/flare", - "version": "1.40.0", + "version": "1.41.0", "description": "Beacon chain debugging tool", "author": "ChainSafe Systems", "license": "Apache-2.0", diff --git a/packages/fork-choice/package.json b/packages/fork-choice/package.json index 2efea39cfddf..3f0d0b227ea0 100644 --- a/packages/fork-choice/package.json +++ b/packages/fork-choice/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.40.0", + "version": "1.41.0", "type": "module", "exports": { ".": { diff --git a/packages/light-client/package.json b/packages/light-client/package.json index 6746541272ed..100c677ff7c5 100644 --- a/packages/light-client/package.json +++ b/packages/light-client/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.40.0", + "version": "1.41.0", "type": "module", "exports": { ".": { diff --git a/packages/logger/package.json b/packages/logger/package.json index 38b241f94ec3..5e05a5d08ab6 100644 --- a/packages/logger/package.json +++ b/packages/logger/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.40.0", + "version": "1.41.0", "type": "module", "exports": { ".": { diff --git a/packages/params/package.json b/packages/params/package.json index 2f7fbbabfda0..d37fe138c3b2 100644 --- a/packages/params/package.json +++ b/packages/params/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/params", - "version": "1.40.0", + "version": "1.41.0", "description": "Chain parameters required for lodestar", "author": "ChainSafe Systems", "license": "Apache-2.0", diff --git a/packages/prover/package.json b/packages/prover/package.json index 5c900d28d600..e087c53f801b 100644 --- a/packages/prover/package.json +++ b/packages/prover/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.40.0", + "version": "1.41.0", "type": "module", "exports": { ".": { diff --git a/packages/reqresp/package.json b/packages/reqresp/package.json index f34eef7498f9..5373f6b8ee4c 100644 --- a/packages/reqresp/package.json +++ b/packages/reqresp/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.40.0", + "version": "1.41.0", "type": "module", "exports": { ".": { diff --git a/packages/spec-test-util/package.json b/packages/spec-test-util/package.json index 5cad567cf9c3..e5449b76fe38 100644 --- a/packages/spec-test-util/package.json +++ b/packages/spec-test-util/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/spec-test-util", - "version": "1.40.0", + "version": "1.41.0", "description": "Spec test suite generator from yaml test files", "author": "ChainSafe Systems", "license": "Apache-2.0", diff --git a/packages/state-transition/package.json b/packages/state-transition/package.json index 937ba9bc2b8c..8ae51f22506c 100644 --- a/packages/state-transition/package.json +++ b/packages/state-transition/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.40.0", + "version": "1.41.0", "type": "module", "exports": { ".": { diff --git a/packages/test-utils/package.json b/packages/test-utils/package.json index 4b267fccb591..0ddd99ed8318 100644 --- a/packages/test-utils/package.json +++ b/packages/test-utils/package.json @@ -1,7 +1,7 @@ { "name": "@lodestar/test-utils", "private": true, - "version": "1.40.0", + "version": "1.41.0", "description": "Test utilities reused across other packages", "author": "ChainSafe Systems", "license": "Apache-2.0", diff --git a/packages/types/package.json b/packages/types/package.json index c54677425b7a..ba62afc42ba0 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.40.0", + "version": "1.41.0", "type": "module", "exports": { ".": { diff --git a/packages/utils/package.json b/packages/utils/package.json index 5c91e5d3aa00..5f832140d0a5 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.40.0", + "version": "1.41.0", "type": "module", "exports": { ".": { diff --git a/packages/validator/package.json b/packages/validator/package.json index 3a3cf095e926..f25a16adda46 100644 --- a/packages/validator/package.json +++ b/packages/validator/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/validator", - "version": "1.40.0", + "version": "1.41.0", "description": "A Typescript implementation of the validator client", "author": "ChainSafe Systems", "license": "Apache-2.0", From 9939b12b53e8c9794eb736ca597f016b5ac66ad5 Mon Sep 17 00:00:00 2001 From: Lodekeeper <258435968+lodekeeper@users.noreply.github.com> Date: Wed, 18 Mar 2026 12:20:02 +0000 Subject: [PATCH 51/63] docs: update docs favicon to current Lodestar logo (#9058) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description Replace outdated circular orb favicon with the current diamond/iridescent Lodestar logo, matching the ENR app (enr.chainsafe.io) branding. ### What changed - `docs/static/images/favicon.ico` — updated to current Lodestar diamond logo (multi-size: 16x16 + 32x32) Preview: https://lodekeeper.github.io/lodestar/ --- *This PR was created with assistance from Lodekeeper 🌟* --------- Co-authored-by: lodekeeper --- docs/static/images/favicon.ico | Bin 41662 -> 6771 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/docs/static/images/favicon.ico b/docs/static/images/favicon.ico index 4e34d94c437f091a09b159f7c48094562622903c..ab0809ad3eaf934268e5ec399a710f62021eca06 100644 GIT binary patch literal 6771 zcmaKvRaDhc)Bk^TNp}bwT98KK&~Rw!5RmTf?(S|5heqixK@m70-AITu2auLVLjFGQ zbMwD=*Sjxf&+J(HotVgdj%3IOo%{5Mv^ z2LNUq0EpL6Q^3Wh!v6P)tE33g`gi}Q0|6%bKTCJ{Z2qq-Qwbui;|Gh@Hdf5kqzKtK z3-Th947YO?}*ux3J z93(`dCAPUZI{h@n-4;YG$~Cc1NQfPg#?%t;Ko=WVVZF6@(PnKdx1!Ym|HgTlMk(AT_o zUkou{m5tAZ%+HWEN&_-LSVHCil!)^yxJ@#RRIe`1HGw0))NT)q6#LWwiy7EClamw9 z?s>{;7M8$JKrhPmUkV8afHV!zh2r42b?q$A1#h}7uSjNuSb51v^D;NWaxBX(3&rY% z^$P&YZVe8tY<-7OuQhg>;P}rI+3@%HvB)+*2NMTdAx?rgM1)&1I`&T#TK?#=KgCr*(b;hX zCF=M4lhoaJ{M~!BmG-sne{oksB&|QpnZ^H8DWD{$25FEni}+u(O#d%U|36yx3<$Ua z0F1-`YPtQLy_@zDsl@(Q*3ld%CxdmNFM|Y0xRl46dBSFaD}x^k?NQRhG&-UPh*ku6 zUKQGxWUQFthJTC>NBu=GB7iy?lY>e$M}>8~ojm!CA|d*gbi9`7r{TKzQ@Worn_vXf z#`Nu!_=eC%m+zDakP7seOcMYJ$`0h@bd9fu8hgc+8x+h~(WNl=PJf6DKd={pXW!v5 z2dAPCgj}C({hq;SLzhBI$76es_l5mUv~wDi0S|TM1}CnKZR7q4d&WVDhf$f@wC;Uq zEwA{kqAUD&dQsHY(9o5w$$NL}ldcHljWRj1b90Zfjr0HuD}Y_h8`RtztX615DZn}W zRBR|?wQxacu+BT225vS_X&0^}#tk!roT?l4zT}xihd?}@obMmxIu&G6csZzAUAE6! zTsC=a`nP}LwI4xNB_K@Vcc$3AG(7f7Gx?eUu0Qf~FN<)*^0H@-4=z~7B*Dy!d_%u& z6W4z5%6z`H2yUsnND7>!{zym5o3A)?pmD#5tk`I4#xt`&{7jJB7~g|l0@brf-p%1k z*3T8xi;RLBy_eE659)P2fO8FvzVQm_E7EMU@=WbFlPaGTwthuxmrZ&>X{qS>(*42E zl{wl)JWHO=8SO1s269LQmk685DYYaJl045@cyE{Q zwz|3X0=F~p-#-imqktr4iOaV^4XA>Pi^j4^yw);0H%%>t9Nk8*CqVaP(Hsyt-|?|G zp|9?jt@yCOg)VbiZ`q|9m-r+5rQ}q&DcAc|!h{adR?xq-R zESYAD2NlV#FcUn`xq>qRLi+{#w=_)*5(g-~YXEs8reBh=T1p+W+VN=4dYzFkF}G{7 zwO%0h3v{%|cPANhpH!r5V|~s76IEC|Bs9WI8g1pv!#nDh8z~iwe4YDVQiE>%LFOwj zjli8~5K{O}!GTc#*UJs%I`&N9M?($_SI^eu_OG~za3@3e#f?xS;o$MFnr_l^``0nz#s&#+zLBgzQ z(7S9ax|#F#r~kA!Nwjean?_VSbevRn8vNaHmifM1)n}C7e7!YI3yGu9zLNx)B)^2OO~tlZcg@xv{Jc0f*QY}WhZpoMuKiS0*LS09XM&^Xt< zWh+H*)8ck`X*bA-w-)7sIA+UG+~N|q{rfCAm!lWHYaLhc9Jrsi|4;JJ?2MT_0I=|8 zzh&v-ar4soa5Wahht`S9(YH&+W~c^PO>{}El&j8EvnCm7#1*#RIANGS zHaTkb>I)J}3CeS+A?AfO8tqji3tYoiwv*AaPXz!csw79Q^B|I<>6zSIL{L^3XfMYqWK*mLGvv8}#v5&5mr~ zfQ8v{`u5;O#M;Md1iu3GkLZW=$N59l>ik&0!qVgvebdRYr@WIMx~1UaQWICg*&Fn%{w=g<&eH0OF`H(3 zT*(|Pg-c<~w3_IXj#mdjC+aCRZ{6OtL3MAZq5{;Z5I>I6i2F`QSj}FeB9+5Ryl&6U z@XIFReeYw;NCByP?~;df^<4mXY@~d;{xCnj)~uw!>C-q(iy#(1S8ni7wY5Cyc?0xi z#cTdhv*yV4P2x*d$-)-!kY4}M;kf_zzhI=7R_;J(P&>!8R>Gmzl^H*6ys;R z@W7i{;rUjThD;C;atlhsia%IkrOi!IBcYzz-BHhx)0Wdhx*vf6{!XCiaj>fS>5BZm z{DoKyf64vB&3TNi*gz6}XAH@WU>Z(#7Nd+3f8pj^((vC1K~KY+vgP%iu>m+}R&S3t5#Wme|9yywJ zSc@B@|0%uXenSM3YG+9HgL?7^UTm}6hY{e$oDJOv@-}-p8M<4qEgBE#G*#BNxcS;H zH2UJr9*uJ~^oS0b-u31M;c}+C!AjfCbga=ld2wZBl$3l;o-|M4wo`_Kp7@`xZ?{Xo zBLjK3Qc_XW@+U{d7H$A)SdHc~0P&l`-blT^wXhHm?YPpK-JUxvb8RhpCLfg>8nxRX z6aR+*{}&?s2Z(w82NB*O|A_?veA53S!Xz}%OLw7gxcA}MldkJb!G)HoODQZWEsCIo zEpjj{j8O(seS2T^9n4@R+My<@y+3=w?rLJm`G>{Vkd0 zk8F%E7;cphUyR(E3ju%%6l{Isw`^Bc6$6~k?hy~u7>vh~V5o6pd*^YWAcf8QZfGY; zSTZj-lh-y$B{tIiTAk>ir}tjBdZN2TH+w|S8&=@JMDHaX?;orqLAa-|l3Vxp+wHU{ zjQ;u>0*(K~A5C!303f0Kc++vJ%Ue__qFo;4s~HxFx_uZQ?6AL5=xCa}MxR#%p}L9(6_4Rq?>du}W}9*BYd) z%Le_!k^IP(*=puBtZi#yz>`(OpM7`1O6T^zLLu?-OT)!+2yIjVc;Dr2vP9F4o8J@L zlHP(~g)HRvFK6z?^14;|P{Jo`HS6wu*JWCt1LB?@B!@su<6-(}cLe zdOV`GRlIC|y*I>QoZ4c6u{Cnikno>k@Ma6IgD>%U>6^*ab+Bn~`N#XVDpSzxCuQcC zhl^jaz3Ru6GIP z>|CAJ?fdmAHDzBEw_nP8+((%>T^ZVoQvMhY&(v*R#ii!Q_?GYism6#PL*Nyj?Af?X zz3Uy>ikj2QhYj{xamQ`vQ$d7 zMPXWVOT|N%^)^|m(B`lH5%0a`D})1yd6_QG89)^&yyUU>jw(A)3M(#hW}nY~tJjB% ze2M2BiLRG$j%3dRoaUgbA!34>ZOlFLuqSDh1AK>9I(hib3^At=?CD|brdD^_mq7*C zyFd1DIRis3=|Mi}T}De+@1TM*|7MMwAH6RQ>6Y3g_j>UnzdgB(jL{>$ya=-lhxB)~ zP5j6y^MrXQ3DaO*etZdDa9(?A5TY%F`1TrEzhr+xRW6SzGai zIdWwPp_zdR2P@R3dbqR`vj@85RTOVHWJZihxrWT4uT^nC1<9J93*!o$>4GaKg>8ca z^%T=&+}Y!hV{&(8?S%R~iuyVD83i`Kc|7yvJfJK7j=-0Q#xk&lRD7PKFqXiP;lq7J zR9V>~16K+jEyQ2JTX*dwp0z2UReqP8Hn6n`Q)B&U7V4X0@1q&J%0yXayA!v|CS1&d zZ&ZL;t_KmTmUFU#)Xm4(dCzqg6AZPpPk5inUzLG{Qf)?Lek+J5s<6EcC#({G^~$mV z+n)Pee(Hv^uI4Zg2&Dw4ZbA5X82Xg#VLW64RVzaomJyXf_9%LM!E1vVB?jboM}ngug~N6G)|J-ePrUVcg6~TkX5~acG1I z4POj;yeI524Yg>5U6f5Y+#s2=@Lmmt1%I4zM(+$+r?h&+np%0MuB3PW%eB#2bazWd1>3kaH|*75>SvX7%{qAMyJA^ud2XG2 z!mOj78DA}zFhi#1zKq)V$&i~w)%9%6`!PHthn>ax2O%`in%jWk?CNbu+J1{gS|!S| z4qsgMIBJ_}{<*6{t9!;CL+5H-*Ek7W{<6eq>8_t03ntcWzZQ#2>|ec}!z8(WGMv1M z8Pc{I$v#g4h58?ho1#jaO9hCbkn4hHEtE#uwCkx&gZ~sHzmqw*K>N4W!@}bt(~ykf zU)*B#xA|Vy-R`KDsEC8$CS8oQLRRx>DK4K|gqcou{G$W2vT_)E z{Hc7Q+)WsZD{{`7oDTz5g<;M(EZN&g|LSUQ8k~K#U*YGdhg{39WjAZnf-=cRb287X zIgk!B(6r0atk6nBrah}3*+fjQa^)iSgwL0TPnYw)Uq3T(=GC+-$InxQtz+S-znp9R zsFhwK$n?VxO%%bFQ5EIDCyZ-pcr{Zmo z5;C+%VQayvjjSp;5^*F1%E;z~JoZ7KI$y^q)I?}hLW0~@lW0@1S_k-G)O-WQxWD)@ zLgH`3PF-%THG6AP@A`)&@7>cHLkwx&R$zNeQ?)Z`=1~pEs>que=Y_|(;F!3K!>jzs z63Fd%eL=%|<5TB{F(|qj5xK*MF5>QNCa_aS@~-|Ao{M1F<*#+E}hu1Tw$!e;mzeYB#IyG-cik zWE3c;4}bA0Y~fF&eyecw0HeX3e=O+$AF=Ygf@scmEW@kWuf=?ncA+OM!;Y~e7SDv- zrfdPfZ*>Rs^N;$&_0aH=N$9^(5>jhf@MtsJOe$$ie(P5^4Q}Wdvaxjg93Rz40B>lc zwhp6K7$NM3=B?ULYS5O;GxwB?S`2w&XB2$1+-RZ=&SYHv)R$=bnykr^En3~sZ%$xC z#>qN58+q70R-vupd0t7&F|W_JUUuYd^!X8f>uh0*3|TovfzE)bjDdtC)>AZuFCDsq zjZEk^->Cf80So|c!qB_>dtJC}%5C9F#R4x#F1qv)xu|2Dhunk!CHBiEo1o!zSzFfT zPMF?mxq;efkiBFkW-NczgWj>Wdo&j$@F6UwR4eJwHvA9sm8VS{P9_@#6_H0A6n`xC z@A-A*;!|BmN4%tmRkquQ)dmltvA9X}hVhe+LQlqjZTU=$#YvOn(?!Ur#0#;?Sutz& z`lHMBOON}7I2p?{%S*ZcTt6USBd!avM=S%p?q@o*&Po%p722{%6@*Q{N~>WJhpSn3 zvo&^nq$16irr_M4PVbpJsfnZ-s!C0E!(({*qFt7?FVXQ`-b^k$=Z?|u+y7)5fRW_S z>`94JM8oX{MN4;uXF>=!1Qj%iaXgX;THa~K2|a-^4fNPv5N)56lzF!CP|BBxQd%X* zFcTr?V!nWMMmVXU$YU=zDg#A}=@)Z!=B+0)Hudd(xe9coRYGH|OthKVHf~Q2c{278 zTq?~irP;&tOk8BfHb!4}WA^u44jYpgP;SpwJ_qHbJdHo(MMnl%@|`C=G8qcF_FV~Tg?hds!^D4_k%p+dQ|o>5RR zcfykW?wDSvqx?RKt&9J;&DMS)oKBb~2}Po)L(oeVzQ}B_yL|exz?H4-m`YR)&DR$6 z+^ihy#fDui%zV$zQSIi|f|Rsl-3b-S1Am6R7^&8(9UAk)EMVux>=NSp+iJeine(%= z$9r_YxB>voCgiZ16W31PQSEc{FGY*aTLiX7e?1zPPc4p%m-9F*&U9c(A{-F$FTBX2 z&!J}S?iUM3<6x09{s6n7xfIO$*}3jrKZ>$cJ|Xg%!ZJzV_1d}kUlTp;FGUCJW5X39 z#+s)S?kH4ThO<8XjP6l09$e(q3G}gsk>m`uamiF8PHB4TC5_uQn0LU9rbxv&7iQx&qxQ&;=3n=h32#lA$fc) z*|z^XVd;5eaP653{;lfCgn?_-=lm%n>W|=OKV>8TdEumjSYE0@tzUgGDCff6$dXhTf$F17cYv*sb157SQ@GXk`?%0Cq% znsl!CSd=P0u#$_)bLgvQ$B*;eM@Czm-GV6?D~|3fv-$ZDtz8t}KjZa}?AFxS$$A7} z)GGE&?O*QnpGA}Q-OTOXjY*ny*6m_o*X^zkh{{BhuB)k~GGH@Q?;y}=A5Zz?mp47* z*}ttY*27UV-qB=~uH|K^~bfr=w-E!WD9!D&%lD!x@=O sn8(j0lEP>wZ43tD1%g&fh4Y@zRQtR(f0g|G(+~2SED9)&Kwi literal 41662 zcmeFaRgk6Gm8Ge9o)?*o**xdtn9Yl3#(K=UyGkvoN>VA6mCO(snHkI+9_|s&%*@Qp z%*@Puo%#B8X20Jo&uT?Ut?DXOx1>_rbfo=xox<<`pMCb;Yp=cbi68vnf6;$euKYm% zf8z(K@jv*%|Nes?{NVr4r7!)URoBJWFaG=_@&`Zo-~QKsxW50w{{a8P_Q4NvaQF$f zLHAGnEPd8p9ID4{zOJscaGRd?587<8voo98TFk)rSM$q=&03qd>AElObdTCT$es-F z_St^;{NC~758lDV5B~>v$Li(BU?I7oR0v(SX7;$!C!pVvGyLXs( ze2LNi38%rnoO?K#%^{vwxcsNTkmdL{xsAgIm*3cP@w$8-XE%NBVqrcF zx&kqt;QtsN-{Qp9iL>xfc5LpkuH;hcR7{9fDJ@@H(DLXOgAOm4vib(YljrPtI&q3lT=HFxWnuUdX>kI~R>nx6EM`c)pa*CT0G zJS%kxq}kJp@w;CzE1&DX@+C$`8;-0jIJt2jyk0_>BW5RS%uzw+lkmXJlTZgAle_zm z{!KRPpW#iHEpEKA8toIa84+wYd0KPeVu?A`pEHLim?OhDchj1mesr_KFcMoV&QdGYia? zPdT!9!Qn$YjQ6gwX8n+^=hvBXu%+J0nYtT~MVGfqRI$OvVw=vs4W-l^;{cvaVMv(pHXhiV+ zKg&k`Bfg2@(|6yPbhN5ovavKzhr6)k;><}_JSP!uocUQPCfIV8=#Tlap4Hls!!MlK z{#NnnySsEfu%Y|DJ5BPLK8xoJJaeS#rarezAnIBfl{VQlJxHL=xr)|DwWMFJXT4_W zwLKahb~ElYMb6bm%I@Zn{;eOe_bVuLDx&B~7!6-Mr2f_uD(qY+`p$`|d(W7;d!H3M zE4F2)L%rYWD{Bbx!~DvLb8j2W8KIm`(Tq7QTJ) zVC=c)=IHZBn6&;I4tF>T_vAQK@kz1C?BS|s4Fknzvs)17x1X}})k7wpK4<2YE$v@k zqwkt6iH}^VyZ4Bi>-VX9>P6s<8uGo%$$MTx@|7aWziXrKMJGeo8d&dMeQl4G$4v}A zYo^tuf$W=gA7}Zoz7`V5zQZGSJ=`&QzS>-|^8btWl>wJjpU%ft<{?L4*};-+ zt5<9|y^Ess}7xztV6y?TP) zm!G&yQy@FNyxxc1rx_I7^Cj8BkCdx^Bz@{YlW_D*;4|7T+p~V>6^lU;96S!;#KB#d z%UPEbW@it~j?XZMxw7l#!CtT{V@}rQ^{2OdOtFT)sJvousd>W`8a+K@uUzcwh0#lO zoQFLJVIIP;dKTphoIEl6dUO8LnImUE_MSKj%Z4*6zi7GX%Fxp%wBNl=#+?8fo<~vq zZHU$^iRgy~Sl@2M-EM^7JFB=}nIzh3jO0hX#9taBVd1}k4u!h;k0q#DRZ60zKCS2la$EIqdJy&0|$?<8o!NT`1B7PYzKX_yEuI2Qa zqbJk0kJuN6-+p?XQ{NEzyJ`w+JI=*%oV!|bV*QeJw_pxD{h5jgW7)%j&KDjufANeA z#d2raAm1{Y^v5~GJ%}YSyptrS9vrTYWAkW=YX9t6%SWt)J8b0Zw_kvTC8w@l9J$+K4Dn&b-x8CX)AaJE z5B^PUviH%|A1!?IWYgrJz2YigfBJxha7Xri+&R&Ga_>53)yL=JKFkjt*?j5G_zNFK zEduD!GZyLNtox4CKlY){CWd&YOadO~Q|Qn{p3@+)4&$OSma%%YN?gDu5uW?F`E23n zwt=VBB(AQDI6a@HxBDOcBix~a_`ph~s0E${@8cY~$upZ}oaHM%FSbZc-X=UsAK#cK z+M<=1mzAVB)DrSAj;t3kw7U8e=MYQ!lW4|WUoz+w$>g&TwqJ#D_^AhHHm<6f{PawZ zIQG@E1_iR|@_-cwEA}E>jB7PH|0Xr$$~$0m;cm~W;?kk&>-}Iij$B?~ z_IkmYXCSA6t{hzVX8wsEa}T{3h=``f#)(|}1Zo}P2!EMCf>$2i_i_k$Rg3?tPOO|J z$#7l6(|ZH=&<(tz4zcv#$L?ZnW4Ex3T)@V0hK}qCN<)j;o*aDbAC11z7`*V2>ztTt&1 za{7_MJ8pAMcyQ3ggC)hH4SyeY{8U%GR1M`3AiuWeNH*C0&Vd!xaB~hG)an`f>}*K6 z<4S>j6roSj$hOTT)GdpTZnbjPd6CcDjRa@y;uw5DYSA9f0Xy9FUdK9k7oTDye$mS` z*UZrtSwQYHPddD;*cs`4ZI8kD$22Fzl4T!Cq-`m$EV~H~?Zqj3k@$QgFXE4Q8Mnt( z&jVhDAMiZkkeK3auG*~-`m~n#;1aR$ouDth~u|OQFy&oe0N7ydC1P^_<4l zV*+=lWo#lg2`k#?W%?1{<{jWwe8eNSLmv8VlT~LTFn*ZukVbM8LsHztwFf)1=KKZg zQ!TIUaa8k|wWzB!ggzrT+MlBAM#7?d2n`v*(r+4z{3TMl_KE4<s>?5#!7)|m`ORBy7sZWZg*3*Si?{pR|V%UA|!Qn-|x3b0* zDNOI`%6_EqOkA)qjgPS==TDpC4Hgz}%v-xnH`+PS=PBG5FH9J!To{bVw7CQEB9QJ5rHm&!oeDVz)ya;7-qLdz=qhs@C}> ztHJLu@AFIAvSLY%_8>f|0gvE(UWE?wByk#A1 zl%F5H&NkD_DK3AK9AEPw=7+mC$1C*mwlv&J@(xyN>_1eUOW+moa85>B6rgNSyM zfA}&KV8N)h|J0wN_#Sv1;A2!xJr^3tT*Q60V*=Pq5_Sy;;UGMLb=}*TUoy3>0TjE1 zkmHt0j%z3pc9FPR#Ss-$MRa&Oubdmm&zK>;W*+}+#U@?%Yd6X4+9PXVnUc0elB;J) ztf@d?aXwY_}_|X0x~)YE1t(9bBe-x)hj*Q6pb8`(Y%FS+6IaF zLloC<5K*#-g?&B2FN+CI%u{YDAX!{kv0tV>PNMvY4HE$=EIUQAp;){Z70#}(j4AAe z-aF7dALrxq$F#|yz1xE(;f;C605&2LISqfsnols>jv>qi<}EV$u$vCv`hMsk;u^W$dC78D%O(K#4nf%{DcjU4>*bcf>W(CLr-AzDSLZ4oE;9mw#VE-Gv@s=Fhyf- z`;PPBFb?uBVetJ}x%Mr_96k|d`iLBV8>*5*=_^mCpr!$ff(6|3r?_9aLEG34wPSmP zRW9*))CvXpo4hFAz%S20q(_r-ej|1M-q=~mXB{1BFUck=EMHhz*f=>4qfMgXRxF2p zs(t-K*l@Hp8}04?DBOyvCFYHO2mj6O&@{F~J=sr5QQoq~m=w;!V|Ql4vnciPqADVi zBIS_MkZ>|(k4!f=$`blWj_c!vPcLp2n*_A)lUcS-c*#Dm+7GCj*`;TAn})g(+Un;B zkLe{hHk-zr5E|us^Rb_^ZTA+dZXdJf_ccc`Z*$&w4|CleSP0_86vX;+A*R!R>_^S4 zO|rjR4SR+19Y5%}%1QON929)ZVSyd<4qvjCd{aGm8@l`-QXA|*OL7vGd2Qr08A&LZ zrmI`f&mWi^2(K0sVcpw=jhd)zU8FR> zoAi`ga`N+NDwhT!$wR!OVs&%%uJjs zlL=olU!BQVYc19F{dm=EQrf&iW#<}03rpOu-^49Vdnsp?w5omr@@9##D>Ii{uqPf&lQU=aS(k=xZx`M)t5P) zk}c-d+ucrO@34&Hy&+nRBXsTjV}0JRy2$c&HHUkdn2jk2>~Y?v_|~W1U+O2EM}Nv; z@P{0P{*;-BXN-A$NPnm$^Ddt=5#xrTGEx1MY-YyHW8h1;h!@h)WP2H%9BdUV75*`IxaN;ab@x%{P^%#9Ep{bIC}Z zA|`Qygw$cmdl#swH(}YaL9q5&V@ngG(+1MZI%uk!WF#@4o{RuyLT)nc@ulMIciOX` zu@Z2LZNHy!68I+PS>JG8@D->MAS(sq56dr zhLBfOd3#W-oI9)dF&m$N(I<-4w5(}^t?e8CTAP?+;(XTPg%LviwLZezfj(45N6``- zOwRL{R5(4M;g$Lbp2?*6mynm%fn8u7xrGaODc;7^Y>?e(BCmCcjP@mZtA6hvQ{fyI!PnEmlWDE7S z*Q4R6gF{m@OB*A^&MvZh^pEjsY<-ok**2y|YB*gjl09-UZ#c?7ZbOfH!&#RgRW=TL zk8{8Oz*hRVY^ZYC;wije9lI)~Tr z=sw_D;U3}1Yt+_{;;H(^HM)Z+>tu41#nU+Zir2SfF2{=uUsq`{9GQ=dV$Cz!ydUo$ z{jaeHJ}GZ(s`p~d6m}99&=Km!aFBk#c&V01OX}U@=nsk|Ji45m@D|duC1J{%q_S&) zsOm++>x`te8flp{Q9QIF>@-1WY8fLX<+LZou~Fv9s?R;9<6bfK{B5Qz-ofZ{hZXP7 zr6Kr$oe0??@-xm`UqRt@7_?OWcGGo_^PNx*H#2BgOycup1Dj=Oj0;00l(k`BK0;B|B%`fU6pyZ8+k3>5nr$+hmndv95RtJ!Mpy$w z(V4=SKE$a0G9)`O=I=$@(*vy{@yF`dqfS&tAqO%oWlU#D2(GjcU72?d>tG_-Hms2*~&f@A2duuhy{ldOGv6v?m?>c1H zQ1Xr#KPjR|4<#{2@R;}YocQGZm=H0pbW4*L%ndHxZb-uGFu z{{<^He}gIZ8YeN=Ia4e-aeG_Zo4=7h?jvXsM=Ad>o5bJmg|N4p$?Rf1#WOvmO%G9H z_(!aJVU(dYBQ?Fflnyj8G2D%DL0rmq73NK0-;GGwR=(ErTyappJMWP=w`-h5TxQSd zJ^9Elr8W79Ai$dp-}ry%3AjePzxIotB|T4WV2W{JrZR@A>O%ao)tf0Jce|N@0ea*`~E&rmAOeM$o znu0yWJxQyio=mCJ3u^2=C~^)}{-_W(ZJ?xdj+Pc9ZWDWij2)7r`pJH9m%gb<>PAPX zZJ#1HzmwdMH2QMm8Hw^@(f={)j+dG8ze(rQPsO?1VCv3au^e`nEvL6Rxrk5U(hJ7E z$61u}MeJp$)Y>(w)-b$Kym9Bu=+D7I1qJi%%#3tUHPuSzyy?OWum7BEZi_D*Afu(5 zp7k*drcn;JDmdG#P~1v|Me%L(LC`Cnyy}{K@doC&`|4T#9s2*|wa(oMe z=PkvBZy5-@N0-ZUI;2+`iG4|DkR=7BwG@~46VWn8S?fI6L#iV;ckv$D=gGhk%|om7 zPnd8mo24+cg^Z{|azdlXunVOt^ez>C*7Vvu`Dc9ulj(b|8`@DhX=0W<)I;k!0rXW0mo{$&x zx=Ewpbc>TQ4UXFXD*4w`5vN_Kca^g7&olHw#CH<$7g%g*Vpl~b%L^qE~bRH4z@cu+aKU; zD^I>s33G~BV;;)!(gv3bBUC-(EL}EHt+RXQ=WO5l_tN>k#rjKOic)kS^5yJvLtc9cSL3VGeu?bGEe38PW{2s-HM0EHq><-$>)!;KJEzEo+;V z%+KZH-P=ynP!Elr)$AP||BgK_J{cVyrM0e?`r$ExHkX)K8Q}C_nA7!G#k_ouW`Z$` z_hQ_U^JeLJ+Vy_%S5%)Uf4N-2=>9pUepfhflV0xLkEI3rDbvdJ(-BVeh1w|IUT0XC zvoXe>uAD$BYD?cJ$SE2T2UCAPyA$>@78h~iZUs}sR;=?)(=NH4CUV!0@NPdQV|<5% zl|#jgY1(V&aLw$ZIJ<}D{1kdh3xpx04S4ny2Aga2d%R@i)tk&Xe9M~shwM4N&yL$$ zoCHW4>ZNt?`#?G%`A5^DG;B>VWb79vtHyj-3%h}w&q(K@c%_D{Z2yRp2EAv!^rCgP z9A{gI2fWQz@HKW{iMO=+ly#3A;_g+i%AZYmU_~r}FLqFpr`8MZe>MIuB z(VEB>wW_BFE^2XUpG;|r{Z(wu)iAbHPR(=~nM17nE5?hTzp}7=TN<@cJjv- zXqg#dbYqxBQwt};Wv82}+t*aftSY~c%P;yLao%msd7m?9U0xjL=^j(RRqT47W5+ku z`@6!H``>+z7RcqOyc|@!PDC`Evmj~H&E8xM@5x#YicSb$?>$t_|anbgaLmS zJ?;}m-LA5c@Fh#$Z?osKDIBQp~=oJ2$Rva0$hWRk% zjbhGMtCTl#RO1yeXdI+zc7&?tV&YmxF|7S=?)Yuvc$fE*F<=k|>!Wnm!16{rhl_C> z?I*}C${%aadM~}txUF)#UPpWtw^Z-6NIQ~um(xt~$S$%+&_j;GK48n^3szpf%eHux zX@|dMMm0sL_HRR`xX8qETE&ah)=N`gok?a}GY)O5#1AborF#rnIl_MF1gF76TBnR; zYcGGL{%3UA1ab8>M1@rSUS7}k&o{Wa6!J220N3gsJSx?zQ=X1pJS1^ZdKuXw!LUQ| z)G~FG^Wxd&$Z4A=F}IYm;tZ;D;;54ju`n&29`~0FL|U+#BaM~v#PTaWkHg<^@ZjHZ zDsJ__7%G<&^{ob;J{9#u1n2ON;P2*%cR-9oG2Rk*it(7x1)=buT zJJqdC)Hio$o;H!yHcaQJ^4h`>hV@!jH*(lsjgXC1)2!deyd-_(i0Yk- zI#K?3+9T}HZp~S#mG1K@r)i31N#C*;XTzfX+w2A1V*Q1!<{EDW2 zCuWtT_`DG+oZzHf;Zh{MQq?L?8}`Xw+NWsYknZ^{yn6R2SM5`4SYuLrXKrqXY*0&9 zdJ2_cN#d^EsS5L@!TSN-vDPevK4r#D+>q_BSato91MiPH5Y{zEOOqiib6Tj_9jn+- zEWA0S_|an}TWF8YN2qqoQ_oL4$6h%LiZ9DMZA>mT(O?*-yr+fu`bzSCOVj=v8~pa` z?y;H7)*+gv7sy{QFf~)g+)NOI>xrCi__8%A{p!4CKE@RvJEeyicISN5gVQP-`K#`; zS$tXL3r-s>*^d#|pY)7%n_sGiz01%!q{JF3%;N$K7fKY2)=IPM#jJ0z8jntR5_U+W*P_)I;BE1Q!| zyKv~c;KKJQpWmfz?vOfzV)6Jkxl`K=Oin3A4N#=mTv??3njb=cK^#p90lFtY%A7Sf z6X2-+)P3<&SELpFTg@WS2*TH z`G)GYlU3CUrY4rQnwc_nvbf$x{puzeGs+306|C&+{r2+zlOK=HFVjCiLGoBHU2DZG zti*7%E*;x~aKf7O+>>Gd;2T}SSff51bcqXYxy$*a;#RZ#rTjWat%_fbUL3}J&OzoK z<#u5a=by9n>ZdF{|BUs(%XEjmq#)o0Ilgv^T{(36`mmTDNPkT=9pi&ME9s_nVim*6 zG8qP~ui{Ny=N>gn2dbsExZbmAjx6kajk8kvjIkDln}iRVh!&o$UO6UreutV_X$=g! zvHmn%TRlxO_+P=@#cT&lR(Uae5TP7o;s4)~i9jG3}~2m4tageA8YvCp&d)?&l~5 z53qeOM$^=VO=iUBP5hJV|6hLb_{>b#$|7s44fM#ibEBE8F2%C7qTIbAtUIG#_lSIC zL>&G_oNBqM`??Fm@{N*v!d}myOB(lrOPm!x;2`%3djTJb7Z$%0pJ`=Sb58HF zr24*DvEc&06a`+voZ_j_zb-odC^;wbDo$2oVTEBHY7fAjn2q3|2bHFp&oyg6!7oiXrI z^|IpDbbxwHstHyVzty)mUsmsLO&X%ZayHlN6=Oze*=S|KI7RiuI4i%!um6`e_%A>2 zF)q=+Jk7*rw(@^Ai{t9oZANiAD_(J2_aJ{UPl+4rw^5C#x_v;oWm0@smGYW!_j&t6 z)u@*^FBKN-lpPB+Pn36?qtx5%%QyBSudw#w$MThTS%`eZXq0+sp0`O4i=a6(n9;g0 zrklbj@0%j2Xpr9VAzDXgDVg6UYFs+A9`%HTi-X7Z{vn6N@I&kV9dhQ5$X?#3X+zx8 z;4JB~U8Z>cKEs^!XPUVv6^^RNz)+LQV1*lX5z-{8=AVhX$6UUH`h_=DTl}5k>$~jP z{+bh~UvL=qmU4w+bG5aeN%^zW6LYs_4u+&n9?%+03q#GNhzm*5TF77Kf-rAoaJDS1 z#IiH{>hl@ba+E_x=v-}~e#^l4_P-pbe#a&%PrBB}SzTyne7uZ})pWL&^Hh)cb2wtd znL#ziv}`fr0Tc4ke$}9(%4Kch%&HVqTQ6g7k&gP}{$~O>6Mt}8@HS`VPdJGEiuF_r zwqE`%orx=h>V9{6F+*v7kA6Z-@W>@u+{O_)Kp8!%EaK1oQE>JM>2`IQw67S@X+aAY|6 zK7)RDrE~jEy|o)Gr`%&b_G@;c?{XIMk@^;2uov@*_z%rH#fkrjzo{6Y+OA(*d7Ctr z^QyxpweCaya1bwhsGqu~wO9({R59y(SN)^4Bxw|)Fl}{ktlm=Bd>ze(Ze~{IRsQ`W zm-{>G{?B}JC=O*{UcI7;0>&o7Ss#z$cuV+g#eyS4jQUm58XARx7j-{_;!kFT^OW1q zr*&@wsuM4ICj+ukgE*WPah~zt2#l72}AIk>H)eE7*P#2h1{xPTC_JZ>1hU_x0_+XNM7!x^J zPvdaTk0W8-gRN>-SMumL4$-AP$L`sG9pC;l=J@_wJL2KTR%%#VDPny!o;_neMyioFguSrK z!YN;K|e8(?8uR7iKF08IgJu$9V4zoHQsr`=bF3uxwJMHoQHDaXozY(#o~b%;-mCDOM&tc z?X{IO#VgrsOnmy9t_AH=qmTMcn$t1Gb6`^3I;>>Fn8o2v3x_AW|I(8Gi{D?`UuDZ! z!LF&4Lu0J?VsTj})tu_Ho7dynm;T{=E>`uA>cc7Rl^MMbsZLeAqV5Y}a^a3z*`ro@ zuu1cuMe_Xu#jyg-SE#N%5ifeI{ki^F`BOQ6Til&dT-|Eu73PD#VL9|Z<1tU^sR*N{ zBZuzt3dSmmX&r1*FJ+X-L4(i0N<+@7~K!kA2Y%)H-Q z7=zyA!1q^ddVDRtgbizt{vF4ux{tI6oH~dL^m`xF$06ZkNcjqL{&nFd@#PEJSM%}< zgLG@ls=uccn~chv7tg#P{rVk0`YD@HR~VOW zqt^3X#>AB^H^;Hun@#3;6*&W=)k_m~JzcFWy{V!dyE>VGYAe`HJi?{Ydw&dz_FjcMAu%scz~~A0SJ6u3yj9 zu0BeRUON?kJGI9y`j|fNYaQOxY@1$f|4R1wPWkjx&O8*y{GPKV9`)Gz?^v?G!S-p^n+7NoPZZXhVSiRaDYFKTure8T|TRt`)dso`iyV68RYbl;-*W&+`t$)q2 z(-)lG{(H?o30nlJcNC|1l>9_zOo07ixZCl$_S|Rso_v4kW5ufL9B$ZfH1EpRj1Ai>t{kc7ZT=(r#qSwQyzu@`FFQNM zoNhG;Hz;>(iyPPymo|4vwh%6#($%G2d#_?r+a2|;-jh#!1Q~CGG91eI{+o!@t7h@MGzn-(e{DW9bIIWZ^=C zRC;xKJk`#$midt7)X$icPnq1`(JaT??1<+$4Ec^7o1byyDtzR9gL7x)trvd<%27n= zJ^l3F${FT}?=^P}>s8RK_pY980%m?CrO|cv=zhY<;&j+4+`tn{0@wvoNCpgd3|u`p+O3MVz<@v4+} z+Y2z)C1NfVJ}pyzOZM0MCumLc^_rpks@8pFYyA~>+LAdR(ATCa6>F2Uze-@g;rre- z;sV*w=UX}QiCq0m1k6OK&Pe5SDT1?&GLDW05xDx(E&Ctzo6Y*EGcX_O+K2N4z3MvG z=NEY5a9evtuNQuMc=7qgrF!V}7>@L@`9vSz+q#Ku$=0G zYSph*dTrLsPeZ)wqdLXU3dJeKqQV$qZt>&7#IzSF{*)+3bSNIxC|@g%P1fpb{}7+d z>xySHiLe}lc{&=iLD!f*H>ImKZeV-6Q8{};cVX16{LepH0kbLT6VrU;`>Y0BWmR?E zUckrfcz(d~lmAP&N99dW#Igdt`CH_mCuu}L#c9__#T>tX*+CW zx7+&ol5)#C%0bF)?c#Ksf35Gmshp#HKCYg}wD7^&P4R~A94x7iZql6O!I17qvG~vR z$9$v(5TCWCe%S02jz-?$bXfIx%TGA3e@hrjeX~wssn&bqQ634Oi0e_!M6d8-jdFO~ zGv!X@YU#c4`&{|^Gt4o1cHtnR^mS>q&n@3lZ6<6WoV)GyH=M1I~kffjRfF^0%->lkT-%w#a&@_1AqC$PW3!>=*XvQ++t3nyLRG z<|SdSDcNM*j|<-FKpfWDb}~ESad%D092he>K419LTI6U|ShtwHrBIHR)h{%Pt6F?t z{FQ2_OpO?Ne#FPQAoyxqZj!v$s{NeangGhipCgYn_+y@0pcWdCu!iOj^&4IAN~(SJH1j znywiftZ5wzpi^8`U%s14x6c_5|2vh>zhF<8Y}4UG4n1y*=YL0^f5x$udfw7mp4t3V zcK=xZp?mGneyI`;>~K&#lfTqkXb-)F@++!!WuGeTpJ9EiPxczqGa1yIn-*qM?RT^- z?t4RXkW2Z@Zb;iJ-TKyMxwO*b9GOr5=yo|*&2bDpVctaTIcb5+-hU$>(L95AisN9# zm7vc#3DykIi+4GF^-K2Db2q8?u;TX_lZg+R4ZWsWNLzZUEyxSDVx}XO>F%8Fn9jkP z+>Q#Cdh(gBN?@=wi}dJ+OjgBX5LRAwdz(e~zvdvvjxG7*ez?xWkYDWSJ&s+z<;?v9 z&hzA_`B(nI4mH|Ss*TPr>h2-gs7dvBr(VUgGb+Eh;1Q=}x0x4;ThZ#Bsvh4`PhvYm z^>L=4WR^4_k!)|(u(j34$pwb|qx?B%NAn`v8^L-n@ywe=@=3j}>#VCKog1^Lebl49 z)1~{JkqxGF_3A#Sb>G8ovV-=|=zr#ZTkgn5^t0M`=b^Gk$iJ6fN&XZdJNUlIuDf&) zp6|2eto4#6!l?Rh-r}!VQ7zJ)XiZ1hFIZ5&da@&o{(*e*`+Ml_YogoGlw;j&@J-dc zQuo9>(;XdjS5?y9mPv780*jgUnpL{OwBy&}!tSvhdrM<^*VOO3tr@vbIMaI_`d#KY z{V`|K!JK7&$4RO3O{;46Qq@M~^6{$s@*8XIG3~1{`T2nSe_C~Wzs>jlvA>b3H3?S< z7{a;cY_`@jg~>JFq?xm6^$AB-IygD`@6Y!;d;O|4i`g{AvA3mKX**qfqBsC!B!{zs zoG)mN4f2O+#oF;u%ojY#sP_Mm{GeU78ugYoYmb{Ev8G)d#*iyE)H@3#V^zk zxy*j-JyxX2HhNxT#pN^B)DN1|zMcuY#b{nQ%@sN`q_c#+x>D(+s;I2*qOQM}TJ`+a zmPUOFCk`Iu51a65m#%Jdh3=6dCY$P{QEwzEKa+u6&67xHxa9c+L--TcLO*9k{PL#e zA`h*9DSN0ND=p7~w3gekAL`sO^<_)cgRZfW9pvY|dJgG#F8BsIWDjxMOsW@k!C{R( z)jFieCnCjp%O10S94-X1E6#mcyyMJ5B2Du(?636vH+#qhpdVjfU~#jV8Dloby9MHA zV#Je0VczoOg6}u4Mk;s6S0;5&6OY6jD9269Kc?j$y*|3naLqobZ!n;rsa34amM$hm zyhM@uC!x|C2dM8B@v-y)(gTHlfXV+$>4HAR==LG2_MftrVo#~t4f2Css3?qMuCmaNxR@Qyh+>qy!7c~q?HZgQ#C+ydmFR4 zo~*?`quKqHwCXy~ChdjJwE0})c^@hs$|oH@(wR{5(=6#^Dl`vYaYOx(&xB93hx!!1 znzhfml=sHv@55e-UwW=l<@XD2abCWn9@Xij7pE)gzb)8uvX_BzHk*~{0{X^dq@yTi z*);p#WRvMb>54a+7#YuHbuNMZ)mTo%s~=7Kus^50u_``i$xg4*Q%oo}Eru&rDThpJ z&wSsn5l_)0jaY@aj(YXjQ>D$yyUB6fha5-W;4Jtfj+B>I9NyPsNGBdE&NfizmpWad z(?=X&t`*&tQPekdP%6FuKzB8Rh7oeRXVo9yAY<{E>h%LkSGLU7Lr3q#=nU<4_0;R9 z4sh$=P!Dleb1o)oHD8j_+fQAso~5gwrKT9#qb(@Z%w}k)NM*jDy>42Vb7fNpU*S5=-oHeENl}!w?(M5h)H@KTwkVoY=&ZKAS=g4SnXCX ztFxT`;vDMYG`E)M$8@APVmF40RguaqHMcn{Q*N(R z?yuLYbke6KdcKSPdG7<+M9vIeyQy28 z`B&K^dv+azdW$2{!5MZ2*jX**Y^#8inTLvvde1%Ou1VQrLOP$R5akcWtwHr?2bAju zTvQ7y_tgp4*E(=ktUg1K`fiywzt>mp`Km7{JiD#=x`hPIq2?*B$KRE%!i~<+-oyZz9Vge^l$U?2I+vu__}Ud8k0(f>eOq@O(4_JRdck_RC&F^;P$zClb19u zbC0zLzhF%{Zr}TTHUh=Z7b>45NUJMt*?FxnRqIpn?b6UFb|H<)d9ARQW|+@gR3l3F zW1b6_7Ds2ZtOrOZ8qCgY5xt82U9+`xElo1Fy88#&2Fi#_wZ=b3}WXVPC3Tt-{^gE@>?$3DLw_sP6laD@-+kR(YHsWG{ALB8w}4# z|647MuI3o~>a!V2_hKX?iI$i|io$~^Ox1Z7KK3ldJ;12>nq7a*bh!PJ&G3iP@yTzb zRXWu(opj6R3*{T_!dBgPIO(z%S0k)BqGxZph#Supw{&LCtj?62^--TYo#_c-^yPAC z3?~>}SfFqAU(O?^>a({Gb@lcoK2qnP zC~hvOm);q zj9@CuUwU_)(Ug@$Sw$;pU8BSeP0=+lM(wm_a;Nqwp3s?S3wzWqoZvmMLv)vB{Hl!R z_~E?=zp+Qpg6T&w`6JxQoYlD8dXlI*Nl8^ueA1# zlto5S84*Txc#>+CV4B0E*-f~m`3d1X@dT?@U$7PNoZWEEKqV^|CTr$c&uA{vdBdf` zR@KVg(~1ov!kN9oFVoVJuDEMXLl|c9C6k*mEG@^dI9E)WVS?7_Ma>uYvAFfmYfO4{ z&w0`cL`_Z0CKIf$v}snVoXPD1VJYdUjWL|9Do-yaskV@R3V53(e~nsx3ZI%D==w+(R0rS8fW?Y)+5^mBEn=sCMhC)f`lA z38Q_Zly*!=FEy@S>>jZbJ2XukO6Pe@^7uXpy*eYUdV_LlcoHio%r`=QcfKMJ7jPiywyu!QRW2yOQ?FZ%aZuRelX-{I*6U-BiO_e@Lz19QID{MzRV$ye}Rw}=6q zLnYsGn>=Q4YnOs?og4MNCTNJxmKIuybD4;6BU!U)L+PdFT@vL#EwvD4y0$836gfV zh@Q16MrJgBHMY#d>LCt}{%IfC+Ayhh8mFX2_a>hjT3%pku}s>LBK3x0MmJAN zq-`&WPa-SLmwL^qS6Avhgcbt@19Mc&t&p^`L+aAe_gBiy0XF^8fYu#imA{B%WeYKx z&BP{F5#*Efd*}bOxcL0U#yOe1QKXKUN=D z`q9KY9O~?fvo6Jr{y^D6Hfa?uHpoBbRi_z~INy_YbiYO#9qENlL&_nOWRA^Kt1}Z0 zP22yZi_9!&Em|kY?`@)2v0`PtRWqH<;wcNo0TjpF>2MMM8s|%eVpeljB@MOH zq%_ZeuXS!+(OlHjApxo(5)DTL%pBt*ZK!+GD*5dTI3?@MkoabuEfT6TMqdH_r0tlPkzKmib}1j(Bco@PhL#bX zCo#ZCX$E~M5oCwy4Eq;WbUJ*)N`$-o`w4U5*V%9rHn}R!S+n*g&r7VRc0Ei`&bZ)* z6Yg-_C?29wpG)6-J{7~ELEN25HS~5CXS+T0FN|v5e2D_($2QGI6dP9mNqa0057W|W zP|WTpVX2+zjTULzJJma`ft_^K7pgy}bd3clZ%FS|r@FLEzMB0VXQ9#yh3s6YsIKh{q9!ftOgs-Vx`6K7x$rjZF2oW|CYh~(i+~>nH^UZYjs|MY%!@l zG#Y+!MxoB_v9-`Ve<*cTMVgI@m;SSh0G%gTQ@o~mv|Z_Kj|d&zC2rz`P|eRIXvX$Z z;Ubht%pvsFva>Kxs3n%`>eCcL-yx_TwOFZaIp-e)<_IjS$vFL6T`@jAET zy-{USp*N5GKNZ9^Hw@w4tsA(0X9iZ*1*S}|L4SJ*`F$IR?q5M9c`5>niHEMcjoP?Q zWNQmhq>4rtV;qeMX=u>v9etv6Z+vwOUVgox6=yV)2l#GA0MmYAWN3`2(xsv=zY^DU z!>DUpKm+_eUweJJi3FwShqBtrLw~0Y*2#ZB+N)?J@EHLJ_;`4O!bd#SX zv2&H&F^^EVz&fNE`f9_oH<}g`5MeNsJF7Zod7v_7I9~!0IGspd%{3D0_V{7#3rf4(Oj8m_x zrxo){lYg*g)lTqUnWWwX|87pUzgTHf_YwZqaK9AV8m|tU`{@q^sSv+N} z|4yt6mYL(33w;L@em^Hwz!h@ueSu-t6-|bpWi6@zofg)j+03ZQu_B|KdN2J8=p7ly ztNx+>173)0(xEZ6yNKQIKV>)Vaq&L1i7hc!ikr_`rV@=E_CJ_d$#l6HN z^cG0aNUr?KNImTe8`AZy(8ms=z&?TU+Pl!zFOXw!2T`|{FvGgN!up3eM=rtu`j}mv zotPzd#aXDwXqF5kGWy;Lk(d$x7BlqYA89XRE99rxVmR#~bNf_qbP#`8~_M z9I!TXx13l2GjrPJFNsCwJ&`D29nnWDul|H_1=cht_|t#A#(7PBfShDZdCQ~&nzSri<$})b|2_ANx5-x+Nr{GW1H{7eUo!e69V8c{M4GFG;Nom9b85mzq zAU8oI?b$H$W64p;^Npt^L!vA|bHxxj8yK4@B3C+jZCZw%sGfG9t9J|u6_ZFN|F`|- zT`b>j#(Wd`VaiPyCy&W36}cRUhZ#w9!IJ4ca>QN42D!qX=6{UGqv4GEGWYvHfdld< zu5Yl0(OHR+<&l&5G5B}~Ei3bw-Ri~0Mh*RY1xWCO-Qg&54QQ}W+{=E>N30URT%C?9PYaOuk&!P7I7A$>hP?hfdP;u$-6rDcs=vU&0I>zE<;Qj@aY2>Li;ta*$K=){+}#`@pF=11w&;Ho zvvM%P+!pI)x9b_B;aa#_7|K#Pa`6#kFi!l_U;*O@83K%#k~_DMeOuE(e&8WAF^6Xx zUPk-CEQ$uH;Zn|hqM2MeH)bKv8$od%xh&HduSzLGos9ZA@v*3gbVF-O4EiEjyP@K~ z8>-^;RAZYl)=hF0FB)@spOt*$J^Z!fVKkn;uZnlBrOiq~FgE`HvbqQGrUpuYeI3R8|B413n{2n7l=>?S$- z3$^r56402M%Q>EcN0w}CN2>6I&yLts=1Og>wKO_*jJngW>*sItepg(U*StcDl|&8W z&t6#96<|$DPJKfqZtHW=E2oYD?cmI$SSSVQFv(4bNh2qkihL-ten^WJ8RMIV(y|B{ z{gZ})UCfL+$@k*KwaP`PY8H`J+l9fl0q#LXsLaX5tyHcb>Wk!yCm$H&l8$WFQ8Ws% z*Tb0O0>@-6l~|Gbbv~;rU?{nT+~yfnaZc2atipO{2T5&PNUh$W%`^jPnFE6C`$$P0fI-uZ=G0>D z4?3iCZLUd9LVIEc>^a3~ktUK#zKr^b)CS73GIq+CeWnG@XnzdIlMjYMQRh;}9U>Qs4{)1+fno5|R9B02mG%!&GAB{dn7Qu0nk#C~gg z>gE1_ReL;KIW?SW#+sJ-?PNV`fg*_SWxba?lQr>HjQBBUDVC!&$U@%BB9u$?(EH{= zuPLQ|QwhxS4g{u*!C+mYe#8jeR1W;6_$gG3yR-~2cHFWCP4O~33wU2L?xV8yqcA-c zoy95KQ&`hY?1n@6J8C(-ja{1yF+%CCd45PSZ5{b;u#dke`X>f;Jx?7Oh)?g`Jf!;cf{LC@(o!3D}4I{O_ z8M#qbB#4tytmpj}ozIj>E;%w%u)%xIp(g*d!iYWQesLRp?B)c zpGRl4k3LL?CUWY&QjN#uw51c&*pH9F9^-{iBnoVK($@Bl#Z+7dH2}4!OSaJuiYIks z3X&CRC{)KGJ1_zMfsIh6jzgHfh^&Hji1T;gW?CZO;1k3Qkf($DjHF-*pQcYiXy|}4 zrW^5D&1kk2qbrX52Wg)WfA=o2D9rO7J@y{8W9ugMD0Dx?YVIZU8@$oXm_ULy8>v|( z&}bW|ku!*_>+=v+uOYO7{6*Zul%;F%FJ-=tdqr5rBD^e*AeK_&D7p;A33)I@TTmgj zpe{TG*1%}S+imC$G-5nB5ldI3*p{Z?X&B=%##rnn7_gBbCX7*!L1Pj|#G1(k+qd)k zUNsNC?!ART*VFgaa_5XrLH~xDmE?YMhA@A2UkIIku@ddv)x@)#F2ufdqJg^vxpw38ph+_8 zf=JVer1*ANjd`@;l;n-QhzGfh)gL_$)_dW|_rWmpEGY(ZXiEZ-n%;;COBV{N=btJ!h*Q_1L~e#Y*i20sBbp5c>Mcpg z?^R4L*fiGsu~tSw4XH5VE|`1U2`nY7sR;Cda&80xiPJPu{;xG(ekpQ%W}V8tx`_%e41ZFD^$i*h9R zWx_0LMX4qaGItAp;W7zH<|F8f$wODZj?1B&I2*ByOQLmhb8aB>)*A6-T*m~%NEH^s zd?gv>s(cJ(G6sC~SQzcVH1W&L?5DD*Jfxd$Jmb0-TgW%aHArb%$7i8C5Slp#mD|u+ ziN`G1f_vmLRMIJ=hBwo8>VQ^AO;v3cZm2Wq6D6Wv!~Ic3+(L9b7JNAd!AZpW7_cYM zU@wSu1_tI8wKC$c$US9<#VlhZLsF?@_T6{i`o8-6oBYmfcI8_WvH<5i z=>d{3A&*6Ca1>&_s7Fq{)Mh^g3cX`cqNt-^Nsdls8{(*eDi586FqHTfql27|)ccBF zgg9yi?_XJlF6IGaO4CTx44;h0?D#C#d}?fGjp3r(Jlxpd!Ln6I^^cHZr1pqp6O!=z zP)9BxH~b!SLTa#j*TE24h5SJBKGJWnhU;MQPDD)zb=Jel?J7@WJS7EdZmfy8L>v;Y z`#?s#7~>M#IuW&=gUOX3Ce~bxr6_XI`+GT@(cdhFe%y!O_w$ZOc6uQwYHCp%h_&!g zVn?*pfb%8aH)G_U#LiqJ22m2uyc27oIQJ)o5wttZgi%MM*+q;}j)n0W^`?Ai)BnzZ zMDHqG&uqqZgB{vSJ^1`zA3!5og(a4pT*h(goGl{q@+3a^XY$K>-aZ+R(H`QIA||1c zO(WQAoa-9rmy&r?6*)ciOAuU|hu&)f`I>%I%G*&P=Gx>@fUMv&H2G0i#?OHI042I) zYINvguoyvY2dxg9-qc9-V|-c1cs;d_4bju~ALdGJ(R%fcu7e&Txi z`|jUA<@bh7)u)%0VSSFU2+WG9El-TtmX=ukXy*B76CX&$*bCvae;A(l({~BgVKzvD zdwwD`oQuRAA1{=831Rj&q2rPk87?Yhgw!LzuNy%D{mAw2g~xwz-w&LDNyV6$aTax+ zBWMe^pNz-K%>`Wdy@#Zr8Dz;H!lrzL^sqq)Kcl8<*aP@_k)t!H3D%ej)O(SC(mxaV zoV#TLDeCy#$orz(leVfmu~tD**b;M2NQgDo6BDk90566d7_2eW2H>&4kDA+&?A5qct z@eR}xFtR=(1(R39nbP(^TTmd{d?Hcj5srLsC3=FD++R${zF31&_d*oMlfTu2u???E zR9w1_o~!-1=XU*MJT}^ym%P}GcK1P)y0)O$uLJ3B#2yC^!W`NI{|~8A;zJF=YdU1R zCZgDt+Rkn&-154B{$MwBUf@398^U@f6P}1-u`g%dMN0dGd-p+D81|(hT#L2XR7S(0 z6O;af+}lb8?p?k5XYuup`}Mm%4}I?LwW1U{SBZ;RXUt&BAi@E0SqEC;+2iT2=;N?Y z9r(jJ|KxHMmM(}e5%47jFL~pxdjtk!iP84fA@9S>sPz%UE=xnwwS1T_PzN%k9aa8) z)T<({MtzsD=B|_R*d3n2Ef+g;tjudGM$sB@9hukasgs@#n{Pd;J#$biPJ!Kx_P3`1 zjRC~JctxV;QZV|iGOpsS!1lE$JmH@GnEBO%XbpDbStlgnJtd*`lu?Nd?%`{Cu1yi) z81|Al#&pu}H3srs`~G#`A3pQW>1CC^>3*_?b=1Ut(r-M-=6y!K>O(#HeJ!!PLNT7u z2Hv?Wg45L(qi#Z$MF*k!Vi?K-1JT2^tm~>jp$AH8VN-M1llW9W=JDNA(R3*v4K8gc z{IL6EJQf-AXmaVrt&4T23ZUO{tq2xD5n_GF3l~a_)+-XUvF`^0ywN}#;tqe;?iz>@ z4_7$+gE$u46FeAe@hAR-8YNIju^&nNs3M$rix{jhw>K4Xp4?)77!4IP+C+a|Kks#Y z`(565s+C?N@p7j#Ur)|<))EAS;z*$|=0<{EVRFWsNt*ZCJ*&z*J}bWNKw z-iOqKdzxz^K9jZ!*8!w*ua6L8_X6wPKEFmySReF71YpkJ9lhd6>eWh+&DezXvKeJQ zg(#5rVOr8lKD?IW;lRr6DGZ#i#vNe`Od<3sg7eWPB=_SrcQjq3=0#vAYwE)=9qLCa zrBAVWnOa@vudsHUImhTgK+Zg(Iu!fKD(bh(v7@2Rfr7P?5kV%Wsqb%Q99|yJUB+70 zLc=n4Ee7Wvu^0P^q2!fg97-v`E^%`^?E5vPocNy$#EfOpUwI#muB>~BB=5B@8ATzL zu(=hY=7U=FhxDF|$NGR11KyP=_^1+9J|$@POXglqEg8mLZwP%b7el?IkWlUcS1=ds zk8Ka;4!L(g#u^9O=Lht~_YABL%1FY7IE*^E22-El%-`%dygV+jnchH(I^rNXvhTng z0i-aW7fByQO#d-bM*k&_Jaz6^rrkdpEyS221a)UFAj{Rrd!M?TM(Pnoy|8x|i7jvV zq7_ww8k9v;!g!e)T5f(U@AswT5qZ94fmkGV)j^+i-rdv#JoBtQ5v<#kRU*SP3w7?Hq-tc0CL#SMW*88S%Zl=pvuput$rUH#;q044h0xA=pmH?1`;t~xhucwCIi6mW-*&;Cj&&xC59}M0 zry$kYAoe~BUC3V~?)QIrj3KbS0iF8HzTt}3gJ`jyK8?*TQ}lI(D5h=+0-eVL;$XdA zpEwI;0Xk=Apm3@u!nf&Rl&Wc2bWy(_80ZCo`#b2fz46!H7tj8mQ>T7>_`J%Cebuvw z@4xxt_}=0F^urhJ!>>L2;?ei*&)RRF{NA(X;TfJEKX~5$(Q*6n^Y#yp+s}XQ@CToN ze){?GQ*R%&MMv#7kJ?8~p5WQ%kAD8tsUJOWKfBt)&;0Ot`{;D1PW|9{`}lM`;j`~g zAOGH|h-Yp4Yuj%>J1#mo{^qmxN%Pc?U)4UjJYL}8@srEr1-`0%a=Em+$A7+kb3+ygZ-q;(V{1e-NE~;Q4&DpR}LP z=jqqBBMz?}FaCjNFL83gN9)PS1s|<9Cl`3M9-Um^(Ry`qLC5Qv{pbQBp8x*I331du zI^oHBd31u4_4McjC+jVq|KN-D`1ybEV*P%0#a~&^pIy Date: Wed, 18 Mar 2026 23:16:59 -0700 Subject: [PATCH 52/63] feat: state cache for ePBS (#8868) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary This PR extends the **Gloas ePBS state cache architecture** to support dual state variants (block state and payload state) by threading the `payloadPresent` flag through the state cache layer, regeneration system, and archive store. ## Context Building on the fork choice changes in `nc/epbs-fc` (which stores checkpoints with payload status), this PR completes the state cache implementation for Gloas ePBS by: 1. Extending all checkpoint cache operations to track `payloadPresent` 2. Updating the regeneration layer to explicitly handle both state variants 3. Propagating payload status from block import through to checkpoint caching ## Key Changes ### 1. State Cache Type System Updates **`packages/beacon-node/src/chain/stateCache/types.ts`** (+208/-111 lines total across files): - Renamed `CheckpointHex` → `CheckpointHexPayload` with required `payloadPresent: boolean` field - Updated `CheckpointStateCache` interface methods to accept `payloadPresent` parameter: - `add(cp, state, payloadPresent)` - explicitly marks state variant when adding to cache - `getLatest(rootHex, maxEpoch, payloadPresent)` - retrieves specific state variant - `getOrReloadLatest(rootHex, maxEpoch, payloadPresent)` - reloads specific state variant from disk - `updatePreComputedCheckpoint(rootHex, epoch, payloadPresent)` - tracks payload status for pre-computed states - Kept `processState()` method signature unchanged (manages both variants internally) ### 2. PersistentCheckpointStateCache Implementation **`packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts`** (~289 lines modified): - Extended cache key format from `"epoch-rootHex"` to `"epoch-rootHex-payloadPresent"` - Updated `toCheckpointHexPayload()` helper to include payload status in keys - Modified all cache operations (add, get, getLatest, etc.) to handle dual state variants - Implemented logic to iterate both `payloadPresent` variants in `processPastEpoch()` for memory management - Updated cache metrics and debugging utilities to reflect dual state architecture ### 3. Regeneration Layer - Dual State Support **`packages/beacon-node/src/chain/regen/interface.ts`** & **`queued.ts`** & **`regen.ts`**: - Added `processPayloadState(payloadState)` method for explicit payload state caching (Gloas-only) - Called after `processExecutionPayloadEnvelope()` when payload is revealed - Complements `processState()` which handles block state caching - Updated `addCheckpointState(cp, state, payloadPresent)` to accept payload flag - Extended `updatePreComputedCheckpoint()` with `payloadPresent` parameter - Modified `getCheckpointState()` and related methods to pass `payloadPresent` through cache lookups ### 4. Block Import - Payload Status Propagation **`packages/beacon-node/src/chain/blocks/importBlock.ts`**: - Derive `payloadPresent` from block type: - **Pre-Gloas**: `payloadPresent = true` (execution payload embedded in block, always FULL variant) - **Post-Gloas**: `payloadPresent = false` (block state only, PENDING/EMPTY variant, payload not yet revealed) - Thread `payloadPresent` through checkpoint caching operations: - `regen.addCheckpointState(cp, checkpointState, payloadPresent)` ### 5. Archive Store - Historical State Management **`packages/beacon-node/src/chain/archiveStore/`**: - Updated `archiveStore.archiveState()` to accept `payloadPresent` parameter - Modified archival strategies (`frequencyStateArchiveStrategy`) to propagate payload status - Ensured historical states maintain proper metadata for state variant tracking ### 6. API & Validation Layer Updates **`packages/beacon-node/src/api/impl/`**: - Updated validator API to retrieve correct state variant with `payloadPresent` flag - Modified beacon state utilities to handle payload-aware checkpoint lookups - Ensured API endpoints return appropriate state variant based on block type ## Technical Details ### Checkpoint Key Format (Post-Gloas) ```typescript // Pre-Gloas (always single variant): "100-0x1234abcd" // epoch-rootHex // Post-Gloas (dual variants): "100-0x1234abcd-false" // epoch-rootHex-payloadPresent (block state) "100-0x1234abcd-true" // epoch-rootHex-payloadPresent (payload state) ``` ### State Variant Semantics - **Block State** (`payloadPresent = false`): State after processing beacon block, before execution payload - Only exists for Gloas blocks (PENDING/EMPTY variants) - **Payload State** (`payloadPresent = true`): State after processing execution payload - Exists for all pre-Gloas blocks (single variant) - Exists for Gloas blocks after payload revelation ## Migration & Compatibility - **Pre-Gloas blocks**: All state cache operations default to `payloadPresent = true` - **Backward compatibility**: Checkpoint cache can handle mix of pre-Gloas and post-Gloas states - **Cache key migration**: Existing cache entries remain valid (treated as `payloadPresent = true`) ## Depends On - `nc/epbs-fc` - Fork choice stores checkpoints with payload status (#8845) --- **AI Disclosure**: This PR was written primarily by Claude Code. --------- Co-authored-by: twoeths <10568965+twoeths@users.noreply.github.com> Co-authored-by: Tuyen Nguyen Co-authored-by: Claude Opus 4.6 Co-authored-by: Cayman --- packages/api/src/beacon/routes/lodestar.ts | 2 + .../src/api/impl/beacon/state/utils.ts | 4 +- .../src/api/impl/validator/index.ts | 10 +- .../src/chain/archiveStore/archiveStore.ts | 10 - .../src/chain/archiveStore/interface.ts | 8 +- .../frequencyStateArchiveStrategy.ts | 13 +- .../chain/archiveStore/utils/archiveBlocks.ts | 60 ++- .../archiveStore/utils/archivePayloads.ts | 15 - .../src/chain/blocks/importBlock.ts | 17 +- packages/beacon-node/src/chain/chain.ts | 35 +- packages/beacon-node/src/chain/interface.ts | 4 +- .../beacon-node/src/chain/prepareNextSlot.ts | 8 +- .../beacon-node/src/chain/regen/errors.ts | 7 +- .../beacon-node/src/chain/regen/interface.ts | 18 +- .../beacon-node/src/chain/regen/queued.ts | 60 ++- packages/beacon-node/src/chain/regen/regen.ts | 44 +- .../src/chain/stateCache/datastore/db.ts | 43 +- .../src/chain/stateCache/datastore/file.ts | 11 +- .../src/chain/stateCache/datastore/types.ts | 5 +- .../chain/stateCache/fifoBlockStateCache.ts | 14 +- .../stateCache/persistentCheckpointsCache.ts | 387 +++++++++++------- .../beacon-node/src/chain/stateCache/types.ts | 26 +- .../persistentCheckpointsCache.test.ts | 195 ++++----- .../test/unit/chain/regen/regen.test.ts | 6 +- .../test/utils/chain/stateCache/datastore.ts | 4 +- .../beacon-node/test/utils/node/simTest.ts | 2 + packages/fork-choice/src/index.ts | 2 +- 27 files changed, 655 insertions(+), 355 deletions(-) delete mode 100644 packages/beacon-node/src/chain/archiveStore/utils/archivePayloads.ts diff --git a/packages/api/src/beacon/routes/lodestar.ts b/packages/api/src/beacon/routes/lodestar.ts index 5236d3d40e38..ea894391f8f2 100644 --- a/packages/api/src/beacon/routes/lodestar.ts +++ b/packages/api/src/beacon/routes/lodestar.ts @@ -547,6 +547,8 @@ export function getDefinitions(_config: ChainForkConfig): RouteDefinitions { + async function waitForCheckpointState(cpHex: CheckpointHexPayload): Promise { const cpState = chain.regen.getCheckpointStateSync(cpHex); if (cpState) { return cpState; @@ -1112,7 +1112,11 @@ export function getValidatorApi( // this is to avoid missed block proposal due to 0 epoch look ahead if (epoch === nextEpoch && toNextEpochMs < prepareNextSlotLookAheadMs) { // wait for maximum 1 slot for cp state which is the timeout of validator api - const cpState = await waitForCheckpointState({rootHex: head.blockRoot, epoch}); + const cpState = await waitForCheckpointState({ + rootHex: head.blockRoot, + epoch, + payloadPresent: head.payloadStatus === PayloadStatus.FULL, + }); if (cpState) { state = cpState; metrics?.duties.requestNextEpochProposalDutiesHit.inc(); diff --git a/packages/beacon-node/src/chain/archiveStore/archiveStore.ts b/packages/beacon-node/src/chain/archiveStore/archiveStore.ts index 812f01034d53..b78330d76313 100644 --- a/packages/beacon-node/src/chain/archiveStore/archiveStore.ts +++ b/packages/beacon-node/src/chain/archiveStore/archiveStore.ts @@ -1,6 +1,5 @@ import {CheckpointWithPayloadStatus} from "@lodestar/fork-choice"; import {LoggerNode} from "@lodestar/logger/node"; -import {ForkSeq} from "@lodestar/params"; import {Checkpoint} from "@lodestar/types/phase0"; import {callFnWhenAwait} from "@lodestar/utils"; import {IBeaconDb} from "../../db/index.js"; @@ -14,7 +13,6 @@ import {HistoricalStateRegen} from "./historicalState/historicalStateRegen.js"; import {ArchiveMode, ArchiveStoreOpts, StateArchiveStrategy} from "./interface.js"; import {FrequencyStateArchiveStrategy} from "./strategies/frequencyStateArchiveStrategy.js"; import {archiveBlocks} from "./utils/archiveBlocks.js"; -import {archiveExecutionPayloadEnvelopes} from "./utils/archivePayloads.js"; import {pruneHistory} from "./utils/pruneHistory.js"; import {updateBackfillRange} from "./utils/updateBackfillRange.js"; @@ -29,7 +27,6 @@ type ArchiveStoreInitOpts = ArchiveStoreOpts & {dbName: string; anchorState: {fi export enum ArchiveStoreTask { ArchiveBlocks = "archive_blocks", - ArchivePayloads = "archive_payloads", PruneHistory = "prune_history", OnFinalizedCheckpoint = "on_finalized_checkpoint", MaybeArchiveState = "maybe_archive_state", @@ -192,7 +189,6 @@ export class ArchiveStore { private processFinalizedCheckpoint = async (finalized: CheckpointWithPayloadStatus): Promise => { try { const finalizedEpoch = finalized.epoch; - const finalizedFork = this.chain.config.getForkSeqAtEpoch(finalizedEpoch); this.logger.verbose("Start processing finalized checkpoint", {epoch: finalizedEpoch, rootHex: finalized.rootHex}); let timer = this.metrics?.processFinalizedCheckpoint.durationByTask.startTimer(); @@ -210,12 +206,6 @@ export class ArchiveStore { ); timer?.({source: ArchiveStoreTask.ArchiveBlocks}); - if (finalizedFork >= ForkSeq.gloas) { - timer = this.metrics?.processFinalizedCheckpoint.durationByTask.startTimer(); - await archiveExecutionPayloadEnvelopes(this.chain, finalized); - timer?.({source: ArchiveStoreTask.ArchivePayloads}); - } - if (this.opts.pruneHistory) { timer = this.metrics?.processFinalizedCheckpoint.durationByTask.startTimer(); await pruneHistory( diff --git a/packages/beacon-node/src/chain/archiveStore/interface.ts b/packages/beacon-node/src/chain/archiveStore/interface.ts index 67054c36463c..25b54c4fa38f 100644 --- a/packages/beacon-node/src/chain/archiveStore/interface.ts +++ b/packages/beacon-node/src/chain/archiveStore/interface.ts @@ -1,4 +1,4 @@ -import {CheckpointWithHex} from "@lodestar/fork-choice"; +import {CheckpointWithPayloadStatus} from "@lodestar/fork-choice"; import {RootHex} from "@lodestar/types"; import {Metrics} from "../../metrics/metrics.js"; @@ -44,9 +44,9 @@ export type FinalizedStats = { export interface StateArchiveStrategy { onCheckpoint(stateRoot: RootHex, metrics?: Metrics | null): Promise; - onFinalizedCheckpoint(finalized: CheckpointWithHex, metrics?: Metrics | null): Promise; - maybeArchiveState(finalized: CheckpointWithHex, metrics?: Metrics | null): Promise; - archiveState(finalized: CheckpointWithHex, metrics?: Metrics | null): Promise; + onFinalizedCheckpoint(finalized: CheckpointWithPayloadStatus, metrics?: Metrics | null): Promise; + maybeArchiveState(finalized: CheckpointWithPayloadStatus, metrics?: Metrics | null): Promise; + archiveState(finalized: CheckpointWithPayloadStatus, metrics?: Metrics | null): Promise; } export interface IArchiveStore { diff --git a/packages/beacon-node/src/chain/archiveStore/strategies/frequencyStateArchiveStrategy.ts b/packages/beacon-node/src/chain/archiveStore/strategies/frequencyStateArchiveStrategy.ts index 3e3d207a40db..09be1942c175 100644 --- a/packages/beacon-node/src/chain/archiveStore/strategies/frequencyStateArchiveStrategy.ts +++ b/packages/beacon-node/src/chain/archiveStore/strategies/frequencyStateArchiveStrategy.ts @@ -1,4 +1,4 @@ -import {CheckpointWithHex} from "@lodestar/fork-choice"; +import {CheckpointWithPayloadStatus} from "@lodestar/fork-choice"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {Epoch, RootHex, Slot} from "@lodestar/types"; @@ -9,6 +9,7 @@ import {AllocSource, BufferPool} from "../../../util/bufferPool.js"; import {getStateSlotFromBytes} from "../../../util/multifork.js"; import {IStateRegenerator} from "../../regen/interface.js"; import {serializeState} from "../../serializeState.js"; +import {fcCheckpointToHexPayload} from "../../stateCache/persistentCheckpointsCache.js"; import {StateArchiveStrategy, StatesArchiveOpts} from "../interface.js"; /** @@ -40,7 +41,7 @@ export class FrequencyStateArchiveStrategy implements StateArchiveStrategy { private readonly bufferPool?: BufferPool | null ) {} - async onFinalizedCheckpoint(_finalized: CheckpointWithHex, _metrics?: Metrics | null): Promise {} + async onFinalizedCheckpoint(_finalized: CheckpointWithPayloadStatus, _metrics?: Metrics | null): Promise {} async onCheckpoint(_stateRoot: RootHex, _metrics?: Metrics | null): Promise {} /** @@ -55,7 +56,7 @@ export class FrequencyStateArchiveStrategy implements StateArchiveStrategy { * epoch - 1024*2 epoch - 1024 epoch - 32 epoch * ``` */ - async maybeArchiveState(finalized: CheckpointWithHex, metrics?: Metrics | null): Promise { + async maybeArchiveState(finalized: CheckpointWithPayloadStatus, metrics?: Metrics | null): Promise { let timer = metrics?.processFinalizedCheckpoint.frequencyStateArchive.startTimer(); const lastStoredSlot = await this.db.stateArchive.lastKey(); timer?.({step: FrequencyStateArchiveStep.LoadLastStoredSlot}); @@ -104,10 +105,12 @@ export class FrequencyStateArchiveStrategy implements StateArchiveStrategy { * Archives finalized states from active bucket to archive bucket. * Only the new finalized state is stored to disk */ - async archiveState(finalized: CheckpointWithHex, metrics?: Metrics | null): Promise { + async archiveState(finalized: CheckpointWithPayloadStatus, metrics?: Metrics | null): Promise { // starting from Mar 2024, the finalized state could be from disk or in memory let timer = metrics?.processFinalizedCheckpoint.frequencyStateArchive.startTimer(); - const finalizedStateOrBytes = await this.regen.getCheckpointStateOrBytes(finalized); + // Convert fork-choice checkpoint to beacon-node checkpoint with payloadPresent + const finalizedHexPayload = fcCheckpointToHexPayload(finalized); + const finalizedStateOrBytes = await this.regen.getCheckpointStateOrBytes(finalizedHexPayload); timer?.({step: FrequencyStateArchiveStep.GetFinalizedState}); const {rootHex} = finalized; diff --git a/packages/beacon-node/src/chain/archiveStore/utils/archiveBlocks.ts b/packages/beacon-node/src/chain/archiveStore/utils/archiveBlocks.ts index f8f62a244aff..61c9aada1fdc 100644 --- a/packages/beacon-node/src/chain/archiveStore/utils/archiveBlocks.ts +++ b/packages/beacon-node/src/chain/archiveStore/utils/archiveBlocks.ts @@ -1,7 +1,7 @@ import path from "node:path"; import {ChainForkConfig} from "@lodestar/config"; import {KeyValue} from "@lodestar/db"; -import {CheckpointWithPayloadStatus, IForkChoice} from "@lodestar/fork-choice"; +import {CheckpointWithPayloadStatus, IForkChoice, PayloadStatus, ProtoBlock} from "@lodestar/fork-choice"; import {ForkSeq, SLOTS_PER_EPOCH} from "@lodestar/params"; import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {Epoch, Slot} from "@lodestar/types"; @@ -66,6 +66,7 @@ export async function archiveBlocks( // NOTE: The finalized block will be exactly the first block of `epoch` or previous const finalizedPostDeneb = finalizedCheckpoint.epoch >= config.DENEB_FORK_EPOCH; const finalizedPostFulu = finalizedCheckpoint.epoch >= config.FULU_FORK_EPOCH; + const finalizedPostGloas = finalizedCheckpoint.epoch >= config.GLOAS_FORK_EPOCH; const finalizedCanonicalBlockRoots: BlockRootSlot[] = finalizedCanonicalBlocks.map((block) => ({ slot: block.slot, @@ -103,6 +104,16 @@ export async function archiveBlocks( ); logger.verbose("Migrated dataColumnSidecars from hot DB to cold DB", {...logCtx, migratedEntries}); } + + if (finalizedPostGloas) { + const migratedEntries = await migrateExecutionPayloadEnvelopesFromHotToColdDb( + config, + db, + logger, + finalizedCanonicalBlocks + ); + logger.verbose("Migrated executionPayloadEnvelopes from hot DB to cold DB", {...logCtx, migratedEntries}); + } } // deleteNonCanonicalBlocks @@ -144,6 +155,11 @@ export async function archiveBlocks( await db.dataColumnSidecar.deleteMany(nonCanonicalBlockRoots); logger.verbose("Deleted non canonical dataColumnSidecars from hot DB", logCtx); } + + if (finalizedPostGloas) { + await db.executionPayloadEnvelope.batchDelete(nonCanonicalBlockRoots); + logger.verbose("Deleted non canonical executionPayloadEnvelopes from hot DB", logCtx); + } } // Delete expired blobs @@ -372,6 +388,48 @@ async function migrateDataColumnSidecarsFromHotToColdDb( return migratedWrappedDataColumns; } +async function migrateExecutionPayloadEnvelopesFromHotToColdDb( + config: ChainForkConfig, + db: IBeaconDb, + logger: Logger, + canonicalBlocks: ProtoBlock[] +): Promise { + let migratedEnvelopes = 0; + + const payloadBlocks = canonicalBlocks.filter( + (block) => config.getForkSeq(block.slot) >= ForkSeq.gloas && block.payloadStatus === PayloadStatus.FULL + ); + if (payloadBlocks.length === 0) return 0; + const blocks = payloadBlocks.map((block) => ({slot: block.slot, root: fromHex(block.blockRoot)})); + + const envelopeEntries: KeyValue[] = []; + const migratedRoots: Uint8Array[] = []; + + const envelopeBytesArray = await Promise.all( + blocks.map((block) => db.executionPayloadEnvelope.getBinary(block.root)) + ); + + for (let i = 0; i < blocks.length; i++) { + const bytes = envelopeBytesArray[i]; + if (bytes !== null) { + envelopeEntries.push({key: blocks[i].slot, value: bytes}); + migratedRoots.push(blocks[i].root); + } else { + logger.debug("Payload in forkchoice but missing in db", {slot: blocks[i].slot, root: toRootHex(blocks[i].root)}); + } + } + + if (envelopeEntries.length > 0) { + await Promise.all([ + db.executionPayloadEnvelopeArchive.batchPutBinary(envelopeEntries), + db.executionPayloadEnvelope.batchDelete(migratedRoots), + ]); + migratedEnvelopes = envelopeEntries.length; + } + + return migratedEnvelopes; +} + /** * ``` * class SignedBeaconBlock(Container): diff --git a/packages/beacon-node/src/chain/archiveStore/utils/archivePayloads.ts b/packages/beacon-node/src/chain/archiveStore/utils/archivePayloads.ts deleted file mode 100644 index 491ae8b74b8d..000000000000 --- a/packages/beacon-node/src/chain/archiveStore/utils/archivePayloads.ts +++ /dev/null @@ -1,15 +0,0 @@ -import {CheckpointWithHex} from "@lodestar/fork-choice"; -import {IBeaconChain} from "../../interface.js"; - -/** - * Archives execution payload envelopes from hot DB to archive DB after finalization. - */ -export async function archiveExecutionPayloadEnvelopes( - chain: IBeaconChain, - _finalized: CheckpointWithHex -): Promise { - const finalizedBlock = chain.forkChoice.getFinalizedBlock(); - if (!finalizedBlock) return; - - // TODO GLOAS: Implement payload envelope archival after epbs fork choice changes are merged -} diff --git a/packages/beacon-node/src/chain/blocks/importBlock.ts b/packages/beacon-node/src/chain/blocks/importBlock.ts index 0d5af688580c..c8df51f91f3a 100644 --- a/packages/beacon-node/src/chain/blocks/importBlock.ts +++ b/packages/beacon-node/src/chain/blocks/importBlock.ts @@ -7,6 +7,7 @@ import { ForkChoiceErrorCode, NotReorgedReason, getSafeExecutionBlockHash, + isGloasBlock, } from "@lodestar/fork-choice"; import {ForkPostAltair, ForkPostElectra, ForkSeq, MAX_SEED_LOOKAHEAD, SLOTS_PER_EPOCH} from "@lodestar/params"; import { @@ -30,7 +31,7 @@ import type {BeaconChain} from "../chain.js"; import {ChainEvent, ReorgEventData} from "../emitter.js"; import {ForkchoiceCaller} from "../forkChoice/index.js"; import {REPROCESS_MIN_TIME_TO_NEXT_SLOT_SEC} from "../reprocess.js"; -import {toCheckpointHex} from "../stateCache/persistentCheckpointsCache.js"; +import {toCheckpointHexPayload} from "../stateCache/persistentCheckpointsCache.js"; import {isBlockInputBlobs, isBlockInputColumns} from "./blockInput/blockInput.js"; import {AttestationImportOpt, FullyVerifiedBlock, ImportBlockOpts} from "./types.js"; import {getCheckpointFromState} from "./utils/checkpoint.js"; @@ -116,7 +117,11 @@ export async function importBlock( // This adds the state necessary to process the next block // Some block event handlers require state being in state cache so need to do this before emitting EventType.block - this.regen.processState(blockRootHex, postState); + // Pre-Gloas: blockSummary.payloadStatus is always FULL, payloadPresent = true + // Post-Gloas: blockSummary.payloadStatus is always PENDING, so payloadPresent = false (block state only, no payload processing yet) + const payloadPresent = !isGloasBlock(blockSummary); + // processState manages both block state and payload state variants together for memory/disk management + this.regen.processBlockState(blockRootHex, postState); this.metrics?.importBlock.bySource.inc({source: source.source}); this.logger.verbose("Added block to forkchoice and state cache", {slot: blockSlot, root: blockRootHex}); @@ -456,12 +461,12 @@ export async function importBlock( // Cache state to preserve epoch transition work const checkpointState = postState; const cp = getCheckpointFromState(checkpointState); - this.regen.addCheckpointState(cp, checkpointState); + this.regen.addCheckpointState(cp, checkpointState, payloadPresent); // consumers should not mutate state ever this.emitter.emit(ChainEvent.checkpoint, cp, checkpointState); // Note: in-lined code from previos handler of ChainEvent.checkpoint - this.logger.verbose("Checkpoint processed", toCheckpointHex(cp)); + this.logger.verbose("Checkpoint processed", toCheckpointHexPayload(cp, payloadPresent)); const activeValidatorsCount = checkpointState.epochCtx.currentShuffling.activeIndices.length; this.metrics?.currentActiveValidators.set(activeValidatorsCount); @@ -479,7 +484,7 @@ export async function importBlock( const justifiedEpoch = justifiedCheckpoint.epoch; const preJustifiedEpoch = parentBlockSummary.justifiedEpoch; if (justifiedEpoch > preJustifiedEpoch) { - this.logger.verbose("Checkpoint justified", toCheckpointHex(justifiedCheckpoint)); + this.logger.verbose("Checkpoint justified", toCheckpointHexPayload(justifiedCheckpoint, payloadPresent)); this.metrics?.previousJustifiedEpoch.set(checkpointState.previousJustifiedCheckpoint.epoch); this.metrics?.currentJustifiedEpoch.set(justifiedCheckpoint.epoch); } @@ -493,7 +498,7 @@ export async function importBlock( state: toRootHex(checkpointState.hashTreeRoot()), executionOptimistic: false, }); - this.logger.verbose("Checkpoint finalized", toCheckpointHex(finalizedCheckpoint)); + this.logger.verbose("Checkpoint finalized", toCheckpointHexPayload(finalizedCheckpoint, payloadPresent)); this.metrics?.finalizedEpoch.set(finalizedCheckpoint.epoch); } } diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index f0a9aeace741..4c41afd33aa6 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -3,11 +3,12 @@ import {PrivateKey} from "@libp2p/interface"; import {CompositeTypeAny, TreeView, Type} from "@chainsafe/ssz"; import {BeaconConfig} from "@lodestar/config"; import { - CheckpointWithHex, CheckpointWithPayloadStatus, IForkChoice, + PayloadStatus, ProtoBlock, UpdateHeadOpt, + getCheckpointPayloadStatus, } from "@lodestar/fork-choice"; import {LoggerNode} from "@lodestar/logger/node"; import { @@ -126,7 +127,7 @@ import {DbCPStateDatastore, checkpointToDatastoreKey} from "./stateCache/datasto import {FileCPStateDatastore} from "./stateCache/datastore/file.js"; import {CPStateDatastore} from "./stateCache/datastore/types.js"; import {FIFOBlockStateCache} from "./stateCache/fifoBlockStateCache.js"; -import {PersistentCheckpointStateCache} from "./stateCache/persistentCheckpointsCache.js"; +import {PersistentCheckpointStateCache, fcCheckpointToHexPayload} from "./stateCache/persistentCheckpointsCache.js"; import {CheckpointStateCache} from "./stateCache/types.js"; import {ValidatorMonitor} from "./validatorMonitor.js"; @@ -375,7 +376,8 @@ export class BeaconChain implements IBeaconChain { const {checkpoint} = computeAnchorCheckpoint(config, anchorState); blockStateCache.add(anchorState); blockStateCache.setHeadState(anchorState); - checkpointStateCache.add(checkpoint, anchorState); + const payloadPresent = getCheckpointPayloadStatus(anchorState, checkpoint.epoch) === PayloadStatus.FULL; + checkpointStateCache.add(checkpoint, anchorState, payloadPresent); const forkChoice = initializeForkChoice( config, @@ -648,15 +650,18 @@ export class BeaconChain implements IBeaconChain { return this.cpStateDatastore.readLatestSafe(); } - const persistedKey = checkpointToDatastoreKey(checkpoint); + // TODO GLOAS: Need to revisit the design of this api. Currently we just retrieve FULL state of the checkpoint for backwards compatibility. + // because pre-gloas we always store FULL checkpoint state. + const persistedKey = checkpointToDatastoreKey(checkpoint, true); return this.cpStateDatastore.read(persistedKey); } getStateByCheckpoint( - checkpoint: CheckpointWithHex + checkpoint: CheckpointWithPayloadStatus ): {state: BeaconStateAllForks; executionOptimistic: boolean; finalized: boolean} | null { // finalized or justified checkpoint states maynot be available with PersistentCheckpointStateCache, use getCheckpointStateOrBytes() api to get Uint8Array - const cachedStateCtx = this.regen.getCheckpointStateSync(checkpoint); + const checkpointHexPayload = fcCheckpointToHexPayload(checkpoint); + const cachedStateCtx = this.regen.getCheckpointStateSync(checkpointHexPayload); if (cachedStateCtx) { const block = this.forkChoice.getBlockDefaultStatus(cachedStateCtx.latestBlockHeader.hashTreeRoot()); const finalizedEpoch = this.forkChoice.getFinalizedCheckpoint().epoch; @@ -671,9 +676,10 @@ export class BeaconChain implements IBeaconChain { } async getStateOrBytesByCheckpoint( - checkpoint: CheckpointWithHex + checkpoint: CheckpointWithPayloadStatus ): Promise<{state: CachedBeaconStateAllForks | Uint8Array; executionOptimistic: boolean; finalized: boolean} | null> { - const cachedStateCtx = await this.regen.getCheckpointStateOrBytes(checkpoint); + const checkpointHexPayload = fcCheckpointToHexPayload(checkpoint); + const cachedStateCtx = await this.regen.getCheckpointStateOrBytes(checkpointHexPayload); if (cachedStateCtx) { const block = this.forkChoice.getBlockDefaultStatus(checkpoint.root); const finalizedEpoch = this.forkChoice.getFinalizedCheckpoint().epoch; @@ -1236,7 +1242,8 @@ export class BeaconChain implements IBeaconChain { checkpoint: CheckpointWithPayloadStatus, blockState: CachedBeaconStateAllForks ): {state: CachedBeaconStateAllForks; stateId: string; shouldWarn: boolean} { - const state = this.regen.getCheckpointStateSync(checkpoint); + const checkpointHexPayload = fcCheckpointToHexPayload(checkpoint); + const state = this.regen.getCheckpointStateSync(checkpointHexPayload); if (state) { return {state, stateId: "checkpoint_state", shouldWarn: false}; } @@ -1363,6 +1370,10 @@ export class BeaconChain implements IBeaconChain { private onClockEpoch(epoch: Epoch): void { this.metrics?.clockEpoch.set(epoch); + if (epoch === this.config.GLOAS_FORK_EPOCH) { + this.regen.upgradeForGloas(epoch); + } + this.seenAttesters.prune(epoch); this.seenAggregators.prune(epoch); this.seenPayloadAttesters.prune(epoch); @@ -1376,7 +1387,7 @@ export class BeaconChain implements IBeaconChain { this.seenContributionAndProof.prune(head.slot); } - private onForkChoiceJustified(this: BeaconChain, cp: CheckpointWithHex): void { + private onForkChoiceJustified(this: BeaconChain, cp: CheckpointWithPayloadStatus): void { this.logger.verbose("Fork choice justified", {epoch: cp.epoch, root: cp.rootHex}); } @@ -1387,7 +1398,7 @@ export class BeaconChain implements IBeaconChain { }); } - private async onForkChoiceFinalized(this: BeaconChain, cp: CheckpointWithHex): Promise { + private async onForkChoiceFinalized(this: BeaconChain, cp: CheckpointWithPayloadStatus): Promise { this.logger.verbose("Fork choice finalized", {epoch: cp.epoch, root: cp.rootHex}); const finalizedSlot = computeStartSlotAtEpoch(cp.epoch); this.seenBlockProposers.prune(finalizedSlot); @@ -1429,7 +1440,7 @@ export class BeaconChain implements IBeaconChain { } } - private async updateValidatorsCustodyRequirement(finalizedCheckpoint: CheckpointWithHex): Promise { + private async updateValidatorsCustodyRequirement(finalizedCheckpoint: CheckpointWithPayloadStatus): Promise { if (this.custodyConfig.targetCustodyGroupCount === this.config.NUMBER_OF_CUSTODY_GROUPS) { // Custody requirements can only be increased, we can disable dynamic custody updates // if the node already maintains custody of all custody groups in case it is configured diff --git a/packages/beacon-node/src/chain/interface.ts b/packages/beacon-node/src/chain/interface.ts index cc300e9a5dd7..d15c9951bfbb 100644 --- a/packages/beacon-node/src/chain/interface.ts +++ b/packages/beacon-node/src/chain/interface.ts @@ -1,6 +1,6 @@ import {CompositeTypeAny, TreeView, Type} from "@chainsafe/ssz"; import {BeaconConfig} from "@lodestar/config"; -import {CheckpointWithHex, IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; +import {CheckpointWithHex, CheckpointWithPayloadStatus, IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; import {BeaconStateAllForks, CachedBeaconStateAllForks, EpochShuffling, PubkeyCache} from "@lodestar/state-transition"; import { BeaconBlock, @@ -192,7 +192,7 @@ export interface IBeaconChain { ): {state: BeaconStateAllForks; executionOptimistic: boolean; finalized: boolean} | null; /** Return state bytes by checkpoint */ getStateOrBytesByCheckpoint( - checkpoint: CheckpointWithHex + checkpoint: CheckpointWithPayloadStatus ): Promise<{state: CachedBeaconStateAllForks | Uint8Array; executionOptimistic: boolean; finalized: boolean} | null>; /** diff --git a/packages/beacon-node/src/chain/prepareNextSlot.ts b/packages/beacon-node/src/chain/prepareNextSlot.ts index 0d84adc771fd..0f4b0d0f6191 100644 --- a/packages/beacon-node/src/chain/prepareNextSlot.ts +++ b/packages/beacon-node/src/chain/prepareNextSlot.ts @@ -1,6 +1,6 @@ import {routes} from "@lodestar/api"; import {ChainForkConfig} from "@lodestar/config"; -import {getSafeExecutionBlockHash} from "@lodestar/fork-choice"; +import {PayloadStatus, getSafeExecutionBlockHash} from "@lodestar/fork-choice"; import {ForkPostBellatrix, ForkSeq, SLOTS_PER_EPOCH, isForkPostBellatrix} from "@lodestar/params"; import { CachedBeaconStateAllForks, @@ -211,7 +211,11 @@ export class PrepareNextSlotScheduler { // + if next slot is a skipped slot, it'd help getting target checkpoint state faster to validate attestations if (isEpochTransition) { this.metrics?.precomputeNextEpochTransition.count.inc({result: "success"}, 1); - const previousHits = this.chain.regen.updatePreComputedCheckpoint(headRoot, nextEpoch); + // Determine payloadPresent from head block's payload status + // Pre-Gloas: payloadStatus is always FULL → payloadPresent = true + // Post-Gloas: FULL → true, EMPTY → false, PENDING → false (conservative, treat as block state) + const payloadPresent = headBlock.payloadStatus === PayloadStatus.FULL; + const previousHits = this.chain.regen.updatePreComputedCheckpoint(headRoot, nextEpoch, payloadPresent); if (previousHits === 0) { this.metrics?.precomputeNextEpochTransition.waste.inc(); } diff --git a/packages/beacon-node/src/chain/regen/errors.ts b/packages/beacon-node/src/chain/regen/errors.ts index eb41e8321da3..6352d765ea11 100644 --- a/packages/beacon-node/src/chain/regen/errors.ts +++ b/packages/beacon-node/src/chain/regen/errors.ts @@ -1,3 +1,4 @@ +import {PayloadStatus} from "@lodestar/fork-choice"; import {Root, RootHex, Slot} from "@lodestar/types"; export enum RegenErrorCode { @@ -9,6 +10,8 @@ export enum RegenErrorCode { BLOCK_NOT_IN_DB = "REGEN_ERROR_BLOCK_NOT_IN_DB", STATE_TRANSITION_ERROR = "REGEN_ERROR_STATE_TRANSITION_ERROR", INVALID_STATE_ROOT = "REGEN_ERROR_INVALID_STATE_ROOT", + UNEXPECTED_PAYLOAD_STATUS = "REGEN_ERROR_UNEXPECTED_PAYLOAD_STATUS", + INTERNAL_ERROR = "REGEN_ERROR_INTERNAL_ERROR", } export type RegenErrorType = @@ -19,7 +22,9 @@ export type RegenErrorType = | {code: RegenErrorCode.TOO_MANY_BLOCK_PROCESSED; stateRoot: RootHex | Root} | {code: RegenErrorCode.BLOCK_NOT_IN_DB; blockRoot: RootHex | Root} | {code: RegenErrorCode.STATE_TRANSITION_ERROR; error: Error} - | {code: RegenErrorCode.INVALID_STATE_ROOT; slot: Slot; expected: RootHex; actual: RootHex}; + | {code: RegenErrorCode.INVALID_STATE_ROOT; slot: Slot; expected: RootHex; actual: RootHex} + | {code: RegenErrorCode.UNEXPECTED_PAYLOAD_STATUS; blockRoot: RootHex | Root; payloadStatus: PayloadStatus} + | {code: RegenErrorCode.INTERNAL_ERROR; message: string}; export class RegenError extends Error { type: RegenErrorType; diff --git a/packages/beacon-node/src/chain/regen/interface.ts b/packages/beacon-node/src/chain/regen/interface.ts index 61b68fa55625..019523a8139f 100644 --- a/packages/beacon-node/src/chain/regen/interface.ts +++ b/packages/beacon-node/src/chain/regen/interface.ts @@ -2,7 +2,7 @@ import {routes} from "@lodestar/api"; import {ProtoBlock} from "@lodestar/fork-choice"; import {CachedBeaconStateAllForks} from "@lodestar/state-transition"; import {BeaconBlock, Epoch, RootHex, Slot, phase0} from "@lodestar/types"; -import {CheckpointHex} from "../stateCache/types.js"; +import {CheckpointHexPayload} from "../stateCache/types.js"; export enum RegenCaller { getDuties = "getDuties", @@ -38,15 +38,21 @@ export interface IStateRegenerator extends IStateRegeneratorInternal { dumpCacheSummary(): routes.lodestar.StateCacheItem[]; getStateSync(stateRoot: RootHex): CachedBeaconStateAllForks | null; getPreStateSync(block: BeaconBlock): CachedBeaconStateAllForks | null; - getCheckpointStateOrBytes(cp: CheckpointHex): Promise; - getCheckpointStateSync(cp: CheckpointHex): CachedBeaconStateAllForks | null; + getCheckpointStateOrBytes(cp: CheckpointHexPayload): Promise; + getCheckpointStateSync(cp: CheckpointHexPayload): CachedBeaconStateAllForks | null; getClosestHeadState(head: ProtoBlock): CachedBeaconStateAllForks | null; pruneOnCheckpoint(finalizedEpoch: Epoch, justifiedEpoch: Epoch, headStateRoot: RootHex): void; pruneOnFinalized(finalizedEpoch: Epoch): void; - processState(blockRootHex: RootHex, postState: CachedBeaconStateAllForks): void; - addCheckpointState(cp: phase0.Checkpoint, item: CachedBeaconStateAllForks): void; + processBlockState(blockRootHex: RootHex, postState: CachedBeaconStateAllForks): void; + processPayloadState(payloadState: CachedBeaconStateAllForks): void; + /** + * payloadPresent is true if this is payload state, false if block state. + * payloadPresent is always true for pre-gloas. + */ + addCheckpointState(cp: phase0.Checkpoint, item: CachedBeaconStateAllForks, payloadPresent: boolean): void; updateHeadState(newHead: ProtoBlock, maybeHeadState: CachedBeaconStateAllForks): void; - updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch): number | null; + updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch, payloadPresent: boolean): number | null; + upgradeForGloas(epoch: Epoch): void; } /** diff --git a/packages/beacon-node/src/chain/regen/queued.ts b/packages/beacon-node/src/chain/regen/queued.ts index 04011d53cbd1..0da06877faa8 100644 --- a/packages/beacon-node/src/chain/regen/queued.ts +++ b/packages/beacon-node/src/chain/regen/queued.ts @@ -1,11 +1,11 @@ import {routes} from "@lodestar/api"; -import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; +import {IForkChoice, PayloadStatus, ProtoBlock} from "@lodestar/fork-choice"; import {CachedBeaconStateAllForks, computeEpochAtSlot} from "@lodestar/state-transition"; import {BeaconBlock, Epoch, RootHex, Slot, isGloasBeaconBlock, phase0} from "@lodestar/types"; -import {Logger, toRootHex} from "@lodestar/utils"; +import {Logger, fromHex, toRootHex} from "@lodestar/utils"; import {Metrics} from "../../metrics/index.js"; import {JobItemQueue} from "../../util/queue/index.js"; -import {BlockStateCache, CheckpointHex, CheckpointStateCache} from "../stateCache/types.js"; +import {BlockStateCache, CheckpointHexPayload, CheckpointStateCache} from "../stateCache/types.js"; import {RegenError, RegenErrorCode} from "./errors.js"; import { IStateRegenerator, @@ -104,9 +104,19 @@ export class QueuedStateRegenerator implements IStateRegenerator { const parentEpoch = computeEpochAtSlot(parentBlock.slot); const blockEpoch = computeEpochAtSlot(block.slot); + // Convert PayloadStatus to payloadPresent boolean + if (parentBlock.payloadStatus === PayloadStatus.PENDING) { + throw new RegenError({ + code: RegenErrorCode.UNEXPECTED_PAYLOAD_STATUS, + blockRoot: block.parentRoot, + payloadStatus: parentBlock.payloadStatus, + }); + } + const payloadPresent = parentBlock.payloadStatus === PayloadStatus.FULL; + // Check the checkpoint cache (if the pre-state is a checkpoint state) if (parentEpoch < blockEpoch) { - const checkpointState = this.checkpointStateCache.getLatest(parentRoot, blockEpoch); + const checkpointState = this.checkpointStateCache.getLatest(parentRoot, blockEpoch, payloadPresent); if (checkpointState && computeEpochAtSlot(checkpointState.slot) === blockEpoch) { return checkpointState; } @@ -125,14 +135,14 @@ export class QueuedStateRegenerator implements IStateRegenerator { return null; } - async getCheckpointStateOrBytes(cp: CheckpointHex): Promise { + async getCheckpointStateOrBytes(cp: CheckpointHexPayload): Promise { return this.checkpointStateCache.getStateOrBytes(cp); } /** * Get checkpoint state from cache */ - getCheckpointStateSync(cp: CheckpointHex): CachedBeaconStateAllForks | null { + getCheckpointStateSync(cp: CheckpointHexPayload): CachedBeaconStateAllForks | null { return this.checkpointStateCache.get(cp); } @@ -140,7 +150,19 @@ export class QueuedStateRegenerator implements IStateRegenerator { * Get state closest to head */ getClosestHeadState(head: ProtoBlock): CachedBeaconStateAllForks | null { - return this.checkpointStateCache.getLatest(head.blockRoot, Infinity) || this.blockStateCache.get(head.stateRoot); + // Convert PayloadStatus to payloadPresent boolean + if (head.payloadStatus === PayloadStatus.PENDING) { + throw new RegenError({ + code: RegenErrorCode.UNEXPECTED_PAYLOAD_STATUS, + blockRoot: fromHex(head.blockRoot), + payloadStatus: head.payloadStatus, + }); + } + const payloadPresent = head.payloadStatus === PayloadStatus.FULL; + return ( + this.checkpointStateCache.getLatest(head.blockRoot, Infinity, payloadPresent) || + this.blockStateCache.get(head.stateRoot) + ); } pruneOnCheckpoint(finalizedEpoch: Epoch, justifiedEpoch: Epoch, headStateRoot: RootHex): void { @@ -153,15 +175,24 @@ export class QueuedStateRegenerator implements IStateRegenerator { this.blockStateCache.deleteAllBeforeEpoch(finalizedEpoch); } - processState(blockRootHex: RootHex, postState: CachedBeaconStateAllForks): void { + processBlockState(blockRootHex: RootHex, postState: CachedBeaconStateAllForks): void { this.blockStateCache.add(postState); this.checkpointStateCache.processState(blockRootHex, postState).catch((e) => { this.logger.debug("Error processing block state", {blockRootHex, slot: postState.slot}, e); }); } - addCheckpointState(cp: phase0.Checkpoint, item: CachedBeaconStateAllForks): void { - this.checkpointStateCache.add(cp, item); + /** + * Process payload state for caching after importing execution payload. + */ + processPayloadState(payloadState: CachedBeaconStateAllForks): void { + // Add payload state to block state cache (keyed by payload state root) + this.blockStateCache.add(payloadState); + } + + // TODO GLOAS: This should also be called when importing execution payload after we implement it + addCheckpointState(cp: phase0.Checkpoint, item: CachedBeaconStateAllForks, payloadPresent: boolean): void { + this.checkpointStateCache.add(cp, item, payloadPresent); } updateHeadState(newHead: ProtoBlock, maybeHeadState: CachedBeaconStateAllForks): void { @@ -197,8 +228,13 @@ export class QueuedStateRegenerator implements IStateRegenerator { } } - updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch): number | null { - return this.checkpointStateCache.updatePreComputedCheckpoint(rootHex, epoch); + updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch, payloadPresent: boolean): number | null { + return this.checkpointStateCache.updatePreComputedCheckpoint(rootHex, epoch, payloadPresent); + } + + upgradeForGloas(epoch: Epoch): void { + this.logger.verbose("Upgrading block state cache for Gloas fork", {epoch}); + this.blockStateCache.upgradeToGloas(); } /** diff --git a/packages/beacon-node/src/chain/regen/regen.ts b/packages/beacon-node/src/chain/regen/regen.ts index 08e580f1295c..8a00364a5f50 100644 --- a/packages/beacon-node/src/chain/regen/regen.ts +++ b/packages/beacon-node/src/chain/regen/regen.ts @@ -1,6 +1,6 @@ import {ChainForkConfig} from "@lodestar/config"; -import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; -import {SLOTS_PER_EPOCH} from "@lodestar/params"; +import {IForkChoice, PayloadStatus, ProtoBlock} from "@lodestar/fork-choice"; +import {ForkSeq, SLOTS_PER_EPOCH} from "@lodestar/params"; import { CachedBeaconStateAllForks, DataAvailabilityStatus, @@ -111,9 +111,20 @@ export class StateRegenerator implements IStateRegeneratorInternal { const {blockRoot} = block; const {checkpointStateCache} = this.modules; const epoch = computeEpochAtSlot(slot); + + // Convert PayloadStatus to payloadPresent boolean + if (block.payloadStatus === PayloadStatus.PENDING) { + throw new RegenError({ + code: RegenErrorCode.UNEXPECTED_PAYLOAD_STATUS, + blockRoot: fromHex(blockRoot), + payloadStatus: block.payloadStatus, + }); + } + const payloadPresent = block.payloadStatus === PayloadStatus.FULL; + const latestCheckpointStateCtx = allowDiskReload - ? await checkpointStateCache.getOrReloadLatest(blockRoot, epoch) - : checkpointStateCache.getLatest(blockRoot, epoch); + ? await checkpointStateCache.getOrReloadLatest(blockRoot, epoch, payloadPresent) + : checkpointStateCache.getLatest(blockRoot, epoch, payloadPresent); // If a checkpoint state exists with the given checkpoint root, it either is in requested epoch // or needs to have empty slots processed until the requested epoch @@ -166,9 +177,19 @@ export class StateRegenerator implements IStateRegeneratorInternal { const lastBlockToReplay = blocksToReplay.at(-1); if (!lastBlockToReplay) continue; const epoch = computeEpochAtSlot(lastBlockToReplay.slot - 1); + + // Convert PayloadStatus to payloadPresent boolean + if (b.payloadStatus === PayloadStatus.PENDING) { + throw new RegenError({ + code: RegenErrorCode.INTERNAL_ERROR, + message: `Unexpected PENDING payloadStatus for ancestor block ${b.blockRoot} at slot ${b.slot}`, + }); + } + const payloadPresent = b.payloadStatus === PayloadStatus.FULL; + state = allowDiskReload - ? await checkpointStateCache.getOrReloadLatest(b.blockRoot, epoch) - : checkpointStateCache.getLatest(b.blockRoot, epoch); + ? await checkpointStateCache.getOrReloadLatest(b.blockRoot, epoch, payloadPresent) + : checkpointStateCache.getLatest(b.blockRoot, epoch, payloadPresent); if (state) { break; } @@ -332,6 +353,11 @@ async function processSlotsByCheckpoint( * emitting "checkpoint" events after every epoch processed. * * Stops processing after no more full epochs can be processed. + * + * Output state variant: + * - Post-Gloas: If slots are processed, returns block state (payloadPresent=false). + * If no slots processed, returns preState as-is (preserves variant). + * - Pre-Gloas: Always payloadPresent=true (no block/payload distinction). */ export async function processSlotsToNearestCheckpoint( modules: { @@ -374,7 +400,11 @@ export async function processSlotsToNearestCheckpoint( // This may becomes the "official" checkpoint state if the 1st block of epoch is skipped const checkpointState = postState; const cp = getCheckpointFromState(checkpointState); - checkpointStateCache.add(cp, checkpointState); + // processSlots() only does epoch transitions, never processes payloads + // Pre-Gloas: payloadPresent is always true (execution payload embedded in block) + // Post-Gloas: result is a block state (payloadPresent=false) + const isPayloadPresent = checkpointState.config.getForkSeq(checkpointState.slot) < ForkSeq.gloas; + checkpointStateCache.add(cp, checkpointState, isPayloadPresent); // consumers should not mutate state ever emitter?.emit(ChainEvent.checkpoint, cp, checkpointState); diff --git a/packages/beacon-node/src/chain/stateCache/datastore/db.ts b/packages/beacon-node/src/chain/stateCache/datastore/db.ts index 64c893c9bdd7..7d49169960d9 100644 --- a/packages/beacon-node/src/chain/stateCache/datastore/db.ts +++ b/packages/beacon-node/src/chain/stateCache/datastore/db.ts @@ -1,6 +1,6 @@ import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {Epoch, phase0, ssz} from "@lodestar/types"; -import {MapDef} from "@lodestar/utils"; +import {MapDef, byteArrayEquals} from "@lodestar/utils"; import {IBeaconDb} from "../../../db/interface.js"; import { getLastProcessedSlotFromBeaconStateSerialized, @@ -14,8 +14,8 @@ import {CPStateDatastore, DatastoreKey} from "./types.js"; export class DbCPStateDatastore implements CPStateDatastore { constructor(private readonly db: IBeaconDb) {} - async write(cpKey: phase0.Checkpoint, stateBytes: Uint8Array): Promise { - const serializedCheckpoint = checkpointToDatastoreKey(cpKey); + async write(cpKey: phase0.Checkpoint, stateBytes: Uint8Array, payloadPresent: boolean): Promise { + const serializedCheckpoint = checkpointToDatastoreKey(cpKey, payloadPresent); await this.db.checkpointState.putBinary(serializedCheckpoint, stateBytes); return serializedCheckpoint; } @@ -40,18 +40,30 @@ export class DbCPStateDatastore implements CPStateDatastore { } } +function extractCheckpointBytes(key: DatastoreKey): Uint8Array { + const fixedSize = ssz.phase0.Checkpoint.minSize; + return key.subarray(0, fixedSize); +} + export function datastoreKeyToCheckpoint(key: DatastoreKey): phase0.Checkpoint { - return ssz.phase0.Checkpoint.deserialize(key); + return ssz.phase0.Checkpoint.deserialize(extractCheckpointBytes(key)); +} + +export function checkpointToDatastoreKey(cp: phase0.Checkpoint, payloadPresent: boolean): DatastoreKey { + const cpBytes = ssz.phase0.Checkpoint.serialize(cp); + const key = new Uint8Array(cpBytes.length + 1); + key.set(cpBytes); + key[cpBytes.length] = payloadPresent ? 1 : 0; + return key; } -export function checkpointToDatastoreKey(cp: phase0.Checkpoint): DatastoreKey { - return ssz.phase0.Checkpoint.serialize(cp); +function isPayloadCheckpointState(key: DatastoreKey): boolean { + return key.at(-1) === 1; } /** - * Get the latest safe checkpoint state the node can use to boot from - * - it should be the checkpoint state that's unique in its epoch - * - its last processed block slot should be at epoch boundary or last slot of previous epoch + * Get the latest "safe" checkpoint state the node can use to boot from + * - its last processed block slot should be at epoch boundary (CRCS) or last slot of previous epoch (PRCS) * - state slot should be at epoch boundary * - state slot should be equal to epoch * SLOTS_PER_EPOCH * @@ -70,9 +82,20 @@ export async function getLatestSafeDatastoreKey( const dataStoreKeyByEpoch: Map = new Map(); for (const [epoch, keys] of checkpointsByEpoch.entries()) { - // only consider epochs with a single checkpoint to avoid ambiguity from forks if (keys.length === 1) { + // PRCS (skipped slot) or CRCS and no payloadPresent + // Pre-gloas always fall into this case dataStoreKeyByEpoch.set(epoch, keys[0]); + } else if (keys.length === 2) { + // CRCS without payload and CRCS with payload + // ie Two keys for the same checkpoint with different payloadPresent suffix (FULL/EMPTY) + // TODO GLOAS: Here we pick FULL key, there is a chance that payload is orphaned hence we not be able to sync + const cp0 = extractCheckpointBytes(keys[0]); + const cp1 = extractCheckpointBytes(keys[1]); + if (byteArrayEquals(cp0, cp1)) { + const fullKey = isPayloadCheckpointState(keys[0]) ? keys[0] : keys[1]; + dataStoreKeyByEpoch.set(epoch, fullKey); + } } } diff --git a/packages/beacon-node/src/chain/stateCache/datastore/file.ts b/packages/beacon-node/src/chain/stateCache/datastore/file.ts index 15ccbfb81f43..6c0391c3c29a 100644 --- a/packages/beacon-node/src/chain/stateCache/datastore/file.ts +++ b/packages/beacon-node/src/chain/stateCache/datastore/file.ts @@ -1,12 +1,13 @@ import path from "node:path"; -import {phase0, ssz} from "@lodestar/types"; +import {phase0} from "@lodestar/types"; import {fromHex, toHex} from "@lodestar/utils"; import {ensureDir, readFile, readFileNames, removeFile, writeIfNotExist} from "../../../util/file.js"; -import {getLatestSafeDatastoreKey} from "./db.js"; +import {checkpointToDatastoreKey, getLatestSafeDatastoreKey} from "./db.js"; import {CPStateDatastore, DatastoreKey} from "./types.js"; const CHECKPOINT_STATES_FOLDER = "checkpoint_states"; -const CHECKPOINT_FILE_NAME_LENGTH = 82; +/** 41 bytes (40 checkpoint + 1 payloadPresent) = 82 hex chars + "0x" prefix = 84 */ +const CHECKPOINT_FILE_NAME_LENGTH = 84; /** * Implementation of CPStateDatastore using file system, this is beneficial for debugging. @@ -28,8 +29,8 @@ export class FileCPStateDatastore implements CPStateDatastore { } } - async write(cpKey: phase0.Checkpoint, stateBytes: Uint8Array): Promise { - const serializedCheckpoint = ssz.phase0.Checkpoint.serialize(cpKey); + async write(cpKey: phase0.Checkpoint, stateBytes: Uint8Array, payloadPresent: boolean): Promise { + const serializedCheckpoint = checkpointToDatastoreKey(cpKey, payloadPresent); const filePath = path.join(this.folderPath, toHex(serializedCheckpoint)); await writeIfNotExist(filePath, stateBytes); return serializedCheckpoint; diff --git a/packages/beacon-node/src/chain/stateCache/datastore/types.ts b/packages/beacon-node/src/chain/stateCache/datastore/types.ts index c63c54cca1d1..de25274539d9 100644 --- a/packages/beacon-node/src/chain/stateCache/datastore/types.ts +++ b/packages/beacon-node/src/chain/stateCache/datastore/types.ts @@ -1,11 +1,12 @@ import {phase0} from "@lodestar/types"; -// With db implementation, persistedKey is serialized data of a checkpoint +// With db implementation, persistedKey is serialized data of a checkpoint + 1 +// ie a fixed size of `ssz.phase0.Checkpoint.minSize + 1` export type DatastoreKey = Uint8Array; // Make this generic to support testing export interface CPStateDatastore { - write: (cpKey: phase0.Checkpoint, stateBytes: Uint8Array) => Promise; + write: (cpKey: phase0.Checkpoint, stateBytes: Uint8Array, payloadPresent: boolean) => Promise; remove: (key: DatastoreKey) => Promise; read: (key: DatastoreKey) => Promise; readLatestSafe: () => Promise; diff --git a/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts b/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts index 37af369e28fd..be3d9b2d3510 100644 --- a/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts +++ b/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts @@ -20,6 +20,11 @@ export type FIFOBlockStateCacheOpts = { * clock slot */ export const DEFAULT_MAX_BLOCK_STATES = 64; +/** + * For Gloas (ePBS), each block can have two states: block state and payload state. + * Double the cache size to maintain the same effective block depth. + */ +export const DEFAULT_MAX_BLOCK_STATES_GLOAS = 128; /** * New implementation of BlockStateCache that keeps the most recent n states consistently @@ -41,10 +46,7 @@ export const DEFAULT_MAX_BLOCK_STATES = 64; * The maintained key order would be: 11 -> 13 -> 12 -> 10, and state 10 will be pruned first. */ export class FIFOBlockStateCache implements BlockStateCache { - /** - * Max number of states allowed in the cache - */ - readonly maxStates: number; + private maxStates: number; private readonly cache: MapTracker; /** @@ -170,6 +172,10 @@ export class FIFOBlockStateCache implements BlockStateCache { } } + upgradeToGloas(): void { + this.maxStates = DEFAULT_MAX_BLOCK_STATES_GLOAS; + } + /** * No need for this implementation * This is only to conform to the old api diff --git a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts index 862b063711f3..4562bac1a52b 100644 --- a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts +++ b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts @@ -1,5 +1,6 @@ import {routes} from "@lodestar/api"; import {BeaconConfig} from "@lodestar/config"; +import {CheckpointWithPayloadStatus} from "@lodestar/fork-choice"; import { CachedBeaconStateAllForks, computeStartSlotAtEpoch, @@ -14,7 +15,7 @@ import {IClock} from "../../util/clock.js"; import {serializeState} from "../serializeState.js"; import {CPStateDatastore, DatastoreKey} from "./datastore/index.js"; import {MapTracker} from "./mapMetrics.js"; -import {BlockStateCache, CacheItemType, CheckpointHex, CheckpointStateCache} from "./types.js"; +import {BlockStateCache, CacheItemType, CheckpointHexPayload, CheckpointStateCache} from "./types.js"; export type PersistentCheckpointStateCacheOpts = { /** Keep max n state epochs in memory, persist the rest to disk */ @@ -54,6 +55,22 @@ type CacheItem = InMemoryCacheItem | PersistedCacheItem; type LoadedStateBytesData = {persistedKey: DatastoreKey; stateBytes: Uint8Array}; +/** Bitmask for tracking which payload variants exist per root in the epochIndex */ +enum PayloadAvailability { + NOT_PRESENT = 1, + PRESENT = 2, +} + +const PAYLOAD_AVAILABILITY_ALL = [PayloadAvailability.NOT_PRESENT, PayloadAvailability.PRESENT] as const; + +function toPayloadAvailability(payloadPresent: boolean): PayloadAvailability { + return payloadPresent ? PayloadAvailability.PRESENT : PayloadAvailability.NOT_PRESENT; +} + +function fromPayloadAvailability(flag: PayloadAvailability): boolean { + return flag === PayloadAvailability.PRESENT; +} + /** * Before n-historical states, lodestar keeps all checkpoint states since finalized * Since Sep 2024, lodestar stores 3 most recent checkpoint states in memory and the rest on disk. The finalized state @@ -106,8 +123,8 @@ const PROCESS_CHECKPOINT_STATES_BPS = 6667; */ export class PersistentCheckpointStateCache implements CheckpointStateCache { private readonly cache: MapTracker; - /** Epoch -> Set */ - private readonly epochIndex = new MapDef>(() => new Set()); + /** Epoch -> Map */ + private readonly epochIndex = new MapDef>(() => new Map()); private readonly config: BeaconConfig; private readonly metrics: Metrics | null | undefined; private readonly logger: Logger; @@ -203,13 +220,18 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { * - Get block for processing * - Regen head state */ - async getOrReload(cp: CheckpointHex): Promise { + async getOrReload(cp: CheckpointHexPayload): Promise { const stateOrStateBytesData = await this.getStateOrLoadDb(cp); if (stateOrStateBytesData === null || isCachedBeaconState(stateOrStateBytesData)) { return stateOrStateBytesData ?? null; } const {persistedKey, stateBytes} = stateOrStateBytesData; - const logMeta = {persistedKey: toHex(persistedKey)}; + const logMeta = { + epoch: cp.epoch, + rootHex: cp.rootHex, + payloadPresent: cp.payloadPresent, + persistedKey: toHex(persistedKey), + }; this.logger.debug("Reload: read state successful", logMeta); this.metrics?.cpStateCache.stateReloadSecFromSlot.observe( this.clock?.secFromSlot(this.clock?.currentSlot ?? 0) ?? 0 @@ -250,7 +272,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { // only remove persisted state once we reload successfully const cpKey = toCacheKey(cp); this.cache.set(cpKey, {type: CacheItemType.inMemory, state: newCachedState, persistedKey}); - this.epochIndex.getOrDefault(cp.epoch).add(cp.rootHex); + this.addToEpochIndex(cp.epoch, cp.rootHex, cp.payloadPresent); // don't prune from memory here, call it at the last 1/3 of slot 0 of an epoch return newCachedState; } catch (e) { @@ -262,7 +284,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { /** * Return either state or state bytes loaded from db. */ - async getStateOrBytes(cp: CheckpointHex): Promise { + async getStateOrBytes(cp: CheckpointHexPayload): Promise { const stateOrLoadedState = await this.getStateOrLoadDb(cp); if (stateOrLoadedState === null || isCachedBeaconState(stateOrLoadedState)) { return stateOrLoadedState; @@ -273,7 +295,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { /** * Return either state or state bytes with persisted key loaded from db. */ - async getStateOrLoadDb(cp: CheckpointHex): Promise { + async getStateOrLoadDb(cp: CheckpointHexPayload): Promise { const cpKey = toCacheKey(cp); const inMemoryState = this.get(cpKey); if (inMemoryState) { @@ -304,7 +326,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { /** * Similar to get() api without reloading from disk */ - get(cpOrKey: CheckpointHex | string): CachedBeaconStateAllForks | null { + get(cpOrKey: CheckpointHexPayload | CacheKey): CachedBeaconStateAllForks | null { this.metrics?.cpStateCache.lookups.inc(); const cpKey = typeof cpOrKey === "string" ? cpOrKey : toCacheKey(cpOrKey); const cacheItem = this.cache.get(cpKey); @@ -330,9 +352,11 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { /** * Add a state of a checkpoint to this cache, prune from memory if necessary. + * @param payloadPresent - For Gloas: true if this is payload state, false if block state. + * Always true for pre-Gloas. */ - add(cp: phase0.Checkpoint, state: CachedBeaconStateAllForks): void { - const cpHex = toCheckpointHex(cp); + add(cp: phase0.Checkpoint, state: CachedBeaconStateAllForks, payloadPresent: boolean): void { + const cpHex = toCheckpointHexPayload(cp, payloadPresent); const key = toCacheKey(cpHex); const cacheItem = this.cache.get(key); this.metrics?.cpStateCache.adds.inc(); @@ -343,27 +367,32 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { this.logger.verbose("Added checkpoint state to memory but a persisted key existed", { epoch: cp.epoch, rootHex: cpHex.rootHex, + payloadPresent, persistedKey: toHex(persistedKey), }); } else { this.cache.set(key, {type: CacheItemType.inMemory, state}); - this.logger.verbose("Added checkpoint state to memory", {epoch: cp.epoch, rootHex: cpHex.rootHex}); + this.logger.verbose("Added checkpoint state to memory", { + epoch: cp.epoch, + rootHex: cpHex.rootHex, + payloadPresent, + }); } - this.epochIndex.getOrDefault(cp.epoch).add(cpHex.rootHex); + this.addToEpochIndex(cp.epoch, cpHex.rootHex, cpHex.payloadPresent); this.prunePersistedStates(); } /** * Searches in-memory state for the latest cached state with a `root` without reload, starting with `epoch` and descending */ - getLatest(rootHex: RootHex, maxEpoch: Epoch): CachedBeaconStateAllForks | null { + getLatest(rootHex: RootHex, maxEpoch: Epoch, payloadPresent: boolean): CachedBeaconStateAllForks | null { // sort epochs in descending order, only consider epochs lte `epoch` const epochs = Array.from(this.epochIndex.keys()) .sort((a, b) => b - a) .filter((e) => e <= maxEpoch); for (const epoch of epochs) { - if (this.epochIndex.get(epoch)?.has(rootHex)) { - const inMemoryClonedState = this.get({rootHex, epoch}); + if (this.hasPayloadVariant(epoch, rootHex, payloadPresent)) { + const inMemoryClonedState = this.get({rootHex, epoch, payloadPresent}); if (inMemoryClonedState) { return inMemoryClonedState; } @@ -379,20 +408,24 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { * - Get block for processing * - Regen head state */ - async getOrReloadLatest(rootHex: RootHex, maxEpoch: Epoch): Promise { + async getOrReloadLatest( + rootHex: RootHex, + maxEpoch: Epoch, + payloadPresent: boolean + ): Promise { // sort epochs in descending order, only consider epochs lte `epoch` const epochs = Array.from(this.epochIndex.keys()) .sort((a, b) => b - a) .filter((e) => e <= maxEpoch); for (const epoch of epochs) { - if (this.epochIndex.get(epoch)?.has(rootHex)) { + if (this.hasPayloadVariant(epoch, rootHex, payloadPresent)) { try { - const state = await this.getOrReload({rootHex, epoch}); + const state = await this.getOrReload({rootHex, epoch, payloadPresent}); if (state) { return state; } } catch (e) { - this.logger.debug("Error get or reload state", {epoch, rootHex}, e as Error); + this.logger.debug("Error get or reload state", {epoch, rootHex, payloadPresent}, e as Error); } } } @@ -400,12 +433,14 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { } /** - * Update the precomputed checkpoint and return the number of his for the + * Update the precomputed checkpoint and return the number of hits for the * previous one (if any). + * @param payloadPresent - For Gloas: true if head block has FULL payload, false if EMPTY. + * Always true for pre-Gloas. */ - updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch): number | null { + updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch, payloadPresent: boolean): number | null { const previousHits = this.preComputedCheckpointHits; - this.preComputedCheckpoint = toCacheKey({rootHex, epoch}); + this.preComputedCheckpoint = toCacheKey({rootHex, epoch, payloadPresent}); this.preComputedCheckpointHits = 0; return previousHits; } @@ -479,6 +514,9 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { * - 2 then we'll persist {root: b2, epoch n-2} checkpoint state to disk, there are also 2 checkpoint states in memory at epoch n, same to the above (maxEpochsInMemory=1) * * As of Mar 2024, it takes <=350ms to persist a holesky state on fast server + * + * For Gloas: Processes both block state and payload state variants together. The decision of which roots to persist/prune + * is based on root canonicality (from state's view), not payload presence. Both variants are managed as a unit. */ async processState(blockRootHex: RootHex, state: CachedBeaconStateAllForks): Promise { let persistCount = 0; @@ -549,7 +587,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { * * Use seed state from the block cache if cannot find any seed states within this cache. */ - findSeedStateToReload(reloadedCp: CheckpointHex): CachedBeaconStateAllForks { + findSeedStateToReload(reloadedCp: CheckpointHexPayload): CachedBeaconStateAllForks { const maxEpoch = Math.max(...Array.from(this.epochIndex.keys())); const reloadedCpSlot = computeStartSlotAtEpoch(reloadedCp.epoch); let firstState: CachedBeaconStateAllForks | null = null; @@ -562,32 +600,35 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { return firstState; } - for (const rootHex of this.epochIndex.get(epoch) || []) { - const cpKey = toCacheKey({rootHex, epoch}); - const cacheItem = this.cache.get(cpKey); - if (cacheItem === undefined) { - // should not happen - continue; - } - if (isInMemoryCacheItem(cacheItem)) { - const {state} = cacheItem; - if (firstState === null) { - firstState = state; + for (const [rootHex, bitmask] of this.epochIndex.get(epoch) || []) { + for (const flag of PAYLOAD_AVAILABILITY_ALL) { + if (!(bitmask & flag)) continue; + const payloadPresent = fromPayloadAvailability(flag); + const cpKey = toCacheKey({rootHex, epoch, payloadPresent}); + const cacheItem = this.cache.get(cpKey); + if (cacheItem === undefined) { + continue; } - const cpLog = {cpEpoch: epoch, cpRoot: rootHex}; - - try { - // amongst states of the same epoch, choose the one with the same view of reloadedCp - if ( - reloadedCpSlot < state.slot && - toRootHex(getBlockRootAtSlot(state, reloadedCpSlot)) === reloadedCp.rootHex - ) { - this.logger.verbose("Reload: use checkpoint state as seed state", {...cpLog, ...logCtx}); - return state; + if (isInMemoryCacheItem(cacheItem)) { + const {state} = cacheItem; + if (firstState === null) { + firstState = state; + } + const cpLog = {cpEpoch: epoch, cpRoot: rootHex, payloadPresent}; + + try { + // amongst states of the same epoch, choose the one with the same view of reloadedCp + if ( + reloadedCpSlot < state.slot && + toRootHex(getBlockRootAtSlot(state, reloadedCpSlot)) === reloadedCp.rootHex + ) { + this.logger.verbose("Reload: use checkpoint state as seed state", {...cpLog, ...logCtx}); + return state; + } + } catch (e) { + // getBlockRootAtSlot may throw error + this.logger.debug("Error finding checkpoint state to reload", {...cpLog, ...logCtx}, e as Error); } - } catch (e) { - // getBlockRootAtSlot may throw error - this.logger.debug("Error finding checkpoint state to reload", {...cpLog, ...logCtx}, e as Error); } } } @@ -604,6 +645,31 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { this.epochIndex.clear(); } + private addToEpochIndex(epoch: Epoch, rootHex: RootHex, payloadPresent: boolean): void { + const rootMap = this.epochIndex.getOrDefault(epoch); + rootMap.set(rootHex, (rootMap.get(rootHex) ?? 0) | toPayloadAvailability(payloadPresent)); + } + + private removeFromEpochIndex(epoch: Epoch, rootHex: RootHex, payloadPresent: boolean): void { + const rootMap = this.epochIndex.get(epoch); + if (rootMap === undefined) return; + const existing = rootMap.get(rootHex); + if (existing === undefined) return; + const updated = existing & ~toPayloadAvailability(payloadPresent); + if (updated === 0) { + rootMap.delete(rootHex); + if (rootMap.size === 0) { + this.epochIndex.delete(epoch); + } + } else { + rootMap.set(rootHex, updated); + } + } + + private hasPayloadVariant(epoch: Epoch, rootHex: RootHex, payloadPresent: boolean): boolean { + return Boolean((this.epochIndex.get(epoch)?.get(rootHex) ?? 0) & toPayloadAvailability(payloadPresent)); + } + /** ONLY FOR DEBUGGING PURPOSES. For lodestar debug API */ dumpSummary(): routes.lodestar.StateCacheItem[] { return Array.from(this.cache.keys()).map((key) => { @@ -682,7 +748,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { const prevEpochRoot = toRootHex(getBlockRootAtSlot(state, epochBoundarySlot - 1)); // for each epoch, usually there are 2 rootHexes respective to the 2 checkpoint states: Previous Root Checkpoint State and Current Root Checkpoint State - const cpRootHexes = this.epochIndex.get(epoch) ?? []; + const cpRootHexMap = this.epochIndex.get(epoch) ?? new Map(); const persistedRootHexes = new Set(); // 1) if there is no CRCS, persist PRCS (block 0 of epoch is skipped). In this case prevEpochRoot === epochBoundaryHex @@ -691,76 +757,81 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { persistedRootHexes.add(epochBoundaryHex); // 3) persist any states with unknown roots to this state - for (const rootHex of cpRootHexes) { + for (const rootHex of cpRootHexMap.keys()) { if (rootHex !== epochBoundaryHex && rootHex !== prevEpochRoot) { persistedRootHexes.add(rootHex); } } - for (const rootHex of cpRootHexes) { - const cpKey = toCacheKey({epoch: epoch, rootHex}); - const cacheItem = this.cache.get(cpKey); - - if (cacheItem !== undefined && isInMemoryCacheItem(cacheItem)) { - let {persistedKey} = cacheItem; - const {state} = cacheItem; - const logMeta = { - stateSlot: state.slot, - rootHex, - epochBoundaryHex, - persistedKey: persistedKey ? toHex(persistedKey) : "", - }; - - if (persistedRootHexes.has(rootHex)) { - if (persistedKey) { - // we don't care if the checkpoint state is already persisted - this.logger.verbose("Pruned checkpoint state from memory but no need to persist", logMeta); - } else { - // persist and do not update epochIndex - this.metrics?.cpStateCache.statePersistSecFromSlot.observe( - this.clock?.secFromSlot(this.clock?.currentSlot ?? 0) ?? 0 - ); - const cpPersist = {epoch: epoch, root: fromHex(rootHex)}; - // It's not sustainable to allocate ~240MB for each state every epoch, so we use buffer pool to reuse the memory. - // As monitored on holesky as of Jan 2024: - // - This does not increase heap allocation while gc time is the same - // - It helps stabilize persist time and save ~300ms in average (1.5s vs 1.2s) - // - It also helps the state reload to save ~500ms in average (4.3s vs 3.8s) - // - Also `serializeState.test.ts` perf test shows a lot of differences allocating ~240MB once vs per state serialization - const timer = this.metrics?.stateSerializeDuration.startTimer({ - source: AllocSource.PERSISTENT_CHECKPOINTS_CACHE_STATE, - }); - persistedKey = await serializeState( - state, - AllocSource.PERSISTENT_CHECKPOINTS_CACHE_STATE, - (stateBytes) => { - timer?.(); - return this.datastore.write(cpPersist, stateBytes); - }, - this.bufferPool - ); + for (const [rootHex, bitmask] of cpRootHexMap) { + for (const flag of PAYLOAD_AVAILABILITY_ALL) { + if (!(bitmask & flag)) continue; + const payloadPresent = fromPayloadAvailability(flag); + const cpKey = toCacheKey({epoch: epoch, rootHex, payloadPresent}); + const cacheItem = this.cache.get(cpKey); - persistCount++; - this.logger.verbose("Pruned checkpoint state from memory and persisted to disk", { - ...logMeta, - persistedKey: toHex(persistedKey), - }); - } - // overwrite cpKey, this means the state is deleted from memory - this.cache.set(cpKey, {type: CacheItemType.persisted, value: persistedKey}); - } else { - if (persistedKey) { - // persisted file will be eventually deleted by the archive task - // this also means the state is deleted from memory + if (cacheItem !== undefined && isInMemoryCacheItem(cacheItem)) { + let {persistedKey} = cacheItem; + const {state} = cacheItem; + const logMeta = { + stateSlot: state.slot, + rootHex, + payloadPresent, + epochBoundaryHex, + persistedKey: persistedKey ? toHex(persistedKey) : "", + }; + + if (persistedRootHexes.has(rootHex)) { + if (persistedKey) { + // we don't care if the checkpoint state is already persisted + this.logger.verbose("Pruned checkpoint state from memory but no need to persist", logMeta); + } else { + // persist and do not update epochIndex + this.metrics?.cpStateCache.statePersistSecFromSlot.observe( + this.clock?.secFromSlot(this.clock?.currentSlot ?? 0) ?? 0 + ); + const cpPersist = {epoch: epoch, root: fromHex(rootHex)}; + // It's not sustainable to allocate ~240MB for each state every epoch, so we use buffer pool to reuse the memory. + // As monitored on holesky as of Jan 2024: + // - This does not increase heap allocation while gc time is the same + // - It helps stabilize persist time and save ~300ms in average (1.5s vs 1.2s) + // - It also helps the state reload to save ~500ms in average (4.3s vs 3.8s) + // - Also `serializeState.test.ts` perf test shows a lot of differences allocating ~240MB once vs per state serialization + const timer = this.metrics?.stateSerializeDuration.startTimer({ + source: AllocSource.PERSISTENT_CHECKPOINTS_CACHE_STATE, + }); + persistedKey = await serializeState( + state, + AllocSource.PERSISTENT_CHECKPOINTS_CACHE_STATE, + (stateBytes) => { + timer?.(); + return this.datastore.write(cpPersist, stateBytes, payloadPresent); + }, + this.bufferPool + ); + + persistCount++; + this.logger.verbose("Pruned checkpoint state from memory and persisted to disk", { + ...logMeta, + persistedKey: toHex(persistedKey), + }); + } + // overwrite cpKey, this means the state is deleted from memory this.cache.set(cpKey, {type: CacheItemType.persisted, value: persistedKey}); - // do not update epochIndex } else { - // delete the state from memory - this.cache.delete(cpKey); - this.epochIndex.get(epoch)?.delete(rootHex); + if (persistedKey) { + // persisted file will be eventually deleted by the archive task + // this also means the state is deleted from memory + this.cache.set(cpKey, {type: CacheItemType.persisted, value: persistedKey}); + // do not update epochIndex + } else { + // delete the state from memory + this.cache.delete(cpKey); + this.removeFromEpochIndex(epoch, rootHex, payloadPresent); + } + this.metrics?.cpStateCache.statePruneFromMemoryCount.inc(); + this.logger.verbose("Pruned checkpoint state from memory", logMeta); } - this.metrics?.cpStateCache.statePruneFromMemoryCount.inc(); - this.logger.verbose("Pruned checkpoint state from memory", logMeta); } } } @@ -773,26 +844,40 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { */ private async deleteAllEpochItems(epoch: Epoch): Promise { let persistCount = 0; - const rootHexes = this.epochIndex.get(epoch) || []; - for (const rootHex of rootHexes) { - const key = toCacheKey({rootHex, epoch}); - const cacheItem = this.cache.get(key); - - if (cacheItem) { - const persistedKey = isPersistedCacheItem(cacheItem) ? cacheItem.value : cacheItem.persistedKey; - if (persistedKey) { - await this.datastore.remove(persistedKey); - persistCount++; - this.metrics?.cpStateCache.persistedStateRemoveCount.inc(); + const rootHexMap = this.epochIndex.get(epoch) || new Map(); + for (const [rootHex, bitmask] of rootHexMap) { + for (const flag of PAYLOAD_AVAILABILITY_ALL) { + if (!(bitmask & flag)) continue; + const payloadPresent = fromPayloadAvailability(flag); + const key = toCacheKey({rootHex, epoch, payloadPresent}); + const cacheItem = this.cache.get(key); + + if (cacheItem) { + const persistedKey = isPersistedCacheItem(cacheItem) ? cacheItem.value : cacheItem.persistedKey; + if (persistedKey) { + await this.datastore.remove(persistedKey); + persistCount++; + this.metrics?.cpStateCache.persistedStateRemoveCount.inc(); + } } + this.cache.delete(key); + this.logger.verbose("Pruned checkpoint state", { + epoch, + rootHex, + payloadPresent, + type: cacheItem ? (isPersistedCacheItem(cacheItem) ? "persisted" : "in-memory") : "missing", + }); } - this.cache.delete(key); } this.epochIndex.delete(epoch); - this.logger.verbose("Pruned checkpoint states for epoch", { + this.logger.verbose("Pruned all checkpoint states for epoch", { epoch, persistCount, - rootHexes: Array.from(rootHexes).join(","), + items: Array.from(rootHexMap.entries()) + .flatMap(([rootHex, bitmask]) => + PAYLOAD_AVAILABILITY_ALL.filter((f) => bitmask & f).map((f) => `${rootHex}:${fromPayloadAvailability(f)}`) + ) + .join(","), }); } @@ -844,29 +929,57 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { } } -export function toCheckpointHex(checkpoint: phase0.Checkpoint): CheckpointHex { +export function toCheckpointHexPayload(checkpoint: phase0.Checkpoint, payloadPresent: boolean): CheckpointHexPayload { return { epoch: checkpoint.epoch, rootHex: toRootHex(checkpoint.root), + payloadPresent, }; } -export function toCheckpointKey(cp: CheckpointHex): string { - return `${cp.rootHex}:${cp.epoch}`; -} +/** + * Convert fork-choice CheckpointWithPayloadStatus to beacon-node CheckpointHexPayload. + * Maps PayloadStatus enum to boolean payloadPresent. + * @throws Error if checkpoint has PENDING payload status (ambiguous which variant to use) + */ +export function fcCheckpointToHexPayload(checkpoint: CheckpointWithPayloadStatus): CheckpointHexPayload { + const PayloadStatus = {PENDING: 0, EMPTY: 1, FULL: 2} as const; -function toCacheKey(cp: CheckpointHex | phase0.Checkpoint): CacheKey { - if (isCheckpointHex(cp)) { - return `${cp.rootHex}_${cp.epoch}`; + if (checkpoint.payloadStatus === PayloadStatus.PENDING) { + throw Error( + `Cannot convert checkpoint with PENDING payload status at epoch ${checkpoint.epoch} root ${checkpoint.rootHex}` + ); } - return `${toRootHex(cp.root)}_${cp.epoch}`; + + return { + epoch: checkpoint.epoch, + rootHex: checkpoint.rootHex, + payloadPresent: checkpoint.payloadStatus === PayloadStatus.FULL, + }; +} + +export function toCheckpointKey(cp: CheckpointHexPayload): string { + return `${cp.rootHex}:${cp.epoch}:${cp.payloadPresent}`; } -function fromCacheKey(key: CacheKey): CheckpointHex { - const [rootHex, epoch] = key.split("_"); +/** + * Convert checkpoint to cache key string. + * Format: `{rootHex}_{epoch}_{payloadPresent}` + */ +function toCacheKey(cp: CheckpointHexPayload): CacheKey { + return `${cp.rootHex}_${cp.epoch}_${cp.payloadPresent}`; +} + +function fromCacheKey(key: CacheKey): CheckpointHexPayload { + const parts = key.split("_"); + const rootHex = parts[0]; + const epoch = Number(parts[1]); + // For backward compatibility with old format (rootHex_epoch), default to true + const payloadPresent = parts.length > 2 ? parts[2] === "true" : true; return { rootHex, - epoch: Number(epoch), + epoch, + payloadPresent, }; } @@ -883,7 +996,3 @@ function isInMemoryCacheItem(cacheItem: CacheItem): cacheItem is InMemoryCacheIt function isPersistedCacheItem(cacheItem: CacheItem): cacheItem is PersistedCacheItem { return cacheItem.type === CacheItemType.persisted; } - -function isCheckpointHex(cp: CheckpointHex | phase0.Checkpoint): cp is CheckpointHex { - return (cp as CheckpointHex).rootHex !== undefined; -} diff --git a/packages/beacon-node/src/chain/stateCache/types.ts b/packages/beacon-node/src/chain/stateCache/types.ts index b16590967c9d..7fc15e31e6df 100644 --- a/packages/beacon-node/src/chain/stateCache/types.ts +++ b/packages/beacon-node/src/chain/stateCache/types.ts @@ -2,7 +2,11 @@ import {routes} from "@lodestar/api"; import {CachedBeaconStateAllForks} from "@lodestar/state-transition"; import {Epoch, RootHex, phase0} from "@lodestar/types"; -export type CheckpointHex = {epoch: Epoch; rootHex: RootHex}; +/** + * Checkpoint hex representation for state cache keys. + * Extends CheckpointWithHex (from fork-choice) with payloadPresent. + */ +export type CheckpointHexPayload = {epoch: Epoch; rootHex: RootHex; payloadPresent: boolean}; /** * Lodestar currently keeps two state caches around. @@ -31,6 +35,8 @@ export interface BlockStateCache { size: number; prune(headStateRootHex: RootHex): void; deleteAllBeforeEpoch(finalizedEpoch: Epoch): void; + /** Upgrade cache capacity for Gloas fork (2x states for block + payload states) */ + upgradeToGloas(): void; dumpSummary(): routes.lodestar.StateCacheItem[]; /** Expose beacon states stored in cache. Use with caution */ getStates(): IterableIterator; @@ -59,13 +65,17 @@ export interface BlockStateCache { */ export interface CheckpointStateCache { init?: () => Promise; - getOrReload(cp: CheckpointHex): Promise; - getStateOrBytes(cp: CheckpointHex): Promise; - get(cpOrKey: CheckpointHex | string): CachedBeaconStateAllForks | null; - add(cp: phase0.Checkpoint, state: CachedBeaconStateAllForks): void; - getLatest(rootHex: RootHex, maxEpoch: Epoch): CachedBeaconStateAllForks | null; - getOrReloadLatest(rootHex: RootHex, maxEpoch: Epoch): Promise; - updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch): number | null; + getOrReload(cp: CheckpointHexPayload): Promise; + getStateOrBytes(cp: CheckpointHexPayload): Promise; + get(cpOrKey: CheckpointHexPayload | string): CachedBeaconStateAllForks | null; + add(cp: phase0.Checkpoint, state: CachedBeaconStateAllForks, payloadPresent: boolean): void; + getLatest(rootHex: RootHex, maxEpoch: Epoch, payloadPresent: boolean): CachedBeaconStateAllForks | null; + getOrReloadLatest( + rootHex: RootHex, + maxEpoch: Epoch, + payloadPresent: boolean + ): Promise; + updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch, payloadPresent: boolean): number | null; prune(finalizedEpoch: Epoch, justifiedEpoch: Epoch): void; pruneFinalized(finalizedEpoch: Epoch): void; processState(blockRootHex: RootHex, state: CachedBeaconStateAllForks): Promise; diff --git a/packages/beacon-node/test/unit-minimal/chain/stateCache/persistentCheckpointsCache.test.ts b/packages/beacon-node/test/unit-minimal/chain/stateCache/persistentCheckpointsCache.test.ts index 9dd74e170f29..4b2ada3b872a 100644 --- a/packages/beacon-node/test/unit-minimal/chain/stateCache/persistentCheckpointsCache.test.ts +++ b/packages/beacon-node/test/unit-minimal/chain/stateCache/persistentCheckpointsCache.test.ts @@ -10,9 +10,9 @@ import {FIFOBlockStateCache} from "../../../../src/chain/index.js"; import {checkpointToDatastoreKey} from "../../../../src/chain/stateCache/datastore/index.js"; import { PersistentCheckpointStateCache, - toCheckpointHex, + toCheckpointHexPayload, } from "../../../../src/chain/stateCache/persistentCheckpointsCache.js"; -import {CheckpointHex} from "../../../../src/chain/stateCache/types.js"; +import {CheckpointHexPayload} from "../../../../src/chain/stateCache/types.js"; import {getTestDatastore} from "../../../utils/chain/stateCache/datastore.js"; import {generateCachedState} from "../../../utils/state.js"; @@ -23,7 +23,10 @@ describe("PersistentCheckpointStateCache", () => { let root0a: Buffer, root0b: Buffer, root1: Buffer, root2: Buffer; let cp0a: phase0.Checkpoint, cp0b: phase0.Checkpoint, cp1: phase0.Checkpoint, cp2: phase0.Checkpoint; - let cp0aHex: CheckpointHex, cp0bHex: CheckpointHex, cp1Hex: CheckpointHex, cp2Hex: CheckpointHex; + let cp0aHex: CheckpointHexPayload, + cp0bHex: CheckpointHexPayload, + cp1Hex: CheckpointHexPayload, + cp2Hex: CheckpointHexPayload; let persistent0bKey: RootHex; const startSlotEpoch20 = computeStartSlotAtEpoch(20); const startSlotEpoch21 = computeStartSlotAtEpoch(21); @@ -53,8 +56,8 @@ describe("PersistentCheckpointStateCache", () => { cp0b = {epoch: 20, root: root0b}; cp1 = {epoch: 21, root: root1}; cp2 = {epoch: 22, root: root2}; - [cp0aHex, cp0bHex, cp1Hex, cp2Hex] = [cp0a, cp0b, cp1, cp2].map((cp) => toCheckpointHex(cp)); - persistent0bKey = toHexString(checkpointToDatastoreKey(cp0b)); + [cp0aHex, cp0bHex, cp1Hex, cp2Hex] = [cp0a, cp0b, cp1, cp2].map((cp) => toCheckpointHexPayload(cp, true)); + persistent0bKey = toHexString(checkpointToDatastoreKey(cp0b, true)); const allStates = [cp0a, cp0b, cp1, cp2] .map((cp) => generateCachedState({slot: cp.epoch * SLOTS_PER_EPOCH})) .map((state, i) => { @@ -104,28 +107,30 @@ describe("PersistentCheckpointStateCache", () => { }, {maxCPStateEpochsInMemory: 2} ); - cache.add(cp0a, states["cp0a"]); - cache.add(cp0b, states["cp0b"]); - cache.add(cp1, states["cp1"]); + cache.add(cp0a, states["cp0a"], true); + cache.add(cp0b, states["cp0b"], true); + cache.add(cp1, states["cp1"], true); }); it("getLatest", () => { // cp0 - expect(cache.getLatest(cp0aHex.rootHex, cp0a.epoch)?.hashTreeRoot()).toEqual(states["cp0a"].hashTreeRoot()); - expect(cache.getLatest(cp0aHex.rootHex, cp0a.epoch + 1)?.hashTreeRoot()).toEqual(states["cp0a"].hashTreeRoot()); - expect(cache.getLatest(cp0aHex.rootHex, cp0a.epoch - 1)?.hashTreeRoot()).toBeUndefined(); + expect(cache.getLatest(cp0aHex.rootHex, cp0a.epoch, true)?.hashTreeRoot()).toEqual(states["cp0a"].hashTreeRoot()); + expect(cache.getLatest(cp0aHex.rootHex, cp0a.epoch + 1, true)?.hashTreeRoot()).toEqual( + states["cp0a"].hashTreeRoot() + ); + expect(cache.getLatest(cp0aHex.rootHex, cp0a.epoch - 1, true)?.hashTreeRoot()).toBeUndefined(); // cp1 - expect(cache.getLatest(cp1Hex.rootHex, cp1.epoch)?.hashTreeRoot()).toEqual(states["cp1"].hashTreeRoot()); - expect(cache.getLatest(cp1Hex.rootHex, cp1.epoch + 1)?.hashTreeRoot()).toEqual(states["cp1"].hashTreeRoot()); - expect(cache.getLatest(cp1Hex.rootHex, cp1.epoch - 1)?.hashTreeRoot()).toBeUndefined(); + expect(cache.getLatest(cp1Hex.rootHex, cp1.epoch, true)?.hashTreeRoot()).toEqual(states["cp1"].hashTreeRoot()); + expect(cache.getLatest(cp1Hex.rootHex, cp1.epoch + 1, true)?.hashTreeRoot()).toEqual(states["cp1"].hashTreeRoot()); + expect(cache.getLatest(cp1Hex.rootHex, cp1.epoch - 1, true)?.hashTreeRoot()).toBeUndefined(); // cp2 - expect(cache.getLatest(cp2Hex.rootHex, cp2.epoch)?.hashTreeRoot()).toBeUndefined(); + expect(cache.getLatest(cp2Hex.rootHex, cp2.epoch, true)?.hashTreeRoot()).toBeUndefined(); }); it("getOrReloadLatest", async () => { - cache.add(cp2, states["cp2"]); + cache.add(cp2, states["cp2"], true); expect(await cache.processState(toHexString(cp2.root), states["cp2"])).toEqual(1); // cp0b is persisted @@ -133,19 +138,21 @@ describe("PersistentCheckpointStateCache", () => { expect(Array.from(fileApisBuffer.keys())).toEqual([persistent0bKey]); // getLatest() does not reload from disk - expect(cache.getLatest(cp0aHex.rootHex, cp0a.epoch)).toBeNull(); - expect(cache.getLatest(cp0bHex.rootHex, cp0b.epoch)).toBeNull(); + expect(cache.getLatest(cp0aHex.rootHex, cp0a.epoch, true)).toBeNull(); + expect(cache.getLatest(cp0bHex.rootHex, cp0b.epoch, true)).toBeNull(); // cp0a has the root from previous epoch so we only prune it from db - expect(await cache.getOrReloadLatest(cp0aHex.rootHex, cp0a.epoch)).toBeNull(); + expect(await cache.getOrReloadLatest(cp0aHex.rootHex, cp0a.epoch, true)).toBeNull(); // but getOrReloadLatest() does for cp0b - expect((await cache.getOrReloadLatest(cp0bHex.rootHex, cp0b.epoch))?.serialize()).toEqual(stateBytes["cp0b"]); - expect((await cache.getOrReloadLatest(cp0bHex.rootHex, cp0b.epoch + 1))?.serialize()).toEqual(stateBytes["cp0b"]); - expect((await cache.getOrReloadLatest(cp0bHex.rootHex, cp0b.epoch - 1))?.serialize()).toBeUndefined(); + expect((await cache.getOrReloadLatest(cp0bHex.rootHex, cp0b.epoch, true))?.serialize()).toEqual(stateBytes["cp0b"]); + expect((await cache.getOrReloadLatest(cp0bHex.rootHex, cp0b.epoch + 1, true))?.serialize()).toEqual( + stateBytes["cp0b"] + ); + expect((await cache.getOrReloadLatest(cp0bHex.rootHex, cp0b.epoch - 1, true))?.serialize()).toBeUndefined(); }); it("pruneFinalized and getStateOrBytes", async () => { - cache.add(cp2, states["cp2"]); + cache.add(cp2, states["cp2"], true); expect(((await cache.getStateOrBytes(cp0bHex)) as CachedBeaconStateAllForks).hashTreeRoot()).toEqual( states["cp0b"].hashTreeRoot() ); @@ -179,9 +186,9 @@ describe("PersistentCheckpointStateCache", () => { }, {maxCPStateEpochsInMemory: 2} ); - cache.add(cp0a, states["cp0a"]); - cache.add(cp0b, states["cp0b"]); - cache.add(cp1, states["cp1"]); + cache.add(cp0a, states["cp0a"], true); + cache.add(cp0b, states["cp0b"], true); + cache.add(cp1, states["cp1"], true); }); // epoch: 19 20 21 22 23 @@ -192,7 +199,7 @@ describe("PersistentCheckpointStateCache", () => { // | // 0a it("single state at lowest memory epoch", async () => { - cache.add(cp2, states["cp2"]); + cache.add(cp2, states["cp2"], true); expect(await cache.processState(toHexString(cp2.root), states["cp2"])).toEqual(1); expect(cache.findSeedStateToReload(cp0aHex)?.hashTreeRoot()).toEqual(states["cp1"].hashTreeRoot()); expect(cache.findSeedStateToReload(cp0bHex)?.hashTreeRoot()).toEqual(states["cp1"].hashTreeRoot()); @@ -208,7 +215,7 @@ describe("PersistentCheckpointStateCache", () => { // ^ ^ // cp1a={0a, 21} {0a, 22}=cp2a it("multiple states at lowest memory epoch", async () => { - cache.add(cp2, states["cp2"]); + cache.add(cp2, states["cp2"], true); expect(await cache.processState(toHexString(cp2.root), states["cp2"])).toEqual(1); const cp1a = {epoch: 21, root: root0a}; @@ -216,14 +223,14 @@ describe("PersistentCheckpointStateCache", () => { cp1aState.slot = 21 * SLOTS_PER_EPOCH; cp1aState.blockRoots.set(startSlotEpoch21 % SLOTS_PER_HISTORICAL_ROOT, root0a); cp1aState.commit(); - cache.add(cp1a, cp1aState); + cache.add(cp1a, cp1aState, true); const cp2a = {epoch: 22, root: root0a}; const cp2aState = cp1aState.clone(); cp2aState.slot = 22 * SLOTS_PER_EPOCH; cp2aState.blockRoots.set(startSlotEpoch22 % SLOTS_PER_HISTORICAL_ROOT, root0a); cp2aState.commit(); - cache.add(cp2a, cp2aState); + cache.add(cp2a, cp2aState, true); const root3 = Buffer.alloc(32, 100); const state3 = cp2aState.clone(); @@ -237,9 +244,9 @@ describe("PersistentCheckpointStateCache", () => { expect(cache.findSeedStateToReload(cp0bHex)?.hashTreeRoot()).toEqual(states["cp1"].hashTreeRoot()); const randomRoot = Buffer.alloc(32, 101); // for other random root it'll pick the first state of epoch 21 which is states["cp1"] - expect(cache.findSeedStateToReload({epoch: 20, rootHex: toHexString(randomRoot)})?.hashTreeRoot()).toEqual( - states["cp1"].hashTreeRoot() - ); + expect( + cache.findSeedStateToReload({epoch: 20, rootHex: toHexString(randomRoot), payloadPresent: true})?.hashTreeRoot() + ).toEqual(states["cp1"].hashTreeRoot()); }); }); @@ -256,9 +263,9 @@ describe("PersistentCheckpointStateCache", () => { }, {maxCPStateEpochsInMemory: 2} ); - cache.add(cp0a, states["cp0a"]); - cache.add(cp0b, states["cp0b"]); - cache.add(cp1, states["cp1"]); + cache.add(cp0a, states["cp0a"], true); + cache.add(cp0b, states["cp0b"], true); + cache.add(cp1, states["cp1"], true); }); // epoch: 19 20 21 22 23 @@ -270,7 +277,7 @@ describe("PersistentCheckpointStateCache", () => { // 0a it("no reorg", async () => { expect(fileApisBuffer.size).toEqual(0); - cache.add(cp2, states["cp2"]); + cache.add(cp2, states["cp2"], true); expect(await cache.processState(toHexString(cp2.root), states["cp2"])).toEqual(1); expect(cache.get(cp2Hex)?.hashTreeRoot()).toEqual(states["cp2"].hashTreeRoot()); expect(fileApisBuffer.size).toEqual(1); @@ -305,7 +312,7 @@ describe("PersistentCheckpointStateCache", () => { it("reorg in same epoch", async () => { // mostly the same to the above test expect(fileApisBuffer.size).toEqual(0); - cache.add(cp2, states["cp2"]); + cache.add(cp2, states["cp2"], true); expect(await cache.processState(toHexString(cp2.root), states["cp2"])).toEqual(1); expect(cache.get(cp2Hex)?.hashTreeRoot()).toEqual(states["cp2"].hashTreeRoot()); expect(fileApisBuffer.size).toEqual(1); @@ -349,7 +356,7 @@ describe("PersistentCheckpointStateCache", () => { // {1a, 22}=cp2a it("reorg 1 epoch", async () => { // process root2 state - cache.add(cp2, states["cp2"]); + cache.add(cp2, states["cp2"], true); expect(await cache.processState(toHexString(cp2.root), states["cp2"])).toEqual(1); await assertPersistedCheckpointState([cp0b], [stateBytes["cp0b"]]); @@ -361,7 +368,7 @@ describe("PersistentCheckpointStateCache", () => { // assuming reorg block is at slot 5 of epoch 21 cp2aState.blockRoots.set((startSlotEpoch21 + 5) % SLOTS_PER_HISTORICAL_ROOT, root1a); cp2aState.blockRoots.set(startSlotEpoch22 % SLOTS_PER_HISTORICAL_ROOT, root1a); - cache.add(cp2a, cp2aState); + cache.add(cp2a, cp2aState, true); // block state of root3 in epoch 22 is built on cp2a const blockStateRoot3 = cp2aState.clone(); @@ -373,7 +380,7 @@ describe("PersistentCheckpointStateCache", () => { await assertPersistedCheckpointState([cp0b], [stateBytes["cp0b"]]); // epoch 22 has 2 checkpoint states expect(cache.get(cp2Hex)).not.toBeNull(); - expect(cache.get(toCheckpointHex(cp2a))).not.toBeNull(); + expect(cache.get(toCheckpointHexPayload(cp2a, true))).not.toBeNull(); // epoch 21 has 1 checkpoint state expect(cache.get(cp1Hex)).not.toBeNull(); // epoch 20 has 0 checkpoint state @@ -393,12 +400,14 @@ describe("PersistentCheckpointStateCache", () => { // cp1a={0a, 21} {0a, 22}=cp2a it("reorg 2 epochs", async () => { // process root2 state - cache.add(cp2, states["cp2"]); + cache.add(cp2, states["cp2"], true); expect(await cache.processState(toHexString(cp2.root), states["cp2"])).toEqual(1); await assertPersistedCheckpointState([cp0b], [stateBytes["cp0b"]]); // reload cp0b from disk - expect((await cache.getOrReload(toCheckpointHex(cp0b)))?.serialize()).toStrictEqual(stateBytes["cp0b"]); + expect((await cache.getOrReload(toCheckpointHexPayload(cp0b, true)))?.serialize()).toStrictEqual( + stateBytes["cp0b"] + ); // regen generates cp1a const root0a = Buffer.alloc(32, 100); @@ -407,14 +416,14 @@ describe("PersistentCheckpointStateCache", () => { cp1aState.slot = 21 * SLOTS_PER_EPOCH; // assuming reorg block is at slot 5 of epoch 20 cp1aState.blockRoots.set((startSlotEpoch20 + 5) % SLOTS_PER_HISTORICAL_ROOT, root0a); - cache.add(cp1a, cp1aState); + cache.add(cp1a, cp1aState, true); // regen generates cp2a const cp2a = {epoch: 22, root: root0a}; const cp2aState = cp1aState.clone(); cp2aState.slot = 22 * SLOTS_PER_EPOCH; cp2aState.blockRoots.set(startSlotEpoch22 % SLOTS_PER_HISTORICAL_ROOT, root0a); - cache.add(cp2a, cp2aState); + cache.add(cp2a, cp2aState, true); // block state of root3 in epoch 22 is built on cp2a const blockStateRoot3 = cp2aState.clone(); @@ -426,9 +435,9 @@ describe("PersistentCheckpointStateCache", () => { await assertPersistedCheckpointState([cp0b], [stateBytes["cp0b"]]); // epoch 21 and 22 have 2 checkpoint states expect(cache.get(cp1Hex)).not.toBeNull(); - expect(cache.get(toCheckpointHex(cp1a))).not.toBeNull(); + expect(cache.get(toCheckpointHexPayload(cp1a, true))).not.toBeNull(); expect(cache.get(cp2Hex)).not.toBeNull(); - expect(cache.get(toCheckpointHex(cp2a))).not.toBeNull(); + expect(cache.get(toCheckpointHexPayload(cp2a, true))).not.toBeNull(); // epoch 20 has 0 checkpoint state expect(cache.get(cp0aHex)).toBeNull(); expect(cache.get(cp0bHex)).toBeNull(); @@ -446,28 +455,28 @@ describe("PersistentCheckpointStateCache", () => { // cp1a={0a, 21} {0a, 22}=cp2a it("reorg 3 epochs, persist cp 0a", async () => { // process root2 state - cache.add(cp2, states["cp2"]); + cache.add(cp2, states["cp2"], true); expect(await cache.processState(toHexString(cp2.root), states["cp2"])).toEqual(1); await assertPersistedCheckpointState([cp0b], [stateBytes["cp0b"]]); // cp0a was pruned from memory and not in disc expect(await cache.getStateOrBytes(cp0aHex)).toBeNull(); // regen needs to regen cp0a - cache.add(cp0a, states["cp0a"]); + cache.add(cp0a, states["cp0a"], true); // regen generates cp1a const cp1a = {epoch: 21, root: root0a}; const cp1aState = generateCachedState({slot: 21 * SLOTS_PER_EPOCH}); cp1aState.blockRoots.set((startSlotEpoch20 - 1) % SLOTS_PER_HISTORICAL_ROOT, root0a); cp1aState.blockRoots.set(startSlotEpoch20 % SLOTS_PER_HISTORICAL_ROOT, root0a); - cache.add(cp1a, cp1aState); + cache.add(cp1a, cp1aState, true); // regen generates cp2a const cp2a = {epoch: 22, root: root0a}; const cp2aState = cp1aState.clone(); cp2aState.slot = 22 * SLOTS_PER_EPOCH; cp2aState.blockRoots.set(startSlotEpoch21 % SLOTS_PER_HISTORICAL_ROOT, root0a); - cache.add(cp2a, cp2aState); + cache.add(cp2a, cp2aState, true); // block state of root3 in epoch 22 is built on cp2a const blockStateRoot3 = cp2aState.clone(); @@ -482,9 +491,9 @@ describe("PersistentCheckpointStateCache", () => { await assertPersistedCheckpointState([cp0b, cp0a], [stateBytes["cp0b"], stateBytes["cp0a"]]); // epoch 21 and 22 have 2 checkpoint states expect(cache.get(cp1Hex)).not.toBeNull(); - expect(cache.get(toCheckpointHex(cp1a))).not.toBeNull(); + expect(cache.get(toCheckpointHexPayload(cp1a, true))).not.toBeNull(); expect(cache.get(cp2Hex)).not.toBeNull(); - expect(cache.get(toCheckpointHex(cp2a))).not.toBeNull(); + expect(cache.get(toCheckpointHexPayload(cp2a, true))).not.toBeNull(); // epoch 20 has 0 checkpoint state expect(cache.get(cp0aHex)).toBeNull(); expect(cache.get(cp0bHex)).toBeNull(); @@ -502,14 +511,14 @@ describe("PersistentCheckpointStateCache", () => { // cp1b={0b, 21} {0b, 22}=cp2b it("reorg 3 epochs, prune but no persist", async () => { // process root2 state - cache.add(cp2, states["cp2"]); + cache.add(cp2, states["cp2"], true); expect(await cache.processState(toHexString(cp2.root), states["cp2"])).toEqual(1); await assertPersistedCheckpointState([cp0b], [stateBytes["cp0b"]]); // cp0a was pruned from memory and not in disc expect(await cache.getStateOrBytes(cp0aHex)).toBeNull(); // regen needs to reload cp0b - cache.add(cp0b, states["cp0b"]); + cache.add(cp0b, states["cp0b"], true); expect(((await cache.getStateOrBytes(cp0bHex)) as CachedBeaconStateAllForks).hashTreeRoot()).toEqual( states["cp0b"].hashTreeRoot() ); @@ -519,14 +528,14 @@ describe("PersistentCheckpointStateCache", () => { const cp1bState = states["cp0b"].clone(); cp1bState.slot = 21 * SLOTS_PER_EPOCH; cp1bState.blockRoots.set(startSlotEpoch21 % SLOTS_PER_HISTORICAL_ROOT, root0b); - cache.add(cp1b, cp1bState); + cache.add(cp1b, cp1bState, true); // regen generates cp2b const cp2b = {epoch: 22, root: root0b}; const cp2bState = cp1bState.clone(); cp2bState.slot = 22 * SLOTS_PER_EPOCH; cp2bState.blockRoots.set(startSlotEpoch22 % SLOTS_PER_HISTORICAL_ROOT, root0b); - cache.add(cp2b, cp2bState); + cache.add(cp2b, cp2bState, true); // block state of root3 in epoch 22 is built on cp2a const blockStateRoot3 = cp2bState.clone(); @@ -540,9 +549,9 @@ describe("PersistentCheckpointStateCache", () => { // epoch 21 and 22 have 2 checkpoint states expect(cache.get(cp1Hex)).not.toBeNull(); - expect(cache.get(toCheckpointHex(cp1b))).not.toBeNull(); + expect(cache.get(toCheckpointHexPayload(cp1b, true))).not.toBeNull(); expect(cache.get(cp2Hex)).not.toBeNull(); - expect(cache.get(toCheckpointHex(cp2b))).not.toBeNull(); + expect(cache.get(toCheckpointHexPayload(cp2b, true))).not.toBeNull(); // epoch 20 has 0 checkpoint state expect(cache.get(cp0aHex)).toBeNull(); expect(cache.get(cp0bHex)).toBeNull(); @@ -562,8 +571,8 @@ describe("PersistentCheckpointStateCache", () => { }, {maxCPStateEpochsInMemory: 1} ); - cache.add(cp0a, states["cp0a"]); - cache.add(cp0b, states["cp0b"]); + cache.add(cp0a, states["cp0a"], true); + cache.add(cp0b, states["cp0b"], true); }); // epoch: 19 20 21 22 23 @@ -575,7 +584,7 @@ describe("PersistentCheckpointStateCache", () => { // 0a it("no reorg", async () => { expect(fileApisBuffer.size).toEqual(0); - cache.add(cp1, states["cp1"]); + cache.add(cp1, states["cp1"], true); expect(await cache.processState(toHexString(cp1.root), states["cp1"])).toEqual(1); expect(cache.get(cp1Hex)?.hashTreeRoot()).toEqual(states["cp1"].hashTreeRoot()); expect(fileApisBuffer.size).toEqual(1); @@ -610,7 +619,7 @@ describe("PersistentCheckpointStateCache", () => { it("reorg in same epoch", async () => { // almost the same to "no reorg" test expect(fileApisBuffer.size).toEqual(0); - cache.add(cp1, states["cp1"]); + cache.add(cp1, states["cp1"], true); expect(await cache.processState(toHexString(cp1.root), states["cp1"])).toEqual(1); expect(cache.get(cp1Hex)?.hashTreeRoot()).toEqual(states["cp1"].hashTreeRoot()); expect(fileApisBuffer.size).toEqual(1); @@ -660,7 +669,7 @@ describe("PersistentCheckpointStateCache", () => { await assertPersistedCheckpointState([], []); // cp1 - cache.add(cp1, states["cp1"]); + cache.add(cp1, states["cp1"], true); expect(await cache.processState(toHexString(cp1.root), states["cp1"])).toEqual(1); expect(cache.get(cp1Hex)?.hashTreeRoot()).toEqual(states["cp1"].hashTreeRoot()); expect(fileApisBuffer.size).toEqual(1); @@ -671,7 +680,7 @@ describe("PersistentCheckpointStateCache", () => { const cp1aState = state1a.clone(); cp1aState.slot = 21 * SLOTS_PER_EPOCH; const cp1a = {epoch: 21, root: root1a}; - cache.add(cp1a, cp1aState); + cache.add(cp1a, cp1aState, true); const blockStateRoot2 = cp1aState.clone(); blockStateRoot2.slot = 21 * SLOTS_PER_EPOCH + 3; const root2 = Buffer.alloc(32, 100); @@ -680,8 +689,8 @@ describe("PersistentCheckpointStateCache", () => { await assertPersistedCheckpointState([cp0b], [stateBytes["cp0b"]]); expect(cache.get(cp1Hex)?.hashTreeRoot()).toEqual(states["cp1"].hashTreeRoot()); // keep these 2 cp states at epoch 21 - expect(cache.get(toCheckpointHex(cp1a))).not.toBeNull(); - expect(cache.get(toCheckpointHex(cp1))).not.toBeNull(); + expect(cache.get(toCheckpointHexPayload(cp1a, true))).not.toBeNull(); + expect(cache.get(toCheckpointHexPayload(cp1, true))).not.toBeNull(); }); // epoch: 19 20 21 22 23 @@ -694,7 +703,7 @@ describe("PersistentCheckpointStateCache", () => { it("reorg 1 epoch, no persist 0b", async () => { expect(fileApisBuffer.size).toEqual(0); // cp1 - cache.add(cp1, states["cp1"]); + cache.add(cp1, states["cp1"], true); expect(await cache.processState(toHexString(cp1.root), states["cp1"])).toEqual(1); expect(cache.get(cp1Hex)?.hashTreeRoot()).toEqual(states["cp1"].hashTreeRoot()); expect(fileApisBuffer.size).toEqual(1); @@ -702,7 +711,7 @@ describe("PersistentCheckpointStateCache", () => { expect(cache.get(cp1Hex)?.hashTreeRoot()).toEqual(states["cp1"].hashTreeRoot()); // simulate regen - cache.add(cp0b, states["cp0b"]); + cache.add(cp0b, states["cp0b"], true); expect(((await cache.getStateOrBytes(cp0bHex)) as CachedBeaconStateAllForks).hashTreeRoot()).toEqual( states["cp0b"].hashTreeRoot() ); @@ -710,7 +719,7 @@ describe("PersistentCheckpointStateCache", () => { const cp1bState = states["cp0b"].clone(); cp1bState.slot = 21 * SLOTS_PER_EPOCH; const cp1b = {epoch: 21, root: root0b}; - cache.add(cp1b, cp1bState); + cache.add(cp1b, cp1bState, true); const blockStateRoot2 = cp1bState.clone(); blockStateRoot2.slot = 21 * SLOTS_PER_EPOCH + 3; const root2 = Buffer.alloc(32, 100); @@ -720,8 +729,8 @@ describe("PersistentCheckpointStateCache", () => { // but cp0b in-memory state is pruned expect(await cache.getStateOrBytes(cp0bHex)).toEqual(stateBytes["cp0b"]); // keep these 2 cp states at epoch 21 - expect(cache.get(toCheckpointHex(cp1b))).not.toBeNull(); - expect(cache.get(toCheckpointHex(cp1))).not.toBeNull(); + expect(cache.get(toCheckpointHexPayload(cp1b, true))).not.toBeNull(); + expect(cache.get(toCheckpointHexPayload(cp1, true))).not.toBeNull(); }); // epoch: 19 20 21 22 23 @@ -748,7 +757,7 @@ describe("PersistentCheckpointStateCache", () => { await assertPersistedCheckpointState([], []); // cp1 - cache.add(cp1, states["cp1"]); + cache.add(cp1, states["cp1"], true); expect(await cache.processState(toHexString(cp1.root), states["cp1"])).toEqual(1); expect(cache.get(cp1Hex)?.hashTreeRoot()).toEqual(states["cp1"].hashTreeRoot()); expect(fileApisBuffer.size).toEqual(1); @@ -762,11 +771,11 @@ describe("PersistentCheckpointStateCache", () => { expect(await cache.getStateOrBytes(cp0aHex)).toBeNull(); // root2, regen cp0a - cache.add(cp0a, states["cp0a"]); + cache.add(cp0a, states["cp0a"], true); const cp1aState = state1a.clone(); cp1aState.slot = 21 * SLOTS_PER_EPOCH; const cp1a = {epoch: 21, root: root1a}; - cache.add(cp1a, cp1aState); + cache.add(cp1a, cp1aState, true); const blockStateRoot2 = cp1aState.clone(); blockStateRoot2.slot = 21 * SLOTS_PER_EPOCH + 3; const root2 = Buffer.alloc(32, 100); @@ -775,8 +784,8 @@ describe("PersistentCheckpointStateCache", () => { await assertPersistedCheckpointState([cp0b, cp0a], [stateBytes["cp0b"], stateBytes["cp0a"]]); expect(cache.get(cp1Hex)?.hashTreeRoot()).toEqual(states["cp1"].hashTreeRoot()); // keep these 2 cp states at epoch 21 - expect(cache.get(toCheckpointHex(cp1a))).not.toBeNull(); - expect(cache.get(toCheckpointHex(cp1))).not.toBeNull(); + expect(cache.get(toCheckpointHexPayload(cp1a, true))).not.toBeNull(); + expect(cache.get(toCheckpointHexPayload(cp1, true))).not.toBeNull(); }); // epoch: 19 20 21 22 23 @@ -790,7 +799,7 @@ describe("PersistentCheckpointStateCache", () => { // cp1a={0a, 21} it("reorg 2 epochs", async () => { // cp1 - cache.add(cp1, states["cp1"]); + cache.add(cp1, states["cp1"], true); expect(await cache.processState(toHexString(cp1.root), states["cp1"])).toEqual(1); expect(cache.get(cp1Hex)?.hashTreeRoot()).toEqual(states["cp1"].hashTreeRoot()); expect(fileApisBuffer.size).toEqual(1); @@ -804,11 +813,11 @@ describe("PersistentCheckpointStateCache", () => { expect(await cache.getStateOrBytes(cp0aHex)).toBeNull(); // root2, regen cp0a - cache.add(cp0a, states["cp0a"]); + cache.add(cp0a, states["cp0a"], true); const cp1aState = states["cp0a"].clone(); cp1aState.slot = 21 * SLOTS_PER_EPOCH; const cp1a = {epoch: 21, root: root0a}; - cache.add(cp1a, cp1aState); + cache.add(cp1a, cp1aState, true); const blockStateRoot2 = cp1aState.clone(); blockStateRoot2.slot = 21 * SLOTS_PER_EPOCH + 3; const root2 = Buffer.alloc(32, 100); @@ -817,8 +826,8 @@ describe("PersistentCheckpointStateCache", () => { await assertPersistedCheckpointState([cp0b, cp0a], [stateBytes["cp0b"], stateBytes["cp0a"]]); expect(cache.get(cp1Hex)?.hashTreeRoot()).toEqual(states["cp1"].hashTreeRoot()); // keep these 2 cp states at epoch 21 - expect(cache.get(toCheckpointHex(cp1a))).not.toBeNull(); - expect(cache.get(toCheckpointHex(cp1))).not.toBeNull(); + expect(cache.get(toCheckpointHexPayload(cp1a, true))).not.toBeNull(); + expect(cache.get(toCheckpointHexPayload(cp1, true))).not.toBeNull(); }); describe("processState, maxEpochsInMemory = 0", () => { @@ -834,8 +843,8 @@ describe("PersistentCheckpointStateCache", () => { }, {maxCPStateEpochsInMemory: 0} ); - cache.add(cp0a, states["cp0a"]); - cache.add(cp0b, states["cp0b"]); + cache.add(cp0a, states["cp0a"], true); + cache.add(cp0b, states["cp0b"], true); }); // epoch: 19 20 21 22 23 @@ -886,7 +895,7 @@ describe("PersistentCheckpointStateCache", () => { expect(await cache.getStateOrBytes(cp0bHex)).toEqual(stateBytes["cp0b"]); // simulate reload cp1b - cache.add(cp0b, states["cp0b"]); + cache.add(cp0b, states["cp0b"], true); expect(((await cache.getStateOrBytes(cp0bHex)) as CachedBeaconStateAllForks).hashTreeRoot()).toEqual( states["cp0b"].hashTreeRoot() ); @@ -931,7 +940,7 @@ describe("PersistentCheckpointStateCache", () => { state1a.slot = 20 * SLOTS_PER_EPOCH + SLOTS_PER_EPOCH + 3; state1a.blockRoots.set(state1a.slot % SLOTS_PER_HISTORICAL_ROOT, root1a); // state transition add to cache - cache.add(cp0b, states["cp0b"]); + cache.add(cp0b, states["cp0b"], true); // do not processState root1a because it's late // no need to reload cp0b because it's available in block state @@ -940,7 +949,7 @@ describe("PersistentCheckpointStateCache", () => { state1b.slot = state1a.slot + 1; state1b.blockRoots.set(state1b.slot % SLOTS_PER_HISTORICAL_ROOT, root1b); // state transition add to cache - cache.add(cp0a, states["cp0a"]); + cache.add(cp0a, states["cp0a"], true); // need to persist 2 checkpoint states expect(await cache.processState(toHexString(root1b), state1b)).toEqual(2); @@ -978,7 +987,7 @@ describe("PersistentCheckpointStateCache", () => { state1b.slot = state1a.slot + 1; state1b.blockRoots.set(state1b.slot % SLOTS_PER_HISTORICAL_ROOT, root1b); // regen should reload cp0a from disk - cache.add(cp0a, states["cp0a"]); + cache.add(cp0a, states["cp0a"], true); expect(await cache.processState(toHexString(root1b), state1b)).toEqual(1); await assertPersistedCheckpointState([cp0b, cp0a], [stateBytes["cp0b"], stateBytes["cp0a"]]); @@ -1002,18 +1011,18 @@ describe("PersistentCheckpointStateCache", () => { expect(await cache.getStateOrBytes(cp0aHex)).toBeNull(); expect(await cache.getStateOrBytes(cp0bHex)).toEqual(stateBytes["cp0b"]); - cache.add(cp1, states["cp1"]); + cache.add(cp1, states["cp1"], true); expect(await cache.processState(toHexString(cp1.root), states["cp1"])).toEqual(1); await assertPersistedCheckpointState([cp0b, cp1], [stateBytes["cp0b"], stateBytes["cp1"]]); // regen should populate cp0a and cp1a checkpoint states - cache.add(cp0a, states["cp0a"]); + cache.add(cp0a, states["cp0a"], true); const cp1a = {epoch: 21, root: root0a}; const cp1aState = states["cp0a"].clone(); cp1aState.blockRoots.set((20 * SLOTS_PER_EPOCH) % SLOTS_PER_HISTORICAL_ROOT, root0a); cp1aState.blockRoots.set((21 * SLOTS_PER_EPOCH) % SLOTS_PER_HISTORICAL_ROOT, root0a); cp1aState.slot = 21 * SLOTS_PER_EPOCH; - cache.add(cp1a, cp1aState); + cache.add(cp1a, cp1aState, true); const root2 = Buffer.alloc(32, 100); const state2 = cp1aState.clone(); @@ -1030,13 +1039,13 @@ describe("PersistentCheckpointStateCache", () => { }); async function assertPersistedCheckpointState(cps: phase0.Checkpoint[], stateBytesArr: Uint8Array[]): Promise { - const persistedKeys = cps.map((cp) => toHexString(checkpointToDatastoreKey(cp))); + const persistedKeys = cps.map((cp) => toHexString(checkpointToDatastoreKey(cp, true))); expect(Array.from(fileApisBuffer.keys())).toStrictEqual(persistedKeys); for (const [i, persistedKey] of persistedKeys.entries()) { expect(fileApisBuffer.get(persistedKey)).toStrictEqual(stateBytesArr[i]); } for (const [i, cp] of cps.entries()) { - const cpHex = toCheckpointHex(cp); + const cpHex = toCheckpointHexPayload(cp, true); expect(await cache.getStateOrBytes(cpHex)).toStrictEqual(stateBytesArr[i]); // simple get() does not reload from disk expect(cache.get(cpHex)).toBeNull(); diff --git a/packages/beacon-node/test/unit/chain/regen/regen.test.ts b/packages/beacon-node/test/unit/chain/regen/regen.test.ts index b2e860d1a17c..1e8ace689d7f 100644 --- a/packages/beacon-node/test/unit/chain/regen/regen.test.ts +++ b/packages/beacon-node/test/unit/chain/regen/regen.test.ts @@ -85,9 +85,9 @@ describe("regen", () => { {maxCPStateEpochsInMemory: 2} ); - cache.add(cp0a, states["cp0a"]); - cache.add(cp0b, states["cp0b"]); - cache.add(cp1, states["cp1"]); + cache.add(cp0a, states["cp0a"], true); + cache.add(cp0b, states["cp0b"], true); + cache.add(cp1, states["cp1"], true); }); /** diff --git a/packages/beacon-node/test/utils/chain/stateCache/datastore.ts b/packages/beacon-node/test/utils/chain/stateCache/datastore.ts index 20d1708c1045..c4bb52acc43c 100644 --- a/packages/beacon-node/test/utils/chain/stateCache/datastore.ts +++ b/packages/beacon-node/test/utils/chain/stateCache/datastore.ts @@ -3,8 +3,8 @@ import {CPStateDatastore, checkpointToDatastoreKey} from "../../../../src/chain/ export function getTestDatastore(fileApisBuffer: Map): CPStateDatastore { const datastore: CPStateDatastore = { - write: (cp, stateBytes) => { - const persistentKey = checkpointToDatastoreKey(cp); + write: (cp, stateBytes, payloadPresent) => { + const persistentKey = checkpointToDatastoreKey(cp, payloadPresent); const stringKey = toHexString(persistentKey); if (!fileApisBuffer.has(stringKey)) { fileApisBuffer.set(stringKey, stateBytes); diff --git a/packages/beacon-node/test/utils/node/simTest.ts b/packages/beacon-node/test/utils/node/simTest.ts index e8a862bc3da6..8af87d6ebe07 100644 --- a/packages/beacon-node/test/utils/node/simTest.ts +++ b/packages/beacon-node/test/utils/node/simTest.ts @@ -63,9 +63,11 @@ export function simTestInfoTracker(bn: BeaconNode, logger: Logger): () => void { if (checkpoint.epoch <= lastSeenEpoch) return; lastSeenEpoch = checkpoint.epoch; + // Pre-Gloas: payloadPresent is always true (execution payload embedded in block) const checkpointState = bn.chain.regen.getCheckpointStateSync({ ...checkpoint, rootHex: toRootHex(checkpoint.root), + payloadPresent: true, }); if (checkpointState == null) { throw Error(`Checkpoint state not found for epoch ${checkpoint.epoch} root ${toRootHex(checkpoint.root)}`); diff --git a/packages/fork-choice/src/index.ts b/packages/fork-choice/src/index.ts index fd141bf7d70e..cf4f75bceffe 100644 --- a/packages/fork-choice/src/index.ts +++ b/packages/fork-choice/src/index.ts @@ -38,5 +38,5 @@ export type { ProtoBlock, ProtoNode, } from "./protoArray/interface.js"; -export {ExecutionStatus, PayloadStatus} from "./protoArray/interface.js"; +export {ExecutionStatus, PayloadStatus, isGloasBlock} from "./protoArray/interface.js"; export {ProtoArray} from "./protoArray/protoArray.js"; From bb16850567eb84a6e62c20482af77993c4c114df Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 19 Mar 2026 11:35:02 +0000 Subject: [PATCH 53/63] feat: update voluntary exit for gloas (#9039) Update code paths outside of state transition (namely gossip and api) to be able to handle builder voluntary exists. image --- .../chain/blocks/verifyBlocksSignatures.ts | 2 +- .../src/chain/validation/voluntaryExit.ts | 3 +- .../src/block/processVoluntaryExit.ts | 131 ++++++++++-------- .../src/signatureSets/index.ts | 5 +- .../src/signatureSets/voluntaryExits.ts | 60 ++++++-- .../unit/signatureSets/signatureSets.test.ts | 1 + 6 files changed, 136 insertions(+), 66 deletions(-) diff --git a/packages/beacon-node/src/chain/blocks/verifyBlocksSignatures.ts b/packages/beacon-node/src/chain/blocks/verifyBlocksSignatures.ts index f547c3d64340..97759b43c519 100644 --- a/packages/beacon-node/src/chain/blocks/verifyBlocksSignatures.ts +++ b/packages/beacon-node/src/chain/blocks/verifyBlocksSignatures.ts @@ -41,7 +41,7 @@ export async function verifyBlocksSignatures( : // // Verify signatures per block to track which block is invalid bls.verifySignatureSets( - getBlockSignatureSets(config, currentSyncCommitteeIndexed, block, indexedAttestationsByBlock[i], { + getBlockSignatureSets(config, currentSyncCommitteeIndexed, preState0, block, indexedAttestationsByBlock[i], { skipProposerSignature: opts.validProposerSignature, }) ); diff --git a/packages/beacon-node/src/chain/validation/voluntaryExit.ts b/packages/beacon-node/src/chain/validation/voluntaryExit.ts index 3779989c87b6..00d1fe8b38e3 100644 --- a/packages/beacon-node/src/chain/validation/voluntaryExit.ts +++ b/packages/beacon-node/src/chain/validation/voluntaryExit.ts @@ -1,4 +1,5 @@ import { + BeaconStateView, VoluntaryExitValidity, getVoluntaryExitSignatureSet, getVoluntaryExitValidity, @@ -59,7 +60,7 @@ async function validateVoluntaryExit( }); } - const signatureSet = getVoluntaryExitSignatureSet(chain.config, state.slot, voluntaryExit); + const signatureSet = getVoluntaryExitSignatureSet(chain.config, new BeaconStateView(state), voluntaryExit); if (!(await chain.bls.verifySignatureSets([signatureSet], {batchable: true, priority: prioritizeBls}))) { throw new VoluntaryExitError(GossipAction.REJECT, { code: VoluntaryExitErrorCode.INVALID_SIGNATURE, diff --git a/packages/state-transition/src/block/processVoluntaryExit.ts b/packages/state-transition/src/block/processVoluntaryExit.ts index 890852dac78d..27f823c1dafb 100644 --- a/packages/state-transition/src/block/processVoluntaryExit.ts +++ b/packages/state-transition/src/block/processVoluntaryExit.ts @@ -1,7 +1,7 @@ -import {PublicKey, Signature, verify} from "@chainsafe/blst"; import {FAR_FUTURE_EPOCH, ForkSeq} from "@lodestar/params"; -import {phase0, ssz} from "@lodestar/types"; +import {phase0} from "@lodestar/types"; import {verifyVoluntaryExitSignature} from "../signatureSets/index.js"; +import {BeaconStateView} from "../stateView/beaconStateView.js"; import {CachedBeaconStateAllForks, CachedBeaconStateElectra, CachedBeaconStateGloas} from "../types.js"; import { convertValidatorIndexToBuilderIndex, @@ -10,7 +10,7 @@ import { isActiveBuilder, isBuilderIndex, } from "../util/gloas.js"; -import {computeSigningRoot, getCurrentEpoch, getPendingBalanceToWithdraw, isActiveValidator} from "../util/index.js"; +import {getPendingBalanceToWithdraw, isActiveValidator} from "../util/index.js"; import {initiateValidatorExit} from "./index.js"; export enum VoluntaryExitValidity { @@ -35,57 +35,20 @@ export function processVoluntaryExit( verifySignature = true ): void { const voluntaryExit = signedVoluntaryExit.message; - const currentEpoch = getCurrentEpoch(state); - // Exits must specify an epoch when they become valid; they are not valid before then - if (currentEpoch < voluntaryExit.epoch) { - throw Error(`Voluntary exit epoch ${voluntaryExit.epoch} is after current epoch ${currentEpoch}`); + const validity = getVoluntaryExitValidity(fork, state, signedVoluntaryExit, verifySignature); + if (validity !== VoluntaryExitValidity.valid) { + throw Error(`Invalid voluntary exit at forkSeq=${fork} reason=${validity}`); } - // Check if this is a builder exit if (fork >= ForkSeq.gloas && isBuilderIndex(voluntaryExit.validatorIndex)) { - const stateGloas = state as CachedBeaconStateGloas; - const builderIndex = convertValidatorIndexToBuilderIndex(voluntaryExit.validatorIndex); - const builder = stateGloas.builders.getReadonly(builderIndex); - - // Verify the builder is active - if (!isActiveBuilder(builder, state.finalizedCheckpoint.epoch)) { - throw Error(`Builder ${builderIndex} is not active`); - } - - // Only exit builder if it has no pending withdrawals in the queue - if (getPendingBalanceToWithdrawForBuilder(stateGloas, builderIndex) !== 0) { - throw Error(`Builder ${builderIndex} has pending withdrawals`); - } - - // Verify signature - if (verifySignature) { - const domain = state.config.getDomainForVoluntaryExit(state.slot); - const signingRoot = computeSigningRoot(ssz.phase0.VoluntaryExit, voluntaryExit, domain); - - try { - const publicKey = PublicKey.fromBytes(builder.pubkey); - const signature = Signature.fromBytes(signedVoluntaryExit.signature, true); - - if (!verify(signingRoot, publicKey, signature)) { - throw Error("BLS verify failed"); - } - } catch (e) { - throw Error(`Builder ${builderIndex} invalid exit signature reason=${(e as Error).message}`); - } - } - - // Initiate builder exit - initiateBuilderExit(stateGloas, builderIndex); + initiateBuilderExit( + state as CachedBeaconStateGloas, + convertValidatorIndexToBuilderIndex(voluntaryExit.validatorIndex) + ); return; } - // Handle validator exit - const validity = getVoluntaryExitValidity(fork, state, signedVoluntaryExit, verifySignature); - if (validity !== VoluntaryExitValidity.valid) { - throw Error(`Invalid voluntary exit at forkSeq=${fork} reason=${validity}`); - } - const validator = state.validators.get(signedVoluntaryExit.message.validatorIndex); initiateValidatorExit(fork, state, validator); } @@ -95,10 +58,74 @@ export function getVoluntaryExitValidity( state: CachedBeaconStateAllForks, signedVoluntaryExit: phase0.SignedVoluntaryExit, verifySignature = true +): VoluntaryExitValidity { + const currentEpoch = state.epochCtx.epoch; + const voluntaryExit = signedVoluntaryExit.message; + + // Exits must specify an epoch when they become valid; they are not valid before then + if (currentEpoch < voluntaryExit.epoch) { + return VoluntaryExitValidity.earlyEpoch; + } + + // Check if this is a builder exit + if (fork >= ForkSeq.gloas && isBuilderIndex(voluntaryExit.validatorIndex)) { + return getBuilderVoluntaryExitValidity(state as CachedBeaconStateGloas, signedVoluntaryExit, verifySignature); + } + + return getValidatorVoluntaryExitValidity(fork, state, signedVoluntaryExit, verifySignature); +} + +function getBuilderVoluntaryExitValidity( + state: CachedBeaconStateGloas, + signedVoluntaryExit: phase0.SignedVoluntaryExit, + verifySignature: boolean +): VoluntaryExitValidity { + const {config, epochCtx} = state; + const builderIndex = convertValidatorIndexToBuilderIndex(signedVoluntaryExit.message.validatorIndex); + + if (builderIndex >= state.builders.length) { + return VoluntaryExitValidity.inactive; + } + + const builder = state.builders.getReadonly(builderIndex); + + // Verify the builder is active + if (!isActiveBuilder(builder, state.finalizedCheckpoint.epoch)) { + return builder.withdrawableEpoch !== FAR_FUTURE_EPOCH + ? VoluntaryExitValidity.alreadyExited + : VoluntaryExitValidity.inactive; + } + + // Only exit builder if it has no pending withdrawals in the queue + if (getPendingBalanceToWithdrawForBuilder(state, builderIndex) !== 0) { + return VoluntaryExitValidity.pendingWithdrawals; + } + + // Verify signature + if ( + verifySignature && + !verifyVoluntaryExitSignature(config, epochCtx.pubkeyCache, new BeaconStateView(state), signedVoluntaryExit) + ) { + return VoluntaryExitValidity.invalidSignature; + } + + return VoluntaryExitValidity.valid; +} + +function getValidatorVoluntaryExitValidity( + fork: ForkSeq, + state: CachedBeaconStateAllForks, + signedVoluntaryExit: phase0.SignedVoluntaryExit, + verifySignature: boolean ): VoluntaryExitValidity { const {config, epochCtx} = state; const voluntaryExit = signedVoluntaryExit.message; - const validator = state.validators.get(voluntaryExit.validatorIndex); + + if (voluntaryExit.validatorIndex >= state.validators.length) { + return VoluntaryExitValidity.inactive; + } + + const validator = state.validators.getReadonly(voluntaryExit.validatorIndex); const currentEpoch = epochCtx.epoch; // verify the validator is active @@ -111,11 +138,6 @@ export function getVoluntaryExitValidity( return VoluntaryExitValidity.alreadyExited; } - // exits must specify an epoch when they become valid; they are not valid before then - if (currentEpoch < voluntaryExit.epoch) { - return VoluntaryExitValidity.earlyEpoch; - } - // verify the validator had been active long enough if (currentEpoch < validator.activationEpoch + config.SHARD_COMMITTEE_PERIOD) { return VoluntaryExitValidity.shortTimeActive; @@ -129,9 +151,10 @@ export function getVoluntaryExitValidity( return VoluntaryExitValidity.pendingWithdrawals; } + // Verify signature if ( verifySignature && - !verifyVoluntaryExitSignature(state.config, epochCtx.pubkeyCache, state.slot, signedVoluntaryExit) + !verifyVoluntaryExitSignature(config, epochCtx.pubkeyCache, new BeaconStateView(state), signedVoluntaryExit) ) { return VoluntaryExitValidity.invalidSignature; } diff --git a/packages/state-transition/src/signatureSets/index.ts b/packages/state-transition/src/signatureSets/index.ts index 8c797e1adb8e..d5bcf44cec85 100644 --- a/packages/state-transition/src/signatureSets/index.ts +++ b/packages/state-transition/src/signatureSets/index.ts @@ -3,6 +3,8 @@ import {ForkSeq} from "@lodestar/params"; import {IndexedAttestation, SignedBeaconBlock, altair, capella} from "@lodestar/types"; import {getSyncCommitteeSignatureSet} from "../block/processSyncCommittee.js"; import {SyncCommitteeCache} from "../cache/syncCommitteeCache.js"; +import {BeaconStateView} from "../stateView/beaconStateView.js"; +import {CachedBeaconStateAllForks} from "../types.js"; import {ISignatureSet} from "../util/index.js"; import {getAttesterSlashingsSignatureSets} from "./attesterSlashings.js"; import {getBlsToExecutionChangeSignatureSets} from "./blsToExecutionChange.js"; @@ -30,6 +32,7 @@ export * from "./voluntaryExits.js"; export function getBlockSignatureSets( config: BeaconConfig, currentSyncCommitteeIndexed: SyncCommitteeCache, + state: CachedBeaconStateAllForks, signedBlock: SignedBeaconBlock, indexedAttestations: IndexedAttestation[], opts?: { @@ -45,7 +48,7 @@ export function getBlockSignatureSets( ...getProposerSlashingsSignatureSets(config, signedBlock), ...getAttesterSlashingsSignatureSets(config, signedBlock), ...getAttestationsSignatureSets(config, signedBlock, indexedAttestations), - ...getVoluntaryExitsSignatureSets(config, signedBlock), + ...getVoluntaryExitsSignatureSets(config, new BeaconStateView(state), signedBlock), ]; if (!opts?.skipProposerSignature) { diff --git a/packages/state-transition/src/signatureSets/voluntaryExits.ts b/packages/state-transition/src/signatureSets/voluntaryExits.ts index 0cedcd399b94..57a517b14da8 100644 --- a/packages/state-transition/src/signatureSets/voluntaryExits.ts +++ b/packages/state-transition/src/signatureSets/voluntaryExits.ts @@ -1,27 +1,56 @@ +import {PublicKey} from "@chainsafe/blst"; import {BeaconConfig} from "@lodestar/config"; +import {ForkSeq} from "@lodestar/params"; import {SignedBeaconBlock, Slot, phase0, ssz} from "@lodestar/types"; import {PubkeyCache} from "../cache/pubkeyCache.js"; +import {IBeaconStateView} from "../stateView/interface.js"; import { ISignatureSet, SignatureSetType, computeSigningRoot, computeStartSlotAtEpoch, + convertValidatorIndexToBuilderIndex, + isBuilderIndex, verifySignatureSet, } from "../util/index.js"; export function verifyVoluntaryExitSignature( config: BeaconConfig, pubkeyCache: PubkeyCache, - stateSlot: Slot, + state: IBeaconStateView, signedVoluntaryExit: phase0.SignedVoluntaryExit ): boolean { - return verifySignatureSet(getVoluntaryExitSignatureSet(config, stateSlot, signedVoluntaryExit), pubkeyCache); + return verifySignatureSet(getVoluntaryExitSignatureSet(config, state, signedVoluntaryExit), pubkeyCache); } /** * Extract signatures to allow validating all block signatures at once */ export function getVoluntaryExitSignatureSet( + config: BeaconConfig, + state: IBeaconStateView, + signedVoluntaryExit: phase0.SignedVoluntaryExit +): ISignatureSet { + const fork = config.getForkSeq(state.slot); + + if (fork >= ForkSeq.gloas && isBuilderVoluntaryExit(signedVoluntaryExit)) { + return getBuilderVoluntaryExitSignatureSet(config, state, signedVoluntaryExit); + } + + return getValidatorVoluntaryExitSignatureSet(config, state.slot, signedVoluntaryExit); +} + +export function getVoluntaryExitsSignatureSets( + config: BeaconConfig, + state: IBeaconStateView, + signedBlock: SignedBeaconBlock +): ISignatureSet[] { + return signedBlock.message.body.voluntaryExits.map((voluntaryExit) => + getVoluntaryExitSignatureSet(config, state, voluntaryExit) + ); +} + +export function getValidatorVoluntaryExitSignatureSet( config: BeaconConfig, stateSlot: Slot, signedVoluntaryExit: phase0.SignedVoluntaryExit @@ -37,11 +66,24 @@ export function getVoluntaryExitSignatureSet( }; } -export function getVoluntaryExitsSignatureSets(config: BeaconConfig, signedBlock: SignedBeaconBlock): ISignatureSet[] { - // the getDomain() api requires the state slot as 1st param, however it's the same to block.slot in state-transition - // and the same epoch when we verify blocks in batch in beacon-node. So we can safely use block.slot here. - const blockSlot = signedBlock.message.slot; - return signedBlock.message.body.voluntaryExits.map((voluntaryExit) => - getVoluntaryExitSignatureSet(config, blockSlot, voluntaryExit) - ); +export function getBuilderVoluntaryExitSignatureSet( + config: BeaconConfig, + state: IBeaconStateView, + signedVoluntaryExit: phase0.SignedVoluntaryExit +): ISignatureSet { + const messageSlot = computeStartSlotAtEpoch(signedVoluntaryExit.message.epoch); + const domain = config.getDomainForVoluntaryExit(state.slot, messageSlot); + const builderIndex = convertValidatorIndexToBuilderIndex(signedVoluntaryExit.message.validatorIndex); + const builder = state.getBuilder(builderIndex); + + return { + type: SignatureSetType.single, + pubkey: PublicKey.fromBytes(builder.pubkey), + signingRoot: computeSigningRoot(ssz.phase0.VoluntaryExit, signedVoluntaryExit.message, domain), + signature: signedVoluntaryExit.signature, + }; +} + +export function isBuilderVoluntaryExit(signedVoluntaryExit: phase0.SignedVoluntaryExit): boolean { + return isBuilderIndex(signedVoluntaryExit.message.validatorIndex); } diff --git a/packages/state-transition/test/unit/signatureSets/signatureSets.test.ts b/packages/state-transition/test/unit/signatureSets/signatureSets.test.ts index 4d79f409e660..fadae9b2d869 100644 --- a/packages/state-transition/test/unit/signatureSets/signatureSets.test.ts +++ b/packages/state-transition/test/unit/signatureSets/signatureSets.test.ts @@ -73,6 +73,7 @@ describe("signatureSets", () => { const signatureSets = getBlockSignatureSets( state.config, state.epochCtx.currentSyncCommitteeIndexed, + state, signedBlock, indexedAttestations ); From 4411584fd816fb0710b9e1bea4ce343988797c70 Mon Sep 17 00:00:00 2001 From: markolazic01 <80317369+markolazic01@users.noreply.github.com> Date: Thu, 19 Mar 2026 19:53:40 +0100 Subject: [PATCH 54/63] chore: use permalinks for spec references (#9065) Closes issue https://github.com/ChainSafe/lodestar/issues/8171 Please let me know if there's anything missing. --------- Co-authored-by: Nico Flaig --- docs/pages/run/beacon-management/syncing.md | 2 +- packages/beacon-node/src/chain/opPools/utils.ts | 2 +- .../src/chain/produceBlock/produceBlockBody.ts | 2 +- .../src/chain/validation/lightClientFinalityUpdate.ts | 2 +- .../src/chain/validation/lightClientOptimisticUpdate.ts | 2 +- packages/beacon-node/src/chain/validatorMonitor.ts | 2 +- packages/beacon-node/src/execution/engine/interface.ts | 4 ++-- packages/beacon-node/src/network/network.ts | 4 ++-- .../beacon-node/src/network/reqresp/ReqRespBeaconNode.ts | 2 +- packages/beacon-node/src/sync/backfill/backfill.ts | 2 +- packages/beacon-node/src/sync/constants.ts | 2 +- .../test/spec/presets/epoch_processing.test.ts | 4 ++-- packages/beacon-node/test/spec/presets/finality.test.ts | 2 +- packages/beacon-node/test/spec/presets/ssz_static.test.ts | 2 +- packages/beacon-node/test/spec/utils/specTestIterator.ts | 2 +- packages/beacon-node/test/spec/utils/sszTestCaseParser.ts | 6 +++--- packages/config/src/chainConfig/configs/mainnet.ts | 2 +- packages/config/src/chainConfig/configs/minimal.ts | 2 +- packages/fork-choice/README.md | 2 +- packages/fork-choice/src/forkChoice/forkChoice.ts | 4 ++-- packages/params/src/presets/mainnet.ts | 2 +- packages/params/src/presets/minimal.ts | 2 +- packages/prover/README.md | 2 +- packages/prover/src/constants.ts | 2 +- packages/spec-test-util/src/sszGeneric.ts | 2 +- packages/state-transition/src/util/execution.ts | 2 +- packages/types/src/deneb/sszTypes.ts | 8 ++++---- specrefs/functions.yml | 2 +- 28 files changed, 37 insertions(+), 37 deletions(-) diff --git a/docs/pages/run/beacon-management/syncing.md b/docs/pages/run/beacon-management/syncing.md index 7dd05c4b4338..024a5a30c332 100644 --- a/docs/pages/run/beacon-management/syncing.md +++ b/docs/pages/run/beacon-management/syncing.md @@ -32,7 +32,7 @@ This is another version of checkpoint sync that allows a node that has not been ## Syncing Lodestar -The implementation of the different syncing styles in Lodestar are actually one of two types under the hood, range sync and unknown-parent sync. Range sync is used when the start point of syncing is known. In the case of historical and checkpoint sync the starting points are well defined, genesis and the last finalized epoch boundary. Snapshot sync is not supported by Lodestar. If the starting point for sync is not known Lodestar must first determine where the starting point is. While the discussion about how that happens is out of scope for this document, the gist is that the beacon node will listen to gossipsub for blocks being broadcast on the network. It will also request [`MetaData`](https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/p2p-interface.md#getmetadata) from its peers and use that to start requesting the correct blocks from the network. +The implementation of the different syncing styles in Lodestar are actually one of two types under the hood, range sync and unknown-parent sync. Range sync is used when the start point of syncing is known. In the case of historical and checkpoint sync the starting points are well defined, genesis and the last finalized epoch boundary. Snapshot sync is not supported by Lodestar. If the starting point for sync is not known Lodestar must first determine where the starting point is. While the discussion about how that happens is out of scope for this document, the gist is that the beacon node will listen to gossipsub for blocks being broadcast on the network. It will also request [`MetaData`](https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/p2p-interface.md#getmetadata-v1) from its peers and use that to start requesting the correct blocks from the network. There are several flags that can be used to configure the sync process. diff --git a/packages/beacon-node/src/chain/opPools/utils.ts b/packages/beacon-node/src/chain/opPools/utils.ts index e136bf1d4094..ea4b2f5a71c3 100644 --- a/packages/beacon-node/src/chain/opPools/utils.ts +++ b/packages/beacon-node/src/chain/opPools/utils.ts @@ -41,7 +41,7 @@ export function isValidBlsToExecutionChangeForBlockInclusion( state: CachedBeaconStateAllForks, signedBLSToExecutionChange: capella.SignedBLSToExecutionChange ): boolean { - // For each condition from https://github.com/ethereum/consensus-specs/blob/dev/specs/capella/beacon-chain.md#new-process_bls_to_execution_change + // For each condition from https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/capella/beacon-chain.md#new-process_bls_to_execution_change // // 1. assert address_change.validator_index < len(state.validators): // If valid before will always be valid in the future, no need to check diff --git a/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts b/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts index 8eaf1bf36791..20a15505bc1d 100644 --- a/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts +++ b/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts @@ -441,7 +441,7 @@ export async function produceBlockBody( parentBlockRoot: toRootHex(parentBlockRoot), feeRecipient, }); - // https://github.com/ethereum/consensus-specs/blob/dev/specs/deneb/validator.md#constructing-the-beaconblockbody + // https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/deneb/validator.md#constructing-the-beaconblockbody const prepareRes = await prepareExecutionPayload( this, this.logger, diff --git a/packages/beacon-node/src/chain/validation/lightClientFinalityUpdate.ts b/packages/beacon-node/src/chain/validation/lightClientFinalityUpdate.ts index e06b61768e5e..58d20a78c8d2 100644 --- a/packages/beacon-node/src/chain/validation/lightClientFinalityUpdate.ts +++ b/packages/beacon-node/src/chain/validation/lightClientFinalityUpdate.ts @@ -6,7 +6,7 @@ import {LightClientError, LightClientErrorCode} from "../errors/lightClientError import {IBeaconChain} from "../interface.js"; import {updateReceivedTooEarly} from "./lightClientOptimisticUpdate.js"; -// https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/p2p-interface.md#light_client_finality_update +// https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/altair/light-client/p2p-interface.md#light_client_finality_update export function validateLightClientFinalityUpdate( config: ChainForkConfig, chain: IBeaconChain, diff --git a/packages/beacon-node/src/chain/validation/lightClientOptimisticUpdate.ts b/packages/beacon-node/src/chain/validation/lightClientOptimisticUpdate.ts index ae9a83086218..7e10d19b4e75 100644 --- a/packages/beacon-node/src/chain/validation/lightClientOptimisticUpdate.ts +++ b/packages/beacon-node/src/chain/validation/lightClientOptimisticUpdate.ts @@ -6,7 +6,7 @@ import {GossipAction} from "../errors/index.js"; import {LightClientError, LightClientErrorCode} from "../errors/lightClientError.js"; import {IBeaconChain} from "../interface.js"; -// https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/p2p-interface.md#light_client_optimistic_update +// https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/altair/light-client/p2p-interface.md#light_client_optimistic_update export function validateLightClientOptimisticUpdate( config: ChainForkConfig, chain: IBeaconChain, diff --git a/packages/beacon-node/src/chain/validatorMonitor.ts b/packages/beacon-node/src/chain/validatorMonitor.ts index 0b7a2c6a0381..0a2689ad5e70 100644 --- a/packages/beacon-node/src/chain/validatorMonitor.ts +++ b/packages/beacon-node/src/chain/validatorMonitor.ts @@ -889,7 +889,7 @@ function renderAttestationSummary( summary: AttestationSummary | undefined, flags: ParticipationFlags ): string { - // Reference https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/beacon-chain.md#get_attestation_participation_flag_indices + // Reference https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/altair/beacon-chain.md#get_attestation_participation_flag_indices // // is_matching_source = data.source == justified_checkpoint // is_matching_target = is_matching_source and data.target.root == get_block_root(state, data.target.epoch) diff --git a/packages/beacon-node/src/execution/engine/interface.ts b/packages/beacon-node/src/execution/engine/interface.ts index 3fdac6c45f32..70555f16ef98 100644 --- a/packages/beacon-node/src/execution/engine/interface.ts +++ b/packages/beacon-node/src/execution/engine/interface.ts @@ -128,7 +128,7 @@ export interface IExecutionEngine { * corresponding state, up to and including finalized_block_hash. * * The call of the notify_forkchoice_updated function maps on the POS_FORKCHOICE_UPDATED event defined in the EIP-3675. - * https://github.com/ethereum/consensus-specs/blob/dev/specs/merge/fork-choice.md#notify_forkchoice_updated + * https://github.com/ethereum/consensus-specs/blob/v1.1.7/specs/merge/fork-choice.md#notify_forkchoice_updated * * Should be called in response to fork-choice head and finalized events */ @@ -145,7 +145,7 @@ export interface IExecutionEngine { * since the corresponding call to prepare_payload method. * * Required for block producing - * https://github.com/ethereum/consensus-specs/blob/dev/specs/merge/validator.md#get_payload + * https://github.com/ethereum/consensus-specs/blob/v1.1.7/specs/merge/validator.md#get_payload */ getPayload( fork: ForkName, diff --git a/packages/beacon-node/src/network/network.ts b/packages/beacon-node/src/network/network.ts index 6afe85dbad2e..396a5e7fc20f 100644 --- a/packages/beacon-node/src/network/network.ts +++ b/packages/beacon-node/src/network/network.ts @@ -721,7 +721,7 @@ export class Network implements INetwork { try { // messages SHOULD be broadcast after SYNC_MESSAGE_DUE_BPS of slot has transpired - // https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/p2p-interface.md#sync-committee + // https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/altair/light-client/p2p-interface.md#sync-committee await this.waitForSyncMessageCutoff(finalityUpdate.signatureSlot); await this.publishLightClientFinalityUpdate(finalityUpdate); } catch (e) { @@ -738,7 +738,7 @@ export class Network implements INetwork { try { // messages SHOULD be broadcast after SYNC_MESSAGE_DUE_BPS of slot has transpired - // https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/p2p-interface.md#sync-committee + // https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/altair/light-client/p2p-interface.md#sync-committee await this.waitForSyncMessageCutoff(optimisticUpdate.signatureSlot); await this.publishLightClientOptimisticUpdate(optimisticUpdate); } catch (e) { diff --git a/packages/beacon-node/src/network/reqresp/ReqRespBeaconNode.ts b/packages/beacon-node/src/network/reqresp/ReqRespBeaconNode.ts index a59a5b5026cf..9a3d5dc83204 100644 --- a/packages/beacon-node/src/network/reqresp/ReqRespBeaconNode.ts +++ b/packages/beacon-node/src/network/reqresp/ReqRespBeaconNode.ts @@ -58,7 +58,7 @@ export type ReqRespBeaconNodeOpts = ReqRespOpts & {disableLightClientServer?: bo * Implementation of Ethereum Consensus p2p Req/Resp domain. * For the spec that this code is based on, see: * https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/p2p-interface.md#the-reqresp-domain - * https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/p2p-interface.md#the-reqresp-domain + * https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/altair/light-client/p2p-interface.md#the-reqresp-domain */ export class ReqRespBeaconNode extends ReqResp { private readonly metadataController: MetadataController; diff --git a/packages/beacon-node/src/sync/backfill/backfill.ts b/packages/beacon-node/src/sync/backfill/backfill.ts index 52b0bd8646e6..c129c17facfd 100644 --- a/packages/beacon-node/src/sync/backfill/backfill.ts +++ b/packages/beacon-node/src/sync/backfill/backfill.ts @@ -127,7 +127,7 @@ export class BackfillSync extends (EventEmitter as {new (): BackfillSyncEmitter} private wsValidated = false; /** - * From https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/weak-subjectivity.md + * From https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/weak-subjectivity.md * * * If diff --git a/packages/beacon-node/src/sync/constants.ts b/packages/beacon-node/src/sync/constants.ts index 7bef3598b181..25e24f620db5 100644 --- a/packages/beacon-node/src/sync/constants.ts +++ b/packages/beacon-node/src/sync/constants.ts @@ -59,7 +59,7 @@ export const BATCH_BUFFER_SIZE = Math.ceil(10 / EPOCHS_PER_BATCH); /** * Maximum number of concurrent requests to perform with a SyncChain. - * This is according to the spec https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/p2p-interface.md + * This is according to the spec https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/p2p-interface.md */ export const MAX_CONCURRENT_REQUESTS = 2; diff --git a/packages/beacon-node/test/spec/presets/epoch_processing.test.ts b/packages/beacon-node/test/spec/presets/epoch_processing.test.ts index d6ba7efefe39..67a5f226957d 100644 --- a/packages/beacon-node/test/spec/presets/epoch_processing.test.ts +++ b/packages/beacon-node/test/spec/presets/epoch_processing.test.ts @@ -55,7 +55,7 @@ const epochTransitionFns: Record = { }; /** - * https://github.com/ethereum/consensus-specs/blob/dev/tests/formats/epoch_processing/README.md + * https://github.com/ethereum/consensus-specs/blob/v1.6.1/tests/formats/epoch_processing/README.md */ type EpochTransitionCacheingTestCase = { meta?: {bls_setting?: bigint}; @@ -87,7 +87,7 @@ const epochProcessing = if (testcase.post === undefined) { // If post.ssz_snappy is not value, the sub-transition processing is aborted - // https://github.com/ethereum/consensus-specs/blob/dev/tests/formats/epoch_processing/README.md#postssz_snappy + // https://github.com/ethereum/consensus-specs/blob/v1.6.1/tests/formats/epoch_processing/README.md#postssz_snappy expect(() => epochTransitionFn(state, epochTransitionCache)).toThrow(); } else { epochTransitionFn(state, epochTransitionCache); diff --git a/packages/beacon-node/test/spec/presets/finality.test.ts b/packages/beacon-node/test/spec/presets/finality.test.ts index ea3a60e212bd..1c86dd6c9cab 100644 --- a/packages/beacon-node/test/spec/presets/finality.test.ts +++ b/packages/beacon-node/test/spec/presets/finality.test.ts @@ -70,7 +70,7 @@ export function generateBlocksSZZTypeMapping(fork: ForkName, n: number): BlocksS * ``` * {blocks_count: 16} * ``` - * https://github.com/ethereum/consensus-specs/blob/dev/tests/formats/finality/README.md + * https://github.com/ethereum/consensus-specs/blob/v1.6.1/tests/formats/finality/README.md */ type FinalityTestCase = { [k: string]: altair.SignedBeaconBlock | unknown | null | undefined; diff --git a/packages/beacon-node/test/spec/presets/ssz_static.test.ts b/packages/beacon-node/test/spec/presets/ssz_static.test.ts index 3a4af2c789a9..0a7d825b6a0f 100644 --- a/packages/beacon-node/test/spec/presets/ssz_static.test.ts +++ b/packages/beacon-node/test/spec/presets/ssz_static.test.ts @@ -19,7 +19,7 @@ import {RunnerType} from "../utils/types.js"; // | serialized.ssz_snappy // | value.yaml // -// Docs: https://github.com/ethereum/consensus-specs/blob/master/tests/formats/ssz_static/core.md +// Docs: https://github.com/ethereum/consensus-specs/blob/v1.6.1/tests/formats/ssz_static/core.md type Types = Record>; diff --git a/packages/beacon-node/test/spec/utils/specTestIterator.ts b/packages/beacon-node/test/spec/utils/specTestIterator.ts index 384eadf81b0b..fead70d52f61 100644 --- a/packages/beacon-node/test/spec/utils/specTestIterator.ts +++ b/packages/beacon-node/test/spec/utils/specTestIterator.ts @@ -103,7 +103,7 @@ export const defaultSkipOpts: SkipOpts = { * tests / mainnet / altair / ssz_static / Validator / ssz_random / case_0/roots.yaml * tests / mainnet / altair / fork / fork / pyspec_tests / altair_fork_random_0/meta.yaml * ``` - * Ref: https://github.com/ethereum/consensus-specs/tree/dev/tests/formats#test-structure + * Ref: https://github.com/ethereum/consensus-specs/blob/v1.6.1/tests/formats/README.md#test-structure */ export function specTestIterator( configDirpath: string, diff --git a/packages/beacon-node/test/spec/utils/sszTestCaseParser.ts b/packages/beacon-node/test/spec/utils/sszTestCaseParser.ts index 90dca88b67ca..f608dd10b768 100644 --- a/packages/beacon-node/test/spec/utils/sszTestCaseParser.ts +++ b/packages/beacon-node/test/spec/utils/sszTestCaseParser.ts @@ -18,7 +18,7 @@ export type ValidTestCaseData = { * | serialized.ssz_snappy * | value.yaml * - * Docs: https://github.com/ethereum/consensus-specs/blob/master/tests/formats/ssz_static/core.md + * Docs: https://github.com/ethereum/consensus-specs/blob/v1.6.1/tests/formats/ssz_static/core.md */ export function parseSszStaticTestcase(dirpath: string): ValidTestCaseData { return parseSszValidTestcase(dirpath, "roots.yaml"); @@ -33,7 +33,7 @@ export function parseSszStaticTestcase(dirpath: string): ValidTestCaseData { * | serialized.ssz_snappy * | value.yaml * - * Docs: https://github.com/ethereum/eth2.0-specs/blob/master/tests/formats/ssz_generic/README.md + * Docs: https://github.com/ethereum/consensus-specs/blob/v1.6.1/tests/formats/ssz_generic/README.md */ export function parseSszGenericValidTestcase(dirpath: string): ValidTestCaseData { return parseSszValidTestcase(dirpath, "meta.yaml"); @@ -71,7 +71,7 @@ export function parseSszValidTestcase(dirpath: string, metaFilename: string): Va * | vec_bool_0 * | serialized.ssz_snappy * - * Docs: https://github.com/ethereum/eth2.0-specs/blob/master/tests/formats/ssz_generic/README.md + * Docs: https://github.com/ethereum/consensus-specs/blob/v1.6.1/tests/formats/ssz_generic/README.md */ export function parseSszGenericInvalidTestcase(dirpath: string) { // The serialized value is stored in serialized.ssz_snappy diff --git a/packages/config/src/chainConfig/configs/mainnet.ts b/packages/config/src/chainConfig/configs/mainnet.ts index 10a2242c6910..db958b3e04e8 100644 --- a/packages/config/src/chainConfig/configs/mainnet.ts +++ b/packages/config/src/chainConfig/configs/mainnet.ts @@ -3,7 +3,7 @@ import {fromHex as b} from "@lodestar/utils"; import {ChainConfig} from "../types.js"; // Mainnet config -// https://github.com/ethereum/consensus-specs/blob/dev/configs/mainnet.yaml +// https://github.com/ethereum/consensus-specs/blob/master/configs/mainnet.yaml export const chainConfig: ChainConfig = { // Extends the mainnet preset diff --git a/packages/config/src/chainConfig/configs/minimal.ts b/packages/config/src/chainConfig/configs/minimal.ts index 200e64ed61af..92d6910b7e35 100644 --- a/packages/config/src/chainConfig/configs/minimal.ts +++ b/packages/config/src/chainConfig/configs/minimal.ts @@ -3,7 +3,7 @@ import {fromHex as b} from "@lodestar/utils"; import {ChainConfig} from "../types.js"; // Minimal config -// https://github.com/ethereum/consensus-specs/blob/dev/configs/minimal.yaml +// https://github.com/ethereum/consensus-specs/blob/master/configs/minimal.yaml export const chainConfig: ChainConfig = { // Extends the minimal preset diff --git a/packages/fork-choice/README.md b/packages/fork-choice/README.md index e51b0aa1d256..eb52ca219b10 100644 --- a/packages/fork-choice/README.md +++ b/packages/fork-choice/README.md @@ -2,7 +2,7 @@ > This package is part of [ChainSafe's Lodestar](https://lodestar.chainsafe.io) project -Lodestar implementation of the [Ethereum Consensus fork choice](https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/fork-choice.md). +Lodestar implementation of the [Ethereum Consensus fork choice](https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/fork-choice.md). ## Usage diff --git a/packages/fork-choice/src/forkChoice/forkChoice.ts b/packages/fork-choice/src/forkChoice/forkChoice.ts index 2d0cef22540c..34416d97066a 100644 --- a/packages/fork-choice/src/forkChoice/forkChoice.ts +++ b/packages/fork-choice/src/forkChoice/forkChoice.ts @@ -429,7 +429,7 @@ export class ForkChoice implements IForkChoice { } // No reorg if parentBlock is "not strong" ie. parentBlock's weight is less than or equal to (REORG_PARENT_WEIGHT_THRESHOLD = 160)% of total attester weight - // https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/fork-choice.md#is_parent_strong + // https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/fork-choice.md#is_parent_strong const parentThreshold = getCommitteeFraction(this.fcStore.justified.totalBalance, { slotsPerEpoch: SLOTS_PER_EPOCH, committeePercent: this.config.REORG_PARENT_WEIGHT_THRESHOLD, @@ -1839,7 +1839,7 @@ export class ForkChoice implements IForkChoice { } } -// Approximate https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/fork-choice.md#calculate_committee_fraction +// Approximate https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/fork-choice.md#calculate_committee_fraction // Calculates proposer boost score when committeePercent = config.PROPOSER_SCORE_BOOST export function getCommitteeFraction( justifiedTotalActiveBalanceByIncrement: number, diff --git a/packages/params/src/presets/mainnet.ts b/packages/params/src/presets/mainnet.ts index 349f0058031e..4ec676b16471 100644 --- a/packages/params/src/presets/mainnet.ts +++ b/packages/params/src/presets/mainnet.ts @@ -1,7 +1,7 @@ import {BeaconPreset} from "../types.js"; // Mainnet preset -// https://github.com/ethereum/consensus-specs/tree/dev/presets/mainnet +// https://github.com/ethereum/consensus-specs/tree/master/presets/mainnet export const mainnetPreset: BeaconPreset = { // Misc diff --git a/packages/params/src/presets/minimal.ts b/packages/params/src/presets/minimal.ts index 42486961ce2d..08080d83f7b7 100644 --- a/packages/params/src/presets/minimal.ts +++ b/packages/params/src/presets/minimal.ts @@ -1,7 +1,7 @@ import {BeaconPreset} from "../types.js"; // Minimal preset -// https://github.com/ethereum/consensus-specs/tree/dev/presets/minimal +// https://github.com/ethereum/consensus-specs/tree/master/presets/minimal export const minimalPreset: BeaconPreset = { // Misc diff --git a/packages/prover/README.md b/packages/prover/README.md index a191b0624639..7bbfda982fc5 100644 --- a/packages/prover/README.md +++ b/packages/prover/README.md @@ -148,7 +148,7 @@ If your project is using some provider type which is not among above list, you h ## What you need -You will need to go over the [specification](https://github.com/ethereum/beacon-apis). You will also need to have a [basic understanding of lightclient](https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/light-client.md). +You will need to go over the [specification](https://github.com/ethereum/beacon-apis). You will also need to have a [basic understanding of lightclient](https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/altair/light-client/light-client.md). ## Getting started diff --git a/packages/prover/src/constants.ts b/packages/prover/src/constants.ts index 5a9eefd0f3ca..e04b13de1cf1 100644 --- a/packages/prover/src/constants.ts +++ b/packages/prover/src/constants.ts @@ -1,4 +1,4 @@ -// https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/p2p-interface.md#configuration +// https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/altair/light-client/p2p-interface.md#configuration export const MAX_REQUEST_LIGHT_CLIENT_UPDATES = 128; export const MAX_PAYLOAD_HISTORY = 32; export const VERIFICATION_FAILED_RESPONSE_CODE = -33091; diff --git a/packages/spec-test-util/src/sszGeneric.ts b/packages/spec-test-util/src/sszGeneric.ts index 1898a11e7870..7901bbfe433d 100644 --- a/packages/spec-test-util/src/sszGeneric.ts +++ b/packages/spec-test-util/src/sszGeneric.ts @@ -42,7 +42,7 @@ export function parseSszValidTestcase(dirpath: string, metaFilename: string): Va * | vec_bool_0 * | serialized.ssz_snappy * - * Docs: https://github.com/ethereum/eth2.0-specs/blob/master/tests/formats/ssz_generic/README.md + * Docs: https://github.com/ethereum/consensus-specs/blob/v1.6.1/tests/formats/ssz_generic/README.md */ export function parseSszGenericInvalidTestcase(dirpath: string) { // The serialized value is stored in serialized.ssz_snappy diff --git a/packages/state-transition/src/util/execution.ts b/packages/state-transition/src/util/execution.ts index 0f5b450f5ca9..5dfcb3a653e8 100644 --- a/packages/state-transition/src/util/execution.ts +++ b/packages/state-transition/src/util/execution.ts @@ -143,7 +143,7 @@ export function executionPayloadToPayloadHeader(fork: ForkSeq, payload: Executio } if (fork >= ForkSeq.deneb) { - // https://github.com/ethereum/consensus-specs/blob/dev/specs/eip4844/beacon-chain.md#process_execution_payload + // https://github.com/ethereum/consensus-specs/blob/v1.3.0-rc.2/specs/eip4844/beacon-chain.md#process_execution_payload (bellatrixPayloadFields as deneb.ExecutionPayloadHeader).blobGasUsed = ( payload as deneb.ExecutionPayloadHeader | deneb.ExecutionPayload ).blobGasUsed; diff --git a/packages/types/src/deneb/sszTypes.ts b/packages/types/src/deneb/sszTypes.ts index 9275ac4c4da5..0e18764c7093 100644 --- a/packages/types/src/deneb/sszTypes.ts +++ b/packages/types/src/deneb/sszTypes.ts @@ -18,10 +18,10 @@ const {UintNum64, Slot, Root, BLSSignature, UintBn64, UintBn256, Bytes32, Bytes4 primitiveSsz; // Polynomial commitments -// https://github.com/ethereum/consensus-specs/blob/dev/specs/eip4844/polynomial-commitments.md +// https://github.com/ethereum/consensus-specs/blob/v1.3.0-rc.2/specs/eip4844/polynomial-commitments.md // Custom types -// https://github.com/ethereum/consensus-specs/blob/dev/specs/eip4844/polynomial-commitments.md#custom-types +// https://github.com/ethereum/consensus-specs/blob/v1.3.0-rc.2/specs/eip4844/polynomial-commitments.md#custom-types export const G1Point = Bytes48; export const G2Point = Bytes96; export const BLSFieldElement = Bytes32; @@ -31,7 +31,7 @@ export const KZGProof = Bytes48; // Beacon chain // Custom types -// https://github.com/ethereum/consensus-specs/blob/dev/specs/eip4844/beacon-chain.md#custom-types +// https://github.com/ethereum/consensus-specs/blob/v1.3.0-rc.2/specs/eip4844/beacon-chain.md#custom-types export const Blob = new ByteVectorType(BYTES_PER_FIELD_ELEMENT * FIELD_ELEMENTS_PER_BLOB); export const Blobs = new ListCompositeType(Blob, MAX_BLOB_COMMITMENTS_PER_BLOCK); @@ -62,7 +62,7 @@ export const BlobIdentifier = new ContainerType( ); // Beacon Chain types -// https://github.com/ethereum/consensus-specs/blob/dev/specs/eip4844/beacon-chain.md#containers +// https://github.com/ethereum/consensus-specs/blob/v1.3.0-rc.2/specs/eip4844/beacon-chain.md#containers export const ExecutionPayload = new ContainerType( { diff --git a/specrefs/functions.yml b/specrefs/functions.yml index 7c53864f5fb6..afa88c57aad5 100644 --- a/specrefs/functions.yml +++ b/specrefs/functions.yml @@ -5484,7 +5484,7 @@ - name: is_parent_strong#phase0 sources: - file: packages/fork-choice/src/forkChoice/forkChoice.ts - search: "// https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/fork-choice.md#is_parent_strong" + search: "// https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/fork-choice.md#is_parent_strong" spec: | def is_parent_strong(store: Store, root: Root) -> bool: From 70938e1eecf87a793e8ac2f4cc0689f2047291e7 Mon Sep 17 00:00:00 2001 From: NC <17676176+ensi321@users.noreply.github.com> Date: Fri, 20 Mar 2026 09:19:37 -0700 Subject: [PATCH 55/63] feat: add gloas execution payload envelope import pipeline (#8962) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Implement the execution payload import pipeline for Gloas (ePBS) - Add `PayloadEnvelopeInput` class to track payload envelope + data columns - Add `importExecutionPayload` function for EL verification and execution payload state transition - Add `SeenPayloadEnvelopeInput` cache for managing payload inputs - Remove `SeenExecutionPayloadEnvelopes` in favour of `SeenPayloadEnvelopeInput` - Integrate with gossip handlers and API for payload publishing ## Background In Gloas (ePBS), beacon blocks and execution payloads have separate validity requirements: - **Beacon blocks** are valid immediately (builder may not reveal payload) - **Execution payloads** require data availability (payload envelope + sampled columns) ## Key Changes ### New Files - `payloadEnvelopeInput/payloadEnvelopeInput.ts` - Tracks bid + envelope + columns with 4-state machine - `importExecutionPayload.ts` - EL verification, signature check, execution payload state transition - `writePayloadEnvelopeInputToDb.ts` - Persists envelope + columns to DB - `seenPayloadEnvelopeInput.ts` - Cache with finalization pruning ### Modified Files - `importBlock.ts` - Creates `PayloadEnvelopeInput` during Gloas block import - `gossipHandlers.ts` - Handles `execution_payload` topic, triggers import on completion - `extractSlotRootFns.ts` - SSZ helpers for payload envelope queueing - `executionPayloadEnvelope.ts` - Validation using block post-state (not head state) - `fork-choice` - Removed `builderIndex`/`blockHashHex` from ProtoBlock (now in PayloadEnvelopeInput) ## TODO - Gloas `data_column_sidecar` gossip handler integration (commented, needs Gloas-specific validation) 🤖 Generated with [Claude Code](https://claude.ai/code) --------- Co-authored-by: Claude Opus 4.6 Co-authored-by: twoeths <10568965+twoeths@users.noreply.github.com> Co-authored-by: Nico Flaig Co-authored-by: Tuyen Nguyen --- packages/api/src/beacon/routes/events.ts | 14 + .../api/test/unit/beacon/oapiSpec.test.ts | 1 - .../api/test/unit/beacon/testData/events.ts | 4 + .../src/api/impl/beacon/blocks/index.ts | 53 ++- .../src/chain/blocks/blockInput/types.ts | 6 +- .../src/chain/blocks/importBlock.ts | 38 +- .../chain/blocks/importExecutionPayload.ts | 241 +++++++++++++ .../blocks/payloadEnvelopeInput/index.ts | 2 + .../payloadEnvelopeInput.ts | 336 ++++++++++++++++++ .../blocks/payloadEnvelopeInput/types.ts | 33 ++ .../chain/blocks/payloadEnvelopeProcessor.ts | 61 ++++ .../beacon-node/src/chain/blocks/types.ts | 8 + .../blocks/writePayloadEnvelopeInputToDb.ts | 55 +++ packages/beacon-node/src/chain/chain.ts | 40 ++- .../chain/errors/executionPayloadEnvelope.ts | 8 +- .../beacon-node/src/chain/forkChoice/index.ts | 10 - packages/beacon-node/src/chain/interface.ts | 9 +- .../chain/produceBlock/computeNewStateRoot.ts | 7 +- .../beacon-node/src/chain/regen/interface.ts | 2 + .../beacon-node/src/chain/seenCache/index.ts | 2 +- .../seenCache/seenExecutionPayloadEnvelope.ts | 34 -- .../chain/seenCache/seenGossipBlockInput.ts | 4 +- .../seenCache/seenPayloadEnvelopeInput.ts | 106 ++++++ .../validation/executionPayloadEnvelope.ts | 63 ++-- .../beacon-node/src/chain/validatorMonitor.ts | 10 + .../src/metrics/metrics/lodestar.ts | 77 ++++ .../network/processor/extractSlotRootFns.ts | 23 +- .../src/network/processor/gossipHandlers.ts | 38 +- packages/beacon-node/src/util/sszBytes.ts | 100 +++++- .../opPools/aggregatedAttestationPool.test.ts | 4 - .../test/spec/presets/operations.test.ts | 5 +- .../test/unit/util/sszBytes.test.ts | 100 ++++++ packages/beacon-node/test/utils/state.ts | 2 - .../test/utils/validationData/attestation.ts | 2 - .../fork-choice/src/forkChoice/forkChoice.ts | 4 - .../fork-choice/src/protoArray/interface.ts | 6 - .../fork-choice/test/perf/forkChoice/util.ts | 2 - .../test/unit/forkChoice/forkChoice.test.ts | 2 - .../unit/forkChoice/getProposerHead.test.ts | 6 - .../shouldOverrideForkChoiceUpdate.test.ts | 6 - .../protoArray/executionStatusUpdates.test.ts | 2 - .../unit/protoArray/getCommonAncestor.test.ts | 4 - .../test/unit/protoArray/gloas.test.ts | 2 - .../test/unit/protoArray/protoArray.test.ts | 6 - .../block/processExecutionPayloadEnvelope.ts | 53 +-- .../signatureSets/executionPayloadEnvelope.ts | 28 +- .../src/stateView/beaconStateView.ts | 13 +- .../src/stateView/interface.ts | 6 +- 48 files changed, 1429 insertions(+), 209 deletions(-) create mode 100644 packages/beacon-node/src/chain/blocks/importExecutionPayload.ts create mode 100644 packages/beacon-node/src/chain/blocks/payloadEnvelopeInput/index.ts create mode 100644 packages/beacon-node/src/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.ts create mode 100644 packages/beacon-node/src/chain/blocks/payloadEnvelopeInput/types.ts create mode 100644 packages/beacon-node/src/chain/blocks/payloadEnvelopeProcessor.ts create mode 100644 packages/beacon-node/src/chain/blocks/writePayloadEnvelopeInputToDb.ts delete mode 100644 packages/beacon-node/src/chain/seenCache/seenExecutionPayloadEnvelope.ts create mode 100644 packages/beacon-node/src/chain/seenCache/seenPayloadEnvelopeInput.ts diff --git a/packages/api/src/beacon/routes/events.ts b/packages/api/src/beacon/routes/events.ts index 4b1dfbe33d60..06ae34700eb9 100644 --- a/packages/api/src/beacon/routes/events.ts +++ b/packages/api/src/beacon/routes/events.ts @@ -88,6 +88,8 @@ export enum EventType { blobSidecar = "blob_sidecar", /** The node has received a valid DataColumnSidecar (from P2P or API) */ dataColumnSidecar = "data_column_sidecar", + /** The node has verified that the execution payload and blobs for a block are available */ + executionPayloadAvailable = "execution_payload_available", } export const eventTypes: {[K in EventType]: K} = { @@ -108,6 +110,7 @@ export const eventTypes: {[K in EventType]: K} = { [EventType.payloadAttributes]: EventType.payloadAttributes, [EventType.blobSidecar]: EventType.blobSidecar, [EventType.dataColumnSidecar]: EventType.dataColumnSidecar, + [EventType.executionPayloadAvailable]: EventType.executionPayloadAvailable, }; export type EventData = { @@ -157,6 +160,10 @@ export type EventData = { [EventType.payloadAttributes]: {version: ForkName; data: SSEPayloadAttributes}; [EventType.blobSidecar]: BlobSidecarSSE; [EventType.dataColumnSidecar]: DataColumnSidecarSSE; + [EventType.executionPayloadAvailable]: { + slot: Slot; + blockRoot: RootHex; + }; }; export type BeaconEvent = {[K in EventType]: {type: K; message: EventData[K]}}[EventType]; @@ -311,6 +318,13 @@ export function getTypeByEvent(config: ChainForkConfig): {[K in EventType]: Type [EventType.payloadAttributes]: WithVersion((fork) => getPostBellatrixForkTypes(fork).SSEPayloadAttributes), [EventType.blobSidecar]: blobSidecarSSE, [EventType.dataColumnSidecar]: dataColumnSidecarSSE, + [EventType.executionPayloadAvailable]: new ContainerType( + { + slot: ssz.Slot, + blockRoot: stringType, + }, + {jsonCase: "eth2"} + ), [EventType.lightClientOptimisticUpdate]: WithVersion( (fork) => getPostAltairForkTypes(fork).LightClientOptimisticUpdate diff --git a/packages/api/test/unit/beacon/oapiSpec.test.ts b/packages/api/test/unit/beacon/oapiSpec.test.ts index 590942f8f992..1e72ddf585cf 100644 --- a/packages/api/test/unit/beacon/oapiSpec.test.ts +++ b/packages/api/test/unit/beacon/oapiSpec.test.ts @@ -78,7 +78,6 @@ runTestCheckAgainstSpec(openApiJson, definitions, testDatas, ignoredOperations, const ignoredTopics: string[] = [ // TODO GLOAS: required by v5.0.0-alpha.0 - "execution_payload_available", "execution_payload_bid", "payload_attestation_message", ]; diff --git a/packages/api/test/unit/beacon/testData/events.ts b/packages/api/test/unit/beacon/testData/events.ts index d9da1503107c..dbab2401a8f8 100644 --- a/packages/api/test/unit/beacon/testData/events.ts +++ b/packages/api/test/unit/beacon/testData/events.ts @@ -274,4 +274,8 @@ export const eventTestData: EventData = { "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505", ], }), + [EventType.executionPayloadAvailable]: { + slot: 10, + blockRoot: "0x9a2fefd2fdb57f74993c7780ea5b9030d2897b615b89f808011ca5aebed54eaf", + }, }; diff --git a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts index 1b45b9457033..ad216e5a3b72 100644 --- a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts @@ -35,6 +35,7 @@ import { } from "@lodestar/types"; import {fromHex, sleep, toHex, toRootHex} from "@lodestar/utils"; import {BlockInputSource, isBlockInputBlobs, isBlockInputColumns} from "../../../../chain/blocks/blockInput/index.js"; +import {PayloadEnvelopeInputSource} from "../../../../chain/blocks/payloadEnvelopeInput/index.js"; import {ImportBlockOpts} from "../../../../chain/blocks/types.js"; import {verifyBlocksInEpoch} from "../../../../chain/blocks/verifyBlock.js"; import {BeaconChain} from "../../../../chain/chain.js"; @@ -48,6 +49,7 @@ import { ProduceFullGloas, } from "../../../../chain/produceBlock/index.js"; import {validateGossipBlock} from "../../../../chain/validation/block.js"; +import {validateApiExecutionPayloadEnvelope} from "../../../../chain/validation/executionPayloadEnvelope.js"; import {OpSource} from "../../../../chain/validatorMonitor.js"; import { computePreFuluKzgCommitmentsInclusionProof, @@ -659,6 +661,8 @@ export function getBeaconBlockApi({ throw new ApiError(400, `Envelope slot ${slot} does not match block slot ${block.slot}`); } + await validateApiExecutionPayloadEnvelope(chain, signedExecutionPayloadEnvelope); + const isSelfBuild = envelope.builderIndex === BUILDER_INDEX_SELF_BUILD; let dataColumnSidecars: gloas.DataColumnSidecars = []; @@ -676,6 +680,7 @@ export function getBeaconBlockApi({ } if (cachedResult.cells && cachedResult.blobsBundle.commitments.length > 0) { + const timer = metrics?.peerDas.dataColumnSidecarComputationTime.startTimer(); const cellsAndProofs = cachedResult.cells.map((rowCells, rowIndex) => ({ cells: rowCells, proofs: cachedResult.blobsBundle.proofs.slice( @@ -685,24 +690,42 @@ export function getBeaconBlockApi({ })); dataColumnSidecars = getDataColumnSidecarsForGloas(slot, envelope.beaconBlockRoot, cellsAndProofs); + timer?.(); } } else { // TODO GLOAS: will this api be used by builders or only for self-building? } - // TODO GLOAS: Verify execution payload envelope signature - // For self-builds, the proposer signs with their own validator key - // For external builders, verify using the builder's registered pubkey - // Use verify_execution_payload_envelope_signature(state, signed_envelope) + // If called near a slot boundary (e.g. late in slot N-1), hold briefly so gossip aligns with slot N. + const msToBlockSlot = computeTimeAtSlot(config, slot, chain.genesisTime) * 1000 - Date.now(); + if (msToBlockSlot <= MAX_API_CLOCK_DISPARITY_MS && msToBlockSlot > 0) { + await sleep(msToBlockSlot); + } - // TODO GLOAS: Process execution payload via state transition - // Call process_execution_payload(state, signed_envelope, execution_engine) + // TODO GLOAS: if block and payload are submitted in parallel, payloadInput may not yet exist. + // A queuing mechanism is needed to handle this case. See https://github.com/ChainSafe/lodestar/issues/8915 + const payloadInput = chain.seenPayloadEnvelopeInputCache.get(blockRootHex); + if (!payloadInput) { + throw new ApiError(404, `PayloadEnvelopeInput not found for block root ${blockRootHex}`); + } - // TODO GLOAS: Update fork choice with the execution payload - // Call on_execution_payload(store, signed_envelope) to update fork choice state + payloadInput.addPayloadEnvelope({ + envelope: signedExecutionPayloadEnvelope, + source: PayloadEnvelopeInputSource.api, + seenTimestampSec, + peerIdStr: undefined, + }); - // TODO GLOAS: Add envelope and data columns to block input via seenBlockInputCache - // and trigger block import (Gloas block import requires both beacon block and envelope) + if (dataColumnSidecars.length > 0) { + for (const columnSidecar of dataColumnSidecars) { + payloadInput.addColumn({ + columnSidecar, + source: PayloadEnvelopeInputSource.api, + seenTimestampSec, + peerIdStr: undefined, + }); + } + } const valLogMeta = { slot, @@ -712,23 +735,19 @@ export function getBeaconBlockApi({ dataColumns: dataColumnSidecars.length, }; - // If called near a slot boundary (e.g. late in slot N-1), hold briefly so gossip aligns with slot N. - const msToBlockSlot = computeTimeAtSlot(config, slot, chain.genesisTime) * 1000 - Date.now(); - if (msToBlockSlot <= MAX_API_CLOCK_DISPARITY_MS && msToBlockSlot > 0) { - await sleep(msToBlockSlot); - } - const delaySec = seenTimestampSec - computeTimeAtSlot(config, slot, chain.genesisTime); metrics?.gossipExecutionPayloadEnvelope.elapsedTimeTillReceived.observe({source: OpSource.api}, delaySec); + chain.validatorMonitor?.registerExecutionPayloadEnvelope(OpSource.api, delaySec, signedExecutionPayloadEnvelope); chain.logger.info("Publishing execution payload envelope", valLogMeta); - // Publish envelope and data columns const publishPromises = [ // Gossip the signed execution payload envelope first () => network.publishSignedExecutionPayloadEnvelope(signedExecutionPayloadEnvelope), // For self-builds, publish all data column sidecars ...dataColumnSidecars.map((dataColumnSidecar) => () => network.publishDataColumnSidecar(dataColumnSidecar)), + // Import execution payload. Signature already verified above + () => chain.processExecutionPayload(payloadInput, {validSignature: true}), ]; const sentPeersArr = await promiseAllMaybeAsync(publishPromises); diff --git a/packages/beacon-node/src/chain/blocks/blockInput/types.ts b/packages/beacon-node/src/chain/blocks/blockInput/types.ts index e932360a9269..06ab97acb45a 100644 --- a/packages/beacon-node/src/chain/blocks/blockInput/types.ts +++ b/packages/beacon-node/src/chain/blocks/blockInput/types.ts @@ -1,5 +1,5 @@ import {ForkName} from "@lodestar/params"; -import {ColumnIndex, RootHex, SignedBeaconBlock, Slot, deneb, fulu} from "@lodestar/types"; +import {ColumnIndex, DataColumnSidecars, RootHex, SignedBeaconBlock, Slot, deneb, fulu} from "@lodestar/types"; import {VersionedHashes} from "../../../execution/index.js"; export enum DAType { @@ -107,9 +107,9 @@ export type MissingColumnMeta = { export interface IDataColumnsInput { readonly slot: Slot; readonly blockRootHex: string; - getCustodyColumns(): fulu.DataColumnSidecars; + getCustodyColumns(): DataColumnSidecars; hasComputedAllData(): boolean; - waitForComputedAllData(timeout: number, signal?: AbortSignal): Promise; + waitForComputedAllData(timeout: number, signal?: AbortSignal): Promise; } /** diff --git a/packages/beacon-node/src/chain/blocks/importBlock.ts b/packages/beacon-node/src/chain/blocks/importBlock.ts index c8df51f91f3a..ef6980e374fe 100644 --- a/packages/beacon-node/src/chain/blocks/importBlock.ts +++ b/packages/beacon-node/src/chain/blocks/importBlock.ts @@ -9,7 +9,14 @@ import { getSafeExecutionBlockHash, isGloasBlock, } from "@lodestar/fork-choice"; -import {ForkPostAltair, ForkPostElectra, ForkSeq, MAX_SEED_LOOKAHEAD, SLOTS_PER_EPOCH} from "@lodestar/params"; +import { + ForkPostAltair, + ForkPostElectra, + ForkPostGloas, + ForkSeq, + MAX_SEED_LOOKAHEAD, + SLOTS_PER_EPOCH, +} from "@lodestar/params"; import { CachedBeaconStateAltair, EpochCache, @@ -21,7 +28,17 @@ import { isStartSlotOfEpoch, isStateValidatorsNodesPopulated, } from "@lodestar/state-transition"; -import {Attestation, BeaconBlock, altair, capella, electra, isGloasBeaconBlock, phase0, ssz} from "@lodestar/types"; +import { + Attestation, + BeaconBlock, + SignedBeaconBlock, + altair, + capella, + electra, + isGloasBeaconBlock, + phase0, + ssz, +} from "@lodestar/types"; import {isErrorAborted, toRootHex} from "@lodestar/utils"; import {ZERO_HASH_HEX} from "../../constants/index.js"; import {callInNextEventLoop} from "../../util/eventLoop.js"; @@ -123,6 +140,23 @@ export async function importBlock( // processState manages both block state and payload state variants together for memory/disk management this.regen.processBlockState(blockRootHex, postState); + // For Gloas blocks, create PayloadEnvelopeInput so it's available for later payload import + if (fork >= ForkSeq.gloas) { + this.seenPayloadEnvelopeInputCache.add({ + blockRootHex, + block: block as SignedBeaconBlock, + sampledColumns: this.custodyConfig.sampledColumns, + custodyColumns: this.custodyConfig.custodyColumns, + timeCreatedSec: fullyVerifiedBlock.seenTimestampSec, + }); + this.logger.debug("Created PayloadEnvelopeInput for block", { + slot: blockSlot, + root: blockRootHex, + source: source.source, + ...(opts.seenTimestampSec !== undefined ? {recvToImport: Date.now() / 1000 - opts.seenTimestampSec} : {}), + }); + } + this.metrics?.importBlock.bySource.inc({source: source.source}); this.logger.verbose("Added block to forkchoice and state cache", {slot: blockSlot, root: blockRootHex}); diff --git a/packages/beacon-node/src/chain/blocks/importExecutionPayload.ts b/packages/beacon-node/src/chain/blocks/importExecutionPayload.ts new file mode 100644 index 000000000000..c7c1234cac24 --- /dev/null +++ b/packages/beacon-node/src/chain/blocks/importExecutionPayload.ts @@ -0,0 +1,241 @@ +import {routes} from "@lodestar/api"; +import {ForkName} from "@lodestar/params"; +import { + BeaconStateView, + CachedBeaconStateGloas, + getExecutionPayloadEnvelopeSignatureSet, +} from "@lodestar/state-transition"; +import {processExecutionPayloadEnvelope} from "@lodestar/state-transition/block"; +import {byteArrayEquals, fromHex, toRootHex} from "@lodestar/utils"; +import {ExecutionPayloadStatus} from "../../execution/index.js"; +import {isQueueErrorAborted} from "../../util/queue/index.js"; +import {BeaconChain} from "../chain.js"; +import {RegenCaller} from "../regen/interface.js"; +import {PayloadEnvelopeInput} from "../seenCache/seenPayloadEnvelopeInput.js"; +import {ImportPayloadOpts} from "./types.js"; + +const EVENTSTREAM_EMIT_RECENT_EXECUTION_PAYLOAD_SLOTS = 64; + +export enum PayloadErrorCode { + EXECUTION_ENGINE_INVALID = "PAYLOAD_ERROR_EXECUTION_ENGINE_INVALID", + EXECUTION_ENGINE_ERROR = "PAYLOAD_ERROR_EXECUTION_ENGINE_ERROR", + BLOCK_NOT_IN_FORK_CHOICE = "PAYLOAD_ERROR_BLOCK_NOT_IN_FORK_CHOICE", + STATE_TRANSITION_ERROR = "PAYLOAD_ERROR_STATE_TRANSITION_ERROR", + INVALID_SIGNATURE = "PAYLOAD_ERROR_INVALID_SIGNATURE", +} + +export type PayloadErrorType = + | { + code: PayloadErrorCode.EXECUTION_ENGINE_INVALID; + execStatus: ExecutionPayloadStatus; + errorMessage: string; + } + | { + code: PayloadErrorCode.EXECUTION_ENGINE_ERROR; + execStatus: ExecutionPayloadStatus; + errorMessage: string; + } + | { + code: PayloadErrorCode.BLOCK_NOT_IN_FORK_CHOICE; + blockRootHex: string; + } + | { + code: PayloadErrorCode.STATE_TRANSITION_ERROR; + message: string; + } + | { + code: PayloadErrorCode.INVALID_SIGNATURE; + }; + +export class PayloadError extends Error { + type: PayloadErrorType; + + constructor(type: PayloadErrorType, message?: string) { + super(message ?? type.code); + this.type = type; + } +} + +/** + * Import an execution payload envelope after all data is available. + * + * This function: + * 1. Gets the ProtoBlock from fork choice + * 2. Applies write-queue backpressure (waitForSpace) early, before verification + * 3. Regenerates the block state + * 4. Runs EL verification (notifyNewPayload) in parallel with signature verification and processExecutionPayloadEnvelope + * 5. Persists verified payload envelope to hot DB + * 6. Updates fork choice + * 7. Caches the post-execution payload state + * 8. Records metrics for column sources + * + */ +export async function importExecutionPayload( + this: BeaconChain, + payloadInput: PayloadEnvelopeInput, + opts: ImportPayloadOpts = {} +): Promise { + const envelope = payloadInput.getPayloadEnvelope(); + const blockRootHex = payloadInput.blockRootHex; + + // 1. Get ProtoBlock for parent root lookup + const protoBlock = this.forkChoice.getBlockHexDefaultStatus(blockRootHex); + if (!protoBlock) { + throw new PayloadError({ + code: PayloadErrorCode.BLOCK_NOT_IN_FORK_CHOICE, + blockRootHex, + }); + } + + // 2. Apply backpressure from the write queue early, before doing verification work. + // The actual DB write is deferred until after verification succeeds. + await this.unfinalizedPayloadEnvelopeWrites.waitForSpace(); + + // 3. Get pre-state for processExecutionPayloadEnvelope + // We need the block state (post-block, pre-payload) to process the envelope + const blockState = (await this.regen.getBlockSlotState( + protoBlock, + protoBlock.slot, + {dontTransferCache: true}, + RegenCaller.processBlock + )) as CachedBeaconStateGloas; + + // 4. Run verification steps in parallel + // Note: No data availability check needed here - importExecutionPayload is only + // called when payloadInput.isComplete() is true, so all data is already available. + const [execResult, signatureValid, postPayloadResult] = await Promise.all([ + this.executionEngine.notifyNewPayload( + ForkName.gloas, + envelope.message.payload, + payloadInput.getVersionedHashes(), + fromHex(protoBlock.parentRoot), + envelope.message.executionRequests + ), + + opts.validSignature === true + ? Promise.resolve(true) + : (async () => { + const signatureSet = getExecutionPayloadEnvelopeSignatureSet( + this.config, + blockState.epochCtx.pubkeyCache, + new BeaconStateView(blockState), + envelope, + payloadInput.proposerIndex + ); + return this.bls.verifySignatureSets([signatureSet]); + })(), + + // Signature verified separately above. + // State root check is done separately below with better error typing (matching block pipeline pattern). + (async () => { + try { + return { + postPayloadState: processExecutionPayloadEnvelope(blockState, envelope, { + verifySignature: false, + verifyStateRoot: false, + }), + }; + } catch (e) { + throw new PayloadError( + { + code: PayloadErrorCode.STATE_TRANSITION_ERROR, + message: (e as Error).message, + }, + `State transition error: ${(e as Error).message}` + ); + } + })(), + ]); + + // 4b. Check signature verification result + if (!signatureValid) { + throw new PayloadError({code: PayloadErrorCode.INVALID_SIGNATURE}); + } + + // 5. Handle EL response + switch (execResult.status) { + case ExecutionPayloadStatus.VALID: + break; + + case ExecutionPayloadStatus.INVALID: + throw new PayloadError({ + code: PayloadErrorCode.EXECUTION_ENGINE_INVALID, + execStatus: execResult.status, + errorMessage: execResult.validationError ?? "", + }); + + case ExecutionPayloadStatus.ACCEPTED: + case ExecutionPayloadStatus.SYNCING: + // TODO GLOAS: Handle optimistic import for payload - for now treat as error + throw new PayloadError({ + code: PayloadErrorCode.EXECUTION_ENGINE_ERROR, + execStatus: execResult.status, + errorMessage: execResult.validationError ?? "EL syncing, payload not yet validated", + }); + + case ExecutionPayloadStatus.INVALID_BLOCK_HASH: + case ExecutionPayloadStatus.ELERROR: + case ExecutionPayloadStatus.UNAVAILABLE: + throw new PayloadError({ + code: PayloadErrorCode.EXECUTION_ENGINE_ERROR, + execStatus: execResult.status, + errorMessage: execResult.validationError ?? "", + }); + } + + // 5b. Verify envelope state root matches post-state + const postPayloadState = postPayloadResult.postPayloadState; + const postPayloadStateRoot = postPayloadState.hashTreeRoot(); + if (!byteArrayEquals(envelope.message.stateRoot, postPayloadStateRoot)) { + throw new PayloadError({ + code: PayloadErrorCode.STATE_TRANSITION_ERROR, + message: `Envelope state root mismatch expected=${toRootHex(envelope.message.stateRoot)} actual=${toRootHex(postPayloadStateRoot)}`, + }); + } + + // 5c. Persist payload envelope to hot DB (performed asynchronously to avoid blocking) + this.unfinalizedPayloadEnvelopeWrites.push(payloadInput).catch((e) => { + if (!isQueueErrorAborted(e)) { + this.logger.error( + "Error pushing payload envelope to unfinalized write queue", + {slot: payloadInput.slot, root: blockRootHex}, + e as Error + ); + } + }); + + // 6. Update fork choice + this.forkChoice.onExecutionPayload( + blockRootHex, + payloadInput.getBlockHashHex(), + envelope.message.payload.blockNumber, + toRootHex(postPayloadStateRoot) + ); + + // 7. Cache payload state + // TODO GLOAS: Enable when PR #8868 merged (adds processPayloadState) + // this.regen.processPayloadState(postPayloadState); + // if epoch boundary also call + // this.regen.addCheckpointState(cp, checkpointState, true); + + // 8. Record metrics for payload envelope and column sources + this.metrics?.importPayload.bySource.inc({source: payloadInput.getPayloadEnvelopeSource().source}); + for (const {source} of payloadInput.getSampledColumnsWithSource()) { + this.metrics?.importPayload.columnsBySource.inc({source}); + } + + this.logger.verbose("Execution payload imported", { + slot: payloadInput.slot, + root: blockRootHex, + blockHash: payloadInput.getBlockHashHex(), + }); + + // 9. Emit event after payload is fully verified and imported to fork choice, only for recent enough payloads + const currentSlot = this.clock.currentSlot; + if (currentSlot - payloadInput.slot < EVENTSTREAM_EMIT_RECENT_EXECUTION_PAYLOAD_SLOTS) { + this.emitter.emit(routes.events.EventType.executionPayloadAvailable, { + slot: payloadInput.slot, + blockRoot: blockRootHex, + }); + } +} diff --git a/packages/beacon-node/src/chain/blocks/payloadEnvelopeInput/index.ts b/packages/beacon-node/src/chain/blocks/payloadEnvelopeInput/index.ts new file mode 100644 index 000000000000..368f98324f22 --- /dev/null +++ b/packages/beacon-node/src/chain/blocks/payloadEnvelopeInput/index.ts @@ -0,0 +1,2 @@ +export * from "./payloadEnvelopeInput.js"; +export * from "./types.js"; diff --git a/packages/beacon-node/src/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.ts b/packages/beacon-node/src/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.ts new file mode 100644 index 000000000000..313b7dce8f74 --- /dev/null +++ b/packages/beacon-node/src/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.ts @@ -0,0 +1,336 @@ +import {NUMBER_OF_COLUMNS} from "@lodestar/params"; +import {ColumnIndex, DataColumnSidecars, RootHex, Slot, ValidatorIndex, deneb, gloas} from "@lodestar/types"; +import {toRootHex, withTimeout} from "@lodestar/utils"; +import {VersionedHashes} from "../../../execution/index.js"; +import {kzgCommitmentToVersionedHash} from "../../../util/blobs.js"; +import {AddPayloadEnvelopeProps, ColumnWithSource, CreateFromBlockProps, SourceMeta} from "./types.js"; + +export type PayloadEnvelopeInputState = + | { + hasPayload: false; + hasAllData: false; + hasComputedAllData: false; + } + | { + hasPayload: false; + hasAllData: true; + hasComputedAllData: boolean; + } + | { + hasPayload: true; + hasAllData: false; + hasComputedAllData: false; + payloadEnvelope: gloas.SignedExecutionPayloadEnvelope; + payloadEnvelopeSource: SourceMeta; + } + | { + hasPayload: true; + hasAllData: true; + hasComputedAllData: boolean; + payloadEnvelope: gloas.SignedExecutionPayloadEnvelope; + payloadEnvelopeSource: SourceMeta; + timeCompleteSec: number; + }; + +type PromiseParts = { + promise: Promise; + resolve: (value: T) => void; + reject: (e: Error) => void; +}; + +function createPromise(): PromiseParts { + let resolve!: (value: T) => void; + let reject!: (e: Error) => void; + const promise = new Promise((_resolve, _reject) => { + resolve = _resolve; + reject = _reject; + }); + return {promise, resolve, reject}; +} + +/** + * Tracks bid + payload envelope + data columns for a Gloas block. + * + * Created during block import from signedExecutionPayloadBid in block body. + * Always has bid (required for creation). + * + * Completion requires: payload envelope + all sampled columns + */ +export class PayloadEnvelopeInput { + readonly blockRootHex: RootHex; + readonly slot: Slot; + readonly proposerIndex: ValidatorIndex; + readonly bid: gloas.ExecutionPayloadBid; + readonly versionedHashes: VersionedHashes; + + private columnsCache = new Map(); + + private readonly sampledColumns: ColumnIndex[]; + private readonly custodyColumns: ColumnIndex[]; + + private timeCreatedSec: number; + + private readonly payloadEnvelopeDataPromise: PromiseParts; + private readonly columnsDataPromise: PromiseParts; + + state: PayloadEnvelopeInputState; + + private constructor(props: { + blockRootHex: RootHex; + slot: Slot; + proposerIndex: ValidatorIndex; + bid: gloas.ExecutionPayloadBid; + sampledColumns: ColumnIndex[]; + custodyColumns: ColumnIndex[]; + timeCreatedSec: number; + }) { + this.blockRootHex = props.blockRootHex; + this.slot = props.slot; + this.proposerIndex = props.proposerIndex; + this.bid = props.bid; + this.versionedHashes = props.bid.blobKzgCommitments.map(kzgCommitmentToVersionedHash); + this.sampledColumns = props.sampledColumns; + this.custodyColumns = props.custodyColumns; + this.timeCreatedSec = props.timeCreatedSec; + this.payloadEnvelopeDataPromise = createPromise(); + this.columnsDataPromise = createPromise(); + + const noBlobs = props.bid.blobKzgCommitments.length === 0; + const noSampledColumns = props.sampledColumns.length === 0; + const hasAllData = noBlobs || noSampledColumns; + + if (hasAllData) { + this.state = {hasPayload: false, hasAllData: true, hasComputedAllData: true}; + this.columnsDataPromise.resolve(this.getSampledColumns()); + } else { + this.state = {hasPayload: false, hasAllData: false, hasComputedAllData: false}; + } + } + + static createFromBlock(props: CreateFromBlockProps): PayloadEnvelopeInput { + const bid = (props.block.message.body as gloas.BeaconBlockBody).signedExecutionPayloadBid.message; + return new PayloadEnvelopeInput({ + blockRootHex: props.blockRootHex, + slot: props.block.message.slot, + proposerIndex: props.block.message.proposerIndex, + bid, + sampledColumns: props.sampledColumns, + custodyColumns: props.custodyColumns, + timeCreatedSec: props.timeCreatedSec, + }); + } + + getBid(): gloas.ExecutionPayloadBid { + return this.bid; + } + + getBuilderIndex(): ValidatorIndex { + return this.bid.builderIndex; + } + + getBlockHashHex(): RootHex { + return toRootHex(this.bid.blockHash); + } + + getBlobKzgCommitments(): deneb.BlobKzgCommitments { + return this.bid.blobKzgCommitments; + } + + addPayloadEnvelope(props: AddPayloadEnvelopeProps): void { + if (this.state.hasPayload) { + throw new Error(`Payload envelope already set for block ${this.blockRootHex}`); + } + if (toRootHex(props.envelope.message.beaconBlockRoot) !== this.blockRootHex) { + throw new Error("Payload envelope beacon_block_root mismatch"); + } + + const source: SourceMeta = { + source: props.source, + seenTimestampSec: props.seenTimestampSec, + peerIdStr: props.peerIdStr, + }; + + if (this.state.hasAllData) { + // Complete state + this.state = { + hasPayload: true, + hasAllData: true, + hasComputedAllData: this.state.hasComputedAllData, + payloadEnvelope: props.envelope, + payloadEnvelopeSource: source, + timeCompleteSec: props.seenTimestampSec, + }; + this.payloadEnvelopeDataPromise.resolve(props.envelope); + } else { + // Has payload, waiting for columns + this.state = { + hasPayload: true, + hasAllData: false, + hasComputedAllData: false, + payloadEnvelope: props.envelope, + payloadEnvelopeSource: source, + }; + } + } + + addColumn(columnWithSource: ColumnWithSource): void { + const {columnSidecar, seenTimestampSec} = columnWithSource; + this.columnsCache.set(columnSidecar.index, columnWithSource); + + const sampledColumns = this.getSampledColumns(); + const hasAllData = + // already hasAllData + this.state.hasAllData || + // has all sampled columns + sampledColumns.length === this.sampledColumns.length || + // has enough columns to reconstruct the rest + this.columnsCache.size >= NUMBER_OF_COLUMNS / 2; + + const hasComputedAllData = + // has all sampled columns + sampledColumns.length === this.sampledColumns.length; + + if (!hasAllData) { + return; + } + + if (hasComputedAllData) { + this.columnsDataPromise.resolve(sampledColumns); + } + + if (this.state.hasPayload) { + // Complete state + this.state = { + hasPayload: true, + hasAllData: true, + hasComputedAllData: hasComputedAllData || this.state.hasComputedAllData, + payloadEnvelope: this.state.payloadEnvelope, + payloadEnvelopeSource: this.state.payloadEnvelopeSource, + timeCompleteSec: seenTimestampSec, + }; + this.payloadEnvelopeDataPromise.resolve(this.state.payloadEnvelope); + } else { + // No payload yet, all data ready + this.state = { + hasPayload: false, + hasAllData: true, + hasComputedAllData: hasComputedAllData || this.state.hasComputedAllData, + }; + } + } + + getVersionedHashes(): VersionedHashes { + return this.versionedHashes; + } + + hasPayloadEnvelope(): boolean { + return this.state.hasPayload; + } + + getPayloadEnvelope(): gloas.SignedExecutionPayloadEnvelope { + if (!this.state.hasPayload) throw new Error("Payload envelope not set"); + return this.state.payloadEnvelope; + } + + getPayloadEnvelopeSource(): SourceMeta { + if (!this.state.hasPayload) throw new Error("Payload envelope source not set"); + return this.state.payloadEnvelopeSource; + } + + getSampledColumns(): gloas.DataColumnSidecars { + const columns: gloas.DataColumnSidecars = []; + for (const index of this.sampledColumns) { + const column = this.columnsCache.get(index); + if (column) { + columns.push(column.columnSidecar); + } + } + return columns; + } + + getSampledColumnsWithSource(): ColumnWithSource[] { + const columns: ColumnWithSource[] = []; + for (const index of this.sampledColumns) { + const column = this.columnsCache.get(index); + if (column) { + columns.push(column); + } + } + return columns; + } + + getCustodyColumns(): gloas.DataColumnSidecars { + const columns: gloas.DataColumnSidecars = []; + for (const index of this.custodyColumns) { + const column = this.columnsCache.get(index); + if (column) { + columns.push(column.columnSidecar); + } + } + return columns; + } + + hasComputedAllData(): boolean { + return this.state.hasComputedAllData; + } + + waitForComputedAllData(timeout: number, signal?: AbortSignal): Promise { + if (this.state.hasComputedAllData) { + return Promise.resolve(this.getSampledColumns()); + } + return withTimeout(() => this.columnsDataPromise.promise, timeout, signal); + } + + getTimeCreated(): number { + return this.timeCreatedSec; + } + + getTimeComplete(): number { + if (!this.state.hasPayload || !this.state.hasAllData) throw new Error("Not yet complete"); + return this.state.timeCompleteSec; + } + + isComplete(): boolean { + return this.state.hasPayload && this.state.hasAllData; + } + + async waitForData(): Promise { + return this.payloadEnvelopeDataPromise.promise; + } + + getSerializedCacheKeys(): object[] { + const objects: object[] = []; + + if (this.state.hasPayload) { + objects.push(this.state.payloadEnvelope); + } + + for (const {columnSidecar} of this.columnsCache.values()) { + objects.push(columnSidecar); + } + + return objects; + } + + getLogMeta(): { + slot: number; + blockRoot: string; + hasPayload: boolean; + hasAllData: boolean; + hasComputedAllData: boolean; + isComplete: boolean; + columnsCount: number; + sampledColumnsCount: number; + } { + return { + slot: this.slot, + blockRoot: this.blockRootHex, + hasPayload: this.state.hasPayload, + hasAllData: this.state.hasAllData, + hasComputedAllData: this.state.hasComputedAllData, + isComplete: this.isComplete(), + columnsCount: this.columnsCache.size, + sampledColumnsCount: this.sampledColumns.length, + }; + } +} diff --git a/packages/beacon-node/src/chain/blocks/payloadEnvelopeInput/types.ts b/packages/beacon-node/src/chain/blocks/payloadEnvelopeInput/types.ts new file mode 100644 index 000000000000..b4683a245b1f --- /dev/null +++ b/packages/beacon-node/src/chain/blocks/payloadEnvelopeInput/types.ts @@ -0,0 +1,33 @@ +import {ForkPostGloas} from "@lodestar/params"; +import {ColumnIndex, RootHex, SignedBeaconBlock, gloas} from "@lodestar/types"; + +export enum PayloadEnvelopeInputSource { + gossip = "gossip", + api = "api", + engine = "engine", + byRange = "req_resp_by_range", + byRoot = "req_resp_by_root", + recovery = "recovery", +} + +export type SourceMeta = { + source: PayloadEnvelopeInputSource; + seenTimestampSec: number; + peerIdStr?: string; +}; + +export type ColumnWithSource = SourceMeta & { + columnSidecar: gloas.DataColumnSidecar; +}; + +export type CreateFromBlockProps = { + blockRootHex: RootHex; + block: SignedBeaconBlock; + sampledColumns: ColumnIndex[]; + custodyColumns: ColumnIndex[]; + timeCreatedSec: number; +}; + +export type AddPayloadEnvelopeProps = SourceMeta & { + envelope: gloas.SignedExecutionPayloadEnvelope; +}; diff --git a/packages/beacon-node/src/chain/blocks/payloadEnvelopeProcessor.ts b/packages/beacon-node/src/chain/blocks/payloadEnvelopeProcessor.ts new file mode 100644 index 000000000000..e50b53bda93e --- /dev/null +++ b/packages/beacon-node/src/chain/blocks/payloadEnvelopeProcessor.ts @@ -0,0 +1,61 @@ +import {Metrics} from "../../metrics/metrics.js"; +import {JobItemQueue} from "../../util/queue/index.js"; +import type {BeaconChain} from "../chain.js"; +import {PayloadEnvelopeInput} from "../seenCache/seenPayloadEnvelopeInput.js"; +import {importExecutionPayload} from "./importExecutionPayload.js"; +import {ImportPayloadOpts} from "./types.js"; + +// TODO GLOAS: Set to be equal to DEFAULT_MAX_PENDING_UNFINALIZED_PAYLOAD_ENVELOPE_WRITES for now +const QUEUE_MAX_LENGTH = 16; + +enum PayloadEnvelopeImportStatus { + queued = "queued", + importing = "importing", + imported = "imported", +} + +/** + * PayloadEnvelopeProcessor processes payload envelope jobs in a queued fashion, one after the other. + */ +export class PayloadEnvelopeProcessor { + readonly jobQueue: JobItemQueue<[PayloadEnvelopeInput, ImportPayloadOpts], void>; + private readonly importStatus = new WeakMap(); + + constructor(chain: BeaconChain, metrics: Metrics | null, signal: AbortSignal) { + this.jobQueue = new JobItemQueue<[PayloadEnvelopeInput, ImportPayloadOpts], void>( + (payloadInput, opts) => { + this.importStatus.set(payloadInput, PayloadEnvelopeImportStatus.importing); + return importExecutionPayload.call(chain, payloadInput, opts); + }, + {maxLength: QUEUE_MAX_LENGTH, noYieldIfOneItem: true, signal}, + metrics?.payloadEnvelopeProcessorQueue ?? undefined + ); + } + + async processPayloadEnvelopeJob(payloadInput: PayloadEnvelopeInput, opts: ImportPayloadOpts = {}): Promise { + if (!payloadInput.isComplete()) { + return; + } + + if (this.importStatus.get(payloadInput) !== undefined) { + return; + } + + await this.jobQueue.waitForSpace(); + + // Re-check after await, as another call may have queued this payload. + if (this.importStatus.get(payloadInput) !== undefined) { + return; + } + + this.importStatus.set(payloadInput, PayloadEnvelopeImportStatus.queued); + + try { + await this.jobQueue.push(payloadInput, opts); + this.importStatus.set(payloadInput, PayloadEnvelopeImportStatus.imported); + } catch (e) { + this.importStatus.delete(payloadInput); + throw e; + } + } +} diff --git a/packages/beacon-node/src/chain/blocks/types.ts b/packages/beacon-node/src/chain/blocks/types.ts index 50ed0076515c..52a6741d29c6 100644 --- a/packages/beacon-node/src/chain/blocks/types.ts +++ b/packages/beacon-node/src/chain/blocks/types.ts @@ -41,6 +41,14 @@ export enum BlobSidecarValidation { Full, } +export type ImportPayloadOpts = { + /** + * Set to true if envelope signature was already verified (e.g., during gossip/API validation). + * When false/undefined, signature will be verified during import. + */ + validSignature?: boolean; +}; + export type ImportBlockOpts = { /** * TEMP: Review if this is safe, Lighthouse always imports attestations even in finalized sync. diff --git a/packages/beacon-node/src/chain/blocks/writePayloadEnvelopeInputToDb.ts b/packages/beacon-node/src/chain/blocks/writePayloadEnvelopeInputToDb.ts new file mode 100644 index 000000000000..425bd83adca8 --- /dev/null +++ b/packages/beacon-node/src/chain/blocks/writePayloadEnvelopeInputToDb.ts @@ -0,0 +1,55 @@ +import {BeaconChain} from "../chain.js"; +import {PayloadEnvelopeInput} from "../seenCache/seenPayloadEnvelopeInput.js"; +import {writeDataColumnsToDb} from "./writeBlockInputToDb.js"; + +/** + * Persists payload envelope data to DB. This operation must be eventually completed if a payload is imported. + * + * TODO GLOAS: Persist envelope metadata (stateRoot, executionRequests, builderIndex, etc.) without the full + * execution payload body — only keep the blockHash reference. The EL already stores the payload. + * See https://github.com/ChainSafe/lodestar/issues/5671 + */ +export async function writePayloadEnvelopeInputToDb( + this: BeaconChain, + payloadInput: PayloadEnvelopeInput +): Promise { + const envelope = payloadInput.getPayloadEnvelope(); + const blockRootHex = payloadInput.blockRootHex; + + const envelopeBytes = this.serializedCache.get(envelope); + const envelopePromise = envelopeBytes + ? this.db.executionPayloadEnvelope.putBinary(this.db.executionPayloadEnvelope.getId(envelope), envelopeBytes) + : this.db.executionPayloadEnvelope.add(envelope); + + // Write envelope and data columns in parallel (reuses shared column writing logic) + await Promise.all([envelopePromise, writeDataColumnsToDb.call(this, payloadInput)]); + this.logger.debug("Persisted payload envelope to db", { + slot: payloadInput.slot, + root: blockRootHex, + }); +} + +export async function persistPayloadEnvelopeInput( + this: BeaconChain, + payloadInput: PayloadEnvelopeInput +): Promise { + await writePayloadEnvelopeInputToDb + .call(this, payloadInput) + .catch((e) => { + this.logger.error( + "Error persisting payload envelope in hot db", + { + slot: payloadInput.slot, + root: payloadInput.blockRootHex, + }, + e + ); + }) + .finally(() => { + this.seenPayloadEnvelopeInputCache.prune(payloadInput.blockRootHex); + this.logger.debug("Pruned payload envelope input", { + slot: payloadInput.slot, + root: payloadInput.blockRootHex, + }); + }); +} diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 4c41afd33aa6..fc6d683c197c 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -84,7 +84,10 @@ import {CheckpointBalancesCache} from "./balancesCache.js"; import {BeaconProposerCache} from "./beaconProposerCache.js"; import {IBlockInput, isBlockInputBlobs, isBlockInputColumns} from "./blocks/blockInput/index.js"; import {BlockProcessor, ImportBlockOpts} from "./blocks/index.js"; +import {PayloadEnvelopeProcessor} from "./blocks/payloadEnvelopeProcessor.js"; +import {ImportPayloadOpts} from "./blocks/types.js"; import {persistBlockInput} from "./blocks/writeBlockInputToDb.js"; +import {persistPayloadEnvelopeInput} from "./blocks/writePayloadEnvelopeInputToDb.js"; import {BlsMultiThreadWorkerPool, BlsSingleThreadVerifier, IBlsVerifier} from "./bls/index.js"; import {ColumnReconstructionTracker} from "./ColumnReconstructionTracker.js"; import {ChainEvent, ChainEventEmitter} from "./emitter.js"; @@ -109,13 +112,14 @@ import {BlockAttributes, produceBlockBody, produceCommonBlockBody} from "./produ import {QueuedStateRegenerator, RegenCaller} from "./regen/index.js"; import {ReprocessController} from "./reprocess.js"; import { + PayloadEnvelopeInput, SeenAggregators, SeenAttesters, SeenBlockProposers, SeenContributionAndProof, SeenExecutionPayloadBids, - SeenExecutionPayloadEnvelopes, SeenPayloadAttesters, + SeenPayloadEnvelopeInput, SeenSyncCommitteeMessages, } from "./seenCache/index.js"; import {SeenAggregatedAttestations} from "./seenCache/seenAggregateAndProof.js"; @@ -148,6 +152,13 @@ const DEFAULT_MAX_CACHED_PRODUCED_RESULTS = 4; */ const DEFAULT_MAX_PENDING_UNFINALIZED_BLOCK_WRITES = 16; +/** + * The maximum number of pending unfinalized payload envelope writes to the database before backpressure is applied. + * Payload envelope write queue entries hold references to payload inputs (including columns), + * keeping them in memory. Keep moderate to avoid OOM during sync. + */ +const DEFAULT_MAX_PENDING_UNFINALIZED_PAYLOAD_ENVELOPE_WRITES = 16; + export class BeaconChain implements IBeaconChain { readonly genesisTime: UintNum64; readonly genesisValidatorsRoot: Root; @@ -172,6 +183,7 @@ export class BeaconChain implements IBeaconChain { readonly reprocessController: ReprocessController; readonly archiveStore: ArchiveStore; readonly unfinalizedBlockWrites: JobItemQueue<[IBlockInput], void>; + readonly unfinalizedPayloadEnvelopeWrites: JobItemQueue<[PayloadEnvelopeInput], void>; // Ops pool readonly attestationPool: AttestationPool; @@ -187,13 +199,13 @@ export class BeaconChain implements IBeaconChain { readonly seenAggregators = new SeenAggregators(); readonly seenPayloadAttesters = new SeenPayloadAttesters(); readonly seenAggregatedAttestations: SeenAggregatedAttestations; - readonly seenExecutionPayloadEnvelopes = new SeenExecutionPayloadEnvelopes(); readonly seenExecutionPayloadBids = new SeenExecutionPayloadBids(); readonly seenBlockProposers = new SeenBlockProposers(); readonly seenSyncCommitteeMessages = new SeenSyncCommitteeMessages(); readonly seenContributionAndProof: SeenContributionAndProof; readonly seenAttestationDatas: SeenAttestationDatas; readonly seenBlockInputCache: SeenBlockInput; + readonly seenPayloadEnvelopeInputCache: SeenPayloadEnvelopeInput; // Seen cache for liveness checks readonly seenBlockAttesters = new SeenBlockAttesters(); @@ -221,6 +233,7 @@ export class BeaconChain implements IBeaconChain { readonly opts: IChainOptions; protected readonly blockProcessor: BlockProcessor; + protected readonly payloadEnvelopeProcessor: PayloadEnvelopeProcessor; protected readonly db: IBeaconDb; // this is only available if nHistoricalStates is enabled private readonly cpStateDatastore?: CPStateDatastore; @@ -334,6 +347,13 @@ export class BeaconChain implements IBeaconChain { metrics, logger, }); + this.seenPayloadEnvelopeInputCache = new SeenPayloadEnvelopeInput({ + chainEvents: emitter, + signal, + serializedCache: this.serializedCache, + metrics, + logger, + }); this._earliestAvailableSlot = anchorState.slot; @@ -411,6 +431,7 @@ export class BeaconChain implements IBeaconChain { this.reprocessController = new ReprocessController(this.metrics); this.blockProcessor = new BlockProcessor(this, metrics, opts, signal); + this.payloadEnvelopeProcessor = new PayloadEnvelopeProcessor(this, metrics, signal); this.forkChoice = forkChoice; this.clock = clock; @@ -447,6 +468,15 @@ export class BeaconChain implements IBeaconChain { metrics?.unfinalizedBlockWritesQueue ); + this.unfinalizedPayloadEnvelopeWrites = new JobItemQueue( + persistPayloadEnvelopeInput.bind(this), + { + maxLength: DEFAULT_MAX_PENDING_UNFINALIZED_PAYLOAD_ENVELOPE_WRITES, + signal, + }, + metrics?.unfinalizedPayloadEnvelopeWritesQueue + ); + // always run PrepareNextSlotScheduler except for fork_choice spec tests if (!opts?.disablePrepareNextSlot) { new PrepareNextSlotScheduler(this, this.config, metrics, this.logger, signal); @@ -477,6 +507,7 @@ export class BeaconChain implements IBeaconChain { // we can abort any ongoing unfinalized block writes. // TODO: persist fork choice to disk and allow unfinalized block writes to complete. this.unfinalizedBlockWrites.dropAllJobs(); + this.unfinalizedPayloadEnvelopeWrites.dropAllJobs(); this.abortController.abort(); } @@ -1010,6 +1041,10 @@ export class BeaconChain implements IBeaconChain { return this.blockProcessor.processBlocksJob(blocks, opts); } + async processExecutionPayload(payloadInput: PayloadEnvelopeInput, opts?: ImportPayloadOpts): Promise { + return this.payloadEnvelopeProcessor.processPayloadEnvelopeJob(payloadInput, opts); + } + getStatus(): Status { const head = this.forkChoice.getHead(); const finalizedCheckpoint = this.forkChoice.getFinalizedCheckpoint(); @@ -1402,7 +1437,6 @@ export class BeaconChain implements IBeaconChain { this.logger.verbose("Fork choice finalized", {epoch: cp.epoch, root: cp.rootHex}); const finalizedSlot = computeStartSlotAtEpoch(cp.epoch); this.seenBlockProposers.prune(finalizedSlot); - this.seenExecutionPayloadEnvelopes.prune(finalizedSlot); // Update validator custody to account for effective balance changes await this.updateValidatorsCustodyRequirement(cp); diff --git a/packages/beacon-node/src/chain/errors/executionPayloadEnvelope.ts b/packages/beacon-node/src/chain/errors/executionPayloadEnvelope.ts index af3a040a8d17..cf7a0fe4a002 100644 --- a/packages/beacon-node/src/chain/errors/executionPayloadEnvelope.ts +++ b/packages/beacon-node/src/chain/errors/executionPayloadEnvelope.ts @@ -4,17 +4,21 @@ import {GossipActionError} from "./gossipValidation.js"; export enum ExecutionPayloadEnvelopeErrorCode { BELONG_TO_FINALIZED_BLOCK = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_BELONG_TO_FINALIZED_BLOCK", BLOCK_ROOT_UNKNOWN = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_BLOCK_ROOT_UNKNOWN", + PARENT_UNKNOWN = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_PARENT_UNKNOWN", + UNKNOWN_BLOCK_STATE = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_UNKNOWN_BLOCK_STATE", ENVELOPE_ALREADY_KNOWN = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_ALREADY_KNOWN", INVALID_BLOCK = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_INVALID_BLOCK", SLOT_MISMATCH = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_SLOT_MISMATCH", BUILDER_INDEX_MISMATCH = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_BUILDER_INDEX_MISMATCH", BLOCK_HASH_MISMATCH = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_BLOCK_HASH_MISMATCH", INVALID_SIGNATURE = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_INVALID_SIGNATURE", - CACHE_FAIL = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_CACHE_FAIL", + PAYLOAD_ENVELOPE_INPUT_MISSING = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_PAYLOAD_ENVELOPE_INPUT_MISSING", } export type ExecutionPayloadEnvelopeErrorType = | {code: ExecutionPayloadEnvelopeErrorCode.BELONG_TO_FINALIZED_BLOCK; envelopeSlot: Slot; finalizedSlot: Slot} | {code: ExecutionPayloadEnvelopeErrorCode.BLOCK_ROOT_UNKNOWN; blockRoot: RootHex} + | {code: ExecutionPayloadEnvelopeErrorCode.PARENT_UNKNOWN; parentRoot: RootHex; slot: Slot} + | {code: ExecutionPayloadEnvelopeErrorCode.UNKNOWN_BLOCK_STATE; blockRoot: RootHex; slot: Slot} | { code: ExecutionPayloadEnvelopeErrorCode.ENVELOPE_ALREADY_KNOWN; blockRoot: RootHex; @@ -33,6 +37,6 @@ export type ExecutionPayloadEnvelopeErrorType = bidBlockHash: RootHex | null; } | {code: ExecutionPayloadEnvelopeErrorCode.INVALID_SIGNATURE} - | {code: ExecutionPayloadEnvelopeErrorCode.CACHE_FAIL; blockRoot: RootHex}; + | {code: ExecutionPayloadEnvelopeErrorCode.PAYLOAD_ENVELOPE_INPUT_MISSING; blockRoot: RootHex}; export class ExecutionPayloadEnvelopeError extends GossipActionError {} diff --git a/packages/beacon-node/src/chain/forkChoice/index.ts b/packages/beacon-node/src/chain/forkChoice/index.ts index f37721a270cb..20b654bd1472 100644 --- a/packages/beacon-node/src/chain/forkChoice/index.ts +++ b/packages/beacon-node/src/chain/forkChoice/index.ts @@ -158,10 +158,6 @@ export function initializeForkChoiceFromFinalizedState( dataAvailabilityStatus: DataAvailabilityStatus.PreData, payloadStatus: isForkPostGloas ? PayloadStatus.PENDING : PayloadStatus.FULL, // TODO GLOAS: Post-gloas how do we know if the checkpoint payload is FULL or EMPTY? - builderIndex: isForkPostGloas ? (state as CachedBeaconStateGloas).latestExecutionPayloadBid.builderIndex : null, - blockHashFromBid: isForkPostGloas - ? toRootHex((state as CachedBeaconStateGloas).latestExecutionPayloadBid.blockHash) - : null, parentBlockHash: isForkPostGloas ? toRootHex((state as CachedBeaconStateGloas).latestBlockHash) : null, }, currentSlot @@ -255,12 +251,6 @@ export function initializeForkChoiceFromUnfinalizedState( dataAvailabilityStatus: DataAvailabilityStatus.PreData, payloadStatus: isForkPostGloas ? PayloadStatus.PENDING : PayloadStatus.FULL, // TODO GLOAS: Post-gloas how do we know if the checkpoint payload is FULL or EMPTY? - builderIndex: isForkPostGloas - ? (unfinalizedState as CachedBeaconStateGloas).latestExecutionPayloadBid.builderIndex - : null, - blockHashFromBid: isForkPostGloas - ? toRootHex((unfinalizedState as CachedBeaconStateGloas).latestExecutionPayloadBid.blockHash) - : null, parentBlockHash: isForkPostGloas ? toRootHex((unfinalizedState as CachedBeaconStateGloas).latestBlockHash) : null, }; diff --git a/packages/beacon-node/src/chain/interface.ts b/packages/beacon-node/src/chain/interface.ts index d15c9951bfbb..f9d78cb8d51a 100644 --- a/packages/beacon-node/src/chain/interface.ts +++ b/packages/beacon-node/src/chain/interface.ts @@ -32,7 +32,7 @@ import {IArchiveStore} from "./archiveStore/interface.js"; import {CheckpointBalancesCache} from "./balancesCache.js"; import {BeaconProposerCache, ProposerPreparationData} from "./beaconProposerCache.js"; import {IBlockInput} from "./blocks/blockInput/index.js"; -import {ImportBlockOpts} from "./blocks/types.js"; +import {ImportBlockOpts, ImportPayloadOpts} from "./blocks/types.js"; import {IBlsVerifier} from "./bls/index.js"; import {ColumnReconstructionTracker} from "./ColumnReconstructionTracker.js"; import {ChainEventEmitter} from "./emitter.js"; @@ -58,7 +58,6 @@ import { SeenBlockProposers, SeenContributionAndProof, SeenExecutionPayloadBids, - SeenExecutionPayloadEnvelopes, SeenPayloadAttesters, SeenSyncCommitteeMessages, } from "./seenCache/index.js"; @@ -66,6 +65,7 @@ import {SeenAggregatedAttestations} from "./seenCache/seenAggregateAndProof.js"; import {SeenAttestationDatas} from "./seenCache/seenAttestationData.js"; import {SeenBlockAttesters} from "./seenCache/seenBlockAttesters.js"; import {SeenBlockInput} from "./seenCache/seenGossipBlockInput.js"; +import {PayloadEnvelopeInput, SeenPayloadEnvelopeInput} from "./seenCache/seenPayloadEnvelopeInput.js"; import {ShufflingCache} from "./shufflingCache.js"; import {ValidatorMonitor} from "./validatorMonitor.js"; @@ -128,13 +128,13 @@ export interface IBeaconChain { readonly seenAggregators: SeenAggregators; readonly seenPayloadAttesters: SeenPayloadAttesters; readonly seenAggregatedAttestations: SeenAggregatedAttestations; - readonly seenExecutionPayloadEnvelopes: SeenExecutionPayloadEnvelopes; readonly seenExecutionPayloadBids: SeenExecutionPayloadBids; readonly seenBlockProposers: SeenBlockProposers; readonly seenSyncCommitteeMessages: SeenSyncCommitteeMessages; readonly seenContributionAndProof: SeenContributionAndProof; readonly seenAttestationDatas: SeenAttestationDatas; readonly seenBlockInputCache: SeenBlockInput; + readonly seenPayloadEnvelopeInputCache: SeenPayloadEnvelopeInput; // Seen cache for liveness checks readonly seenBlockAttesters: SeenBlockAttesters; @@ -242,6 +242,9 @@ export interface IBeaconChain { /** Process a chain of blocks until complete */ processChainSegment(blocks: IBlockInput[], opts?: ImportBlockOpts): Promise; + /** Process execution payload envelope: verify, import to fork choice, and persist to DB */ + processExecutionPayload(payloadInput: PayloadEnvelopeInput, opts?: ImportPayloadOpts): Promise; + getStatus(): Status; recomputeForkChoiceHead(caller: ForkchoiceCaller): ProtoBlock; diff --git a/packages/beacon-node/src/chain/produceBlock/computeNewStateRoot.ts b/packages/beacon-node/src/chain/produceBlock/computeNewStateRoot.ts index 466b42d6e0f9..f150c0275f54 100644 --- a/packages/beacon-node/src/chain/produceBlock/computeNewStateRoot.ts +++ b/packages/beacon-node/src/chain/produceBlock/computeNewStateRoot.ts @@ -73,7 +73,12 @@ export function computeEnvelopeStateRoot( }; const processEnvelopeTimer = metrics?.blockPayload.executionPayloadEnvelopeProcessingTime.startTimer(); - const postEnvelopeState = processExecutionPayloadEnvelope(postBlockState, signedEnvelope, false, { + const postEnvelopeState = processExecutionPayloadEnvelope(postBlockState, signedEnvelope, { + // Signature is zero-ed (G2_POINT_AT_INFINITY), skip verification + verifySignature: false, + // State root is being computed here, the envelope doesn't have it yet + verifyStateRoot: false, + // Preserve cache in source state, since the resulting state is not added to the state cache dontTransferCache: true, }); processEnvelopeTimer?.(); diff --git a/packages/beacon-node/src/chain/regen/interface.ts b/packages/beacon-node/src/chain/regen/interface.ts index 019523a8139f..d3f41e638f2b 100644 --- a/packages/beacon-node/src/chain/regen/interface.ts +++ b/packages/beacon-node/src/chain/regen/interface.ts @@ -9,8 +9,10 @@ export enum RegenCaller { processBlock = "processBlock", produceBlock = "produceBlock", validateGossipBlock = "validateGossipBlock", + validateGossipPayloadEnvelope = "validateGossipPayloadEnvelope", validateGossipBlob = "validateGossipBlob", validateGossipDataColumn = "validateGossipDataColumn", + validateGossipExecutionPayloadEnvelope = "validateGossipExecutionPayloadEnvelope", precomputeEpoch = "precomputeEpoch", predictProposerHead = "predictProposerHead", produceAttestationData = "produceAttestationData", diff --git a/packages/beacon-node/src/chain/seenCache/index.ts b/packages/beacon-node/src/chain/seenCache/index.ts index f16ae79f7f2e..23782e14385b 100644 --- a/packages/beacon-node/src/chain/seenCache/index.ts +++ b/packages/beacon-node/src/chain/seenCache/index.ts @@ -3,5 +3,5 @@ export {SeenBlockProposers} from "./seenBlockProposers.js"; export {SeenSyncCommitteeMessages} from "./seenCommittee.js"; export {SeenContributionAndProof} from "./seenCommitteeContribution.js"; export {SeenExecutionPayloadBids} from "./seenExecutionPayloadBids.js"; -export {SeenExecutionPayloadEnvelopes} from "./seenExecutionPayloadEnvelope.js"; export {SeenBlockInput} from "./seenGossipBlockInput.js"; +export {PayloadEnvelopeInput, SeenPayloadEnvelopeInput} from "./seenPayloadEnvelopeInput.js"; diff --git a/packages/beacon-node/src/chain/seenCache/seenExecutionPayloadEnvelope.ts b/packages/beacon-node/src/chain/seenCache/seenExecutionPayloadEnvelope.ts deleted file mode 100644 index cbd389d29449..000000000000 --- a/packages/beacon-node/src/chain/seenCache/seenExecutionPayloadEnvelope.ts +++ /dev/null @@ -1,34 +0,0 @@ -import {RootHex, Slot} from "@lodestar/types"; - -/** - * Cache to prevent processing multiple execution payload envelopes for the same block root. - * Only one builder qualifies to submit an execution payload for a given slot. - * We only keep track of envelopes of unfinalized slots. - * [IGNORE] The node has not seen another valid `SignedExecutionPayloadEnvelope` for this block root. - */ -export class SeenExecutionPayloadEnvelopes { - private readonly slotByBlockRoot = new Map(); - private finalizedSlot: Slot = 0; - - isKnown(blockRoot: RootHex): boolean { - return this.slotByBlockRoot.has(blockRoot); - } - - add(blockRoot: RootHex, slot: Slot): void { - if (slot < this.finalizedSlot) { - throw Error(`slot ${slot} < finalizedSlot ${this.finalizedSlot}`); - } - - this.slotByBlockRoot.set(blockRoot, slot); - } - - prune(finalizedSlot: Slot): void { - this.finalizedSlot = finalizedSlot; - - for (const [blockRoot, slot] of this.slotByBlockRoot.entries()) { - if (slot < finalizedSlot) { - this.slotByBlockRoot.delete(blockRoot); - } - } - } -} diff --git a/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts b/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts index 7570c2a03ffd..336256678962 100644 --- a/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts +++ b/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts @@ -180,7 +180,7 @@ export class SeenBlockInput { blockInput = this.blockInputs.get(parentRootHex ?? ""); parentRootHex = blockInput?.parentRootHex; } - this.logger?.debug(`BlockInputCache.prune deleted ${deletedCount} cached BlockInputs`); + this.logger?.debug("BlockInputCache.prune deleted cached BlockInputs", {deletedCount}); this.pruneToMaxSize(); } @@ -193,7 +193,7 @@ export class SeenBlockInput { this.evictBlockInput(blockInput); } } - this.logger?.debug(`BlockInputCache.onFinalized deleted ${deletedCount} cached BlockInputs`); + this.logger?.debug("BlockInputCache.onFinalized deleted cached BlockInputs", {deletedCount}); this.pruneToMaxSize(); }; diff --git a/packages/beacon-node/src/chain/seenCache/seenPayloadEnvelopeInput.ts b/packages/beacon-node/src/chain/seenCache/seenPayloadEnvelopeInput.ts new file mode 100644 index 000000000000..1932a4c38f42 --- /dev/null +++ b/packages/beacon-node/src/chain/seenCache/seenPayloadEnvelopeInput.ts @@ -0,0 +1,106 @@ +import {CheckpointWithHex} from "@lodestar/fork-choice"; +import {computeStartSlotAtEpoch} from "@lodestar/state-transition"; +import {RootHex} from "@lodestar/types"; +import {Logger} from "@lodestar/utils"; +import {Metrics} from "../../metrics/metrics.js"; +import {SerializedCache} from "../../util/serializedCache.js"; +import {CreateFromBlockProps, PayloadEnvelopeInput} from "../blocks/payloadEnvelopeInput/index.js"; +import {ChainEvent, ChainEventEmitter} from "../emitter.js"; + +export type {PayloadEnvelopeInputState} from "../blocks/payloadEnvelopeInput/index.js"; +export {PayloadEnvelopeInput} from "../blocks/payloadEnvelopeInput/index.js"; + +export type SeenPayloadEnvelopeInputModules = { + chainEvents: ChainEventEmitter; + signal: AbortSignal; + serializedCache: SerializedCache; + metrics: Metrics | null; + logger?: Logger; +}; + +/** + * Cache for tracking PayloadEnvelopeInput instances, keyed by beacon block root. + * + * Created during block import when a block is processed. + * Pruned on finalization and after payload is written to DB. + */ +export class SeenPayloadEnvelopeInput { + private readonly chainEvents: ChainEventEmitter; + private readonly signal: AbortSignal; + private readonly serializedCache: SerializedCache; + private readonly metrics: Metrics | null; + private readonly logger?: Logger; + private payloadInputs = new Map(); + + constructor({chainEvents, signal, serializedCache, metrics, logger}: SeenPayloadEnvelopeInputModules) { + this.chainEvents = chainEvents; + this.signal = signal; + this.serializedCache = serializedCache; + this.metrics = metrics; + this.logger = logger; + + if (metrics) { + metrics.seenCache.payloadEnvelopeInput.count.addCollect(() => { + metrics.seenCache.payloadEnvelopeInput.count.set(this.payloadInputs.size); + metrics.seenCache.payloadEnvelopeInput.serializedObjectRefs.set( + Array.from(this.payloadInputs.values()).reduce( + (count, payloadInput) => count + payloadInput.getSerializedCacheKeys().length, + 0 + ) + ); + }); + } + + this.chainEvents.on(ChainEvent.forkChoiceFinalized, this.onFinalized); + this.signal.addEventListener("abort", () => { + this.chainEvents.off(ChainEvent.forkChoiceFinalized, this.onFinalized); + }); + } + + private onFinalized = (checkpoint: CheckpointWithHex): void => { + // Prune all entries with slot < finalized slot + const finalizedSlot = computeStartSlotAtEpoch(checkpoint.epoch); + let deletedCount = 0; + for (const [, input] of this.payloadInputs) { + if (input.slot < finalizedSlot) { + this.evictPayloadInput(input); + deletedCount++; + } + } + this.logger?.debug("SeenPayloadEnvelopeInput.onFinalized deleted cached entries", {deletedCount}); + }; + + add(props: CreateFromBlockProps): PayloadEnvelopeInput { + if (this.payloadInputs.has(props.blockRootHex)) { + throw new Error(`PayloadEnvelopeInput already exists for block ${props.blockRootHex}`); + } + const input = PayloadEnvelopeInput.createFromBlock(props); + this.payloadInputs.set(props.blockRootHex, input); + this.metrics?.seenCache.payloadEnvelopeInput.created.inc(); + return input; + } + + get(blockRootHex: RootHex): PayloadEnvelopeInput | undefined { + return this.payloadInputs.get(blockRootHex); + } + + has(blockRootHex: RootHex): boolean { + return this.payloadInputs.has(blockRootHex); + } + + prune(blockRootHex: RootHex): void { + const payloadInput = this.payloadInputs.get(blockRootHex); + if (payloadInput) { + this.evictPayloadInput(payloadInput); + } + } + + size(): number { + return this.payloadInputs.size; + } + + private evictPayloadInput(payloadInput: PayloadEnvelopeInput): void { + this.serializedCache.delete(payloadInput.getSerializedCacheKeys()); + this.payloadInputs.delete(payloadInput.blockRootHex); + } +} diff --git a/packages/beacon-node/src/chain/validation/executionPayloadEnvelope.ts b/packages/beacon-node/src/chain/validation/executionPayloadEnvelope.ts index 467a9676f2a3..c9715d59cb68 100644 --- a/packages/beacon-node/src/chain/validation/executionPayloadEnvelope.ts +++ b/packages/beacon-node/src/chain/validation/executionPayloadEnvelope.ts @@ -1,14 +1,15 @@ -import {PublicKey} from "@chainsafe/blst"; +import {PayloadStatus} from "@lodestar/fork-choice"; import { + BeaconStateView, CachedBeaconStateGloas, computeStartSlotAtEpoch, - createSingleSignatureSetFromComponents, - getExecutionPayloadEnvelopeSigningRoot, + getExecutionPayloadEnvelopeSignatureSet, } from "@lodestar/state-transition"; import {gloas} from "@lodestar/types"; import {toRootHex} from "@lodestar/utils"; import {ExecutionPayloadEnvelopeError, ExecutionPayloadEnvelopeErrorCode, GossipAction} from "../errors/index.js"; import {IBeaconChain} from "../index.js"; +import {RegenCaller} from "../regen/index.js"; export async function validateApiExecutionPayloadEnvelope( chain: IBeaconChain, @@ -47,7 +48,9 @@ async function validateExecutionPayloadEnvelope( // [IGNORE] The node has not seen another valid // `SignedExecutionPayloadEnvelope` for this block root from this builder. - if (chain.seenExecutionPayloadEnvelopes.isKnown(blockRootHex)) { + const envelopeBlock = chain.forkChoice.getBlockHex(blockRootHex, PayloadStatus.FULL); + const payloadInput = chain.seenPayloadEnvelopeInputCache.get(blockRootHex); + if (envelopeBlock || payloadInput?.hasPayloadEnvelope()) { throw new ExecutionPayloadEnvelopeError(GossipAction.IGNORE, { code: ExecutionPayloadEnvelopeErrorCode.ENVELOPE_ALREADY_KNOWN, blockRoot: blockRootHex, @@ -55,6 +58,14 @@ async function validateExecutionPayloadEnvelope( }); } + if (!payloadInput) { + // PayloadEnvelopeInput should have been created during block import + throw new ExecutionPayloadEnvelopeError(GossipAction.IGNORE, { + code: ExecutionPayloadEnvelopeErrorCode.PAYLOAD_ENVELOPE_INPUT_MISSING, + blockRoot: blockRootHex, + }); + } + // [IGNORE] The envelope is from a slot greater than or equal to the latest finalized slot -- i.e. validate that `envelope.slot >= compute_start_slot_at_epoch(store.finalized_checkpoint.epoch)` const finalizedCheckpoint = chain.forkChoice.getFinalizedCheckpoint(); const finalizedSlot = computeStartSlotAtEpoch(finalizedCheckpoint.epoch); @@ -79,45 +90,47 @@ async function validateExecutionPayloadEnvelope( }); } - if (block.builderIndex == null || block.blockHashFromBid == null) { - // This indicates this block is a pre-gloas block which is wrong - throw new ExecutionPayloadEnvelopeError(GossipAction.IGNORE, { - code: ExecutionPayloadEnvelopeErrorCode.CACHE_FAIL, - blockRoot: blockRootHex, - }); - } - // [REJECT] `envelope.builder_index == bid.builder_index` - if (envelope.builderIndex !== block.builderIndex) { + if (envelope.builderIndex !== payloadInput.getBuilderIndex()) { throw new ExecutionPayloadEnvelopeError(GossipAction.REJECT, { code: ExecutionPayloadEnvelopeErrorCode.BUILDER_INDEX_MISMATCH, envelopeBuilderIndex: envelope.builderIndex, - bidBuilderIndex: block.builderIndex, + bidBuilderIndex: payloadInput.getBuilderIndex(), }); } // [REJECT] `payload.block_hash == bid.block_hash` - if (toRootHex(payload.blockHash) !== block.blockHashFromBid) { + if (toRootHex(payload.blockHash) !== payloadInput.getBlockHashHex()) { throw new ExecutionPayloadEnvelopeError(GossipAction.REJECT, { code: ExecutionPayloadEnvelopeErrorCode.BLOCK_HASH_MISMATCH, envelopeBlockHash: toRootHex(payload.blockHash), - bidBlockHash: block.blockHashFromBid, + bidBlockHash: payloadInput.getBlockHashHex(), }); } - // [REJECT] `signed_execution_payload_envelope.signature` is valid with respect to the builder's public key. - const state = chain.getHeadState() as CachedBeaconStateGloas; - const signatureSet = createSingleSignatureSetFromComponents( - PublicKey.fromBytes(state.builders.getReadonly(envelope.builderIndex).pubkey), - getExecutionPayloadEnvelopeSigningRoot(chain.config, envelope), - executionPayloadEnvelope.signature + // Get the post block state which is the pre-payload state to verify the builder's signature. + const blockState = await chain.regen + .getState(block.stateRoot, RegenCaller.validateGossipPayloadEnvelope) + .catch(() => { + throw new ExecutionPayloadEnvelopeError(GossipAction.IGNORE, { + code: ExecutionPayloadEnvelopeErrorCode.UNKNOWN_BLOCK_STATE, + blockRoot: blockRootHex, + slot: envelope.slot, + }); + }); + + const state = blockState as CachedBeaconStateGloas; + const signatureSet = getExecutionPayloadEnvelopeSignatureSet( + chain.config, + chain.pubkeyCache, + new BeaconStateView(state), + executionPayloadEnvelope, + payloadInput.proposerIndex ); - if (!(await chain.bls.verifySignatureSets([signatureSet]))) { + if (!(await chain.bls.verifySignatureSets([signatureSet], {verifyOnMainThread: true}))) { throw new ExecutionPayloadEnvelopeError(GossipAction.REJECT, { code: ExecutionPayloadEnvelopeErrorCode.INVALID_SIGNATURE, }); } - - chain.seenExecutionPayloadEnvelopes.add(blockRootHex, envelope.slot); } diff --git a/packages/beacon-node/src/chain/validatorMonitor.ts b/packages/beacon-node/src/chain/validatorMonitor.ts index 0a2689ad5e70..45c88e42a99b 100644 --- a/packages/beacon-node/src/chain/validatorMonitor.ts +++ b/packages/beacon-node/src/chain/validatorMonitor.ts @@ -23,6 +23,7 @@ import { ValidatorIndex, altair, deneb, + gloas, } from "@lodestar/types"; import {LogData, LogHandler, LogLevel, Logger, MapDef, MapDefMax, prettyPrintIndices, toRootHex} from "@lodestar/utils"; import {GENESIS_SLOT} from "../constants/constants.js"; @@ -61,6 +62,11 @@ export type ValidatorMonitor = { ): void; registerBeaconBlock(src: OpSource, delaySec: Seconds, block: BeaconBlock): void; registerBlobSidecar(src: OpSource, seenTimestampSec: Seconds, blob: deneb.BlobSidecar): void; + registerExecutionPayloadEnvelope( + src: OpSource, + delaySec: Seconds, + envelope: gloas.SignedExecutionPayloadEnvelope + ): void; registerImportedBlock(block: BeaconBlock, data: {proposerBalanceDelta: number}): void; onPoolSubmitUnaggregatedAttestation( seenTimestampSec: number, @@ -450,6 +456,10 @@ export function createValidatorMonitor( //TODO: freetheblobs }, + registerExecutionPayloadEnvelope(_src, _delaySec, _envelope) { + // TODO GLOAS: implement execution payload envelope monitoring + }, + registerImportedBlock(block, {proposerBalanceDelta}) { const validator = validators.get(block.proposerIndex); if (validator) { diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index 32955d5a9e4d..f08a46608956 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -3,6 +3,7 @@ import {NotReorgedReason} from "@lodestar/fork-choice"; import {ArchiveStoreTask} from "../../chain/archiveStore/archiveStore.js"; import {FrequencyStateArchiveStep} from "../../chain/archiveStore/strategies/frequencyStateArchiveStrategy.js"; import {BlockInputSource} from "../../chain/blocks/blockInput/index.js"; +import {PayloadEnvelopeInputSource} from "../../chain/blocks/payloadEnvelopeInput/index.js"; import {JobQueueItemType} from "../../chain/bls/index.js"; import {AttestationErrorCode, BlockErrorCode} from "../../chain/errors/index.js"; import { @@ -237,6 +238,56 @@ export function createLodestarMetrics( }), }, + payloadEnvelopeProcessorQueue: { + length: register.gauge({ + name: "lodestar_payload_envelope_processor_queue_length", + help: "Count of total payload envelope processor queue length", + }), + droppedJobs: register.gauge({ + name: "lodestar_payload_envelope_processor_queue_dropped_jobs_total", + help: "Count of total payload envelope processor queue dropped jobs", + }), + jobTime: register.histogram({ + name: "lodestar_payload_envelope_processor_queue_job_time_seconds", + help: "Time to process payload envelope processor queue job in seconds", + buckets: [0.01, 0.1, 1, 4, 12], + }), + jobWaitTime: register.histogram({ + name: "lodestar_payload_envelope_processor_queue_job_wait_time_seconds", + help: "Time from job added to the payload envelope processor queue to starting in seconds", + buckets: [0.01, 0.1, 1, 4, 12], + }), + concurrency: register.gauge({ + name: "lodestar_payload_envelope_processor_queue_concurrency", + help: "Current concurrency of payload envelope processor queue", + }), + }, + + unfinalizedPayloadEnvelopeWritesQueue: { + length: register.gauge({ + name: "lodestar_unfinalized_payload_envelope_writes_queue_length", + help: "Count of total unfinalized payload envelope writes queue length", + }), + droppedJobs: register.gauge({ + name: "lodestar_unfinalized_payload_envelope_writes_queue_dropped_jobs_total", + help: "Count of total unfinalized payload envelope writes queue dropped jobs", + }), + jobTime: register.histogram({ + name: "lodestar_unfinalized_payload_envelope_writes_queue_job_time_seconds", + help: "Time to process unfinalized payload envelope writes queue job in seconds", + buckets: [0.01, 0.1, 1, 4, 12], + }), + jobWaitTime: register.histogram({ + name: "lodestar_unfinalized_payload_envelope_writes_queue_job_wait_time_seconds", + help: "Time from job added to the unfinalized payload envelope writes queue to starting in seconds", + buckets: [0.01, 0.1, 1, 4, 12], + }), + concurrency: register.gauge({ + name: "lodestar_unfinalized_payload_envelope_writes_queue_concurrency", + help: "Current concurrency of unfinalized payload envelope writes queue", + }), + }, + engineHttpProcessorQueue: { length: register.gauge({ name: "lodestar_engine_http_processor_queue_length", @@ -925,6 +976,18 @@ export function createLodestarMetrics( labelNames: ["reason"], }), }, + importPayload: { + bySource: register.gauge<{source: PayloadEnvelopeInputSource}>({ + name: "lodestar_import_payload_by_source_total", + help: "Total number of imported execution payload envelopes by source", + labelNames: ["source"], + }), + columnsBySource: register.gauge<{source: PayloadEnvelopeInputSource}>({ + name: "lodestar_import_payload_columns_by_source_total", + help: "Total number of payload-attached columns (sampled columns for Gloas) by source", + labelNames: ["source"], + }), + }, engineNotifyNewPayloadResult: register.gauge<{result: ExecutionPayloadStatus}>({ name: "lodestar_execution_engine_notify_new_payload_result_total", help: "The total result of calling notifyNewPayload execution engine api", @@ -1495,6 +1558,20 @@ export function createLodestarMetrics( help: "Number of BlockInputs created via a data column being seen first", }), }, + payloadEnvelopeInput: { + count: register.gauge({ + name: "lodestar_seen_payload_envelope_input_cache_size", + help: "Number of cached PayloadEnvelopeInputs", + }), + serializedObjectRefs: register.gauge({ + name: "lodestar_seen_payload_envelope_input_cache_serialized_object_refs", + help: "Number of serialized-cache object refs retained by cached PayloadEnvelopeInputs", + }), + created: register.counter({ + name: "lodestar_seen_payload_envelope_input_cache_items_created_total", + help: "Number of PayloadEnvelopeInputs created", + }), + }, }, processFinalizedCheckpoint: { diff --git a/packages/beacon-node/src/network/processor/extractSlotRootFns.ts b/packages/beacon-node/src/network/processor/extractSlotRootFns.ts index 1cf09716bbc6..304e2c54f236 100644 --- a/packages/beacon-node/src/network/processor/extractSlotRootFns.ts +++ b/packages/beacon-node/src/network/processor/extractSlotRootFns.ts @@ -1,11 +1,14 @@ -import {ForkName} from "@lodestar/params"; +import {ForkName, isForkPostGloas} from "@lodestar/params"; import {SlotOptionalRoot, SlotRootHex} from "@lodestar/types"; import { + getBeaconBlockRootFromDataColumnSidecarSerialized, + getBeaconBlockRootFromExecutionPayloadEnvelopeSerialized, getBlockRootFromBeaconAttestationSerialized, getBlockRootFromSignedAggregateAndProofSerialized, getSlotFromBeaconAttestationSerialized, getSlotFromBlobSidecarSerialized, getSlotFromDataColumnSidecarSerialized, + getSlotFromExecutionPayloadEnvelopeSerialized, getSlotFromSignedAggregateAndProofSerialized, getSlotFromSignedBeaconBlockSerialized, } from "../../util/sszBytes.js"; @@ -52,13 +55,23 @@ export function createExtractBlockSlotRootFns(): ExtractSlotRootFns { } return {slot}; }, - [GossipType.data_column_sidecar]: (data: Uint8Array): SlotOptionalRoot | null => { - const slot = getSlotFromDataColumnSidecarSerialized(data); - + [GossipType.data_column_sidecar]: (data: Uint8Array, fork: ForkName): SlotOptionalRoot | null => { + const slot = getSlotFromDataColumnSidecarSerialized(data, fork); if (slot === null) { return null; } - return {slot}; + + const root = isForkPostGloas(fork) ? getBeaconBlockRootFromDataColumnSidecarSerialized(data) : null; + return root !== null ? {slot, root} : {slot}; + }, + [GossipType.execution_payload]: (data: Uint8Array): SlotRootHex | null => { + const slot = getSlotFromExecutionPayloadEnvelopeSerialized(data); + const root = getBeaconBlockRootFromExecutionPayloadEnvelopeSerialized(data); + + if (slot === null || root === null) { + return null; + } + return {slot, root}; }, }; } diff --git a/packages/beacon-node/src/network/processor/gossipHandlers.ts b/packages/beacon-node/src/network/processor/gossipHandlers.ts index b022a1866470..2dc0aab0f262 100644 --- a/packages/beacon-node/src/network/processor/gossipHandlers.ts +++ b/packages/beacon-node/src/network/processor/gossipHandlers.ts @@ -30,6 +30,7 @@ import { IBlockInput, isBlockInputColumns, } from "../../chain/blocks/blockInput/index.js"; +import {PayloadEnvelopeInputSource} from "../../chain/blocks/payloadEnvelopeInput/index.js"; import {BlobSidecarValidation} from "../../chain/blocks/types.js"; import {ChainEvent} from "../../chain/emitter.js"; import { @@ -42,6 +43,8 @@ import { BlockGossipError, DataColumnSidecarErrorCode, DataColumnSidecarGossipError, + ExecutionPayloadEnvelopeError, + ExecutionPayloadEnvelopeErrorCode, GossipAction, GossipActionError, SyncCommitteeError, @@ -616,6 +619,13 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand }); }); } + + // TODO GLOAS: In Gloas, also add column to PayloadEnvelopeInput and notify the payload processor: + // const payloadInput = chain.seenPayloadEnvelopeInput.get(blockRootHex); + // if (payloadInput) { + // payloadInput.addColumn({columnSidecar, source: BlockInputSource.gossip, seenTimestampSec, peerIdStr}); + // chain.processExecutionPayload(payloadInput, {validSignature: true}); + // } }, [GossipType.beacon_aggregate_and_proof]: async ({ @@ -826,17 +836,43 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand [GossipType.execution_payload]: async ({ gossipData, topic, + peerIdStr, seenTimestampSec, }: GossipHandlerParamGeneric) => { const {serializedData} = gossipData; const executionPayloadEnvelope = sszDeserialize(topic, serializedData); + // TODO GLOAS: handle BLOCK_ROOT_UNKNOWN error to trigger sync await validateGossipExecutionPayloadEnvelope(chain, executionPayloadEnvelope); const slot = executionPayloadEnvelope.message.slot; const delaySec = seenTimestampSec - computeTimeAtSlot(config, slot, chain.genesisTime); metrics?.gossipExecutionPayloadEnvelope.elapsedTimeTillReceived.observe({source: OpSource.gossip}, delaySec); + chain.validatorMonitor?.registerExecutionPayloadEnvelope(OpSource.gossip, delaySec, executionPayloadEnvelope); + + const blockRootHex = toRootHex(executionPayloadEnvelope.message.beaconBlockRoot); + const payloadInput = chain.seenPayloadEnvelopeInputCache.get(blockRootHex); + + if (!payloadInput) { + // This shouldn't happen because beacon block should have been imported and thus payload input should have been created. + throw new ExecutionPayloadEnvelopeError(GossipAction.REJECT, { + code: ExecutionPayloadEnvelopeErrorCode.PAYLOAD_ENVELOPE_INPUT_MISSING, + blockRoot: blockRootHex, + }); + } - // TODO GLOAS: Handle valid envelope. Need an import flow that calls `processExecutionPayloadEnvelope` and fork choice + chain.serializedCache.set(executionPayloadEnvelope, serializedData); + + payloadInput.addPayloadEnvelope({ + envelope: executionPayloadEnvelope, + source: PayloadEnvelopeInputSource.gossip, + seenTimestampSec, + peerIdStr, + }); + + // TODO GLOAS: Emit execution_payload_gossip event for gossip receipt. + chain.processExecutionPayload(payloadInput, {validSignature: true}).catch((e) => { + chain.logger.debug("Error processing execution payload from gossip", {slot, root: blockRootHex}, e as Error); + }); }, [GossipType.payload_attestation_message]: async ({ gossipData, diff --git a/packages/beacon-node/src/util/sszBytes.ts b/packages/beacon-node/src/util/sszBytes.ts index 8ba5ddf3291e..d33bd898033d 100644 --- a/packages/beacon-node/src/util/sszBytes.ts +++ b/packages/beacon-node/src/util/sszBytes.ts @@ -8,6 +8,7 @@ import { ForkSeq, MAX_COMMITTEES_PER_SLOT, isForkPostElectra, + isForkPostGloas, } from "@lodestar/params"; import {BLSSignature, CommitteeIndex, RootHex, Slot, ValidatorIndex, ssz} from "@lodestar/types"; @@ -398,23 +399,102 @@ export function getSlotFromBlobSidecarSerialized(data: Uint8Array): Slot | null } /** + * Pre-Gloas DataColumnSidecar: * { - index: ColumnIndex [ fixed - 8 bytes], - column: DataColumn BYTES_PER_FIELD_ELEMENT * FIELD_ELEMENTS_PER_CELL * , - kzgCommitments: denebSsz.BlobKzgCommitments, - kzgProofs: denebSsz.KZGProofs, - signedBlockHeader: phase0Ssz.SignedBeaconBlockHeader, - kzgCommitmentsInclusionProof: KzgCommitmentsInclusionProof, + * index: ColumnIndex [fixed - 8 bytes], + * column: DataColumn (offset - 4 bytes), + * kzgCommitments: (offset - 4 bytes), + * kzgProofs: (offset - 4 bytes), + * signedBlockHeader: (offset - 4 bytes) -> slot at variable offset after fixed header + * kzgCommitmentsInclusionProof: (offset - 4 bytes), + * } + * Post-Gloas DataColumnSidecar: + * { + * index: ColumnIndex [8 bytes], + * column: DataColumn (offset - 4 bytes), + * kzgProofs: (offset - 4 bytes), + * slot: Slot [8 bytes] - at offset 16, + * beaconBlockRoot: Root [32 bytes] - at offset 24, + * } + */ +const SLOT_BYTES_POSITION_IN_SIGNED_DATA_COLUMN_SIDECAR_PRE_GLOAS = 20; +const SLOT_BYTES_POSITION_IN_SIGNED_DATA_COLUMN_SIDECAR_POST_GLOAS = 16; +const BEACON_BLOCK_ROOT_POSITION_IN_GLOAS_DATA_COLUMN_SIDECAR = 24; + +export function getSlotFromDataColumnSidecarSerialized(data: Uint8Array, fork: ForkName): Slot | null { + const offset = isForkPostGloas(fork) + ? SLOT_BYTES_POSITION_IN_SIGNED_DATA_COLUMN_SIDECAR_POST_GLOAS + : SLOT_BYTES_POSITION_IN_SIGNED_DATA_COLUMN_SIDECAR_PRE_GLOAS; + + if (data.length < offset + SLOT_SIZE) { + return null; } + + return getSlotFromOffset(data, offset); +} + +export function getBeaconBlockRootFromDataColumnSidecarSerialized(data: Uint8Array): RootHex | null { + if (data.length < BEACON_BLOCK_ROOT_POSITION_IN_GLOAS_DATA_COLUMN_SIDECAR + ROOT_SIZE) { + return null; + } + + blockRootBuf.set( + data.subarray( + BEACON_BLOCK_ROOT_POSITION_IN_GLOAS_DATA_COLUMN_SIDECAR, + BEACON_BLOCK_ROOT_POSITION_IN_GLOAS_DATA_COLUMN_SIDECAR + ROOT_SIZE + ) + ); + return "0x" + blockRootBuf.toString("hex"); +} + +/** + * SignedExecutionPayloadEnvelope SSZ Layout: + * ├─ 4 bytes: message offset (points to byte 100) + * ├─ 96 bytes: signature + * └─ ExecutionPayloadEnvelope (starts at byte 100): + * ├─ 4 bytes: payload offset + * ├─ 4 bytes: executionRequests offset + * ├─ 8 bytes: builderIndex (offset 108-115) + * ├─ 32 bytes: beaconBlockRoot (offset 116-147) + * ├─ 8 bytes: slot (offset 148-155) + * └─ 32 bytes: stateRoot (offset 156-187) */ +const SIGNED_EXECUTION_PAYLOAD_ENVELOPE_MESSAGE_OFFSET = 4; +const SIGNED_EXECUTION_PAYLOAD_ENVELOPE_SIGNATURE_SIZE = 96; +const EXECUTION_PAYLOAD_ENVELOPE_PAYLOAD_OFFSET = 4; +const EXECUTION_PAYLOAD_ENVELOPE_REQUESTS_OFFSET = 4; +const EXECUTION_PAYLOAD_ENVELOPE_BUILDER_INDEX_SIZE = 8; + +const BEACON_BLOCK_ROOT_OFFSET_IN_SIGNED_EXECUTION_PAYLOAD_ENVELOPE = + SIGNED_EXECUTION_PAYLOAD_ENVELOPE_MESSAGE_OFFSET + + SIGNED_EXECUTION_PAYLOAD_ENVELOPE_SIGNATURE_SIZE + + EXECUTION_PAYLOAD_ENVELOPE_PAYLOAD_OFFSET + + EXECUTION_PAYLOAD_ENVELOPE_REQUESTS_OFFSET + + EXECUTION_PAYLOAD_ENVELOPE_BUILDER_INDEX_SIZE; // 116 + +const SLOT_OFFSET_IN_SIGNED_EXECUTION_PAYLOAD_ENVELOPE = + BEACON_BLOCK_ROOT_OFFSET_IN_SIGNED_EXECUTION_PAYLOAD_ENVELOPE + ROOT_SIZE; // 148 + +export function getSlotFromExecutionPayloadEnvelopeSerialized(data: Uint8Array): Slot | null { + if (data.length < SLOT_OFFSET_IN_SIGNED_EXECUTION_PAYLOAD_ENVELOPE + SLOT_SIZE) { + return null; + } -const SLOT_BYTES_POSITION_IN_SIGNED_DATA_COLUMN_SIDECAR = 20; -export function getSlotFromDataColumnSidecarSerialized(data: Uint8Array): Slot | null { - if (data.length < SLOT_BYTES_POSITION_IN_SIGNED_DATA_COLUMN_SIDECAR + SLOT_SIZE) { + return getSlotFromOffset(data, SLOT_OFFSET_IN_SIGNED_EXECUTION_PAYLOAD_ENVELOPE); +} + +export function getBeaconBlockRootFromExecutionPayloadEnvelopeSerialized(data: Uint8Array): RootHex | null { + if (data.length < BEACON_BLOCK_ROOT_OFFSET_IN_SIGNED_EXECUTION_PAYLOAD_ENVELOPE + ROOT_SIZE) { return null; } - return getSlotFromOffset(data, SLOT_BYTES_POSITION_IN_SIGNED_DATA_COLUMN_SIDECAR); + blockRootBuf.set( + data.subarray( + BEACON_BLOCK_ROOT_OFFSET_IN_SIGNED_EXECUTION_PAYLOAD_ENVELOPE, + BEACON_BLOCK_ROOT_OFFSET_IN_SIGNED_EXECUTION_PAYLOAD_ENVELOPE + ROOT_SIZE + ) + ); + return "0x" + blockRootBuf.toString("hex"); } /** diff --git a/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts b/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts index 1136d16ab590..192d96f29d7a 100644 --- a/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts +++ b/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts @@ -71,8 +71,6 @@ describe(`getAttestationsForBlock vc=${vc}`, () => { parentBlockHash: null, payloadStatus: 2, // PayloadStatus.FULL - builderIndex: null, - blockHashFromBid: null, }, originalState.slot ); @@ -101,8 +99,6 @@ describe(`getAttestationsForBlock vc=${vc}`, () => { parentBlockHash: null, payloadStatus: 2, // PayloadStatus.FULL - builderIndex: null, - blockHashFromBid: null, }, slot, null diff --git a/packages/beacon-node/test/spec/presets/operations.test.ts b/packages/beacon-node/test/spec/presets/operations.test.ts index 6b8d7a90300a..48fd66e28c50 100644 --- a/packages/beacon-node/test/spec/presets/operations.test.ts +++ b/packages/beacon-node/test/spec/presets/operations.test.ts @@ -78,7 +78,10 @@ const operationFns: Record> = ): CachedBeaconStateAllForks | void => { const fork = state.config.getForkSeq(state.slot); if (fork >= ForkSeq.gloas) { - return blockFns.processExecutionPayloadEnvelope(state as CachedBeaconStateGloas, testCase.signed_envelope, true); + return blockFns.processExecutionPayloadEnvelope(state as CachedBeaconStateGloas, testCase.signed_envelope, { + verifySignature: true, + verifyStateRoot: true, + }); } blockFns.processExecutionPayload(fork, state as CachedBeaconStateBellatrix, testCase.body, { executionPayloadStatus: testCase.execution.execution_valid diff --git a/packages/beacon-node/test/unit/util/sszBytes.test.ts b/packages/beacon-node/test/unit/util/sszBytes.test.ts index 141aa2eca196..e30bde3735ff 100644 --- a/packages/beacon-node/test/unit/util/sszBytes.test.ts +++ b/packages/beacon-node/test/unit/util/sszBytes.test.ts @@ -26,6 +26,8 @@ import { getAttDataFromSignedAggregateAndProofPhase0, getAttDataFromSingleAttestationSerialized, getAttesterIndexFromSingleAttestationSerialized, + getBeaconBlockRootFromDataColumnSidecarSerialized, + getBeaconBlockRootFromExecutionPayloadEnvelopeSerialized, getBlobKzgCommitmentsCountFromSignedBeaconBlockSerialized, getBlockRootFromAttestationSerialized, getBlockRootFromSignedAggregateAndProofSerialized, @@ -38,6 +40,8 @@ import { getSlotFromAttestationSerialized, getSlotFromBeaconStateSerialized, getSlotFromBlobSidecarSerialized, + getSlotFromDataColumnSidecarSerialized, + getSlotFromExecutionPayloadEnvelopeSerialized, getSlotFromSignedAggregateAndProofSerialized, getSlotFromSignedBeaconBlockSerialized, getSlotFromSingleAttestationSerialized, @@ -478,3 +482,99 @@ function blobSidecarFromValues(slot: Slot): deneb.BlobSidecar { blobSidecar.signedBlockHeader.message.slot = slot; return blobSidecar; } + +describe("SignedExecutionPayloadEnvelope SSZ serialized picking", () => { + const testCases: {slot: Slot; blockRoot: RootHex}[] = [ + {slot: 0, blockRoot: "0x" + "00".repeat(32)}, + {slot: 1_000_000, blockRoot: "0x" + "aa".repeat(32)}, + {slot: 4_294_967_295, blockRoot: "0x" + "ff".repeat(32)}, // max uint32 + ]; + + for (const {slot, blockRoot} of testCases) { + it(`slot=${slot}`, () => { + const envelope = ssz.gloas.SignedExecutionPayloadEnvelope.defaultValue(); + envelope.message.slot = slot; + envelope.message.beaconBlockRoot = fromHex(blockRoot); + const bytes = ssz.gloas.SignedExecutionPayloadEnvelope.serialize(envelope); + + expect(getSlotFromExecutionPayloadEnvelopeSerialized(bytes)).toBe(slot); + expect(getBeaconBlockRootFromExecutionPayloadEnvelopeSerialized(bytes)).toBe(blockRoot); + }); + } + + it("getSlotFromExecutionPayloadEnvelopeSerialized - invalid data", () => { + // Slot is at offset 148, need at least 156 bytes + const invalidSizes = [0, 50, 100, 155]; + for (const size of invalidSizes) { + expect(getSlotFromExecutionPayloadEnvelopeSerialized(Buffer.alloc(size))).toBeNull(); + } + }); + + it("getBeaconBlockRootFromExecutionPayloadEnvelopeSerialized - invalid data", () => { + // Block root is at offset 116, need at least 148 bytes + const invalidSizes = [0, 50, 100, 147]; + for (const size of invalidSizes) { + expect(getBeaconBlockRootFromExecutionPayloadEnvelopeSerialized(Buffer.alloc(size))).toBeNull(); + } + }); +}); + +describe("DataColumnSidecar SSZ serialized picking (fork-aware)", () => { + describe("Fulu (pre-Gloas)", () => { + const testCases: {slot: Slot}[] = [{slot: 0}, {slot: 500_000}, {slot: 4_294_967_295}]; + + for (const {slot} of testCases) { + it(`slot=${slot}`, () => { + const sidecar = ssz.fulu.DataColumnSidecar.defaultValue(); + sidecar.signedBlockHeader.message.slot = slot; + const bytes = ssz.fulu.DataColumnSidecar.serialize(sidecar); + + expect(getSlotFromDataColumnSidecarSerialized(bytes, ForkName.fulu)).toBe(slot); + }); + } + + it("getSlotFromDataColumnSidecarSerialized - invalid data", () => { + // Slot is at offset 20 for pre-Gloas, need at least 28 bytes + const invalidSizes = [0, 10, 27]; + for (const size of invalidSizes) { + expect(getSlotFromDataColumnSidecarSerialized(Buffer.alloc(size), ForkName.fulu)).toBeNull(); + } + }); + }); + + describe("Gloas", () => { + const testCases: {slot: Slot; blockRoot: RootHex}[] = [ + {slot: 0, blockRoot: "0x" + "00".repeat(32)}, + {slot: 600_000, blockRoot: "0x" + "bb".repeat(32)}, + {slot: 4_294_967_295, blockRoot: "0x" + "ff".repeat(32)}, + ]; + + for (const {slot, blockRoot} of testCases) { + it(`slot=${slot}`, () => { + const sidecar = ssz.gloas.DataColumnSidecar.defaultValue(); + sidecar.slot = slot; + sidecar.beaconBlockRoot = fromHex(blockRoot); + const bytes = ssz.gloas.DataColumnSidecar.serialize(sidecar); + + expect(getSlotFromDataColumnSidecarSerialized(bytes, ForkName.gloas)).toBe(slot); + expect(getBeaconBlockRootFromDataColumnSidecarSerialized(bytes)).toBe(blockRoot); + }); + } + + it("getSlotFromDataColumnSidecarSerialized - invalid data", () => { + // Slot is at offset 16 for Gloas, need at least 24 bytes + const invalidSizes = [0, 10, 23]; + for (const size of invalidSizes) { + expect(getSlotFromDataColumnSidecarSerialized(Buffer.alloc(size), ForkName.gloas)).toBeNull(); + } + }); + + it("getBeaconBlockRootFromDataColumnSidecarSerialized - invalid data", () => { + // Block root is at offset 24 for Gloas, need at least 56 bytes + const invalidSizes = [0, 20, 55]; + for (const size of invalidSizes) { + expect(getBeaconBlockRootFromDataColumnSidecarSerialized(Buffer.alloc(size))).toBeNull(); + } + }); + }); +}); diff --git a/packages/beacon-node/test/utils/state.ts b/packages/beacon-node/test/utils/state.ts index af7791611087..d5ee1595e637 100644 --- a/packages/beacon-node/test/utils/state.ts +++ b/packages/beacon-node/test/utils/state.ts @@ -176,7 +176,5 @@ export const zeroProtoBlock: ProtoBlock = { ...{executionPayloadBlockHash: null, executionStatus: ExecutionStatus.PreMerge}, dataAvailabilityStatus: DataAvailabilityStatus.PreData, payloadStatus: PayloadStatus.FULL, - builderIndex: null, - blockHashFromBid: null, parentBlockHash: null, }; diff --git a/packages/beacon-node/test/utils/validationData/attestation.ts b/packages/beacon-node/test/utils/validationData/attestation.ts index 1e57551cd8e8..f54ea3748d9b 100644 --- a/packages/beacon-node/test/utils/validationData/attestation.ts +++ b/packages/beacon-node/test/utils/validationData/attestation.ts @@ -82,8 +82,6 @@ export function getAttestationValidData(opts: AttestationValidDataOpts): { parentBlockHash: null, payloadStatus: 2, // PayloadStatus.FULL - builderIndex: null, - blockHashFromBid: null, }; const shufflingCache = new ShufflingCache(null, null, {}, [ diff --git a/packages/fork-choice/src/forkChoice/forkChoice.ts b/packages/fork-choice/src/forkChoice/forkChoice.ts index 34416d97066a..44f717003753 100644 --- a/packages/fork-choice/src/forkChoice/forkChoice.ts +++ b/packages/fork-choice/src/forkChoice/forkChoice.ts @@ -834,10 +834,6 @@ export class ForkChoice implements IForkChoice { }), payloadStatus: isGloasBeaconBlock(block) ? PayloadStatus.PENDING : PayloadStatus.FULL, - builderIndex: isGloasBeaconBlock(block) ? block.body.signedExecutionPayloadBid.message.builderIndex : null, - blockHashFromBid: isGloasBeaconBlock(block) - ? toRootHex(block.body.signedExecutionPayloadBid.message.blockHash) - : null, parentBlockHash: parentHashHex, }; diff --git a/packages/fork-choice/src/protoArray/interface.ts b/packages/fork-choice/src/protoArray/interface.ts index 5c42e6f33c03..02e9890297fc 100644 --- a/packages/fork-choice/src/protoArray/interface.ts +++ b/packages/fork-choice/src/protoArray/interface.ts @@ -127,12 +127,6 @@ export type ProtoBlock = BlockExtraMeta & { /** Payload status for this node (Gloas fork). Always FULL in pre-gloas */ payloadStatus: PayloadStatus; - // GLOAS: The followings are from bids. They are null in pre-gloas - // Used for execution payload gossip validation - builderIndex: number | null; - // Used for execution payload gossip validation. Not to be confused with executionPayloadBlockHash - blockHashFromBid: RootHex | null; - // Used to determine if this block extends EMPTY or FULL parent variant // Spec: gloas/fork-choice.md#new-get_parent_payload_status parentBlockHash: RootHex | null; diff --git a/packages/fork-choice/test/perf/forkChoice/util.ts b/packages/fork-choice/test/perf/forkChoice/util.ts index 9cc0cfe9a1fe..85d1918abc4a 100644 --- a/packages/fork-choice/test/perf/forkChoice/util.ts +++ b/packages/fork-choice/test/perf/forkChoice/util.ts @@ -110,8 +110,6 @@ export function initializeForkChoice(opts: Opts): ForkChoice { parentBlockHash: null, payloadStatus: PayloadStatus.FULL, - builderIndex: null, - blockHashFromBid: null, }; protoArr.onBlock(block, block.slot, null); diff --git a/packages/fork-choice/test/unit/forkChoice/forkChoice.test.ts b/packages/fork-choice/test/unit/forkChoice/forkChoice.test.ts index c2ab249d3093..df68c53632dd 100644 --- a/packages/fork-choice/test/unit/forkChoice/forkChoice.test.ts +++ b/packages/fork-choice/test/unit/forkChoice/forkChoice.test.ts @@ -133,8 +133,6 @@ describe("Forkchoice", () => { parentBlockHash: null, payloadStatus: PayloadStatus.FULL, - builderIndex: null, - blockHashFromBid: null, }; }; diff --git a/packages/fork-choice/test/unit/forkChoice/getProposerHead.test.ts b/packages/fork-choice/test/unit/forkChoice/getProposerHead.test.ts index 0d556343e5b7..8c89b5d35dba 100644 --- a/packages/fork-choice/test/unit/forkChoice/getProposerHead.test.ts +++ b/packages/fork-choice/test/unit/forkChoice/getProposerHead.test.ts @@ -52,8 +52,6 @@ describe("Forkchoice / GetProposerHead", () => { parentBlockHash: null, payloadStatus: PayloadStatus.FULL, - builderIndex: null, - blockHashFromBid: null, }; const baseHeadBlock: ProtoBlockWithWeight = { @@ -82,8 +80,6 @@ describe("Forkchoice / GetProposerHead", () => { parentBlockHash: null, payloadStatus: PayloadStatus.FULL, - builderIndex: null, - blockHashFromBid: null, }; const baseParentHeadBlock: ProtoBlockWithWeight = { @@ -111,8 +107,6 @@ describe("Forkchoice / GetProposerHead", () => { parentBlockHash: null, payloadStatus: PayloadStatus.FULL, - builderIndex: null, - blockHashFromBid: null, }; const fcStore: IForkChoiceStore = { diff --git a/packages/fork-choice/test/unit/forkChoice/shouldOverrideForkChoiceUpdate.test.ts b/packages/fork-choice/test/unit/forkChoice/shouldOverrideForkChoiceUpdate.test.ts index a613962a0740..4cd4544c0240 100644 --- a/packages/fork-choice/test/unit/forkChoice/shouldOverrideForkChoiceUpdate.test.ts +++ b/packages/fork-choice/test/unit/forkChoice/shouldOverrideForkChoiceUpdate.test.ts @@ -52,8 +52,6 @@ describe("Forkchoice / shouldOverrideForkChoiceUpdate", () => { parentBlockHash: null, payloadStatus: PayloadStatus.FULL, - builderIndex: null, - blockHashFromBid: null, }; const baseHeadBlock: ProtoBlockWithWeight = { @@ -82,8 +80,6 @@ describe("Forkchoice / shouldOverrideForkChoiceUpdate", () => { parentBlockHash: null, payloadStatus: PayloadStatus.FULL, - builderIndex: null, - blockHashFromBid: null, }; const baseParentHeadBlock: ProtoBlockWithWeight = { @@ -111,8 +107,6 @@ describe("Forkchoice / shouldOverrideForkChoiceUpdate", () => { parentBlockHash: null, payloadStatus: PayloadStatus.FULL, - builderIndex: null, - blockHashFromBid: null, }; const fcStore: IForkChoiceStore = { diff --git a/packages/fork-choice/test/unit/protoArray/executionStatusUpdates.test.ts b/packages/fork-choice/test/unit/protoArray/executionStatusUpdates.test.ts index 9e8b79261ad0..8a0a544baeb4 100644 --- a/packages/fork-choice/test/unit/protoArray/executionStatusUpdates.test.ts +++ b/packages/fork-choice/test/unit/protoArray/executionStatusUpdates.test.ts @@ -124,8 +124,6 @@ function setupForkChoice(): ProtoArray { parentBlockHash: null, payloadStatus: PayloadStatus.FULL, - builderIndex: null, - blockHashFromBid: null, }, block.slot, null diff --git a/packages/fork-choice/test/unit/protoArray/getCommonAncestor.test.ts b/packages/fork-choice/test/unit/protoArray/getCommonAncestor.test.ts index c72f97ebf65a..e8349f4ea426 100644 --- a/packages/fork-choice/test/unit/protoArray/getCommonAncestor.test.ts +++ b/packages/fork-choice/test/unit/protoArray/getCommonAncestor.test.ts @@ -48,8 +48,6 @@ describe("getCommonAncestor", () => { parentBlockHash: null, payloadStatus: PayloadStatus.FULL, - builderIndex: null, - blockHashFromBid: null, }, 0 ); @@ -79,8 +77,6 @@ describe("getCommonAncestor", () => { parentBlockHash: null, payloadStatus: PayloadStatus.FULL, - builderIndex: null, - blockHashFromBid: null, }, block.slot, null diff --git a/packages/fork-choice/test/unit/protoArray/gloas.test.ts b/packages/fork-choice/test/unit/protoArray/gloas.test.ts index b1e5bb84d684..8648633e725d 100644 --- a/packages/fork-choice/test/unit/protoArray/gloas.test.ts +++ b/packages/fork-choice/test/unit/protoArray/gloas.test.ts @@ -53,8 +53,6 @@ describe("Gloas Fork Choice", () => { dataAvailabilityStatus: DataAvailabilityStatus.Available, parentBlockHash: parentBlockHash === undefined ? null : parentBlockHash, payloadStatus: PayloadStatus.FULL, - builderIndex: null, - blockHashFromBid: null, }; } diff --git a/packages/fork-choice/test/unit/protoArray/protoArray.test.ts b/packages/fork-choice/test/unit/protoArray/protoArray.test.ts index 61567ea73204..f23332f7a292 100644 --- a/packages/fork-choice/test/unit/protoArray/protoArray.test.ts +++ b/packages/fork-choice/test/unit/protoArray/protoArray.test.ts @@ -37,8 +37,6 @@ describe("ProtoArray", () => { parentBlockHash: null, payloadStatus: PayloadStatus.FULL, - builderIndex: null, - blockHashFromBid: null, }, genesisSlot ); @@ -68,8 +66,6 @@ describe("ProtoArray", () => { parentBlockHash: null, payloadStatus: PayloadStatus.FULL, - builderIndex: null, - blockHashFromBid: null, }, genesisSlot + 1, null @@ -100,8 +96,6 @@ describe("ProtoArray", () => { parentBlockHash: null, payloadStatus: PayloadStatus.FULL, - builderIndex: null, - blockHashFromBid: null, }, genesisSlot + 1, null diff --git a/packages/state-transition/src/block/processExecutionPayloadEnvelope.ts b/packages/state-transition/src/block/processExecutionPayloadEnvelope.ts index 50fd56edcc7b..5330e41129b9 100644 --- a/packages/state-transition/src/block/processExecutionPayloadEnvelope.ts +++ b/packages/state-transition/src/block/processExecutionPayloadEnvelope.ts @@ -1,19 +1,18 @@ -import {PublicKey, Signature, verify} from "@chainsafe/blst"; -import { - BUILDER_INDEX_SELF_BUILD, - DOMAIN_BEACON_BUILDER, - SLOTS_PER_EPOCH, - SLOTS_PER_HISTORICAL_ROOT, -} from "@lodestar/params"; +import {SLOTS_PER_EPOCH, SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params"; import {gloas, ssz} from "@lodestar/types"; import {byteArrayEquals, toHex, toRootHex} from "@lodestar/utils"; +import {getExecutionPayloadEnvelopeSignatureSet} from "../signatureSets/executionPayloadEnvelope.js"; +import {BeaconStateView} from "../stateView/beaconStateView.js"; import {CachedBeaconStateGloas} from "../types.js"; -import {computeSigningRoot, computeTimeAtSlot} from "../util/index.js"; +import {computeTimeAtSlot} from "../util/index.js"; +import {verifySignatureSet} from "../util/signatureSets.js"; import {processConsolidationRequest} from "./processConsolidationRequest.js"; import {processDepositRequest} from "./processDepositRequest.js"; import {processWithdrawalRequest} from "./processWithdrawalRequest.js"; export type ProcessExecutionPayloadEnvelopeOpts = { + verifySignature?: boolean; + verifyStateRoot?: boolean; dontTransferCache?: boolean; }; @@ -23,14 +22,14 @@ export type ProcessExecutionPayloadEnvelopeOpts = { export function processExecutionPayloadEnvelope( state: CachedBeaconStateGloas, signedEnvelope: gloas.SignedExecutionPayloadEnvelope, - verify: boolean, opts?: ProcessExecutionPayloadEnvelopeOpts ): CachedBeaconStateGloas { + const {verifySignature = true, verifyStateRoot = true} = opts ?? {}; const envelope = signedEnvelope.message; const payload = envelope.payload; const fork = state.config.getForkSeq(envelope.slot); - if (verify && !verifyExecutionPayloadEnvelopeSignature(state, signedEnvelope)) { + if (verifySignature && !verifyExecutionPayloadEnvelopeSignature(state, signedEnvelope)) { throw Error(`Execution payload envelope has invalid signature builderIndex=${envelope.builderIndex}`); } @@ -70,7 +69,7 @@ export function processExecutionPayloadEnvelope( postState.commit(); - if (verify && !byteArrayEquals(envelope.stateRoot, postState.hashTreeRoot())) { + if (verifyStateRoot && !byteArrayEquals(envelope.stateRoot, postState.hashTreeRoot())) { throw new Error( `Envelope's state root does not match state envelope=${toRootHex(envelope.stateRoot)} state=${toRootHex(postState.hashTreeRoot())}` ); @@ -160,28 +159,12 @@ function verifyExecutionPayloadEnvelopeSignature( state: CachedBeaconStateGloas, signedEnvelope: gloas.SignedExecutionPayloadEnvelope ): boolean { - const builderIndex = signedEnvelope.message.builderIndex; - - const domain = state.config.getDomain(state.slot, DOMAIN_BEACON_BUILDER); - const signingRoot = computeSigningRoot(ssz.gloas.ExecutionPayloadEnvelope, signedEnvelope.message, domain); - - try { - let publicKey: PublicKey; - - if (builderIndex === BUILDER_INDEX_SELF_BUILD) { - const validatorIndex = state.latestBlockHeader.proposerIndex; - const proposerPubkey = state.epochCtx.pubkeyCache.get(validatorIndex); - if (!proposerPubkey) { - return false; - } - publicKey = proposerPubkey; - } else { - publicKey = PublicKey.fromBytes(state.builders.getReadonly(builderIndex).pubkey); - } - const signature = Signature.fromBytes(signedEnvelope.signature, true); - - return verify(signingRoot, publicKey, signature); - } catch (_e) { - return false; // Catch all BLS errors: failed key validation, failed signature validation, invalid signature - } + const signatureSet = getExecutionPayloadEnvelopeSignatureSet( + state.config, + state.epochCtx.pubkeyCache, + new BeaconStateView(state), + signedEnvelope, + state.latestBlockHeader.proposerIndex + ); + return verifySignatureSet(signatureSet); } diff --git a/packages/state-transition/src/signatureSets/executionPayloadEnvelope.ts b/packages/state-transition/src/signatureSets/executionPayloadEnvelope.ts index 40c21f25fd77..6107ef8eceae 100644 --- a/packages/state-transition/src/signatureSets/executionPayloadEnvelope.ts +++ b/packages/state-transition/src/signatureSets/executionPayloadEnvelope.ts @@ -1,7 +1,11 @@ +import {PublicKey} from "@chainsafe/blst"; import {BeaconConfig} from "@lodestar/config"; -import {DOMAIN_BEACON_BUILDER} from "@lodestar/params"; -import {gloas, ssz} from "@lodestar/types"; +import {BUILDER_INDEX_SELF_BUILD, DOMAIN_BEACON_BUILDER} from "@lodestar/params"; +import {ValidatorIndex, gloas, ssz} from "@lodestar/types"; +import {PubkeyCache} from "../cache/pubkeyCache.js"; +import {IBeaconStateView} from "../stateView/interface.js"; import {computeSigningRoot} from "../util/index.js"; +import {type SingleSignatureSet, createSingleSignatureSetFromComponents} from "../util/signatureSets.js"; export function getExecutionPayloadEnvelopeSigningRoot( config: BeaconConfig, @@ -11,3 +15,23 @@ export function getExecutionPayloadEnvelopeSigningRoot( return computeSigningRoot(ssz.gloas.ExecutionPayloadEnvelope, envelope, domain); } + +export function getExecutionPayloadEnvelopeSignatureSet( + config: BeaconConfig, + pubkeyCache: PubkeyCache, + state: IBeaconStateView, + signedEnvelope: gloas.SignedExecutionPayloadEnvelope, + proposerIndex: ValidatorIndex +): SingleSignatureSet { + const envelope = signedEnvelope.message; + const pubkey = + envelope.builderIndex === BUILDER_INDEX_SELF_BUILD + ? pubkeyCache.getOrThrow(proposerIndex) + : PublicKey.fromBytes(state.getBuilder(envelope.builderIndex).pubkey); + + return createSingleSignatureSetFromComponents( + pubkey, + getExecutionPayloadEnvelopeSigningRoot(config, envelope), + signedEnvelope.signature + ); +} diff --git a/packages/state-transition/src/stateView/beaconStateView.ts b/packages/state-transition/src/stateView/beaconStateView.ts index 47f7f772c111..12ce6ce0457f 100644 --- a/packages/state-transition/src/stateView/beaconStateView.ts +++ b/packages/state-transition/src/stateView/beaconStateView.ts @@ -28,6 +28,7 @@ import { } from "@lodestar/types"; import {Checkpoint, Fork} from "@lodestar/types/phase0"; import {processExecutionPayloadEnvelope} from "../block/index.js"; +import {ProcessExecutionPayloadEnvelopeOpts} from "../block/processExecutionPayloadEnvelope.js"; import {VoluntaryExitValidity, getVoluntaryExitValidity} from "../block/processVoluntaryExit.js"; import {getExpectedWithdrawals} from "../block/processWithdrawals.js"; import {EffectiveBalanceIncrements} from "../cache/effectiveBalanceIncrements.js"; @@ -761,12 +762,20 @@ export class BeaconStateView implements IBeaconStateView { return new BeaconStateView(newState); } - processExecutionPayloadEnvelope(signedEnvelope: gloas.SignedExecutionPayloadEnvelope, verify: boolean): void { + processExecutionPayloadEnvelope( + signedEnvelope: gloas.SignedExecutionPayloadEnvelope, + opts?: ProcessExecutionPayloadEnvelopeOpts + ): BeaconStateView { const fork = this.config.getForkName(this.cachedState.slot); if (!isForkPostGloas(fork)) { throw Error(`processExecutionPayloadEnvelope is only available for gloas+ forks, got fork=${fork}`); } - processExecutionPayloadEnvelope(this.cachedState as CachedBeaconStateGloas, signedEnvelope, verify); + const postPayloadState = processExecutionPayloadEnvelope( + this.cachedState as CachedBeaconStateGloas, + signedEnvelope, + opts + ); + return new BeaconStateView(postPayloadState); } } diff --git a/packages/state-transition/src/stateView/interface.ts b/packages/state-transition/src/stateView/interface.ts index bef62b4bcfdb..9ab6b56ff0e1 100644 --- a/packages/state-transition/src/stateView/interface.ts +++ b/packages/state-transition/src/stateView/interface.ts @@ -23,6 +23,7 @@ import { rewards, } from "@lodestar/types"; import {Checkpoint, Fork} from "@lodestar/types/phase0"; +import {ProcessExecutionPayloadEnvelopeOpts} from "../block/processExecutionPayloadEnvelope.js"; import {VoluntaryExitValidity} from "../block/processVoluntaryExit.js"; import {EffectiveBalanceIncrements} from "../cache/effectiveBalanceIncrements.js"; import {EpochTransitionCacheOpts} from "../cache/epochTransitionCache.js"; @@ -209,5 +210,8 @@ export interface IBeaconStateView { epochTransitionCacheOpts?: EpochTransitionCacheOpts & {dontTransferCache?: boolean}, modules?: StateTransitionModules ): IBeaconStateView; - processExecutionPayloadEnvelope(signedEnvelope: gloas.SignedExecutionPayloadEnvelope, verify: boolean): void; + processExecutionPayloadEnvelope( + signedEnvelope: gloas.SignedExecutionPayloadEnvelope, + opts?: ProcessExecutionPayloadEnvelopeOpts + ): IBeaconStateView; } From f6213da56d6c22db1e22e413719ee78a3e04999e Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Fri, 20 Mar 2026 11:26:44 -0500 Subject: [PATCH 56/63] feat: schedule Fulu fork for Gnosis mainnet (#9074) Schedule Fulu fork for Gnosis mainnet. - Epoch: 1714688 - Timestamp: 1776168380 - UTC: Tue Apr 14 2026 12:06:20 Matches: gnosischain/configs#50, gnosischain/specs#92 --------- Co-authored-by: Nico Flaig Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- packages/config/src/chainConfig/networks/gnosis.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/config/src/chainConfig/networks/gnosis.ts b/packages/config/src/chainConfig/networks/gnosis.ts index 3a8a711c9be5..4116447466cb 100644 --- a/packages/config/src/chainConfig/networks/gnosis.ts +++ b/packages/config/src/chainConfig/networks/gnosis.ts @@ -57,7 +57,7 @@ export const gnosisChainConfig: ChainConfig = { ELECTRA_FORK_EPOCH: 1337856, // 2025-04-30T14:03:40.000Z // Fulu FULU_FORK_VERSION: b("0x06000064"), - FULU_FORK_EPOCH: Infinity, + FULU_FORK_EPOCH: 1714688, // 2026-04-14T12:06:20.000Z // Gloas GLOAS_FORK_VERSION: b("0x07000064"), GLOAS_FORK_EPOCH: Infinity, From 755283262084c67ab836c58abeacb181be73e594 Mon Sep 17 00:00:00 2001 From: Lodekeeper <258435968+lodekeeper@users.noreply.github.com> Date: Fri, 20 Mar 2026 16:47:48 +0000 Subject: [PATCH 57/63] fix: override MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS for Gnosis (#9075) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem The Gnosis chain config inherits mainnet's `MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS: 4096`, but the [official Gnosis config](https://github.com/gnosischain/configs/blob/main/mainnet/config.yaml) specifies `16384`. With Gnosis's faster block times (5s slots, 16 slots/epoch), this drastically shortens the data column retention window: | Config | Epochs | Retention | |--------|--------|-----------| | Mainnet (4096 × 32 slots × 12s) | 4096 | **~18.2 days** | | Gnosis with wrong 4096 (4096 × 16 slots × 5s) | 4096 | **~3.7 days** ❌ | | Gnosis with correct 16384 (16384 × 16 slots × 5s) | 16384 | **~15.2 days** ✅ | ### Impact (once Fulu activates on Gnosis) 1. **Premature pruning**: Lodestar prunes data columns after ~3.7 days instead of ~15 days 2. **Refuses serving**: `DataColumnSidecarsByRoot` handler rejects requests for columns older than ~3.7 days (`packages/beacon-node/src/network/reqresp/handlers/dataColumnSidecarsByRoot.ts:22-27`) 3. **Archive pruning**: Block archiver prunes data column sidecars too aggressively (`packages/beacon-node/src/chain/archiveStore/utils/archiveBlocks.ts:171-195`) 4. **Interop mismatch**: Other clients using the correct 16384 would retain data for ~15 days, causing Lodestar to diverge from the network ### Evidence - Official Gnosis config: [`MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS: 16384`](https://github.com/gnosischain/configs/blob/main/mainnet/config.yaml) - The existing `MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 16384` override is already present in `gnosis.ts` (line 35) — this is the same pattern ## Fix Add `MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS: 16384` to the Gnosis chain config, matching the official Gnosis config and the existing blob sidecars override. Flagged by @chatgpt-codex-connector in https://github.com/ChainSafe/lodestar/pull/9074#discussion_r2966702513 Co-authored-by: lodekeeper --- packages/config/src/chainConfig/networks/gnosis.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/config/src/chainConfig/networks/gnosis.ts b/packages/config/src/chainConfig/networks/gnosis.ts index 4116447466cb..2f97347f7caa 100644 --- a/packages/config/src/chainConfig/networks/gnosis.ts +++ b/packages/config/src/chainConfig/networks/gnosis.ts @@ -33,6 +33,7 @@ export const gnosisChainConfig: ChainConfig = { // Networking MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 16384, + MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS: 16384, // Dec 8, 2021, 13:00 UTC MIN_GENESIS_TIME: 1638968400, From 1d50253953d319f830f00d306bede7927097d01a Mon Sep 17 00:00:00 2001 From: Lodekeeper <258435968+lodekeeper@users.noreply.github.com> Date: Fri, 20 Mar 2026 18:08:52 +0000 Subject: [PATCH 58/63] docs: improve AGENTS.md with code quality rules and structural cleanup (#9077) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Improve AGENTS.md to better guide AI coding agents and human contributors. Adds new rules, consolidates duplicated guidance, and removes redundant sections. ### New content - **Critical rules section** at the top — quick-reference checklist for the most important rules (target branch, pre-push checks, `.js` imports, no `any`, no `lib/` edits, follow patterns, structured logging, incremental commits) - **Expanded `as any` guidance** — explicitly bans `(obj as any).field` in both production and test code, with `biome-ignore` escape hatch pattern - **`.js` import examples** — adds ✅/❌ examples showing correct `.js` vs wrong `.ts` extensions, clarifies this only applies to TS source files - **Pre-push checklist** — dedicated section: `pnpm lint`, `pnpm check-types`, `pnpm docs:lint`, no `lib/` edits ### Structural cleanup - **Removed "Common pitfalls" section** — rules moved into relevant sections (imports → Import organization, lint → Pre-push checklist, force push → PR etiquette, etc.) - **Removed "Important notes" section** — content merged into existing sections or the new Critical rules header - **Consolidated "Running specific tests"** — removed duplicate test commands already covered in Build commands - **Condensed "Style learnings from reviews"** — collapsed verbose code examples into concise bullet points - **Deduplicated fork list** — referenced from Architecture patterns instead of repeating ### Triggered by - `as any` in tests: https://github.com/ChainSafe/lodestar/pull/9068#discussion_r2966999528 - `.ts` imports: https://github.com/ChainSafe/lodestar/pull/8857#discussion_r2966970293 --------- Co-authored-by: lodekeeper --- AGENTS.md | 162 +++++++++++++++--------------------------------------- 1 file changed, 44 insertions(+), 118 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 19e3420ca11c..8aac83cac823 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,5 +1,16 @@ # AGENTS.md +## Critical rules + +- **Target branch:** `unstable` (never `stable`) +- **Pre-push:** run `pnpm lint`, `pnpm check-types`, `pnpm test:unit` before every push +- **Relative imports:** use `.js` extension in TypeScript ESM imports +- **No `any`:** avoid `any` / `as any`; use proper types or justified `biome-ignore` +- **No `lib/` edits:** never edit `packages/*/lib/` — these are build outputs +- **Follow existing patterns** before introducing new abstractions +- **Structured logging** with specific error codes (not generic `Error`) +- **Incremental commits** after review starts — do not force push unless maintainer requests it + ## Project overview Lodestar is a TypeScript implementation of the Ethereum consensus client @@ -103,8 +114,12 @@ Lodestar uses [Biome](https://biomejs.dev/) for linting and formatting. - **Naming**: `camelCase` for functions/variables, `PascalCase` for classes, `UPPER_SNAKE_CASE` for constants - **Quotes**: Use double quotes (`"`) not single quotes -- **Types**: All functions must have explicit parameter and return types -- **No `any`**: Avoid TypeScript `any` type +- **Types**: Prefer explicit types on public APIs and complex functions +- **No `any` or `as any`**: Do not use `any` type or `as any` assertions to bypass + the type system. In production code, find the proper type or interface. In test code, + use public APIs rather than accessing private fields via `as any`. If genuinely + unavoidable, add a suppression with the full rule ID and justification: + `// biome-ignore lint/suspicious/noExplicitAny: ` - **Private fields**: No underscore prefix (use `private dirty`, not `private _dirty`) - **Named exports only**: No default exports @@ -117,10 +132,17 @@ Imports are auto-sorted by Biome in this order: 3. `@chainsafe/*` and `@lodestar/*` packages 4. Relative paths -Always use `.js` extension for relative imports (even for `.ts` files): +In TypeScript source and test files, use `.js` extension for relative ESM imports +(even though source files are `.ts`). This is required for Node.js ESM resolution. +This rule does **not** apply to non-TS files (e.g., `package.json`, `.mjs` config). ```typescript +// ✅ Correct import {something} from "./utils.js"; +import {IBeaconStateView} from "../stateView/interface.js"; + +// ❌ Wrong — will break at runtime +import {something} from "./utils.ts"; ``` ### Comments @@ -258,25 +280,8 @@ for (const block of blocks) { ### Running specific tests -Use vitest project filters for targeted test runs: - -```bash -# Unit tests only (from repo root) -pnpm vitest run --project unit test/unit/chain/validation/block.test.ts - -# With pattern matching -pnpm vitest run --project unit -t "should reject" - -# From package directory (no project filter needed) -cd packages/beacon-node -pnpm vitest run test/unit/chain/validation/block.test.ts -``` - -For spec tests with minimal preset (faster): - -```bash -LODESTAR_PRESET=minimal pnpm vitest run --config vitest.spec.config.ts -``` +See **Build commands** above for all test invocations. Use `--project unit` +for targeted runs and `LODESTAR_PRESET=minimal` for faster spec tests. ## Pull request guidelines @@ -321,11 +326,20 @@ refactor(reqresp)!: support byte based handlers ### PR etiquette - Keep PRs as drafts until ready for review -- Don't force push after review starts (use incremental commits) -- Close stale PRs rather than letting them sit +- Avoid force push after review starts unless a maintainer requests it (use incremental commits) +- Flag stale PRs to maintainers rather than letting them sit indefinitely - Respond to review feedback promptly — reply to every comment, including bot reviewers - When updating based on feedback, respond in-thread to acknowledge +## Pre-push checklist + +Before pushing any commit, verify: + +1. `pnpm lint` — Biome enforces formatting; CI catches failures but wastes a round-trip +2. `pnpm check-types` — catch type errors before CI +3. `pnpm docs:lint` — if you edited any `.md` files, check Prettier formatting +4. No edits in `packages/*/lib/` — these are build outputs; edit `src/` instead + ## Common tasks ### Adding a new feature @@ -341,7 +355,7 @@ refactor(reqresp)!: support byte based handlers 1. Write a failing test that reproduces the bug 2. Fix the bug 3. Verify the test passes -4. Run full test suite: `pnpm test:unit` +4. Run checks: `pnpm lint`, `pnpm check-types`, `pnpm test:unit` ### Adding a new SSZ type @@ -360,47 +374,10 @@ refactor(reqresp)!: support byte based handlers ## Style learnings from reviews -### Prefer inline logic over helper functions - -For simple validation logic, inline the check rather than creating a helper: - -```typescript -// Preferred -if (error.code === RegenErrorCode.BLOCK_NOT_IN_FORKCHOICE) { - return GossipAction.REJECT; -} - -// Avoid (unless logic is complex and reused) -function shouldReject(error: Error): boolean { - return error.code === RegenErrorCode.BLOCK_NOT_IN_FORKCHOICE; -} -``` - -### Match existing comment style - -When adding comments to containers or functions modified across forks, -follow the existing style in that file. Don't add unnecessary markers. - -### Error handling patterns - -Use specific error codes when available: - -```typescript -// Preferred -throw new BlockError(block, {code: BlockErrorCode.PARENT_UNKNOWN}); - -// Avoid generic errors when specific ones exist -throw new Error("Parent not found"); -``` - -### Config value coercion - -When reading optional config values, handle undefined explicitly: - -```typescript -const peers = config.directPeers ?? []; -const trimmed = value?.trim() ?? ""; -``` +- **Prefer inline logic** over single-use helper functions for simple checks +- **Match existing patterns** in the file you're modifying (comments, structure) +- **Use specific error codes** (`BlockErrorCode.PARENT_UNKNOWN`) over generic `Error` +- **Handle undefined** explicitly: `config.directPeers ?? []`, `value?.trim() ?? ""` ## Implementing consensus specs @@ -422,8 +399,7 @@ When implementing changes from the consensus specs, the mapping is typically: ### Fork organization -Specs and code are organized by fork: `phase0`, `altair`, `bellatrix`, -`capella`, `deneb`, `electra`, `fulu`, `gloas`. +Forks follow the progression defined in **Architecture patterns > Fork-aware code** above. - **@lodestar/types/src/** - Each fork has its own directory with SSZ type definitions - **@lodestar/state-transition/src/block/** - Block processing functions @@ -431,55 +407,5 @@ Specs and code are organized by fork: `phase0`, `altair`, `bellatrix`, - **@lodestar/state-transition/src/epoch/** - Epoch processing functions - **@lodestar/state-transition/src/slot/** - Slot processing functions -## Important notes - -### Default branch is `unstable` - -All PRs should target `unstable`. The `stable` branch is for releases only -(see RELEASE.md for details). - -### Spec tests require download - -Before running `pnpm test:spec`, download test vectors: - -```bash -pnpm download-spec-tests -``` - -### E2E tests require Docker - -Start the e2e environment before running e2e tests: - -```bash -./scripts/run_e2e_env.sh start -pnpm test:e2e -./scripts/run_e2e_env.sh stop -``` - -### Generated files - -Do not edit files in `packages/*/lib/` - these are build outputs. -Edit source files in `packages/*/src/` instead. - -### Consensus spec references - The `specrefs/` directory contains pinned consensus spec versions. When implementing spec changes, reference the exact spec version. - -## Common pitfalls - -- **Forgetting `pnpm lint` before pushing**: Biome enforces formatting. Always - run it before committing. CI will catch it, but it wastes a round-trip. -- **Forgetting `pnpm docs:lint` after editing docs**: Markdown files are - formatted by Prettier. Run `pnpm docs:lint` (or `pnpm docs:lint:fix` to - auto-fix) before pushing changes to `.md` files. -- **Editing `lib/` instead of `src/`**: Files in `packages/*/lib/` are build - outputs. Always edit in `packages/*/src/`. -- **Stale fork choice head**: After modifying proto-array execution status, - the cached head from `getHead()` is stale. Call `recomputeForkChoiceHead()`. -- **Holding state references**: Beacon state objects are large. Don't store - references beyond their immediate use — let them be garbage collected. -- **Missing `.js` extension**: Relative imports must use `.js` even though - source files are `.ts`. This is required for Node.js ESM resolution. -- **Force pushing after review**: Never force push once a reviewer has started. - Use incremental commits — reviewers track changes between reviews. From 7b5ce3932d498aed0bc57896cd4d21cb7d4fdb84 Mon Sep 17 00:00:00 2001 From: Lodekeeper <258435968+lodekeeper@users.noreply.github.com> Date: Fri, 20 Mar 2026 22:10:51 +0000 Subject: [PATCH 59/63] chore: enable forceJsExtensions in biome lint config (#9079) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description Enable `forceJsExtensions: true` in the Biome `useImportExtensions` rule. This makes Biome flag `.ts` import extensions and enforce `.js`, preventing a recurring bug where AI-assisted code uses `.ts` instead of `.js` in relative imports. ### Context Raised by @nflaig in [PR #8857 review](https://github.com/ChainSafe/lodestar/pull/8857#discussion_r2966970293) — a `.ts` import was introduced by the refactor. The existing rule had `forceJsExtensions: false` which only checks that an extension exists, not that it's `.js` specifically. Flipping to `true` catches this at lint time. No code changes needed — the codebase already uses `.js` everywhere. Lint passes cleanly. > This PR was implemented with AI assistance (Claude). Co-authored-by: lodekeeper --- biome.jsonc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/biome.jsonc b/biome.jsonc index f58cdb488846..72e43151af09 100644 --- a/biome.jsonc +++ b/biome.jsonc @@ -72,7 +72,7 @@ "useImportExtensions": { "level": "error", "options": { - "forceJsExtensions": false + "forceJsExtensions": true } }, "useParseIntRadix": { From d73eccfa569e3253cd51a93ec23042c6adc243cb Mon Sep 17 00:00:00 2001 From: markolazic01 <80317369+markolazic01@users.noreply.github.com> Date: Sat, 21 Mar 2026 13:19:50 +0100 Subject: [PATCH 60/63] feat: add --http.requestTimeout flag to configure vc http request timeout (#9086) Closes #7806 --- packages/cli/src/cmds/validator/handler.ts | 1 + packages/cli/src/cmds/validator/options.ts | 7 +++++++ packages/validator/src/validator.ts | 7 ++++++- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/cmds/validator/handler.ts b/packages/cli/src/cmds/validator/handler.ts index dd92257ed785..f2aafef98717 100644 --- a/packages/cli/src/cmds/validator/handler.ts +++ b/packages/cli/src/cmds/validator/handler.ts @@ -164,6 +164,7 @@ export async function validatorHandler(args: IValidatorCliArgs & GlobalArgs): Pr globalInit: { requestWireFormat: parseWireFormat(args, "http.requestWireFormat"), responseWireFormat: parseWireFormat(args, "http.responseWireFormat"), + timeoutMs: args["http.requestTimeout"], headers: {"User-Agent": `Lodestar/${version}`}, }, }, diff --git a/packages/cli/src/cmds/validator/options.ts b/packages/cli/src/cmds/validator/options.ts index 191aff6adcd8..a73b3821b118 100644 --- a/packages/cli/src/cmds/validator/options.ts +++ b/packages/cli/src/cmds/validator/options.ts @@ -60,6 +60,7 @@ export type IValidatorCliArgs = AccountValidatorArgs & "http.requestWireFormat"?: string; "http.responseWireFormat"?: string; + "http.requestTimeout"?: number; "clock.skipSlots"?: boolean; @@ -333,6 +334,12 @@ export const validatorOptions: CliCommandOptions = { group: "http", }, + "http.requestTimeout": { + type: "number", + description: "Timeout in milliseconds for HTTP requests to the beacon node", + group: "http", + }, + "clock.skipSlots": { hidden: true, description: "Skip slots when tasks take more than one slot to run", diff --git a/packages/validator/src/validator.ts b/packages/validator/src/validator.ts index 02b5de07c5f7..eca4e47103ab 100644 --- a/packages/validator/src/validator.ts +++ b/packages/validator/src/validator.ts @@ -179,7 +179,11 @@ export class Validator { urls: typeof clientOrUrls === "string" ? [clientOrUrls] : clientOrUrls, // Validator would need the beacon to respond within the slot // See https://github.com/ChainSafe/lodestar/issues/5315 for rationale - globalInit: {timeoutMs: config.SLOT_DURATION_MS, signal: controller.signal, ...globalInit}, + globalInit: { + signal: controller.signal, + ...globalInit, + timeoutMs: globalInit?.timeoutMs ?? config.SLOT_DURATION_MS, + }, }, {config, logger, metrics: metrics?.restApiClient} ); @@ -296,6 +300,7 @@ export class Validator { urls: urls.map(toPrintableUrl).toString(), requestWireFormat: globalInit?.requestWireFormat ?? defaultInit.requestWireFormat, responseWireFormat: globalInit?.responseWireFormat ?? defaultInit.responseWireFormat, + requestTimeoutMs: globalInit?.timeoutMs ?? config.SLOT_DURATION_MS, }); } else { api = clientOrUrls; From bfeb9e7e157d9526d77842558a34940ac549669a Mon Sep 17 00:00:00 2001 From: twoeths <10568965+twoeths@users.noreply.github.com> Date: Mon, 23 Mar 2026 21:04:18 +0700 Subject: [PATCH 61/63] chore: clean up UnknownBlockSync (#9069) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Motivation** found during review #9034 - `bestPeerForBlockInput()` is not used anywhere after [PR-8200](https://github.com/ChainSafe/lodestar/pull/8200/changes#diff-593e2f511966ad9d8cb737688f30aadb67c896c68766d1b09a33cf1e251f7860L649) - We have to handle `pendingColumns` as `Set | null` for pre-fulu; we can simplify to `Set` since fulu is the active fork already **Description** - Remove dead `bestPeerForBlockInput` method from `UnknownBlockPeerBalancer` and its test - Fix stale comments referencing non-existent functions (`fetchUnknownBlockRoot`, `fetchUnavailableBlockInput`) and a wrong field name (`peerIdStrs` → `peerIdStrings`) - Narrow `pendingColumns` type from `Set | null` to `Set` in `bestPeerForPendingColumns` and `filterPeers`, removing the pre-fulu null path **AI Assistance Disclosure** Used Claude Code to identify and implement this cleanup. --------- Co-authored-by: Tuyen Nguyen Co-authored-by: Claude Sonnet 4.6 --- packages/beacon-node/src/sync/unknownBlock.ts | 58 +++---------------- .../test/unit/sync/unknownBlock.test.ts | 43 +------------- 2 files changed, 10 insertions(+), 91 deletions(-) diff --git a/packages/beacon-node/src/sync/unknownBlock.ts b/packages/beacon-node/src/sync/unknownBlock.ts index ad3e5cbc2ad9..fa520bdbcae0 100644 --- a/packages/beacon-node/src/sync/unknownBlock.ts +++ b/packages/beacon-node/src/sync/unknownBlock.ts @@ -481,7 +481,7 @@ export class BlockInputSync { * From a set of shuffled peers: * - fetch the block * - from deneb, fetch all missing blobs - * - from peerDAS, fetch sampled colmns + * - from peerDAS, fetch sampled columns * TODO: this means we only have block root, and nothing else. Consider to reflect this in the function name * prefulu, will attempt a max of `MAX_ATTEMPTS_PER_BLOCK` on different peers, postfulu we may attempt more as defined in `getMaxDownloadAttempts()` function * Also verifies the received block root + returns the peer that provided the block for future downscoring. @@ -489,10 +489,7 @@ export class BlockInputSync { private async fetchBlockInput(cacheItem: BlockInputSyncCacheItem): Promise { const rootHex = getBlockInputSyncCacheItemRootHex(cacheItem); const excludedPeers = new Set(); - const defaultPendingColumns = - this.config.getForkSeq(this.chain.clock.currentSlot) >= ForkSeq.fulu - ? new Set(this.network.custodyConfig.sampledColumns) - : null; + const defaultPendingColumns = new Set(this.network.custodyConfig.sampledColumns); const fetchStartSec = Date.now() / 1000; let slot = isPendingBlockInput(cacheItem) ? cacheItem.blockInput.slot : undefined; @@ -506,14 +503,10 @@ export class BlockInputSync { isPendingBlockInput(cacheItem) && isBlockInputColumns(cacheItem.blockInput) ? new Set(cacheItem.blockInput.getMissingSampledColumnMeta().missing) : defaultPendingColumns; - // pendingDataColumns is null pre-fulu const peerMeta = this.peerBalancer.bestPeerForPendingColumns(pendingColumns, excludedPeers); if (peerMeta === null) { // no more peer with needed columns to try, throw error - let message = `Error fetching UnknownBlockRoot slot=${slot} root=${rootHex} after ${i}: cannot find peer`; - if (pendingColumns) { - message += ` with needed columns=${prettyPrintIndices(Array.from(pendingColumns))}`; - } + const message = `Error fetching UnknownBlockRoot slot=${slot} root=${rootHex} after ${i}: cannot find peer with needed columns=${prettyPrintIndices(Array.from(pendingColumns))}`; this.metrics?.blockInputSync.fetchTimeSec.observe( {result: FetchResult.FailureTriedAllPeers}, Date.now() / 1000 - fetchStartSec @@ -650,7 +643,7 @@ export class BlockInputSync { // TODO(fulu): why is this commented out here? // // this.knownBadBlocks.add(block.blockRootHex); - // for (const peerIdStr of block.peerIdStrs) { + // for (const peerIdStr of block.peerIdStrings) { // // TODO: Refactor peerRpcScores to work with peerIdStr only // this.network.reportPeer(peerIdStr, PeerAction.LowToleranceError, "BadBlockByRoot"); // } @@ -729,11 +722,11 @@ export class UnknownBlockPeerBalancer { } /** - * called from fetchUnknownBlockRoot() where we only have block root and nothing else + * called from fetchBlockInput() where we only have block root and nothing else * excludedPeers are the peers that we requested already so we don't want to try again * pendingColumns is empty for prefulu, or the 1st time we we download a block by root */ - bestPeerForPendingColumns(pendingColumns: Set | null, excludedPeers: Set): PeerSyncMeta | null { + bestPeerForPendingColumns(pendingColumns: Set, excludedPeers: Set): PeerSyncMeta | null { const eligiblePeers = this.filterPeers(pendingColumns, excludedPeers); if (eligiblePeers.length === 0) { return null; @@ -750,37 +743,6 @@ export class UnknownBlockPeerBalancer { return this.peersMeta.get(bestPeerId) ?? null; } - /** - * called from fetchUnavailableBlockInput() where we have either BlockInput or NullBlockInput - * excludedPeers are the peers that we requested already so we don't want to try again - */ - bestPeerForBlockInput(blockInput: IBlockInput, excludedPeers: Set): PeerSyncMeta | null { - const eligiblePeers: PeerIdStr[] = []; - - if (isBlockInputColumns(blockInput)) { - const pendingDataColumns: Set = new Set(blockInput.getMissingSampledColumnMeta().missing); - // there could be no pending column in case when block is still missing - eligiblePeers.push(...this.filterPeers(pendingDataColumns, excludedPeers)); - } else { - // prefulu - eligiblePeers.push(...this.filterPeers(null, excludedPeers)); - } - - if (eligiblePeers.length === 0) { - return null; - } - - const sortedEligiblePeers = sortBy( - shuffle(eligiblePeers), - // prefer peers with least active req - (peerId) => this.activeRequests.get(peerId) ?? 0 - ); - - const bestPeerId = sortedEligiblePeers[0]; - this.onRequest(bestPeerId); - return this.peersMeta.get(bestPeerId) ?? null; - } - /** * Consumers don't need to call this method directly, it is called internally by bestPeer*() methods * make this public for testing @@ -804,8 +766,7 @@ export class UnknownBlockPeerBalancer { return totalActiveRequests; } - // pendingDataColumns could be null for prefulu - private filterPeers(pendingDataColumns: Set | null, excludedPeers: Set): PeerIdStr[] { + private filterPeers(pendingDataColumns: Set, excludedPeers: Set): PeerIdStr[] { let maxColumnCount = 0; const considerPeers: {peerId: PeerIdStr; columnCount: number}[] = []; for (const [peerId, syncMeta] of this.peersMeta.entries()) { @@ -820,13 +781,12 @@ export class UnknownBlockPeerBalancer { continue; } - if (pendingDataColumns === null || pendingDataColumns.size === 0) { - // prefulu, no pending columns + if (pendingDataColumns.size === 0) { considerPeers.push({peerId, columnCount: 0}); continue; } - // postfulu, find peers that have custody columns that we need + // find peers that have custody columns that we need const {custodyColumns: peerColumns} = syncMeta; // check if the peer has all needed columns // get match diff --git a/packages/beacon-node/test/unit/sync/unknownBlock.test.ts b/packages/beacon-node/test/unit/sync/unknownBlock.test.ts index 53dbf6cbc589..96700c05e041 100644 --- a/packages/beacon-node/test/unit/sync/unknownBlock.test.ts +++ b/packages/beacon-node/test/unit/sync/unknownBlock.test.ts @@ -5,10 +5,9 @@ import {createChainForkConfig} from "@lodestar/config"; import {config as minimalConfig} from "@lodestar/config/default"; import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; import {testLogger} from "@lodestar/logger/test-utils"; -import {ForkName} from "@lodestar/params"; import {ssz} from "@lodestar/types"; import {notNullish, sleep} from "@lodestar/utils"; -import {BlockInputColumns, BlockInputPreData} from "../../../src/chain/blocks/blockInput/blockInput.js"; +import {BlockInputPreData} from "../../../src/chain/blocks/blockInput/blockInput.js"; import {BlockInputSource} from "../../../src/chain/blocks/blockInput/types.js"; import {BlockError, BlockErrorCode} from "../../../src/chain/errors/blockError.js"; import {ChainEvent, ChainEventEmitter, IBeaconChain} from "../../../src/chain/index.js"; @@ -22,7 +21,6 @@ import {CustodyConfig} from "../../../src/util/dataColumns.js"; import {PeerIdStr} from "../../../src/util/peerId.js"; import {ClockStopped} from "../../mocks/clock.js"; import {MockedBeaconChain, getMockedBeaconChain} from "../../mocks/mockedBeaconChain.js"; -import {generateBlockWithColumnSidecars} from "../../utils/blocksAndData.js"; import {getRandPeerIdStr, getRandPeerSyncMeta} from "../../utils/peer.js"; describe("sync by UnknownBlockSync", {timeout: 20_000}, () => { @@ -355,7 +353,6 @@ describe("UnknownBlockSync", () => { }); describe("UnknownBlockPeerBalancer", async () => { - const custodyConfig = {sampledColumns: [0, 1, 2, 3]} as CustodyConfig; const peer0 = await getRandPeerSyncMeta("peer-0"); const peer1 = await getRandPeerSyncMeta("peer-1"); const peer2 = await getRandPeerSyncMeta("peer-2"); @@ -414,44 +411,6 @@ describe("UnknownBlockPeerBalancer", async () => { peers[i].custodyColumns = groups; } - const signedBlock = ssz.fulu.SignedBeaconBlock.defaultValue(); - signedBlock.message.body.blobKzgCommitments = [ssz.fulu.KZGCommitment.defaultValue()]; - const {block, rootHex, columnSidecars} = generateBlockWithColumnSidecars({forkName: ForkName.fulu}); - const blockInput = BlockInputColumns.createFromBlock({ - block: block, - blockRootHex: rootHex, - forkName: ForkName.fulu, - daOutOfRange: false, - source: BlockInputSource.gossip, - seenTimestampSec: Math.floor(Date.now() / 1000), - custodyColumns: custodyConfig.custodyColumns, - sampledColumns: custodyConfig.sampledColumns, - }); - - // test cases rely on first 2 columns being known, the rest unknown - for (const sidecar of columnSidecars.slice(0, 2)) { - blockInput.addColumn({ - columnSidecar: sidecar, - blockRootHex: rootHex, - seenTimestampSec: Math.floor(Date.now() / 1000), - source: BlockInputSource.gossip, - }); - } - - it(`bestPeerForBlockInput - test case ${testCaseIndex}`, () => { - for (const [i, activeRequest] of activeRequests.entries()) { - for (let j = 0; j < activeRequest; j++) { - peerBalancer.onRequest(peers[i].peerId); - } - } - const peer = peerBalancer.bestPeerForBlockInput(blockInput, new Set(excludedPeers)); - if (bestPeer) { - expect(peer).toEqual(bestPeer); - } else { - expect(peer).toBeNull(); - } - }); - it(`bestPeerForPendingColumns - test case ${testCaseIndex}`, () => { for (const [i, activeRequest] of activeRequests.entries()) { for (let j = 0; j < activeRequest; j++) { From 5f9f015475ed1936937abc7d5385f0fd8f6cec91 Mon Sep 17 00:00:00 2001 From: Lodekeeper <258435968+lodekeeper@users.noreply.github.com> Date: Mon, 23 Mar 2026 14:06:02 +0000 Subject: [PATCH 62/63] fix: improve sync aggregate participation (#9070) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation Fixes #8294 — Two gossip-layer bugs reduce sync aggregate participation. The effect is most visible in small devnets (~64% instead of 100%) but also affects mainnet, just with lower probability due to the larger validator set. ## Root Cause ### 1. `fastMsgIdFn` cross-topic dedup gossipsub's `fastMsgIdFn` only hashes `rpcMsg.data` without the topic. `SyncCommitteeMessage` is published to multiple `sync_committee_{subnet}` topics with identical data, so receiving nodes deduplicate the message after processing it on the first topic — dropping it for all other subnets. **Fix:** Use a topic-derived xxhash seed (cached per topic) to differentiate identical data across topics. Zero allocation on the hot path. ### 2. `getIndexInSubcommittee` returns first match only When a validator appears multiple times in the same subcommittee, only the first position is tracked in the gossip handler. The API path (`submitPoolSyncCommitteeSignatures`) correctly iterates all positions, but the gossip path does not. **Fix:** Return all matching positions and add the message to the pool for each one. ## Kurtosis Results (4 nodes × 64 validators) | Metric | Before | After | |--------|--------|-------| | Participation | 328/512 (64.1%) | 512/512 (100.0%) | | Consistency | Every block identical | Every block identical | Equivalent fix in Prysm: [OffchainLabs/prysm#15608](https://github.com/OffchainLabs/prysm/pull/15608) > **Note:** This is an AI-generated PR created with assistance from Lodekeeper 🌟 --------- Co-authored-by: lodekeeper Co-authored-by: Nico Flaig --- .../src/chain/validation/syncCommittee.ts | 29 ++++++++++--------- .../src/network/gossip/encoding.ts | 16 ++++++++++ .../src/network/processor/gossipHandlers.ts | 13 +++++---- 3 files changed, 38 insertions(+), 20 deletions(-) diff --git a/packages/beacon-node/src/chain/validation/syncCommittee.ts b/packages/beacon-node/src/chain/validation/syncCommittee.ts index d1b5c0b31e84..e1a7fcd820d9 100644 --- a/packages/beacon-node/src/chain/validation/syncCommittee.ts +++ b/packages/beacon-node/src/chain/validation/syncCommittee.ts @@ -15,12 +15,12 @@ export async function validateGossipSyncCommittee( chain: IBeaconChain, syncCommittee: altair.SyncCommitteeMessage, subnet: SubnetID -): Promise<{indexInSubcommittee: IndexInSubcommittee}> { +): Promise<{indicesInSubcommittee: IndexInSubcommittee[]}> { const {slot, validatorIndex, beaconBlockRoot} = syncCommittee; const messageRoot = toRootHex(beaconBlockRoot); const headState = chain.getHeadState(); - const indexInSubcommittee = validateGossipSyncCommitteeExceptSig(chain, headState, subnet, syncCommittee); + const indicesInSubcommittee = validateGossipSyncCommitteeExceptSig(chain, headState, subnet, syncCommittee); // [IGNORE] The signature's slot is for the current slot, i.e. sync_committee_signature.slot == current_slot. // > Checked in validateGossipSyncCommitteeExceptSig() @@ -68,7 +68,7 @@ export async function validateGossipSyncCommittee( // Register this valid item as seen chain.seenSyncCommitteeMessages.add(slot, subnet, validatorIndex, messageRoot); - return {indexInSubcommittee}; + return {indicesInSubcommittee}; } export async function validateApiSyncCommittee( @@ -105,7 +105,7 @@ export function validateGossipSyncCommitteeExceptSig( headState: CachedBeaconStateAllForks, subnet: SubnetID, data: Pick -): IndexInSubcommittee { +): IndexInSubcommittee[] { const {slot, validatorIndex} = data; // [IGNORE] The signature's slot is for the current slot, i.e. sync_committee_signature.slot == current_slot. // (with a MAXIMUM_GOSSIP_CLOCK_DISPARITY allowance) @@ -127,26 +127,27 @@ export function validateGossipSyncCommitteeExceptSig( // [REJECT] The subnet_id is valid for the given validator, i.e. subnet_id in compute_subnets_for_sync_committee(state, sync_committee_signature.validator_index). // Note this validation implies the validator is part of the broader current sync committee along with the correct subcommittee. - const indexInSubcommittee = getIndexInSubcommittee(headState, subnet, data); - if (indexInSubcommittee === null) { + const indicesInSubcommittee = getIndicesInSubcommittee(headState, subnet, data); + if (indicesInSubcommittee === null) { throw new SyncCommitteeError(GossipAction.REJECT, { code: SyncCommitteeErrorCode.VALIDATOR_NOT_IN_SYNC_COMMITTEE, validatorIndex, }); } - return indexInSubcommittee; + return indicesInSubcommittee; } /** - * Returns the IndexInSubcommittee of the given `subnet`. - * Returns `null` if not part of the sync committee or not part of the given `subnet` + * Returns all IndexInSubcommittee positions of the given `subnet`. + * Returns `null` if not part of the sync committee or not part of the given `subnet`. + * A validator may appear multiple times in the same subcommittee. */ -function getIndexInSubcommittee( +function getIndicesInSubcommittee( headState: CachedBeaconStateAllForks, subnet: SubnetID, data: Pick -): IndexInSubcommittee | null { +): IndexInSubcommittee[] | null { const syncCommittee = headState.epochCtx.getIndexedSyncCommittee(data.slot); const indexesInCommittee = syncCommittee.validatorIndexMap.get(data.validatorIndex); if (indexesInCommittee === undefined) { @@ -154,12 +155,12 @@ function getIndexInSubcommittee( return null; } + const indices: IndexInSubcommittee[] = []; for (const indexInCommittee of indexesInCommittee) { if (Math.floor(indexInCommittee / SYNC_COMMITTEE_SUBNET_SIZE) === subnet) { - return indexInCommittee % SYNC_COMMITTEE_SUBNET_SIZE; + indices.push(indexInCommittee % SYNC_COMMITTEE_SUBNET_SIZE); } } - // Not part of this specific subnet - return null; + return indices.length > 0 ? indices : null; } diff --git a/packages/beacon-node/src/network/gossip/encoding.ts b/packages/beacon-node/src/network/gossip/encoding.ts index 9b495804c44e..9b3f658ad073 100644 --- a/packages/beacon-node/src/network/gossip/encoding.ts +++ b/packages/beacon-node/src/network/gossip/encoding.ts @@ -24,12 +24,28 @@ const decoder = new snappyWasm.Decoder(); // Shared buffer to convert msgId to string const sharedMsgIdBuf = Buffer.alloc(20); +// Cache topic -> seed to avoid per-message allocations on the hot path. +// Topics are a fixed set per fork (changes only at fork boundaries). +const topicSeedCache = new Map(); + /** * The function used to generate a gossipsub message id * We use the first 8 bytes of SHA256(data) for content addressing */ export function fastMsgIdFn(rpcMsg: RPC.Message): string { if (rpcMsg.data) { + if (rpcMsg.topic) { + // Use topic-derived seed to prevent cross-topic deduplication of identical messages. + // SyncCommitteeMessages are published to multiple sync_committee_{subnet} topics with + // identical data, so hashing only the data incorrectly deduplicates across subnets. + // See https://github.com/ChainSafe/lodestar/issues/8294 + let topicSeed = topicSeedCache.get(rpcMsg.topic); + if (topicSeed === undefined) { + topicSeed = xxhash.h64Raw(Buffer.from(rpcMsg.topic), h64Seed); + topicSeedCache.set(rpcMsg.topic, topicSeed); + } + return xxhash.h64Raw(rpcMsg.data, topicSeed).toString(16); + } return xxhash.h64Raw(rpcMsg.data, h64Seed).toString(16); } return "0000000000000000"; diff --git a/packages/beacon-node/src/network/processor/gossipHandlers.ts b/packages/beacon-node/src/network/processor/gossipHandlers.ts index 2dc0aab0f262..b15af398c82b 100644 --- a/packages/beacon-node/src/network/processor/gossipHandlers.ts +++ b/packages/beacon-node/src/network/processor/gossipHandlers.ts @@ -777,9 +777,9 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand const {serializedData} = gossipData; const syncCommittee = sszDeserialize(topic, serializedData); const {subnet} = topic; - let indexInSubcommittee = 0; + let indicesInSubcommittee: number[] = [0]; try { - indexInSubcommittee = (await validateGossipSyncCommittee(chain, syncCommittee, subnet)).indexInSubcommittee; + indicesInSubcommittee = (await validateGossipSyncCommittee(chain, syncCommittee, subnet)).indicesInSubcommittee; } catch (e) { if (e instanceof SyncCommitteeError && e.action === GossipAction.REJECT) { chain.persistInvalidSszValue(ssz.altair.SyncCommitteeMessage, syncCommittee, "gossip_reject"); @@ -787,11 +787,12 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand throw e; } - // Handler - + // Handler — add for ALL positions this validator holds in the subcommittee try { - const insertOutcome = chain.syncCommitteeMessagePool.add(subnet, syncCommittee, indexInSubcommittee); - metrics?.opPool.syncCommitteeMessagePoolInsertOutcome.inc({insertOutcome}); + for (const indexInSubcommittee of indicesInSubcommittee) { + const insertOutcome = chain.syncCommitteeMessagePool.add(subnet, syncCommittee, indexInSubcommittee); + metrics?.opPool.syncCommitteeMessagePoolInsertOutcome.inc({insertOutcome}); + } } catch (e) { logger.debug("Error adding to syncCommittee pool", {subnet}, e as Error); } From 5007d8c6d6c1f38bb20c2a727f121d20e6db3221 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Mon, 23 Mar 2026 18:44:32 +0000 Subject: [PATCH 63/63] refactor: avoid falsy zero check in indexed attestation bounds validation (#9089) This doesn't fix a real bug, we are passing the test added already without any changes, so https://github.com/ChainSafe/lodestar/pull/9088 doesn't make sense. But we can simplify the code and avoid the extra `indices.at(-1)` since `prev` is already the highest index. --- .../src/block/isValidIndexedAttestation.ts | 5 ++--- .../test/unit/block/isValidIndexedAttestation.test.ts | 10 ++++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/state-transition/src/block/isValidIndexedAttestation.ts b/packages/state-transition/src/block/isValidIndexedAttestation.ts index e13203d76fb7..4015708efe2c 100644 --- a/packages/state-transition/src/block/isValidIndexedAttestation.ts +++ b/packages/state-transition/src/block/isValidIndexedAttestation.ts @@ -74,9 +74,8 @@ export function isValidIndexedAttestationIndices( prev = index; } - // check if indices are out of bounds, by checking the highest index (since it is sorted) - const lastIndex = indices.at(-1); - if (lastIndex && lastIndex >= validatorsLen) { + // check if indices are out of bounds, `prev` is the highest index since indices are sorted + if (prev >= validatorsLen) { return false; } diff --git a/packages/state-transition/test/unit/block/isValidIndexedAttestation.test.ts b/packages/state-transition/test/unit/block/isValidIndexedAttestation.test.ts index 2ee5197e61e3..0c977dfeadcf 100644 --- a/packages/state-transition/test/unit/block/isValidIndexedAttestation.test.ts +++ b/packages/state-transition/test/unit/block/isValidIndexedAttestation.test.ts @@ -28,11 +28,21 @@ describe("validate indexed attestation", () => { expectedValue: false, name: "should return invalid indexed attestation - indexes not sorted", }, + { + indices: [0, 0], + expectedValue: false, + name: "should return invalid indexed attestation - duplicate indexes", + }, { indices: [0, 1, 2, 3], expectedValue: true, name: "should return valid indexed attestation", }, + { + indices: [0], + expectedValue: true, + name: "should return valid indexed attestation - single index 0", + }, ]; it.each(testValues)("$name", ({indices, expectedValue}) => {