From 768de0e90740149f7aff0578f7d36b22e4ebb9e4 Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Wed, 29 Jul 2026 08:02:47 -0500 Subject: [PATCH 01/49] fix(discv4): scope pending requests to the destination peer Packet hashes alias across peers (deterministic signatures, 1s Expiration granularity), so a response from one node could resolve a request sent to another. Key pending requests by hash + destination node ID and hold a slice per key so concurrent waiters don't orphan each other. Reject a second in-flight FINDNODE to the same peer: NEIGHBORS carries no reply token, so two cannot be told apart. Harden ENRRESPONSE: require a matching pending request, require the record's key to derive the sender's node ID, and install via seq-monotonic UpdateENR so a replay cannot roll a node back to an older record. Drop the unused map-returning Stats methods. --- discv4/node/node.go | 18 ++ discv4/protocol/handler.go | 173 +++++++++----- discv4/protocol/pending_neighbors_test.go | 26 ++- discv4/protocol/pending_request_test.go | 261 ++++++++++++++++++++++ discv4/protocol/response_delivery_test.go | 15 +- discv4/service.go | 16 -- 6 files changed, 426 insertions(+), 83 deletions(-) create mode 100644 discv4/protocol/pending_request_test.go diff --git a/discv4/node/node.go b/discv4/node/node.go index c85fa6b..4761b9b 100644 --- a/discv4/node/node.go +++ b/discv4/node/node.go @@ -192,6 +192,24 @@ func (n *Node) SetENR(record *enr.Record) { n.mu.Unlock() } +// UpdateENR installs the record only if it is newer than the current one, so +// a replayed response cannot roll the node back to an older record. +func (n *Node) UpdateENR(record *enr.Record) bool { + if record == nil { + return false + } + + n.mu.Lock() + defer n.mu.Unlock() + + if n.enr != nil && record.Seq() <= n.enr.Seq() { + return false + } + n.enr = record + + return true +} + // statsRef returns the current shared stats pointer for use outside the lock. func (n *Node) statsRef() *stats.SharedStats { n.mu.RLock() diff --git a/discv4/protocol/handler.go b/discv4/protocol/handler.go index fc1c72b..2cf19a0 100644 --- a/discv4/protocol/handler.go +++ b/discv4/protocol/handler.go @@ -61,9 +61,11 @@ type Handler struct { nodesMu sync.RWMutex nodes map[node.ID]*node.Node - // Pending requests (hash -> PendingRequest) + // Pending requests, keyed by packet hash + destination node ID: the hash + // alone aliases across peers (deterministic signatures, 1s Expiration + // granularity), and identical requests to one peer share a key's slice. requestsMu sync.RWMutex - requests map[string]*PendingRequest + requests map[string][]*PendingRequest // Pending multi-packet FINDNODE responses pendingNeighborsMu sync.RWMutex @@ -223,7 +225,7 @@ func NewHandler(ctx context.Context, config HandlerConfig, transport Transport) ctx: ctx, transport: transport, nodes: make(map[node.ID]*node.Node), - requests: make(map[string]*PendingRequest), + requests: make(map[string][]*PendingRequest), pendingNeighbors: make(map[string]*PendingNeighborsResponse), localENR: config.LocalENR, } @@ -382,9 +384,8 @@ func (h *Handler) handlePong(fromNode *node.Node, from *net.UDPAddr, pong *Pong) h.config.OnPongReceived(fromNode, pong.To.IP, pong.To.UDP) } - // Match to pending request - req := h.getPendingRequest(string(pong.ReplyTok)) - if req != nil { + // Match to pending requests + for _, req := range h.getPendingRequests(pong.ReplyTok, fromNode.ID()) { h.deliverResponse(req, pong) } @@ -463,7 +464,7 @@ func (h *Handler) handleNeighbors(fromNode *node.Node, from *net.UDPAddr, neighb // a delivered one. Room is reserved before decoding, so records past the // cap are never persisted in the global node map, even when packets are // dispatched concurrently. - key := string(matchedReq.RequestHash) + key := requestKey(matchedReq.RequestHash, matchedReq.ToNode.ID()) h.pendingNeighborsMu.Lock() pending := h.pendingNeighbors[key] @@ -576,12 +577,31 @@ func (h *Handler) handleENRResponse(fromNode *node.Node, from *net.UDPAddr, resp "enr_seq": resp.Record.Seq(), }).Debug("Received ENRRESPONSE") - // Update node's ENR - fromNode.SetENR(resp.Record) + // Only a response to a request we actually sent to this peer may touch any + // state: ENRRESPONSE carries no expiration, so an unsolicited replay could + // otherwise roll the node back to an older record. + reqs := h.getPendingRequests(resp.ReplyTok, fromNode.ID()) + if len(reqs) == 0 { + return nil + } + + // Bind the record to the sender's identity before installing it, so a + // matched response cannot attach another node's ENR to this node. + if resp.Record == nil { + return nil + } + pub := resp.Record.PublicKey() + if pub == nil || node.PubkeyToID(pub) != fromNode.ID() { + logrus.WithFields(logrus.Fields{ + "from": from.String(), + "node_id": fmt.Sprintf("%x", fromNode.IDBytes()[:8]), + }).Debug("Dropping ENRRESPONSE: record does not match sender identity") + return nil + } + + fromNode.UpdateENR(resp.Record) - // Match to pending request - req := h.getPendingRequest(string(resp.ReplyTok)) - if req != nil { + for _, req := range reqs { h.deliverResponse(req, resp.Record) } @@ -614,8 +634,11 @@ func (h *Handler) Ping(n *node.Node) (*Pong, error) { } // Register pending request; removal is deferred so every exit path clears it. - req := h.addPendingRequest(hash, n, PingPacket) - defer h.removePendingRequest(string(hash)) + req, err := h.addPendingRequest(hash, n, PingPacket) + if err != nil { + return nil, err + } + defer h.removePendingRequest(req) // Send packet if err := h.transport.SendTo(packet, n.Addr()); err != nil { @@ -674,8 +697,11 @@ func (h *Handler) Findnode(n *node.Node, target []byte) ([]*node.Node, error) { // Register pending request. Removal is deferred so every exit path clears // it: a completed request left in the map keeps matching later NEIGHBORS // from that node and reopens collection windows until cleanup runs. - req := h.addPendingRequest(hash, n, FindnodePacket) - defer h.removePendingRequest(string(hash)) + req, err := h.addPendingRequest(hash, n, FindnodePacket) + if err != nil { + return nil, err + } + defer h.removePendingRequest(req) // Send packet if err := h.transport.SendTo(packet, n.Addr()); err != nil { @@ -725,8 +751,11 @@ func (h *Handler) RequestENR(n *node.Node) (*enr.Record, error) { } // Register pending request; removal is deferred so every exit path clears it. - pendingReq := h.addPendingRequest(hash, n, ENRRequestPacket) - defer h.removePendingRequest(string(hash)) + pendingReq, err := h.addPendingRequest(hash, n, ENRRequestPacket) + if err != nil { + return nil, err + } + defer h.removePendingRequest(pendingReq) // Send packet if err := h.transport.SendTo(packet, n.Addr()); err != nil { @@ -904,8 +933,16 @@ func (h *Handler) AllNodes() []*node.Node { // Request Tracking -// addPendingRequest registers a new pending request. -func (h *Handler) addPendingRequest(hash []byte, toNode *node.Node, packetType byte) *PendingRequest { +// requestKey scopes a pending request to its destination, since the packet +// hash alone aliases across peers (see the requests field). +func requestKey(hash []byte, id node.ID) string { + return string(hash) + string(id[:]) +} + +// addPendingRequest registers a new pending request. A second FINDNODE to a +// peer with one already in flight is rejected: NEIGHBORS carries no reply +// token, so two in-flight FINDNODEs to one peer cannot be told apart. +func (h *Handler) addPendingRequest(hash []byte, toNode *node.Node, packetType byte) (*PendingRequest, error) { req := &PendingRequest{ RequestHash: hash, ToNode: toNode, @@ -916,37 +953,68 @@ func (h *Handler) addPendingRequest(hash []byte, toNode *node.Node, packetType b } h.requestsMu.Lock() - h.requests[string(hash)] = req - h.requestsMu.Unlock() + defer h.requestsMu.Unlock() + + if packetType == FindnodePacket && h.pendingFindnodeLocked(toNode.ID()) != nil { + return nil, fmt.Errorf("findnode already in flight to %x", toNode.IDBytes()[:8]) + } - return req + key := requestKey(hash, toNode.ID()) + h.requests[key] = append(h.requests[key], req) + + return req, nil } -// getPendingRequest retrieves a pending request by hash. -func (h *Handler) getPendingRequest(hash string) *PendingRequest { +// getPendingRequests returns the pending requests matching a reply token and +// its sender, so a response can only resolve requests sent to that peer. +func (h *Handler) getPendingRequests(replyTok []byte, id node.ID) []*PendingRequest { h.requestsMu.RLock() defer h.requestsMu.RUnlock() - return h.requests[hash] + return append([]*PendingRequest(nil), h.requests[requestKey(replyTok, id)]...) } -// findPendingFindnode returns a pending FINDNODE request awaiting a response +// findPendingFindnode returns the pending FINDNODE request awaiting a response // from the given node, or nil if none exists. func (h *Handler) findPendingFindnode(id node.ID) *PendingRequest { h.requestsMu.RLock() defer h.requestsMu.RUnlock() - for _, req := range h.requests { - if req.PacketType == FindnodePacket && req.ToNode != nil && req.ToNode.ID() == id { - return req + return h.pendingFindnodeLocked(id) +} + +func (h *Handler) pendingFindnodeLocked(id node.ID) *PendingRequest { + for _, reqs := range h.requests { + for _, req := range reqs { + if req.PacketType == FindnodePacket && req.ToNode != nil && req.ToNode.ID() == id { + return req + } } } return nil } -// removePendingRequest removes a pending request. -func (h *Handler) removePendingRequest(hash string) { +// removePendingRequest removes one pending request, leaving other waiters on +// the same key in place so one caller's cleanup cannot orphan another's. +func (h *Handler) removePendingRequest(req *PendingRequest) { + if req == nil || req.ToNode == nil { + return + } + key := requestKey(req.RequestHash, req.ToNode.ID()) + h.requestsMu.Lock() - delete(h.requests, hash) - h.requestsMu.Unlock() + defer h.requestsMu.Unlock() + + reqs := h.requests[key] + for i, r := range reqs { + if r == req { + reqs = append(reqs[:i], reqs[i+1:]...) + break + } + } + if len(reqs) == 0 { + delete(h.requests, key) + } else { + h.requests[key] = reqs + } } // deliverResponse hands a response to a waiting request without blocking. @@ -986,9 +1054,17 @@ func (h *Handler) cleanup() { // Clean up expired requests h.requestsMu.Lock() - for hash, req := range h.requests { - if now.After(req.Timeout) { - delete(h.requests, hash) + for key, reqs := range h.requests { + kept := reqs[:0] + for _, req := range reqs { + if !now.After(req.Timeout) { + kept = append(kept, req) + } + } + if len(kept) == 0 { + delete(h.requests, key) + } else { + h.requests[key] = kept } } h.requestsMu.Unlock() @@ -1078,7 +1154,10 @@ func (h *Handler) GetStats() HandlerStats { knownNodes := len(h.nodes) h.nodesMu.RUnlock() h.requestsMu.RLock() - pendingRequests := len(h.requests) + pendingRequests := 0 + for _, reqs := range h.requests { + pendingRequests += len(reqs) + } h.requestsMu.RUnlock() h.pendingNeighborsMu.RLock() pendingNeighbors := len(h.pendingNeighbors) @@ -1101,24 +1180,6 @@ func (h *Handler) GetStats() HandlerStats { } } -// Stats returns current statistics as a map, for callers that render it -// generically. -func (h *Handler) Stats() map[string]interface{} { - s := h.GetStats() - return map[string]interface{}{ - "packets_received": s.PacketsReceived, - "packets_sent": s.PacketsSent, - "invalid_packets": s.InvalidPackets, - "expired_packets": s.ExpiredPackets, - "unbonded_findnode": s.UnbondedFindnode, - "findnode_requests_recv": s.FindnodeRequestsRecv, - "findnode_responses_recv": s.FindnodeResponsesRecv, - "known_nodes": s.KnownNodes, - "pending_requests": s.PendingRequests, - "pending_neighbors": s.PendingNeighbors, - } -} - // LocalRecord returns the ENR the handler currently advertises. func (h *Handler) LocalRecord() *enr.Record { h.localENRMu.RLock() diff --git a/discv4/protocol/pending_neighbors_test.go b/discv4/protocol/pending_neighbors_test.go index b811080..e8853b1 100644 --- a/discv4/protocol/pending_neighbors_test.go +++ b/discv4/protocol/pending_neighbors_test.go @@ -70,7 +70,9 @@ func TestNeighborsAccumulationCapped(t *testing.T) { defer cancel() from := makeDiscv4Node(t) - h.addPendingRequest([]byte("req"), from, FindnodePacket) + if _, err := h.addPendingRequest([]byte("req"), from, FindnodePacket); err != nil { + t.Fatalf("addPendingRequest: %v", err) + } // Pre-build packets so no slow key generation happens between the calls and // the read (the delivery goroutine deletes the entry after the window). @@ -85,7 +87,7 @@ func TestNeighborsAccumulationCapped(t *testing.T) { } h.pendingNeighborsMu.RLock() - pending := h.pendingNeighbors["req"] + pending := h.pendingNeighbors[requestKey([]byte("req"), from.ID())] h.pendingNeighborsMu.RUnlock() if pending == nil { t.Fatal("expected a pending entry for the matched FINDNODE") @@ -102,7 +104,10 @@ func TestNeighborsDeliveredToWaiter(t *testing.T) { defer cancel() from := makeDiscv4Node(t) - req := h.addPendingRequest([]byte("req"), from, FindnodePacket) + req, err := h.addPendingRequest([]byte("req"), from, FindnodePacket) + if err != nil { + t.Fatalf("addPendingRequest: %v", err) + } if err := h.handleNeighbors(from, from.Addr(), makeNeighbors(t, 5)); err != nil { t.Fatal(err) @@ -152,7 +157,9 @@ func TestNeighborsCapAppliesBeforeNodePersistence(t *testing.T) { defer cancel() from := makeDiscv4Node(t) - h.addPendingRequest([]byte("req"), from, FindnodePacket) + if _, err := h.addPendingRequest([]byte("req"), from, FindnodePacket); err != nil { + t.Fatalf("addPendingRequest: %v", err) + } if err := h.handleNeighbors(from, from.Addr(), makeNeighbors(t, maxNeighborsPerResponse+20)); err != nil { t.Fatal(err) @@ -250,7 +257,9 @@ func TestNeighborsPersistenceCapExactUnderConcurrency(t *testing.T) { defer cancel() from := makeDiscv4Node(t) - h.addPendingRequest([]byte("req"), from, FindnodePacket) + if _, err := h.addPendingRequest([]byte("req"), from, FindnodePacket); err != nil { + t.Fatalf("addPendingRequest: %v", err) + } packets := make([]*Neighbors, 6) for i := range packets { @@ -285,7 +294,10 @@ func TestNeighborsAfterDeliveryPersistNothing(t *testing.T) { defer cancel() from := makeDiscv4Node(t) - req := h.addPendingRequest([]byte("req"), from, FindnodePacket) + req, err := h.addPendingRequest([]byte("req"), from, FindnodePacket) + if err != nil { + t.Fatalf("addPendingRequest: %v", err) + } if err := h.handleNeighbors(from, from.Addr(), makeNeighbors(t, 2)); err != nil { t.Fatal(err) @@ -311,7 +323,7 @@ func TestNeighborsAfterDeliveryPersistNothing(t *testing.T) { t.Fatalf("a post-delivery packet persisted %d records, want 0", after-before) } h.pendingNeighborsMu.RLock() - pending := h.pendingNeighbors["req"] + pending := h.pendingNeighbors[requestKey([]byte("req"), from.ID())] h.pendingNeighborsMu.RUnlock() if pending == nil || !pending.Closed || len(pending.Nodes) != 2 { t.Fatalf("tombstone state = %+v, want closed with the delivered 2 nodes", pending) diff --git a/discv4/protocol/pending_request_test.go b/discv4/protocol/pending_request_test.go new file mode 100644 index 0000000..2eababe --- /dev/null +++ b/discv4/protocol/pending_request_test.go @@ -0,0 +1,261 @@ +package protocol + +import ( + "crypto/ecdsa" + "net" + "testing" + "time" + + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethpandaops/bootnodoor/discv4/node" + "github.com/ethpandaops/bootnodoor/enr" +) + +func makeKeyedNode(t *testing.T, port int) (*node.Node, *ecdsa.PrivateKey) { + t.Helper() + key, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("generate key: %v", err) + } + return node.New(&key.PublicKey, &net.UDPAddr{IP: net.IPv4(1, 2, 3, 4), Port: port}), key +} + +func signedV4Record(t *testing.T, key *ecdsa.PrivateKey, seq uint64) *enr.Record { + t.Helper() + rec := enr.New() + if err := rec.Set("ip", net.IPv4(1, 2, 3, 4)); err != nil { + t.Fatalf("set ip: %v", err) + } + if err := rec.Set("udp", uint16(30303)); err != nil { + t.Fatalf("set udp: %v", err) + } + rec.SetSeq(seq) + if err := rec.Sign(key); err != nil { + t.Fatalf("sign: %v", err) + } + return rec +} + +func expectResponse(t *testing.T, req *PendingRequest) interface{} { + t.Helper() + select { + case resp := <-req.ResponseChan: + return resp + case <-time.After(2 * time.Second): + t.Fatal("expected a delivered response") + return nil + } +} + +func expectNoResponse(t *testing.T, req *PendingRequest) { + t.Helper() + select { + case resp := <-req.ResponseChan: + t.Fatalf("unexpected response delivered: %v", resp) + default: + } +} + +// TestIdenticalRequestsToDifferentPeersDoNotAlias covers the collision that +// poisoned live routing tables: ENRREQUEST carries only a 1s-granularity +// expiration and signatures are deterministic, so same-second requests to +// different peers share a packet hash. A response from one peer must resolve +// only that peer's request and must not touch the other node's ENR. +func TestIdenticalRequestsToDifferentPeersDoNotAlias(t *testing.T) { + h, cancel := newTestHandler(t) + defer cancel() + + nodeA, _ := makeKeyedNode(t, 30301) + nodeB, keyB := makeKeyedNode(t, 30302) + hash := []byte("same-second-packet") + + reqA, err := h.addPendingRequest(hash, nodeA, ENRRequestPacket) + if err != nil { + t.Fatalf("addPendingRequest A: %v", err) + } + reqB, err := h.addPendingRequest(hash, nodeB, ENRRequestPacket) + if err != nil { + t.Fatalf("addPendingRequest B: %v", err) + } + + recB := signedV4Record(t, keyB, 7) + if err := h.handleENRResponse(nodeB, nodeB.Addr(), &ENRResponse{ReplyTok: hash, Record: recB}); err != nil { + t.Fatalf("handleENRResponse: %v", err) + } + + if got := expectResponse(t, reqB); got != recB { + t.Fatalf("request B received %v, want node B's record", got) + } + expectNoResponse(t, reqA) + if nodeA.ENR() != nil { + t.Fatal("node B's response installed a record on node A") + } + if nodeB.ENR() != recB { + t.Fatal("node B's record was not installed on node B") + } +} + +// TestSamePeerDuplicateRequestsBothComplete covers concurrent identical +// requests to one peer (the lookup fires RequestENR per neighbor before +// dedup): every waiter gets the response, and removing one request must not +// orphan the other's pending entry. +func TestSamePeerDuplicateRequestsBothComplete(t *testing.T) { + h, cancel := newTestHandler(t) + defer cancel() + + n, key := makeKeyedNode(t, 30301) + hash := []byte("same-second-packet") + + req1, err := h.addPendingRequest(hash, n, ENRRequestPacket) + if err != nil { + t.Fatalf("addPendingRequest 1: %v", err) + } + req2, err := h.addPendingRequest(hash, n, ENRRequestPacket) + if err != nil { + t.Fatalf("addPendingRequest 2: %v", err) + } + + h.removePendingRequest(req1) + if got := len(h.getPendingRequests(hash, n.ID())); got != 1 { + t.Fatalf("after removing one duplicate, %d pending remain, want 1", got) + } + + rec := signedV4Record(t, key, 3) + if err := h.handleENRResponse(n, n.Addr(), &ENRResponse{ReplyTok: hash, Record: rec}); err != nil { + t.Fatalf("handleENRResponse: %v", err) + } + if got := expectResponse(t, req2); got != rec { + t.Fatalf("surviving request received %v, want the record", got) + } +} + +// TestUnsolicitedENRResponseDoesNotMutate covers replay: ENRRESPONSE has no +// expiration, so a response matching no pending request must not touch the +// node's ENR at all. +func TestUnsolicitedENRResponseDoesNotMutate(t *testing.T) { + h, cancel := newTestHandler(t) + defer cancel() + + n, key := makeKeyedNode(t, 30301) + newer := signedV4Record(t, key, 9) + n.SetENR(newer) + + older := signedV4Record(t, key, 2) + if err := h.handleENRResponse(n, n.Addr(), &ENRResponse{ReplyTok: []byte("nothing-pending"), Record: older}); err != nil { + t.Fatalf("handleENRResponse: %v", err) + } + if n.ENR() != newer { + t.Fatal("unsolicited response replaced the node's ENR") + } +} + +// TestStaleENRResponseNotInstalled covers rollback through a matched request: +// an equal-or-lower-sequence record must not replace a newer one. +func TestStaleENRResponseNotInstalled(t *testing.T) { + h, cancel := newTestHandler(t) + defer cancel() + + n, key := makeKeyedNode(t, 30301) + newer := signedV4Record(t, key, 9) + n.SetENR(newer) + + hash := []byte("pending") + req, err := h.addPendingRequest(hash, n, ENRRequestPacket) + if err != nil { + t.Fatalf("addPendingRequest: %v", err) + } + + stale := signedV4Record(t, key, 9) + if err := h.handleENRResponse(n, n.Addr(), &ENRResponse{ReplyTok: hash, Record: stale}); err != nil { + t.Fatalf("handleENRResponse: %v", err) + } + if n.ENR() != newer { + t.Fatal("equal-sequence response replaced the node's ENR") + } + if got := expectResponse(t, req); got != stale { + t.Fatalf("waiter received %v, want the response record", got) + } +} + +// TestMismatchedIdentityENRResponseDropped: a matched response whose record is +// signed by a different key must neither install nor be delivered. +func TestMismatchedIdentityENRResponseDropped(t *testing.T) { + h, cancel := newTestHandler(t) + defer cancel() + + n, _ := makeKeyedNode(t, 30301) + _, otherKey := makeKeyedNode(t, 30302) + + hash := []byte("pending") + req, err := h.addPendingRequest(hash, n, ENRRequestPacket) + if err != nil { + t.Fatalf("addPendingRequest: %v", err) + } + + foreign := signedV4Record(t, otherKey, 5) + if err := h.handleENRResponse(n, n.Addr(), &ENRResponse{ReplyTok: hash, Record: foreign}); err != nil { + t.Fatalf("handleENRResponse: %v", err) + } + if n.ENR() != nil { + t.Fatal("foreign record was installed") + } + expectNoResponse(t, req) +} + +// TestIdenticalFindnodeToDifferentPeersSeparateAccumulators: colliding +// FINDNODE hashes to different peers must keep separate NEIGHBORS +// accumulators and deliver each peer's response to its own request. +func TestIdenticalFindnodeToDifferentPeersSeparateAccumulators(t *testing.T) { + h, cancel := newTestHandler(t) + defer cancel() + + nodeA, _ := makeKeyedNode(t, 30301) + nodeB, _ := makeKeyedNode(t, 30302) + hash := []byte("same-target-same-second") + + reqA, err := h.addPendingRequest(hash, nodeA, FindnodePacket) + if err != nil { + t.Fatalf("addPendingRequest A: %v", err) + } + reqB, err := h.addPendingRequest(hash, nodeB, FindnodePacket) + if err != nil { + t.Fatalf("addPendingRequest B: %v", err) + } + + if err := h.handleNeighbors(nodeA, nodeA.Addr(), makeNeighbors(t, 2)); err != nil { + t.Fatalf("handleNeighbors: %v", err) + } + + nodes, ok := expectResponse(t, reqA).([]*node.Node) + if !ok || len(nodes) != 2 { + t.Fatalf("request A received %v, want 2 nodes", nodes) + } + expectNoResponse(t, reqB) + + h.pendingNeighborsMu.RLock() + _, sharedKey := h.pendingNeighbors[string(hash)] + bEntry := h.pendingNeighbors[requestKey(hash, nodeB.ID())] + h.pendingNeighborsMu.RUnlock() + if sharedKey { + t.Fatal("accumulator stored under the bare hash key") + } + if bEntry != nil { + t.Fatal("node A's NEIGHBORS created an accumulator for node B's request") + } +} + +// TestSecondFindnodeToSamePeerRejected: NEIGHBORS has no reply token, so two +// in-flight FINDNODEs to one peer cannot be told apart and the second must be +// refused. +func TestSecondFindnodeToSamePeerRejected(t *testing.T) { + h, cancel := newTestHandler(t) + defer cancel() + + n, _ := makeKeyedNode(t, 30301) + if _, err := h.addPendingRequest([]byte("hash-1"), n, FindnodePacket); err != nil { + t.Fatalf("first findnode: %v", err) + } + if _, err := h.addPendingRequest([]byte("hash-2"), n, FindnodePacket); err == nil { + t.Fatal("second in-flight findnode to the same peer was accepted") + } +} diff --git a/discv4/protocol/response_delivery_test.go b/discv4/protocol/response_delivery_test.go index c2052a5..1e85de8 100644 --- a/discv4/protocol/response_delivery_test.go +++ b/discv4/protocol/response_delivery_test.go @@ -21,7 +21,10 @@ func TestDeliverResponseNeverBlocks(t *testing.T) { h, cancel := newTestHandler(t) defer cancel() - req := h.addPendingRequest([]byte("reqhash"), nil, PingPacket) + req, err := h.addPendingRequest([]byte("reqhash"), makeDiscv4Node(t), PingPacket) + if err != nil { + t.Fatalf("addPendingRequest: %v", err) + } const dups = 200 var wg sync.WaitGroup @@ -63,7 +66,11 @@ func TestDuplicateResponsesWaiterGetsOneNoLeak(t *testing.T) { defer cancel() hash := []byte("reqhash") - req := h.addPendingRequest(hash, nil, PingPacket) + to := makeDiscv4Node(t) + req, err := h.addPendingRequest(hash, to, PingPacket) + if err != nil { + t.Fatalf("addPendingRequest: %v", err) + } got := make(chan interface{}, 1) var waiter sync.WaitGroup @@ -71,7 +78,7 @@ func TestDuplicateResponsesWaiterGetsOneNoLeak(t *testing.T) { go func() { defer waiter.Done() resp := <-req.ResponseChan - h.removePendingRequest(string(hash)) + h.removePendingRequest(req) got <- resp }() @@ -81,7 +88,7 @@ func TestDuplicateResponsesWaiterGetsOneNoLeak(t *testing.T) { for i := 0; i < dups; i++ { go func() { defer wg.Done() - if r := h.getPendingRequest(string(hash)); r != nil { + for _, r := range h.getPendingRequests(hash, to.ID()) { h.deliverResponse(r, "pong") } }() diff --git a/discv4/service.go b/discv4/service.go index abc2168..567e376 100644 --- a/discv4/service.go +++ b/discv4/service.go @@ -382,22 +382,6 @@ func (s *Service) Handler() *protocol.Handler { return s.handler } -// Statistics - -// Stats returns service statistics. -func (s *Service) Stats() map[string]interface{} { - handler := s.Handler() - if handler == nil { - return map[string]interface{}{} - } - - stats := handler.Stats() - - // Note: Transport stats are not included since transport is managed externally - - return stats -} - // Utility Methods // ParseNodeFromEnode creates a node from an enode:// URL. From 4d1f995371d09d1323f7f14d924585c912e33811 Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Wed, 29 Jul 2026 08:02:53 -0500 Subject: [PATCH 02/49] fix(discv5): pair the UDP port with its address family An IPv6-only record advertising udp6 but no udp was rejected outright, and UpdateENR silently kept a stale address when the new record's family differed. Extract udpEndpoint so both paths pair ip with udp and ip6 with udp6, falling back to udp for dual-stack records. --- discv5/node/node.go | 55 ++++++++++----------- discv5/node/node_test.go | 104 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+), 29 deletions(-) diff --git a/discv5/node/node.go b/discv5/node/node.go index 4eebd88..183d69f 100644 --- a/discv5/node/node.go +++ b/discv5/node/node.go @@ -107,25 +107,9 @@ func New(record *enr.Record) (*Node, error) { } id := PubkeyToID(pubKey) - // Extract IP address - ip := record.IP() - if ip == nil { - ip = record.IP6() - } - if ip == nil { - return nil, fmt.Errorf("node: ENR missing IP address") - } - - // Extract UDP port - udpPort := record.UDP() - if udpPort == 0 { - return nil, fmt.Errorf("node: ENR missing UDP port") - } - - // Create UDP address - addr := &net.UDPAddr{ - IP: ip, - Port: int(udpPort), + addr, err := udpEndpoint(record) + if err != nil { + return nil, err } // Extract optional TCP port @@ -144,6 +128,27 @@ func New(record *enr.Record) (*Node, error) { }, nil } +// udpEndpoint extracts the discovery endpoint from a record, keeping the port +// paired with its address family: ip goes with udp, ip6 with udp6 (falling +// back to udp, which dual-stack records share across both families). +func udpEndpoint(record *enr.Record) (*net.UDPAddr, error) { + ip := record.IP() + port := record.UDP() + if ip == nil { + ip = record.IP6() + if p := record.UDP6(); p != 0 { + port = p + } + } + if ip == nil { + return nil, fmt.Errorf("node: ENR missing IP address") + } + if port == 0 { + return nil, fmt.Errorf("node: ENR missing UDP port") + } + return &net.UDPAddr{IP: ip, Port: int(port)}, nil +} + // ID returns the node's unique identifier. func (n *Node) ID() ID { return n.id @@ -282,16 +287,8 @@ func (n *Node) UpdateENR(newRecord *enr.Record) bool { n.record = newRecord // Update network address if changed - ip := newRecord.IP() - if ip == nil { - ip = newRecord.IP6() - } - udpPort := newRecord.UDP() - if ip != nil && udpPort != 0 { - n.addr = &net.UDPAddr{ - IP: ip, - Port: int(udpPort), - } + if addr, err := udpEndpoint(newRecord); err == nil { + n.addr = addr } n.tcpPort = newRecord.TCP() diff --git a/discv5/node/node_test.go b/discv5/node/node_test.go index 8f9d8c1..3d4d0a3 100644 --- a/discv5/node/node_test.go +++ b/discv5/node/node_test.go @@ -31,6 +31,110 @@ func signedRecord(t *testing.T, seq uint64, port uint16) *enr.Record { return rec } +// TestNewIPv6OnlyRecord covers records carrying only ip6/udp6: the port must +// fall back to "udp6" or every IPv6-only bootnode is rejected. +func TestNewIPv6OnlyRecord(t *testing.T) { + key, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("generate key: %v", err) + } + rec := enr.New() + ip6 := net.ParseIP("2001:db8::1") + if err := rec.Set("ip6", ip6); err != nil { + t.Fatalf("set ip6: %v", err) + } + if err := rec.Set("udp6", uint16(30304)); err != nil { + t.Fatalf("set udp6: %v", err) + } + rec.SetSeq(1) + if err := rec.Sign(key); err != nil { + t.Fatalf("sign: %v", err) + } + + n, err := New(rec) + if err != nil { + t.Fatalf("New rejected IPv6-only record: %v", err) + } + if got := n.UDPPort(); got != 30304 { + t.Fatalf("UDPPort() = %d, want 30304", got) + } + if !n.IP().Equal(ip6) { + t.Fatalf("IP() = %v, want %v", n.IP(), ip6) + } +} + +// TestNewMismatchedFamilyRejected covers a record with an IPv4 address but +// only an IPv6 port: no complete endpoint exists, so pairing them would send +// packets to an unrelated port. +func TestNewMismatchedFamilyRejected(t *testing.T) { + key, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("generate key: %v", err) + } + rec := enr.New() + if err := rec.Set("ip", net.IPv4(3, 3, 3, 3)); err != nil { + t.Fatalf("set ip: %v", err) + } + if err := rec.Set("udp6", uint16(30304)); err != nil { + t.Fatalf("set udp6: %v", err) + } + rec.SetSeq(1) + if err := rec.Sign(key); err != nil { + t.Fatalf("sign: %v", err) + } + + if _, err := New(rec); err == nil { + t.Fatal("New accepted an ip record with only a udp6 port") + } +} + +// TestUpdateENRRefreshesIPv6Endpoint covers refreshing a node with a +// higher-sequence IPv6-only record: the address must follow the record. +func TestUpdateENRRefreshesIPv6Endpoint(t *testing.T) { + key, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("generate key: %v", err) + } + rec := enr.New() + if err := rec.Set("ip", net.IPv4(3, 3, 3, 3)); err != nil { + t.Fatalf("set ip: %v", err) + } + if err := rec.Set("udp", uint16(30303)); err != nil { + t.Fatalf("set udp: %v", err) + } + rec.SetSeq(1) + if err := rec.Sign(key); err != nil { + t.Fatalf("sign: %v", err) + } + n, err := New(rec) + if err != nil { + t.Fatalf("New: %v", err) + } + + ip6 := net.ParseIP("2001:db8::2") + newRec := enr.New() + if err := newRec.Set("ip6", ip6); err != nil { + t.Fatalf("set ip6: %v", err) + } + if err := newRec.Set("udp6", uint16(30305)); err != nil { + t.Fatalf("set udp6: %v", err) + } + newRec.SetSeq(2) + if err := newRec.Sign(key); err != nil { + t.Fatalf("sign: %v", err) + } + + if !n.UpdateENR(newRec) { + t.Fatal("UpdateENR rejected higher-sequence record") + } + if got := n.UDPPort(); got != 30305 { + t.Fatalf("UDPPort() = %d after update, want 30305", got) + } + if !n.IP().Equal(ip6) { + t.Fatalf("IP() = %v after update, want %v", n.IP(), ip6) + } +} + // TestNodeConcurrentENRAndStatsAccess exercises UpdateENR and SetStats against // every reader of the guarded fields. Under the race detector it fails if any // access to record, addr, tcpPort or the stats pointer is unsynchronized. From 8ccef33d7b82c4674a1379efffa569093e29377a Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Wed, 29 Jul 2026 08:03:04 -0500 Subject: [PATCH 03/49] fix(bootnode): gate admission counters on the layer entry, tear down on New failure Records without an eth entry are consensus nodes, not wrong-fork execution nodes; counting them made the EL rejection counter track cross-layer traffic, which on a dual-layer network is most of what arrives. Centralize the gate in ENRManager.RecordELAdmission and mirror it in the CL filter, where every EL node was inflating totalChecks. New() now unwinds via a single deferred guard that also cancels the context: a late failure previously leaked the NodeDB queue processors and handler cleanup goroutines hanging off s.ctx. Extract the two lookup admission closures and the repeated bootnode table-add/persist block; reuse discv5/session.Stats instead of redeclaring it. --- bootnode/clconfig/filter.go | 19 +- bootnode/clconfig/filter_test.go | 79 +++++++ bootnode/enr.go | 10 + bootnode/service.go | 362 ++++++++++++++----------------- bootnode/service_test.go | 25 +++ bootnode/stats.go | 12 +- services/lookup_test.go | 6 +- services/ping.go | 18 +- 8 files changed, 301 insertions(+), 230 deletions(-) diff --git a/bootnode/clconfig/filter.go b/bootnode/clconfig/filter.go index fb90ac2..ba07897 100644 --- a/bootnode/clconfig/filter.go +++ b/bootnode/clconfig/filter.go @@ -124,23 +124,28 @@ func (f *ForkDigestFilter) SetLogger(logger Logger) { // ResponseFilter: filter.ResponseFilter(), // }) func (f *ForkDigestFilter) Filter(record *enr.Record) bool { - f.mu.Lock() - f.totalChecks++ - f.mu.Unlock() - - // Get eth2 field from ENR + // No eth2 entry means an execution node, not an invalid consensus node: + // reject without moving counters (mirrors RecordELAdmission's eth gate). + // A present but undecodable entry is a broken consensus node: that counts. var eth2Data []byte if err := record.Get("eth2", ð2Data); err != nil { - // No eth2 field, reject + if !record.Has("eth2") { + return false + } f.mu.Lock() + f.totalChecks++ f.rejectedInvalid++ if f.logger != nil { - f.logger.Debugf("Rejected node: no eth2 field in ENR") + f.logger.Debugf("Rejected node: undecodable eth2 field - %v", err) } f.mu.Unlock() return false } + f.mu.Lock() + f.totalChecks++ + f.mu.Unlock() + // Parse fork digest (first 4 bytes only) forkDigest, err := ParseETH2Field(eth2Data) if err != nil { diff --git a/bootnode/clconfig/filter_test.go b/bootnode/clconfig/filter_test.go index 1c5eb0e..dbc9035 100644 --- a/bootnode/clconfig/filter_test.go +++ b/bootnode/clconfig/filter_test.go @@ -2,8 +2,12 @@ package clconfig import ( "math" + "net" "testing" "time" + + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethpandaops/bootnodoor/enr" ) func TestNextForkInfoReturnsUpcomingFork(t *testing.T) { @@ -71,6 +75,81 @@ func TestNextForkInfoFallsBackToFarFuture(t *testing.T) { } } +// TestFilterSkipsRecordsWithoutEth2: a record with no eth2 entry is an +// execution node, not an invalid consensus node, so it must not move any +// counter (mirrors the EL side's RecordELAdmission gate). A malformed eth2 +// entry still counts as invalid. +func TestFilterSkipsRecordsWithoutEth2(t *testing.T) { + cfg := &Config{ + SecondsPerSlot: 12, + customSlotsPerEpoch: 32, + genesisForkVersion: [4]byte{0x00, 0x00, 0x00, 0x01}, + forks: []forkDefinition{ + {name: "Altair", epoch: 0, parsedVersion: [4]byte{0x01, 0x00, 0x00, 0x00}}, + }, + } + cfg.SetGenesisTime(uint64(time.Now().Unix()) - 60) + filter := NewForkDigestFilter(cfg, time.Hour) + + key, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("generate key: %v", err) + } + noEth2 := enr.New() + if err := noEth2.Set("ip", net.IPv4(1, 2, 3, 4)); err != nil { + t.Fatalf("set ip: %v", err) + } + noEth2.SetSeq(1) + if err := noEth2.Sign(key); err != nil { + t.Fatalf("sign: %v", err) + } + + if filter.Filter(noEth2) { + t.Fatal("record without eth2 passed the CL filter") + } + stats := filter.GetStats() + if stats.TotalChecks != 0 || stats.RejectedInvalid != 0 { + t.Fatalf("no-eth2 record moved counters: checks=%d rejectedInvalid=%d, want 0/0", + stats.TotalChecks, stats.RejectedInvalid) + } + + malformed := enr.New() + if err := malformed.Set("eth2", []byte{0x01, 0x02}); err != nil { + t.Fatalf("set eth2: %v", err) + } + malformed.SetSeq(1) + if err := malformed.Sign(key); err != nil { + t.Fatalf("sign: %v", err) + } + + if filter.Filter(malformed) { + t.Fatal("malformed eth2 passed the CL filter") + } + stats = filter.GetStats() + if stats.TotalChecks != 1 || stats.RejectedInvalid != 1 { + t.Fatalf("malformed eth2 counters: checks=%d rejectedInvalid=%d, want 1/1", + stats.TotalChecks, stats.RejectedInvalid) + } + + undecodable := enr.New() + if err := undecodable.Set("eth2", []uint64{1, 2}); err != nil { + t.Fatalf("set eth2: %v", err) + } + undecodable.SetSeq(1) + if err := undecodable.Sign(key); err != nil { + t.Fatalf("sign: %v", err) + } + + if filter.Filter(undecodable) { + t.Fatal("undecodable eth2 passed the CL filter") + } + stats = filter.GetStats() + if stats.TotalChecks != 2 || stats.RejectedInvalid != 2 { + t.Fatalf("undecodable eth2 counters: checks=%d rejectedInvalid=%d, want 2/2", + stats.TotalChecks, stats.RejectedInvalid) + } +} + // TestForkDigestFilterUpdateTransition verifies Update() tracks an epoch // crossing: the current digest swaps, the old digest lands in the grace map // and stays accepted, and the recomputed eth2 field reflects the new digest. diff --git a/bootnode/enr.go b/bootnode/enr.go index 50066ed..f6a566e 100644 --- a/bootnode/enr.go +++ b/bootnode/enr.go @@ -228,6 +228,16 @@ func (m *ENRManager) GetELFilter() *elconfig.ForkFilter { return m.elFilter } +// RecordELAdmission records an EL admission decision on the filter stats. +// Records without an eth entry are consensus nodes, not wrong-fork execution +// nodes, and are not counted (see services.AdmissionRejectedLayer). +func (m *ENRManager) RecordELAdmission(record *enr.Record, accepted bool, forkID elconfig.ForkID) { + if m.elFilter == nil || record == nil || !record.Has("eth") { + return + } + m.elFilter.RecordAdmission(accepted, forkID) +} + // GetCLFilter returns the CL fork digest filter (may be nil). func (m *ENRManager) GetCLFilter() *clconfig.ForkDigestFilter { return m.clFilter diff --git a/bootnode/service.go b/bootnode/service.go index 9029e43..8e91bc4 100644 --- a/bootnode/service.go +++ b/bootnode/service.go @@ -120,13 +120,23 @@ func New(cfg *Config) (*Service, error) { t.Close() } } + + // Everything created below hangs off s.ctx (NodeDB queue processors, + // protocol-handler cleanup goroutines), so cancelling tears it all down. + ok := false + defer func() { + if !ok { + cancel() + closeTransports() + } + }() + for _, id := range s.identities { if transports[id.bindPort] == nil { // JoinHostPort so an IPv6 bind addr becomes [::]:port, not :::port. listenAddr := net.JoinHostPort(cfg.BindIP.String(), fmt.Sprintf("%d", id.bindPort)) t, terr := transport.NewUDPTransport(&transport.Config{ListenAddr: listenAddr, Logger: cfg.Logger}) if terr != nil { - closeTransports() return nil, fmt.Errorf("failed to create UDP transport on port %d: %w", id.bindPort, terr) } cfg.Logger.WithField("address", listenAddr).Info("listening for discovery") @@ -143,7 +153,6 @@ func New(cfg *Config) (*Service, error) { localNode, nerr := createLocalNode(cfg, id.key, id.enrIP, id.enrIP6, id.enrPort, storedENR) if nerr != nil { - closeTransports() return nil, fmt.Errorf("failed to create local node: %w", nerr) } id.localNode = localNode @@ -191,14 +200,12 @@ func New(cfg *Config) (*Service, error) { if cfg.HasEL() { s.elTable, err = s.createTable(s.elIdentity().localNode.ID(), s.elNodeDB, "EL") if err != nil { - closeTransports() return nil, fmt.Errorf("failed to create EL table: %w", err) } } if cfg.HasCL() { s.clTable, err = s.createTable(s.clIdentity().localNode.ID(), s.clNodeDB, "CL") if err != nil { - closeTransports() return nil, fmt.Errorf("failed to create CL table: %w", err) } } @@ -207,7 +214,6 @@ func New(cfg *Config) (*Service, error) { if cfg.EnableDiscv5 { for _, id := range s.identities { if ierr := s.initDiscv5(id); ierr != nil { - closeTransports() return nil, fmt.Errorf("failed to initialize discv5: %w", ierr) } } @@ -217,12 +223,6 @@ func New(cfg *Config) (*Service, error) { // Create the discv4 service (EL-only) on the EL identity. if cfg.EnableDiscv4 { if ierr := s.initDiscv4(s.elIdentity()); ierr != nil { - for _, id := range s.identities { - if id.discv5Service != nil { - id.discv5Service.Stop() - } - } - closeTransports() return nil, fmt.Errorf("failed to initialize discv4: %w", ierr) } } @@ -253,84 +253,8 @@ func New(cfg *Config) (*Service, error) { Layer: db.LayerEL, Alpha: 3, LookupTimeout: 30 * time.Second, - OnNodeFound: func(n *nodes.Node) services.AdmissionResult { - // Filter by fork ID before adding to table - if n.Record() != nil && s.enrManager != nil { - isEL, forkID := s.enrManager.FilterELNode(n.Record()) - if !isEL { - // A record with no eth entry is a consensus node, not an - // execution node on the wrong fork. Counting those as - // fork rejections would make a healthy dual-layer network - // look like a fork-compatibility failure. - if !n.Record().Has("eth") { - if err := cfg.Database.StoreBadNode(n.IDBytes(), db.LayerEL, "not_el"); err != nil { - cfg.Logger.WithError(err).Debug("failed to store bad node") - } - return services.AdmissionRejectedLayer - } - if elFilter := s.enrManager.GetELFilter(); elFilter != nil { - elFilter.RecordAdmission(false, forkID) - } - cfg.Logger.WithFields(logrus.Fields{ - "peerID": n.PeerID(), - "eth": forkID.String(), - }).Debug("EL lookup admission rejected: incompatible fork id") - // Mark as bad node - if err := cfg.Database.StoreBadNode(n.IDBytes(), db.LayerEL, "invalid_fork_id"); err != nil { - cfg.Logger.WithError(err).Debug("failed to store bad node") - } - return services.AdmissionRejectedFilter - } - if elFilter := s.enrManager.GetELFilter(); elFilter != nil { - elFilter.RecordAdmission(true, forkID) - } - } - - // If node was discovered via v4 (only has v4 support), immediately test for v5 support - if n.HasV4() && !n.HasV5() && s.getV5Handler() != nil { - record := n.Record() - if record != nil { - // Try to create v5 node from ENR - v5Node, err := nodes.NewV5NodeFromRecord(record) - if err == nil && s.getV5Handler() != nil { - // Ping on v5 to test support - start := time.Now() - respChan, err := s.getV5Handler().SendPing(v5Node) - if err == nil { - resp := <-respChan - rtt := time.Since(start) - if resp.Error == nil { - // v5 ping succeeded - add v5 support - n.SetV5(v5Node) - cfg.Logger.WithFields(logrus.Fields{ - "peerID": n.PeerID(), - "addr": n.Addr(), - "rtt": rtt, - }).Debug("discovered v5 support on v4-discovered node") - - // Queue protocol support update (SetV5 already marked it dirty) - if s.elNodeDB != nil { - if err := s.elNodeDB.QueueUpdate(n); err != nil { - cfg.Logger.WithError(err).Debug("failed to queue node for protocol support update") - } - } - } - } - } - } - } - - // Attempt to add to EL table - if !s.elTable.Add(n) { - return services.AdmissionRejectedPool - } - // Remove from bad nodes list if it was previously bad - if err := cfg.Database.RemoveBadNode(n.IDBytes(), db.LayerEL); err != nil { - cfg.Logger.WithError(err).Debug("failed to remove from bad nodes") - } - return services.AdmissionAccepted - }, - Logger: cfg.Logger.WithField("service", "el-lookup"), + OnNodeFound: s.admitELLookupNode, + Logger: cfg.Logger.WithField("service", "el-lookup"), }) } @@ -356,39 +280,13 @@ func New(cfg *Config) (*Service, error) { Layer: db.LayerCL, Alpha: 3, LookupTimeout: 30 * time.Second, - OnNodeFound: func(n *nodes.Node) services.AdmissionResult { - // Filter by fork digest before adding to table - if n.Record() != nil && s.enrManager != nil { - if !s.enrManager.FilterCLNode(n.Record()) { - // No eth2 entry means an execution node, not a consensus - // node on the wrong digest; keep the two distinguishable. - if !n.Record().Has("eth2") { - if err := cfg.Database.StoreBadNode(n.IDBytes(), db.LayerCL, "not_cl"); err != nil { - cfg.Logger.WithError(err).Debug("failed to store bad node") - } - return services.AdmissionRejectedLayer - } - // Mark as bad node - if err := cfg.Database.StoreBadNode(n.IDBytes(), db.LayerCL, "invalid_fork_digest"); err != nil { - cfg.Logger.WithError(err).Debug("failed to store bad node") - } - return services.AdmissionRejectedFilter - } - } - // Attempt to add to CL table - if !s.clTable.Add(n) { - return services.AdmissionRejectedPool - } - // Remove from bad nodes list if it was previously bad - if err := cfg.Database.RemoveBadNode(n.IDBytes(), db.LayerCL); err != nil { - cfg.Logger.WithError(err).Debug("failed to remove from bad nodes") - } - return services.AdmissionAccepted - }, - Logger: cfg.Logger.WithField("service", "cl-lookup"), + OnNodeFound: s.admitCLLookupNode, + Logger: cfg.Logger.WithField("service", "cl-lookup"), }) } + ok = true + return s, nil } @@ -935,29 +833,17 @@ func (s *Service) connectELBootnodes() { // connectELBootnodeENR connects to an EL bootnode via ENR. func (s *Service) connectELBootnodeENR(record *enr.Record) { - // Convert to v5 node + // Convert to v5 node; this also rejects records missing an IP or UDP port. v5, err := v5node.New(record) if err != nil { s.config.Logger.WithError(err).Warn("failed to create v5 node from ENR") return } - // Verify ENR has required fields (IP and port) - if record.IP() == nil && record.IP6() == nil { - s.config.Logger.Warn("bootnode ENR missing IP address, skipping") - return - } - if record.UDP() == 0 { - s.config.Logger.Warn("bootnode ENR missing UDP port, skipping") - return - } - // Filter by fork ID before adding if s.enrManager != nil { isEL, forkID := s.enrManager.FilterELNode(record) - if elFilter := s.enrManager.GetELFilter(); elFilter != nil { - elFilter.RecordAdmission(isEL, forkID) - } + s.enrManager.RecordELAdmission(record, isEL, forkID) if !isEL { s.config.Logger.WithFields(logrus.Fields{ "nodeID": fmt.Sprintf("%x", v5.ID().Bytes()[:8]), @@ -967,23 +853,8 @@ func (s *Service) connectELBootnodeENR(record *enr.Record) { } } - // Create generic node and add to table genericNode := nodes.NewFromV5(v5, s.elNodeDB) - if s.elTable != nil { - if !s.elTable.Add(genericNode) { - s.config.Logger.Debug("ENR bootnode not admitted to table, not persisting") - return - } - s.config.Logger.Info("added ENR bootnode to table") - - // Persist to database - if s.elNodeDB != nil { - genericNode.MarkDirty(nodes.DirtyFull) - if err := s.elNodeDB.QueueUpdate(genericNode); err != nil { - s.config.Logger.WithError(err).Debug("failed to queue bootnode for database update") - } - } - } + s.addBootnodeToTable(s.elTable, s.elNodeDB, genericNode, s.config.Logger.WithField("layer", "EL")) } // connectELBootnodeEnode connects to an EL bootnode via enode URL. @@ -1027,9 +898,7 @@ func (s *Service) connectELBootnodeEnode(enodeURL *enode.Enode) { // Filter by fork ID before adding if s.enrManager != nil { isEL, forkID := s.enrManager.FilterELNode(enrRecord) - if elFilter := s.enrManager.GetELFilter(); elFilter != nil { - elFilter.RecordAdmission(isEL, forkID) - } + s.enrManager.RecordELAdmission(enrRecord, isEL, forkID) if !isEL { s.config.Logger.WithFields(logrus.Fields{ "nodeID": fmt.Sprintf("%x", nodeID[:8]), @@ -1045,21 +914,8 @@ func (s *Service) connectELBootnodeEnode(enodeURL *enode.Enode) { // Track successful ENR exchange genericNode.IncrementSuccess() - if s.elTable != nil { - if !s.elTable.Add(genericNode) { - s.config.Logger.WithField("nodeID", fmt.Sprintf("%x", nodeID[:8])).Debug("enode bootnode not admitted to table, not persisting") - return - } - s.config.Logger.WithField("nodeID", fmt.Sprintf("%x", nodeID[:8])).Info("added enode bootnode to table") - - // Persist to database - if s.elNodeDB != nil { - genericNode.MarkDirty(nodes.DirtyFull) - if err := s.elNodeDB.QueueUpdate(genericNode); err != nil { - s.config.Logger.WithError(err).Debug("failed to queue bootnode for database update") - } - } - } + s.addBootnodeToTable(s.elTable, s.elNodeDB, genericNode, + s.config.Logger.WithField("layer", "EL").WithField("nodeID", fmt.Sprintf("%x", nodeID[:8]))) } // connectCLBootnodes connects to CL bootnodes (ENR only). @@ -1073,7 +929,7 @@ func (s *Service) connectCLBootnodes() { continue } - // Convert to v5 node to get node ID + // Convert to v5 node; this also rejects records missing an IP or UDP port. v5, err := v5node.New(record) if err != nil { s.config.Logger.WithError(err).Warn("failed to create v5 node from ENR") @@ -1082,40 +938,38 @@ func (s *Service) connectCLBootnodes() { nodeID := v5.ID() - // Verify ENR has required fields (IP and port) - if record.IP() == nil && record.IP6() == nil { - s.config.Logger.WithField("nodeID", fmt.Sprintf("%x", nodeID[:8])).Warn("CL bootnode ENR missing IP address, skipping") - continue - } - if record.UDP() == 0 { - s.config.Logger.WithField("nodeID", fmt.Sprintf("%x", nodeID[:8])).Warn("CL bootnode ENR missing UDP port, skipping") - continue - } - // Filter by fork digest before adding if s.enrManager != nil && !s.enrManager.FilterCLNode(record) { s.config.Logger.WithField("nodeID", fmt.Sprintf("%x", nodeID[:8])).Warn("CL bootnode ENR has invalid fork digest, skipping") continue } - // Create generic node and add to table genericNode := nodes.NewFromV5(v5, s.clNodeDB) - if s.clTable != nil { - if !s.clTable.Add(genericNode) { - s.config.Logger.WithField("nodeID", fmt.Sprintf("%x", nodeID[:8])).Debug("CL bootnode not admitted to table, not persisting") - continue - } - s.config.Logger.WithField("nodeID", fmt.Sprintf("%x", nodeID[:8])).Info("added CL ENR bootnode to table") + s.addBootnodeToTable(s.clTable, s.clNodeDB, genericNode, + s.config.Logger.WithField("layer", "CL").WithField("nodeID", fmt.Sprintf("%x", nodeID[:8]))) + } +} - // Persist to database - if s.clNodeDB != nil { - genericNode.MarkDirty(nodes.DirtyFull) - if err := s.clNodeDB.QueueUpdate(genericNode); err != nil { - s.config.Logger.WithError(err).Debug("failed to queue bootnode for database update") - } - } +// addBootnodeToTable admits a configured bootnode to a routing table and +// persists it, reporting whether it was admitted. +func (s *Service) addBootnodeToTable(table *nodes.FlatTable, nodeDB *nodes.NodeDB, n *nodes.Node, logger logrus.FieldLogger) bool { + if table == nil { + return false + } + if !table.Add(n) { + logger.Debug("bootnode not admitted to table, not persisting") + return false + } + logger.Info("added bootnode to table") + + if nodeDB != nil { + n.MarkDirty(nodes.DirtyFull) + if err := nodeDB.QueueUpdate(n); err != nil { + logger.WithError(err).Debug("failed to queue bootnode for database update") } } + + return true } // loadStoredENR loads the stored ENR from database. @@ -1328,6 +1182,115 @@ func (s *Service) requestENRV4(n *v4node.Node) { }() } +// admitELLookupNode decides admission of a lookup-discovered node to the EL +// table. +func (s *Service) admitELLookupNode(n *nodes.Node) services.AdmissionResult { + if n.Record() != nil && s.enrManager != nil { + isEL, forkID := s.enrManager.FilterELNode(n.Record()) + s.enrManager.RecordELAdmission(n.Record(), isEL, forkID) + if !isEL { + // A record with no eth entry is a consensus node, not an + // execution node on the wrong fork. + if !n.Record().Has("eth") { + if err := s.config.Database.StoreBadNode(n.IDBytes(), db.LayerEL, "not_el"); err != nil { + s.config.Logger.WithError(err).Debug("failed to store bad node") + } + return services.AdmissionRejectedLayer + } + s.config.Logger.WithFields(logrus.Fields{ + "peerID": n.PeerID(), + "eth": forkID.String(), + }).Debug("EL lookup admission rejected: incompatible fork id") + if err := s.config.Database.StoreBadNode(n.IDBytes(), db.LayerEL, "invalid_fork_id"); err != nil { + s.config.Logger.WithError(err).Debug("failed to store bad node") + } + return services.AdmissionRejectedFilter + } + } + + if n.HasV4() && !n.HasV5() { + s.probeV5Support(n) + } + + if !s.elTable.Add(n) { + return services.AdmissionRejectedPool + } + if err := s.config.Database.RemoveBadNode(n.IDBytes(), db.LayerEL); err != nil { + s.config.Logger.WithError(err).Debug("failed to remove from bad nodes") + } + return services.AdmissionAccepted +} + +// admitCLLookupNode decides admission of a lookup-discovered node to the CL +// table. +func (s *Service) admitCLLookupNode(n *nodes.Node) services.AdmissionResult { + if n.Record() != nil && s.enrManager != nil { + if !s.enrManager.FilterCLNode(n.Record()) { + // No eth2 entry means an execution node, not a consensus + // node on the wrong digest; keep the two distinguishable. + if !n.Record().Has("eth2") { + if err := s.config.Database.StoreBadNode(n.IDBytes(), db.LayerCL, "not_cl"); err != nil { + s.config.Logger.WithError(err).Debug("failed to store bad node") + } + return services.AdmissionRejectedLayer + } + if err := s.config.Database.StoreBadNode(n.IDBytes(), db.LayerCL, "invalid_fork_digest"); err != nil { + s.config.Logger.WithError(err).Debug("failed to store bad node") + } + return services.AdmissionRejectedFilter + } + } + + if !s.clTable.Add(n) { + return services.AdmissionRejectedPool + } + if err := s.config.Database.RemoveBadNode(n.IDBytes(), db.LayerCL); err != nil { + s.config.Logger.WithError(err).Debug("failed to remove from bad nodes") + } + return services.AdmissionAccepted +} + +// probeV5Support pings a v4-discovered node over discv5 and, on success, +// attaches v5 support so lookups prefer the richer protocol. +func (s *Service) probeV5Support(n *nodes.Node) { + handler := s.getV5Handler() + if handler == nil { + return + } + record := n.Record() + if record == nil { + return + } + v5Node, err := nodes.NewV5NodeFromRecord(record) + if err != nil { + return + } + + start := time.Now() + respChan, err := handler.SendPing(v5Node) + if err != nil { + return + } + resp := <-respChan + if resp.Error != nil { + return + } + + n.SetV5(v5Node) + s.config.Logger.WithFields(logrus.Fields{ + "peerID": n.PeerID(), + "addr": n.Addr(), + "rtt": time.Since(start), + }).Debug("discovered v5 support on v4-discovered node") + + // Queue protocol support update (SetV5 already marked it dirty) + if s.elNodeDB != nil { + if err := s.elNodeDB.QueueUpdate(n); err != nil { + s.config.Logger.WithError(err).Debug("failed to queue node for protocol support update") + } + } +} + // checkAndAddNodeV4 adds a discv4 node to the EL table after filtering. func (s *Service) checkAndAddNodeV4(n *v4node.Node) bool { // Ensure we have an ENR for filtering @@ -1352,11 +1315,7 @@ func (s *Service) checkAndAddNodeV4(n *v4node.Node) bool { // Filter the node using ENR manager (EL-only for discv4) if s.enrManager != nil { filter, forkID := s.enrManager.FilterELNode(n.ENR()) - if n.ENR().Has("eth") { - if elFilter := s.enrManager.GetELFilter(); elFilter != nil { - elFilter.RecordAdmission(filter, forkID) - } - } + s.enrManager.RecordELAdmission(n.ENR(), filter, forkID) if !filter { s.config.Logger.WithFields(logrus.Fields{ "nodeID": fmt.Sprintf("%x", n.IDBytes()[:8]), @@ -1393,14 +1352,7 @@ func (s *Service) checkAndAddNode(n *v5node.Node) bool { // Determine layer isEL, elForkID := s.enrManager.FilterELNode(n.Record()) isCL := s.enrManager.FilterCLNode(n.Record()) - // Only an execution record is an execution admission decision; counting - // consensus nodes here made the rejection counter track cross-layer - // traffic, which on a dual-layer network is most of what arrives. - if n.Record().Has("eth") { - if elFilter := s.enrManager.GetELFilter(); elFilter != nil { - elFilter.RecordAdmission(isEL, elForkID) - } - } + s.enrManager.RecordELAdmission(n.Record(), isEL, elForkID) // Add to appropriate table(s) added := false diff --git a/bootnode/service_test.go b/bootnode/service_test.go index dd4d267..5fdbd6a 100644 --- a/bootnode/service_test.go +++ b/bootnode/service_test.go @@ -90,6 +90,31 @@ func TestUpdateENR_DropsUnservedFields(t *testing.T) { } } +// A record without an eth entry is a consensus node, not a wrong-fork +// execution node, so it must not move the EL admission counters. +func TestRecordELAdmission_SkipsRecordsWithoutEth(t *testing.T) { + cfg := &Config{Logger: quietLogger(), ELConfig: &elconfig.ChainConfig{}, ELGenesisHash: [32]byte{1, 2, 3}, ELGenesisTime: 1000} + key := mustKey(t) + ln, err := createLocalNode(cfg, key, net.ParseIP("1.2.3.4"), nil, 9000, nil) + if err != nil { + t.Fatalf("createLocalNode: %v", err) + } + m := NewENRManager(cfg, key, ln, true, false) + + clOnly := storedENRWith(t, key, map[string][]byte{"eth2": {0xaa, 0xbb, 0xcc, 0xdd}}) + m.RecordELAdmission(clOnly, false, elconfig.ForkID{}) + if got := m.GetELFilter().GetStats().TotalChecks; got != 0 { + t.Fatalf("TotalChecks = %d after CL-only record, want 0", got) + } + + elRec := storedENRWith(t, key, map[string][]byte{"eth": {0x01, 0x02, 0x03, 0x04}}) + m.RecordELAdmission(elRec, false, elconfig.ForkID{}) + stats := m.GetELFilter().GetStats() + if stats.TotalChecks != 1 || stats.Rejected != 1 { + t.Fatalf("stats = %+v after eth record, want 1 check / 1 rejection", stats) + } +} + // newTestService builds a minimal Service with the given identities and an // in-memory database, enough to exercise updateENRWithDiscoveredIP. func newTestService(t *testing.T, ids []*identity) *Service { diff --git a/bootnode/stats.go b/bootnode/stats.go index 2478a40..7362070 100644 --- a/bootnode/stats.go +++ b/bootnode/stats.go @@ -4,6 +4,7 @@ import ( "time" v4protocol "github.com/ethpandaops/bootnodoor/discv4/protocol" + "github.com/ethpandaops/bootnodoor/discv5/session" "github.com/ethpandaops/bootnodoor/services" "github.com/ethpandaops/bootnodoor/transport" ) @@ -18,11 +19,11 @@ type Stats struct { Discv5 Discv5Stats Discv4 v4protocol.HandlerStats HasV4 bool - Sessions SessionStats + Sessions session.Stats Packets transport.MetricsSnapshot } -// Discv5Stats is the per-identity discv5 handler counters, summed. +// Discv5Stats is the deliberate subset of protocol.HandlerStats the web UI renders, summed per identity. type Discv5Stats struct { PacketsReceived int PacketsSent int @@ -33,13 +34,6 @@ type Discv5Stats struct { PendingChallenges int } -// SessionStats is the discv5 session cache totals, summed per identity. -type SessionStats struct { - Total int - Active int - Expired int -} - // GetStats returns a snapshot of the service's discovery counters. func (s *Service) GetStats() Stats { var out Stats diff --git a/services/lookup_test.go b/services/lookup_test.go index 29164c0..00f47fb 100644 --- a/services/lookup_test.go +++ b/services/lookup_test.go @@ -288,7 +288,11 @@ func TestPingServiceStatsRace(t *testing.T) { defer writers.Done() for j := 0; j < 200; j++ { ps.countPingSent() - ps.countProtocol(j%2 == 0) + if j%2 == 0 { + ps.countV5Ping() + } else { + ps.countV4Ping() + } ps.countPong(time.Millisecond) ps.countTimeout() } diff --git a/services/ping.go b/services/ping.go index 087dbbb..8098c68 100644 --- a/services/ping.go +++ b/services/ping.go @@ -65,7 +65,7 @@ func (ps *PingService) Ping(n *nodedb.Node) (bool, time.Duration, error) { // Try discv5 first if available if v5Node := n.V5(); v5Node != nil && ps.v5Handler != nil { - ps.countProtocol(true) + ps.countV5Ping() respChan, err := ps.v5Handler.SendPing(v5Node) if err != nil { // Failed to send ping - only increment failure if no v4 fallback available @@ -136,7 +136,7 @@ func (ps *PingService) Ping(n *nodedb.Node) (bool, time.Duration, error) { // Try discv4 fallback if available if v4Node := n.V4(); v4Node != nil && ps.v4Service != nil { - ps.countProtocol(false) + ps.countV4Ping() pong, err := ps.v4Service.Ping(v4Node) rtt := time.Since(start) @@ -441,13 +441,15 @@ func (ps *PingService) countPingSent() { ps.mu.Unlock() } -func (ps *PingService) countProtocol(isV5 bool) { +func (ps *PingService) countV5Ping() { ps.mu.Lock() - if isV5 { - ps.pingsV5++ - } else { - ps.pingsV4++ - } + ps.pingsV5++ + ps.mu.Unlock() +} + +func (ps *PingService) countV4Ping() { + ps.mu.Lock() + ps.pingsV4++ ps.mu.Unlock() } From e7dce7f5f4359539056dd3052b3b3420fc1400cf Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Wed, 29 Jul 2026 08:05:40 -0500 Subject: [PATCH 04/49] fix(enr): satisfy rlp.Encoder and enforce the size limit on ingest EncodeRLP had the signature ([]byte, error) instead of (io.Writer) error, so it never satisfied rlp.Encoder. Every field of Record is unexported, so a nested Record fell through to struct encoding and serialized as c0 -- go-ethereum rejected our discv4 ENRRESPONSE with "record contains less than two list elements". discv5 was unaffected because Nodes.Encode calls the byte-slice encoder by hand. Rename the byte-slice form to EncodeRLPBytes (symmetric with DecodeRLPBytes) and add a real EncodeRLP. The receiver stays a pointer, unlike go-ethereum's value receiver, because Record holds a mutex. DecodeRLPBytes also now rejects payloads over 300 bytes, mirroring geth's decodeRecord. encode() already enforced this outbound, but ingest wrote straight to the raw cache, so an oversized wire record could be stored and re-served -- reachable since relayed records became admissible. --- bootnode/service.go | 2 +- discv5/protocol/handler.go | 2 +- discv5/protocol/handler_test.go | 2 +- discv5/protocol/messages.go | 4 +- enr/encoding.go | 24 ++++- enr/encoding_test.go | 170 ++++++++++++++++++++++++++++++++ enr/record.go | 4 +- enr/record_test.go | 4 +- nodes/nodedb.go | 4 +- 9 files changed, 202 insertions(+), 14 deletions(-) create mode 100644 enr/encoding_test.go diff --git a/bootnode/service.go b/bootnode/service.go index 8e91bc4..3d3c5a9 100644 --- a/bootnode/service.go +++ b/bootnode/service.go @@ -984,7 +984,7 @@ func (s *Service) loadStoredENR(key string) (*enr.Record, error) { // storeENR stores an identity's ENR to the database under its state key. func (s *Service) storeENR(key string, record *enr.Record) error { - data, err := record.EncodeRLP() + data, err := record.EncodeRLPBytes() if err != nil { return err } diff --git a/discv5/protocol/handler.go b/discv5/protocol/handler.go index 476553c..4405f2a 100644 --- a/discv5/protocol/handler.go +++ b/discv5/protocol/handler.go @@ -648,7 +648,7 @@ func (h *Handler) handleWHOAREYOUPacket(packet *Packet, from *net.UDPAddr, local var enrBytes []byte localENR := h.config.LocalNode.Record() if packet.Challenge.ENRSeq == 0 || packet.Challenge.ENRSeq < localENR.Seq() { - enrBytes, err = localENR.EncodeRLP() + enrBytes, err = localENR.EncodeRLPBytes() if err != nil { h.config.Logger.WithError(err).Warn("handler: failed to encode ENR") } else { diff --git a/discv5/protocol/handler_test.go b/discv5/protocol/handler_test.go index c507385..111e7e3 100644 --- a/discv5/protocol/handler_test.go +++ b/discv5/protocol/handler_test.go @@ -22,7 +22,7 @@ func TestResolveHandshakeSender(t *testing.T) { t.Fatalf("create node: %v", err) } - encoded, err := record.EncodeRLP() + encoded, err := record.EncodeRLPBytes() if err != nil { t.Fatalf("encode record: %v", err) } diff --git a/discv5/protocol/messages.go b/discv5/protocol/messages.go index 9a095d2..b9f0be1 100644 --- a/discv5/protocol/messages.go +++ b/discv5/protocol/messages.go @@ -169,7 +169,7 @@ func (n *Nodes) Encode() ([]byte, error) { // Encode each ENR record and wrap in rlp.RawValue to prevent double-encoding records := make([]interface{}, len(n.Records)) for i, record := range n.Records { - encoded, err := record.EncodeRLP() + encoded, err := record.EncodeRLPBytes() if err != nil { return nil, fmt.Errorf("failed to encode ENR %d: %w", i, err) } @@ -248,7 +248,7 @@ func (r *RegTopic) Type() byte { // Encode returns the RLP encoding of the REGTOPIC message func (r *RegTopic) Encode() ([]byte, error) { - enrBytes, err := r.ENR.EncodeRLP() + enrBytes, err := r.ENR.EncodeRLPBytes() if err != nil { return nil, fmt.Errorf("failed to encode ENR: %w", err) } diff --git a/enr/encoding.go b/enr/encoding.go index de89888..95627c8 100644 --- a/enr/encoding.go +++ b/enr/encoding.go @@ -9,7 +9,7 @@ import ( "github.com/ethereum/go-ethereum/rlp" ) -// EncodeRLP returns the RLP encoding of the record. +// EncodeRLPBytes returns the RLP encoding of the record. // // The encoding format is: [signature, seq, k1, v1, k2, v2, ...] // where keys are sorted lexicographically. @@ -18,7 +18,7 @@ import ( // when the record is modified. // // Returns ErrRecordTooLarge if the encoded record exceeds 300 bytes. -func (r *Record) EncodeRLP() ([]byte, error) { +func (r *Record) EncodeRLPBytes() ([]byte, error) { r.mu.Lock() defer r.mu.Unlock() @@ -38,6 +38,20 @@ func (r *Record) EncodeRLP() ([]byte, error) { return encoded, nil } +// EncodeRLP implements rlp.Encoder. Without it a nested Record encodes as an +// empty list, since every field is unexported. Receiver must stay a pointer +// (unlike go-ethereum's value receiver): Record holds a mutex. +func (r *Record) EncodeRLP(w io.Writer) error { + encoded, err := r.EncodeRLPBytes() + if err != nil { + return err + } + + _, err = w.Write(encoded) + + return err +} + // DecodeRLPBytes decodes an RLP-encoded record from a byte slice. // // The input must be a valid RLP list containing: @@ -54,6 +68,10 @@ func (r *Record) EncodeRLP() ([]byte, error) { // // Handle error // } func (r *Record) DecodeRLPBytes(data []byte) error { + if len(data) > MaxRecordSize { + return ErrRecordTooLarge + } + r.mu.Lock() defer r.mu.Unlock() @@ -139,7 +157,7 @@ func (r *Record) DecodeRLP(s *rlp.Stream) error { // // Example output: "enr:-IS4QHCYrYZ..." func (r *Record) EncodeBase64() (string, error) { - encoded, err := r.EncodeRLP() + encoded, err := r.EncodeRLPBytes() if err != nil { return "", err } diff --git a/enr/encoding_test.go b/enr/encoding_test.go new file mode 100644 index 0000000..4e927cf --- /dev/null +++ b/enr/encoding_test.go @@ -0,0 +1,170 @@ +package enr + +import ( + "bytes" + "crypto/ecdsa" + "net" + "testing" + + "github.com/ethereum/go-ethereum/crypto" + gethenr "github.com/ethereum/go-ethereum/p2p/enr" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/rlp" +) + +// enrResponse mirrors discv4's ENRRESPONSE layout: a Record nested in a struct +// alongside other fields, which is the shape that encoded as an empty list. +type enrResponse struct { + ReplyTok []byte + Record *Record +} + +func signedRecord(t *testing.T) (*Record, *ecdsa.PrivateKey) { + t.Helper() + + privKey, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("Failed to generate key: %v", err) + } + + record := New() + record.Set("ip", net.IPv4(192, 168, 1, 1)) + record.Set("udp", uint16(9000)) + + if err := record.Sign(privKey); err != nil { + t.Fatalf("Failed to sign record: %v", err) + } + + return record, privKey +} + +// TestNestedRecordEncoding tests that a Record nested in a struct serializes as +// its own record encoding rather than an empty list. +func TestNestedRecordEncoding(t *testing.T) { + record, _ := signedRecord(t) + + want, err := record.EncodeRLPBytes() + if err != nil { + t.Fatalf("Failed to encode record: %v", err) + } + + encoded, err := rlp.EncodeToBytes(&enrResponse{ReplyTok: []byte{0xaa}, Record: record}) + if err != nil { + t.Fatalf("Failed to encode response: %v", err) + } + + if !bytes.Contains(encoded, want) { + t.Fatalf("Nested record not present in encoding: got %x, want it to contain %x", encoded, want) + } + + var decoded enrResponse + if err := rlp.DecodeBytes(encoded, &decoded); err != nil { + t.Fatalf("Failed to decode response: %v", err) + } + + if decoded.Record.UDP() != record.UDP() { + t.Errorf("UDP port mismatch: got %d, want %d", decoded.Record.UDP(), record.UDP()) + } + + if decoded.Record.Seq() != record.Seq() { + t.Errorf("Sequence mismatch: got %d, want %d", decoded.Record.Seq(), record.Seq()) + } +} + +// TestRecordSliceEncoding tests that records in a slice each carry their own +// encoding. +func TestRecordSliceEncoding(t *testing.T) { + record, _ := signedRecord(t) + + want, err := record.EncodeRLPBytes() + if err != nil { + t.Fatalf("Failed to encode record: %v", err) + } + + encoded, err := rlp.EncodeToBytes([]*Record{record}) + if err != nil { + t.Fatalf("Failed to encode slice: %v", err) + } + + if !bytes.Contains(encoded, want) { + t.Fatalf("Record not present in slice encoding: got %x, want it to contain %x", encoded, want) + } +} + +// TestNestedRecordDecodesInGoEthereum tests that a nested record survives +// go-ethereum's decoder, which rejected our ENRRESPONSE with "record contains +// less than two list elements". +func TestNestedRecordDecodesInGoEthereum(t *testing.T) { + record, privKey := signedRecord(t) + + encoded, err := rlp.EncodeToBytes(&enrResponse{ReplyTok: []byte{0xaa}, Record: record}) + if err != nil { + t.Fatalf("Failed to encode response: %v", err) + } + + var decoded struct { + ReplyTok []byte + Record gethenr.Record + } + if err := rlp.DecodeBytes(encoded, &decoded); err != nil { + t.Fatalf("go-ethereum failed to decode response: %v", err) + } + + n, err := enode.New(enode.ValidSchemes, &decoded.Record) + if err != nil { + t.Fatalf("go-ethereum failed to build node from record: %v", err) + } + + if n.ID() != enode.PubkeyToIDV4(&privKey.PublicKey) { + t.Errorf("Node ID mismatch: got %v, want %v", n.ID(), enode.PubkeyToIDV4(&privKey.PublicKey)) + } + + if n.UDP() != int(record.UDP()) { + t.Errorf("UDP port mismatch: got %d, want %d", n.UDP(), record.UDP()) + } +} + +// TestDecodeRejectsOversizedRecord tests that the 300-byte limit is enforced on +// ingest, not just on encode. +func TestDecodeRejectsOversizedRecord(t *testing.T) { + oversized, err := rlp.EncodeToBytes([]interface{}{ + make([]byte, 64), + uint64(1), + "padding", + make([]byte, MaxRecordSize), + }) + if err != nil { + t.Fatalf("Failed to build oversized payload: %v", err) + } + + if len(oversized) <= MaxRecordSize { + t.Fatalf("Payload is %d bytes, expected more than %d", len(oversized), MaxRecordSize) + } + + if err := New().DecodeRLPBytes(oversized); err != ErrRecordTooLarge { + t.Errorf("DecodeRLPBytes error = %v, want %v", err, ErrRecordTooLarge) + } + + if _, err := Load(oversized); err != ErrRecordTooLarge { + t.Errorf("Load error = %v, want %v", err, ErrRecordTooLarge) + } +} + +// TestDecodeAcceptsRecordAtSizeLimit tests that the size check does not reject +// compliant records. +func TestDecodeAcceptsRecordAtSizeLimit(t *testing.T) { + record, _ := signedRecord(t) + + encoded, err := record.EncodeRLPBytes() + if err != nil { + t.Fatalf("Failed to encode record: %v", err) + } + + if len(encoded) > MaxRecordSize { + t.Fatalf("Record is %d bytes, expected at most %d", len(encoded), MaxRecordSize) + } + + if err := New().DecodeRLPBytes(encoded); err != nil { + t.Errorf("DecodeRLPBytes error = %v, want nil", err) + } +} diff --git a/enr/record.go b/enr/record.go index 3584821..e04e230 100644 --- a/enr/record.go +++ b/enr/record.go @@ -112,7 +112,7 @@ func (r *Record) SetSeq(seq uint64) { // clone.Set("ip", newIP) func (r *Record) Clone() (*Record, error) { // Encode the current record to RLP bytes - data, err := r.EncodeRLP() + data, err := r.EncodeRLPBytes() if err != nil { return nil, fmt.Errorf("failed to encode record for cloning: %w", err) } @@ -583,7 +583,7 @@ func (r *Record) encode() ([]byte, error) { // This is useful for interoperability with go-ethereum's p2p stack. // Returns nil if the record cannot be converted (missing required fields). func (r *Record) ToEnode() *enode.Node { - encoded, err := r.EncodeRLP() + encoded, err := r.EncodeRLPBytes() if err != nil { return nil } diff --git a/enr/record_test.go b/enr/record_test.go index 541a297..fabc6a4 100644 --- a/enr/record_test.go +++ b/enr/record_test.go @@ -43,7 +43,7 @@ func TestRecordEncoding(t *testing.T) { t.Fatalf("Failed to sign record: %v", err) } - encoded, err := original.EncodeRLP() + encoded, err := original.EncodeRLPBytes() if err != nil { t.Fatalf("Failed to encode record: %v", err) } @@ -269,6 +269,6 @@ func BenchmarkRecordEncoding(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - record.EncodeRLP() + record.EncodeRLPBytes() } } diff --git a/nodes/nodedb.go b/nodes/nodedb.go index 1fcad76..3fc452e 100644 --- a/nodes/nodedb.go +++ b/nodes/nodedb.go @@ -297,7 +297,7 @@ func (ndb *NodeDB) updateNodeENRTx(tx *sqlx.Tx, n *Node) error { } } - enrBytes, err := n.ENR().EncodeRLP() + enrBytes, err := n.ENR().EncodeRLPBytes() if err != nil { return fmt.Errorf("failed to encode ENR: %w", err) } @@ -335,7 +335,7 @@ func (ndb *NodeDB) upsertNodeTx(tx *sqlx.Tx, n *Node) error { port := n.Addr().Port seq := n.ENR().Seq() - enrBytes, err := n.ENR().EncodeRLP() + enrBytes, err := n.ENR().EncodeRLPBytes() if err != nil { return fmt.Errorf("failed to encode ENR: %w", err) } From f50f721f3d7522443c3eb06b0177925b2b6a733a Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Wed, 29 Jul 2026 03:48:21 +0200 Subject: [PATCH 05/49] bootnode: add --serve-all to disable classification and serve every peer Adds an opt-in rendezvous mode. With --serve-all the bootnode skips EL/CL classification and fork-ID/digest filtering entirely: every discovered node is pooled into every enabled table (regardless of eth/eth2 fields) and served to every requester. Turns bootnodoor into a plain discv5 rendezvous, like a stock geth bootnode. Default off; classification + fork filtering unchanged. (cherry picked from commit c9aed7eb210c7dd502f18fec7c7b1d2b5a0e9f36) Adapted to this branch, which refactored the paths the original patched: - The two OnNodeFound closures are now admitELLookupNode/admitCLLookupNode and return services.AdmissionResult, so the guard moved into those methods. - Gate checkAndAddNodeV4 too. The original missed the discv4 path, so discv4-discovered nodes stayed fork-filtered and were recorded as bad nodes, contradicting "every discovered node is pooled". - Gate the three configured-seed paths. Fork-filtering a seed under serve-all can reject the only seed, leaving the table empty so discovery never starts. - Skip the filters under serve-all instead of overriding their results. This branch records admission stats inside the filters, so calling them would move the counters for decisions never made and report rejections of nodes that were in fact admitted. - Keep onNodeSeen's EL-xor-CL short circuit: computing both up front made the CL filter run for every EL node, drifting its counters in classified mode. Also dedupe onFindNodeV5 results when serving both layers. A node can sit in both tables, so the concatenated response returned it twice. Serve-all makes that systematic, but it already affected dual-stack peers on a shared identity, so the fix is not gated on the flag. Known limitation: a lookup result is still admitted only to its own layer's table. With separate EL and CL identities a peer found via one identity is not served to the other's requesters until it is seen directly. --- bootnode/config.go | 6 ++ bootnode/service.go | 77 +++++++++++++++++++----- bootnode/service_test.go | 123 +++++++++++++++++++++++++++++++++++++++ cmd/bootnodoor/main.go | 7 +++ 4 files changed, 197 insertions(+), 16 deletions(-) diff --git a/bootnode/config.go b/bootnode/config.go index f507abb..71904be 100644 --- a/bootnode/config.go +++ b/bootnode/config.go @@ -112,6 +112,12 @@ type Config struct { // EnableDiscv5 enables Discovery v5 protocol (default: true) EnableDiscv5 bool + // ServeAll disables EL/CL classification and fork-ID/digest filtering. Every + // discovered node is pooled (into every enabled table) and served to every + // requester, turning the bootnode into a plain discv5 rendezvous that relays + // all peers regardless of eth/eth2 fields. Default: false (classify + filter). + ServeAll bool + // SessionLifetime is the discv5 session lifetime (default: 12 hours) SessionLifetime time.Duration diff --git a/bootnode/service.go b/bootnode/service.go index 3d3c5a9..3c84890 100644 --- a/bootnode/service.go +++ b/bootnode/service.go @@ -840,8 +840,9 @@ func (s *Service) connectELBootnodeENR(record *enr.Record) { return } - // Filter by fork ID before adding - if s.enrManager != nil { + // Filter by fork ID before adding. Serve-all must not drop a configured seed: + // rejecting the only seed leaves the table empty, so discovery never starts. + if !s.config.ServeAll && s.enrManager != nil { isEL, forkID := s.enrManager.FilterELNode(record) s.enrManager.RecordELAdmission(record, isEL, forkID) if !isEL { @@ -896,7 +897,7 @@ func (s *Service) connectELBootnodeEnode(enodeURL *enode.Enode) { v4Node.SetENR(enrRecord) // Filter by fork ID before adding - if s.enrManager != nil { + if !s.config.ServeAll && s.enrManager != nil { isEL, forkID := s.enrManager.FilterELNode(enrRecord) s.enrManager.RecordELAdmission(enrRecord, isEL, forkID) if !isEL { @@ -939,7 +940,7 @@ func (s *Service) connectCLBootnodes() { nodeID := v5.ID() // Filter by fork digest before adding - if s.enrManager != nil && !s.enrManager.FilterCLNode(record) { + if !s.config.ServeAll && s.enrManager != nil && !s.enrManager.FilterCLNode(record) { s.config.Logger.WithField("nodeID", fmt.Sprintf("%x", nodeID[:8])).Warn("CL bootnode ENR has invalid fork digest, skipping") continue } @@ -1007,16 +1008,29 @@ func (s *Service) onNodeSeen(n *v5node.Node, timestamp time.Time) { if s.enrManager != nil { nodeID := n.ID() - if isEL, _ := s.enrManager.FilterELNode(n.Record()); isEL && s.elTable != nil && s.elNodeDB != nil { - // Look up the generic node from the table + // Serve-all pools a node into every table, so refresh last-seen wherever it + // actually lives rather than by classification. Classified mode is + // EL-xor-CL, so the CL filter only runs when the EL one declines. + var isEL, isCL bool + if s.config.ServeAll { + isEL = s.elTable != nil + isCL = s.clTable != nil + } else { + isEL, _ = s.enrManager.FilterELNode(n.Record()) + if !isEL { + isCL = s.enrManager.FilterCLNode(n.Record()) + } + } + + if isEL && s.elTable != nil && s.elNodeDB != nil { if genericNode := s.elTable.Get(nodeID); genericNode != nil { genericNode.SetLastSeen(timestamp) // This marks it dirty // Get falls back to the DB, so Add re-admits demoted nodes s.elTable.Add(genericNode) s.elNodeDB.QueueUpdate(genericNode) } - } else if s.enrManager.FilterCLNode(n.Record()) && s.clTable != nil && s.clNodeDB != nil { - // Look up the generic node from the table + } + if isCL && s.clTable != nil && s.clNodeDB != nil { if genericNode := s.clTable.Get(nodeID); genericNode != nil { genericNode.SetLastSeen(timestamp) // This marks it dirty s.clTable.Add(genericNode) @@ -1037,7 +1051,8 @@ func (s *Service) onFindNodeV5(id *identity, msg *v5protocol.FindNode, sourceNod // A shared identity serves both layers under one ID, so classify a known // requester by its ENR and serve only its layer(s); an unclassifiable known // peer gets nothing, an unknown one (no ENR yet) falls through to both. - if id.servesEL && id.servesCL && sourceNode != nil && s.enrManager != nil { + // Serve-all skips this: every requester gets nodes from every served layer. + if !s.config.ServeAll && id.servesEL && id.servesCL && sourceNode != nil && s.enrManager != nil { sourceRecord := sourceNode.Record() serveEL, _ = s.enrManager.FilterELNode(sourceRecord) serveCL = s.enrManager.FilterCLNode(sourceRecord) @@ -1050,6 +1065,12 @@ func (s *Service) onFindNodeV5(id *identity, msg *v5protocol.FindNode, sourceNod allNodes = append(allNodes, s.clTable.GetNodesByDistance(localID, msg.Distances, 8)...) } + // A node can sit in both tables (any dual-stack peer, and every peer under + // serve-all), so serving both layers would return it twice. + if serveEL && serveCL { + allNodes = dedupeByID(allNodes) + } + // Filter nodes based on protocol support (only return v5-capable nodes) // and apply LAN-aware filtering filteredNodes := s.filterNodesForRequester(allNodes, requester, true) @@ -1185,7 +1206,7 @@ func (s *Service) requestENRV4(n *v4node.Node) { // admitELLookupNode decides admission of a lookup-discovered node to the EL // table. func (s *Service) admitELLookupNode(n *nodes.Node) services.AdmissionResult { - if n.Record() != nil && s.enrManager != nil { + if !s.config.ServeAll && n.Record() != nil && s.enrManager != nil { isEL, forkID := s.enrManager.FilterELNode(n.Record()) s.enrManager.RecordELAdmission(n.Record(), isEL, forkID) if !isEL { @@ -1224,7 +1245,7 @@ func (s *Service) admitELLookupNode(n *nodes.Node) services.AdmissionResult { // admitCLLookupNode decides admission of a lookup-discovered node to the CL // table. func (s *Service) admitCLLookupNode(n *nodes.Node) services.AdmissionResult { - if n.Record() != nil && s.enrManager != nil { + if !s.config.ServeAll && n.Record() != nil && s.enrManager != nil { if !s.enrManager.FilterCLNode(n.Record()) { // No eth2 entry means an execution node, not a consensus // node on the wrong digest; keep the two distinguishable. @@ -1313,7 +1334,7 @@ func (s *Service) checkAndAddNodeV4(n *v4node.Node) bool { alreadyKnown := s.elTable.Get(n.ID()) != nil // Filter the node using ENR manager (EL-only for discv4) - if s.enrManager != nil { + if !s.config.ServeAll && s.enrManager != nil { filter, forkID := s.enrManager.FilterELNode(n.ENR()) s.enrManager.RecordELAdmission(n.ENR(), filter, forkID) if !filter { @@ -1349,10 +1370,18 @@ func (s *Service) checkAndAddNode(n *v5node.Node) bool { return false } - // Determine layer - isEL, elForkID := s.enrManager.FilterELNode(n.Record()) - isCL := s.enrManager.FilterCLNode(n.Record()) - s.enrManager.RecordELAdmission(n.Record(), isEL, elForkID) + // Serve-all skips the filters rather than overriding their results: calling + // them would move admission counters for decisions never made. + var isEL, isCL bool + if s.config.ServeAll { + isEL = s.elTable != nil + isCL = s.clTable != nil + } else { + var elForkID elconfig.ForkID + isEL, elForkID = s.enrManager.FilterELNode(n.Record()) + isCL = s.enrManager.FilterCLNode(n.Record()) + s.enrManager.RecordELAdmission(n.Record(), isEL, elForkID) + } // Add to appropriate table(s) added := false @@ -1372,6 +1401,22 @@ func (s *Service) checkAndAddNode(n *v5node.Node) bool { return added } +// dedupeByID drops repeat node IDs, keeping the first occurrence. +func dedupeByID(nodeList []*nodes.Node) []*nodes.Node { + seen := make(map[[32]byte]struct{}, len(nodeList)) + out := nodeList[:0] + for _, n := range nodeList { + id := n.ID() + if _, dup := seen[id]; dup { + continue + } + seen[id] = struct{}{} + out = append(out, n) + } + + return out +} + // filterNodesForRequester applies LAN-aware and protocol filtering. func (s *Service) filterNodesForRequester(nodeList []*nodes.Node, requester *net.UDPAddr, needsV5 bool) []*nodes.Node { requesterIsLAN := v5node.IsLANAddress(requester.IP) diff --git a/bootnode/service_test.go b/bootnode/service_test.go index 5fdbd6a..0a47731 100644 --- a/bootnode/service_test.go +++ b/bootnode/service_test.go @@ -1,6 +1,7 @@ package bootnode import ( + "context" "crypto/ecdsa" "fmt" "net" @@ -11,7 +12,9 @@ import ( "github.com/ethpandaops/bootnodoor/bootnode/clconfig" "github.com/ethpandaops/bootnodoor/bootnode/elconfig" "github.com/ethpandaops/bootnodoor/db" + v5node "github.com/ethpandaops/bootnodoor/discv5/node" "github.com/ethpandaops/bootnodoor/enr" + "github.com/ethpandaops/bootnodoor/nodes" "github.com/ethpandaops/bootnodoor/services" "github.com/sirupsen/logrus" ) @@ -632,3 +635,123 @@ func mustChainConfig(t *testing.T, jsonCfg string) *elconfig.ChainConfig { } return cfg } + +// newServeAllTestService builds a Service with an EL table and ENR manager, +// enough to exercise checkAndAddNode admission. +func newServeAllTestService(t *testing.T, serveAll bool) *Service { + t.Helper() + + logger := quietLogger() + database := db.NewDatabase(&db.SqliteDatabaseConfig{File: ":memory:", MaxOpenConns: 5, MaxIdleConns: 2}, logger) + if err := database.Init(); err != nil { + t.Fatalf("db init: %v", err) + } + t.Cleanup(func() { database.Close() }) + if err := database.ApplyEmbeddedDbSchema(-2); err != nil { + t.Fatalf("db schema: %v", err) + } + + cfg := &Config{ + Logger: logger, + Database: database, + ELConfig: &elconfig.ChainConfig{}, + ELGenesisHash: [32]byte{1, 2, 3}, + ELGenesisTime: 1000, + ServeAll: serveAll, + } + + key := mustKey(t) + ln, err := createLocalNode(cfg, key, net.ParseIP("1.2.3.4"), nil, 9000, nil) + if err != nil { + t.Fatalf("createLocalNode: %v", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + s := &Service{config: cfg, ctx: ctx, enrManager: NewENRManager(cfg, key, ln, true, false)} + s.elNodeDB = nodes.NewNodeDB(ctx, database, db.LayerEL, logger) + s.elTable, err = s.createTable(ln.ID(), s.elNodeDB, "EL") + if err != nil { + t.Fatalf("createTable: %v", err) + } + + return s +} + +// A node advertising neither eth nor eth2 is unclassifiable: rejected in the +// default classified mode, admitted under --serve-all. +func TestServeAll_AdmitsUnclassifiableNode(t *testing.T) { + for _, tc := range []struct { + name string + serveAll bool + wantAdded bool + }{ + {name: "classified rejects", serveAll: false, wantAdded: false}, + {name: "serve-all admits", serveAll: true, wantAdded: true}, + } { + t.Run(tc.name, func(t *testing.T) { + s := newServeAllTestService(t, tc.serveAll) + + record := storedENRWith(t, mustKey(t), nil) + n, err := v5node.New(record) + if err != nil { + t.Fatalf("v5node.New: %v", err) + } + + if got := s.checkAndAddNode(n); got != tc.wantAdded { + t.Fatalf("checkAndAddNode = %v, want %v", got, tc.wantAdded) + } + }) + } +} + +// Serve-all makes no fork-based admission decision, so it must not move the EL +// admission counters. +func TestServeAll_LeavesAdmissionCountersUntouched(t *testing.T) { + s := newServeAllTestService(t, true) + + record := storedENRWith(t, mustKey(t), map[string][]byte{"eth": {0x01, 0x02, 0x03, 0x04}}) + n, err := v5node.New(record) + if err != nil { + t.Fatalf("v5node.New: %v", err) + } + + s.checkAndAddNode(n) + + if got := s.enrManager.GetELFilter().GetStats().TotalChecks; got != 0 { + t.Fatalf("TotalChecks = %d under serve-all, want 0", got) + } +} + +// A node present in both tables must be served once, not once per table. +func TestDedupeByID(t *testing.T) { + s := newServeAllTestService(t, true) + + rec := storedENRWith(t, mustKey(t), nil) + v5, err := v5node.New(rec) + if err != nil { + t.Fatalf("v5node.New: %v", err) + } + + a := nodes.NewFromV5(v5, s.elNodeDB) + b := nodes.NewFromV5(v5, s.elNodeDB) + other := nodes.NewFromV5(mustV5Node(t), s.elNodeDB) + + got := dedupeByID([]*nodes.Node{a, b, other, a}) + if len(got) != 2 { + t.Fatalf("dedupeByID returned %d nodes, want 2", len(got)) + } + if got[0].ID() != a.ID() || got[1].ID() != other.ID() { + t.Errorf("dedupeByID did not keep first occurrences in order") + } +} + +func mustV5Node(t *testing.T) *v5node.Node { + t.Helper() + n, err := v5node.New(storedENRWith(t, mustKey(t), nil)) + if err != nil { + t.Fatalf("v5node.New: %v", err) + } + return n +} diff --git a/cmd/bootnodoor/main.go b/cmd/bootnodoor/main.go index ac4e822..75a1d11 100644 --- a/cmd/bootnodoor/main.go +++ b/cmd/bootnodoor/main.go @@ -74,6 +74,9 @@ var ( enableEL bool enableCL bool + // Rendezvous mode + serveAll bool + // WebUI flags enableWebUI bool webUIHost string @@ -148,6 +151,9 @@ func init() { rootCmd.Flags().BoolVar(&enableEL, "enable-el", true, "Enable Execution Layer support (discv4 + discv5)") rootCmd.Flags().BoolVar(&enableCL, "enable-cl", true, "Enable Consensus Layer support (discv5)") + // Rendezvous mode + rootCmd.Flags().BoolVar(&serveAll, "serve-all", false, "Disable EL/CL classification and fork-ID filtering: pool and serve every discovered node to everyone (plain discv5 rendezvous)") + // WebUI rootCmd.Flags().BoolVar(&enableWebUI, "web-ui", false, "Enable web UI") rootCmd.Flags().StringVar(&webUIHost, "web-host", "0.0.0.0", "Web UI host") @@ -518,6 +524,7 @@ func runBootnode(cmd *cobra.Command, args []string) error { config.ENRPort = enrUDPPort config.EnableDiscv4 = enableDiscv4 config.EnableDiscv5 = enableDiscv5 + config.ServeAll = serveAll config.MaxActiveNodes = maxActiveNodes config.MaxNodesPerIP = maxNodesPerIP config.Logger = logger From 49aac528e3851eb8756ec6779c2cf467962d2c99 Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Wed, 29 Jul 2026 08:20:17 -0500 Subject: [PATCH 06/49] style(enr): fix import ordering in encoding test --- enr/encoding_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/enr/encoding_test.go b/enr/encoding_test.go index 4e927cf..977dde5 100644 --- a/enr/encoding_test.go +++ b/enr/encoding_test.go @@ -7,8 +7,8 @@ import ( "testing" "github.com/ethereum/go-ethereum/crypto" - gethenr "github.com/ethereum/go-ethereum/p2p/enr" "github.com/ethereum/go-ethereum/p2p/enode" + gethenr "github.com/ethereum/go-ethereum/p2p/enr" "github.com/ethereum/go-ethereum/rlp" ) From 6856adcd54fac12e2ceabea10c7445e3757f7a07 Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Wed, 29 Jul 2026 10:25:29 -0500 Subject: [PATCH 07/49] refactor: reuse and simplification cleanups - elconfig: drop the duplicate crc32 walk (sums) and read checksums off allForkIDs; remove blockHead and the sentry special case, which cannot affect any validate() outcome under the static head stance. Verified equivalent by differential test across every schedule shape, time head and perturbed fork ID. Copy the slice out of GetAllForkIDs now that validate depends on it. - clconfig: slices.Sort/Compact for the boundary walk; GetAllForkDigests projects from GetAllForkDigestInfos instead of duplicating it. - stats: drop Discv5Stats.PacketsReceived/PacketsSent, summed but never read (the UI takes packet totals from transport metrics). - lookup: slices.Contains for the local-identity check. - Fork names: strings.ToUpper/Join instead of hand-rolled byte arithmetic that corrupts any name not starting a-z. --- bootnode/clconfig/config.go | 35 +++++++--------- bootnode/elconfig/filter.go | 84 +++++++++++-------------------------- bootnode/stats.go | 4 -- services/lookup.go | 8 +--- 4 files changed, 43 insertions(+), 88 deletions(-) diff --git a/bootnode/clconfig/config.go b/bootnode/clconfig/config.go index 53ba155..6b4754d 100644 --- a/bootnode/clconfig/config.go +++ b/bootnode/clconfig/config.go @@ -13,6 +13,7 @@ import ( "fmt" "math" "os" + "slices" "sort" "strings" "time" @@ -549,22 +550,19 @@ func (c *Config) currentEpochNow() (uint64, bool) { // genesis, each registered fork (including BPO pseudo-forks), and each blob // schedule boundary. Sorted ascending, deduplicated. func (c *Config) forkBoundaryEpochs() []uint64 { - seen := map[uint64]bool{0: true} epochs := []uint64{0} - add := func(epoch uint64) { - if epoch != math.MaxUint64 && !seen[epoch] { - seen[epoch] = true - epochs = append(epochs, epoch) - } - } for _, fork := range c.getForks() { - add(fork.epoch) + if fork.epoch != math.MaxUint64 { + epochs = append(epochs, fork.epoch) + } } for _, entry := range c.BlobSchedule { - add(entry.Epoch) + if entry.Epoch != math.MaxUint64 { + epochs = append(epochs, entry.Epoch) + } } - sort.Slice(epochs, func(i, j int) bool { return epochs[i] < epochs[j] }) - return epochs + slices.Sort(epochs) + return slices.Compact(epochs) } // GetCurrentForkDigest returns the fork digest for the current epoch. @@ -659,14 +657,13 @@ type ForkDigestInfo struct { // for same-epoch intermediate forks (never current on the wire) are // intentionally not included. func (c *Config) GetAllForkDigests() []ForkDigest { - var digests []ForkDigest - seen := make(map[ForkDigest]bool) - for _, epoch := range c.forkBoundaryEpochs() { - digest := c.GetForkDigestForEpoch(epoch) - if !seen[digest] { - seen[digest] = true - digests = append(digests, digest) - } + infos := c.GetAllForkDigestInfos() + if len(infos) == 0 { + return nil + } + digests := make([]ForkDigest, 0, len(infos)) + for _, info := range infos { + digests = append(digests, info.Digest) } return digests } diff --git a/bootnode/elconfig/filter.go b/bootnode/elconfig/filter.go index de69f2c..b4424a9 100644 --- a/bootnode/elconfig/filter.go +++ b/bootnode/elconfig/filter.go @@ -2,8 +2,9 @@ package elconfig import ( "fmt" - "hash/crc32" "math" + "slices" + "strings" "sync" "time" ) @@ -31,19 +32,11 @@ type ForkFilter struct { // special casing. forks []uint64 - // numBlockForks is the boundary index separating block forks from time - // forks in forks, including go-ethereum's rule that the sentry counts as - // a block fork when the chain has no time forks at all. + // numBlockForks indexes the first time fork in forks. Every block fork is + // passed under the static head stance, so validation starts scanning here. numBlockForks int - // blockHead is the static block head: at or past every canonical block - // fork, before the sentry. - blockHead uint64 - - // sums[i] is the checksum after passing the first i fork boundaries. - sums [][4]byte - - // allForkIDs contains the complete canonical list for display + // allForkIDs[i].Hash is the checksum after passing the first i boundaries. allForkIDs []ForkID // Admission outcomes, recorded by the admission call sites only (the @@ -79,38 +72,15 @@ func NewForkFilter(genesisHash [32]byte, config *ChainConfig, genesisTime uint64 forksByBlock, forksByTime := GatherForks(config, genesisTime) forks := append(append([]uint64{}, forksByBlock...), forksByTime...) - sums := make([][4]byte, len(forks)+1) - hash := crc32.ChecksumIEEE(genesisHash[:]) - sums[0] = checksumToBytes(hash) - for i, fork := range forks { - hash = checksumUpdate(hash, fork) - sums[i+1] = checksumToBytes(hash) - } - - blockHead := uint64(0) - if len(forksByBlock) > 0 { - blockHead = forksByBlock[len(forksByBlock)-1] - } - - numBlockForks := len(forksByBlock) forks = append(forks, math.MaxUint64) - if len(forksByTime) == 0 { - // In purely block based forks, keep the sentry out of timestamp - // territory (go-ethereum's rule). - numBlockForks++ - } - - allForkIDs := ComputeAllForkIDs(genesisHash, forksByBlock, forksByTime) return &ForkFilter{ genesisHash: genesisHash, chainConfig: config, genesisTime: genesisTime, forks: forks, - numBlockForks: numBlockForks, - blockHead: blockHead, - sums: sums, - allForkIDs: allForkIDs, + numBlockForks: len(forksByBlock), + allForkIDs: ComputeAllForkIDs(genesisHash, forksByBlock, forksByTime), } } @@ -137,16 +107,12 @@ func (f *ForkFilter) Filter(id ForkID) bool { // be completed with locally known future forks: accept (we are syncing). // 4. Reject in all other cases. func (f *ForkFilter) validate(id ForkID, now uint64) error { - for i, fork := range f.forks { - head := f.blockHead - if i >= f.numBlockForks { - head = now - } - if head >= fork { + for i := f.numBlockForks; i < len(f.forks); i++ { + if now >= f.forks[i] { continue } // Found the first unpassed fork, check the remote against it (rule #1). - if f.sums[i] == id.Hash { + if f.allForkIDs[i].Hash == id.Hash { // A remote-announced fork we have already passed means the remote // is stale (rule #1a). Every unpassed fork here is time-scheduled // (block forks are all passed under the static stance), so the @@ -158,7 +124,7 @@ func (f *ForkFilter) validate(id ForkID, now uint64) error { } // Different fork state: subset means the remote is syncing (rule #2). for j := 0; j < i; j++ { - if f.sums[j] == id.Hash { + if f.allForkIDs[j].Hash == id.Hash { if f.forks[j] != id.Next { return fmt.Errorf("remote is stale: subset checksum with next %d, want %d", id.Next, f.forks[j]) } @@ -166,8 +132,8 @@ func (f *ForkFilter) validate(id ForkID, now uint64) error { } } // Superset means we would be the one syncing (rule #3). - for j := i + 1; j < len(f.sums); j++ { - if f.sums[j] == id.Hash { + for j := i + 1; j < len(f.allForkIDs); j++ { + if f.allForkIDs[j].Hash == id.Hash { return nil } } @@ -177,8 +143,9 @@ func (f *ForkFilter) validate(id ForkID, now uint64) error { return nil } -// RecordAdmission records an admission decision for the stats surface. Call -// this from admission paths only, never from layer classification. +// RecordAdmission records an admission decision for the stats surface. Its only +// caller is ENRManager.AdmitELNode, which owns the eth-entry gate; the pure +// predicate path (ClassifyELNode) must never reach here. func (f *ForkFilter) RecordAdmission(acceptedNode bool, id ForkID) { f.statsMu.Lock() defer f.statsMu.Unlock() @@ -203,9 +170,11 @@ func (f *ForkFilter) GetStats() FilterStats { } } -// GetAllForkIDs returns all valid fork IDs for debugging. +// GetAllForkIDs returns all valid fork IDs for debugging. Copied because +// validate reads allForkIDs to decide admission; mutating it would corrupt +// peer filtering. func (f *ForkFilter) GetAllForkIDs() []ForkID { - return f.allForkIDs + return slices.Clone(f.allForkIDs) } // GetCurrentForkID calculates the current fork ID based on chain state. @@ -273,19 +242,16 @@ func (f *ForkFilter) GetAllForkIDsWithNames() []ForkIDWithName { if i+1 >= len(f.allForkIDs) { break } - name := "" - for j, n := range b.names { + names := make([]string, 0, len(b.names)) + for _, n := range b.names { if len(n) > 0 { - n = string(n[0]-32) + n[1:] - } - if j > 0 { - name += "/" + n = strings.ToUpper(n[:1]) + n[1:] } - name += n + names = append(names, n) } result = append(result, ForkIDWithName{ ForkID: f.allForkIDs[i+1], - Name: name, + Name: strings.Join(names, "/"), Activation: b.value, IsTime: b.isTime, }) diff --git a/bootnode/stats.go b/bootnode/stats.go index 7362070..12bfe5e 100644 --- a/bootnode/stats.go +++ b/bootnode/stats.go @@ -25,8 +25,6 @@ type Stats struct { // Discv5Stats is the deliberate subset of protocol.HandlerStats the web UI renders, summed per identity. type Discv5Stats struct { - PacketsReceived int - PacketsSent int InvalidPackets int FilteredResponses int FindNodeReceived int @@ -72,8 +70,6 @@ func (s *Service) GetStats() Stats { if id.discv5Service != nil { if h := id.discv5Service.Handler(); h != nil { d := h.GetStats() - out.Discv5.PacketsReceived += d.PacketsReceived - out.Discv5.PacketsSent += d.PacketsSent out.Discv5.InvalidPackets += d.InvalidPackets out.Discv5.FilteredResponses += d.FilteredResponses out.Discv5.FindNodeReceived += d.FindNodeReceived diff --git a/services/lookup.go b/services/lookup.go index 2f062b0..e23fd46 100644 --- a/services/lookup.go +++ b/services/lookup.go @@ -13,6 +13,7 @@ import ( "crypto/rand" "fmt" mathrand "math/rand" + "slices" "sort" "sync" "time" @@ -123,12 +124,7 @@ type Config struct { // isLocal reports whether id belongs to one of our own identities. The // parameter is the raw array so both discv4 and discv5 node IDs can be passed. func (ls *LookupService) isLocal(id [32]byte) bool { - for _, local := range ls.config.LocalIDs { - if local == id { - return true - } - } - return false + return slices.Contains(ls.config.LocalIDs, id) } // discoveries accumulates the records observed during a single lookup, keeping From c3d9e1649ef846cd60d47b1a49dd491a7c638d0e Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Wed, 29 Jul 2026 10:26:10 -0500 Subject: [PATCH 08/49] fix(discovery): require endpoint proof before a response mutates state An off-path attacker could move the ENR this bootnode publishes, and a bonded peer could reflect amplified NEIGHBORS traffic at a third party. Both follow from the same gap: nothing proved a peer was reachable at the address it claimed. discv4 PONG: side effects (bond, external-IP vote, ENR refresh) ran before any request match, so any well-formed PONG applied them. They now require a match that is - source-bound: the PONG source IP must equal the address the PING was sent to. PendingRequest snapshots that IP at send time, because getOrCreateNode rewrites ToNode.Addr() from every inbound packet, including the spoofed one this check exists to catch. - PING-typed: getPendingRequests ignored PacketType, and peers know the hashes of packets we sent them, so an ENRREQUEST hash matched as a reply token. - consumed once: the entry was removed by the caller, so for the 500ms Ping() wait every replayed PONG re-cast the IP vote. Varying the spoofed source reached MinReports/MinDistinctIPs off one exchange. discv4 bonds: tracked per proven IP rather than per node ID, so a bond earned at one address no longer serves requests from another. Ports are excluded, matching go-ethereum (checkBond passes ip.Addr(), dropping the port) and avoiding a false negative when a NAT mapping rotates inside the 24h bond. Per-IP rather than a single address so dual-stack peers keep both bonds. handlePing no longer grants a bond for merely receiving a PING: we pong whatever source the packet claimed, which proves nothing, and that was the remaining route to bonding a victim's address. discv5: MatchResponse validated only requestID and node ID, so a PONG could match and consume a pending FINDNODE, firing the IP-vote path and stranding the lookup whose channel it closed. It now checks the response against the stored request; handlePong returns early when unmatched. IP discovery: consensus counted repeat reports from one peer as independent, measuring independence only by spoofable source IP, so a single node ID with three spoofed sources met both thresholds. Distinct reporter node IDs are now required too, and the reporter key is the full ID rather than an 8-byte prefix that let two peers count as one. CL fork filter: Filter was both predicate and stats recorder, and ran on per-packet classification paths, so the UI's Fork Filter card counted packets rather than admissions. Split into a pure Matches and a counting Admit over a shared classify, mirrored as ClassifyCLNode/AdmitCLNode and ClassifyELNode/AdmitELNode; RecordELAdmission and its five-site paired-call contract are gone. Renaming made every call site a compile error rather than a silent behaviour change. classify holds the lock once, which fixes a concurrent map read/write against Update() on oldForkDigests (an unrecoverable abort, not an error) and makes TotalChecks always equal the sum of the buckets. Also: count FINDNODE responses after the unsolicited gate, not before. --- bootnode/clconfig/admit_test.go | 187 ++++++++++++++++ bootnode/clconfig/filter.go | 238 ++++++++++----------- bootnode/clconfig/filter_test.go | 12 +- bootnode/clfilter_test.go | 117 ++++++++++ bootnode/enr.go | 146 +++++++------ bootnode/service.go | 173 +++++++-------- bootnode/service_test.go | 31 +-- discv4/node/node.go | 41 +++- discv4/protocol/endpoint_proof_test.go | 248 ++++++++++++++++++++++ discv4/protocol/handler.go | 138 +++++++++--- discv4/protocol/handler_test.go | 2 +- discv4/protocol/pending_neighbors_test.go | 2 +- discv5/protocol/handler.go | 8 +- discv5/protocol/request.go | 29 +++ discv5/protocol/request_match_test.go | 57 +++++ services/ipdiscovery.go | 30 +-- services/ipdiscovery_test.go | 70 ++++++ 17 files changed, 1178 insertions(+), 351 deletions(-) create mode 100644 bootnode/clconfig/admit_test.go create mode 100644 bootnode/clfilter_test.go create mode 100644 discv4/protocol/endpoint_proof_test.go create mode 100644 discv5/protocol/request_match_test.go create mode 100644 services/ipdiscovery_test.go diff --git a/bootnode/clconfig/admit_test.go b/bootnode/clconfig/admit_test.go new file mode 100644 index 0000000..96b22a3 --- /dev/null +++ b/bootnode/clconfig/admit_test.go @@ -0,0 +1,187 @@ +package clconfig + +import ( + "net" + "sync" + "testing" + "time" + + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethpandaops/bootnodoor/enr" +) + +func admitTestFilter(t *testing.T) *ForkDigestFilter { + t.Helper() + + cfg := &Config{ + SecondsPerSlot: 12, + customSlotsPerEpoch: 32, + genesisForkVersion: [4]byte{0x00, 0x00, 0x00, 0x01}, + forks: []forkDefinition{ + {name: "Altair", epoch: 0, parsedVersion: [4]byte{0x01, 0x00, 0x00, 0x00}}, + }, + } + cfg.SetGenesisTime(uint64(time.Now().Unix()) - 60) + return NewForkDigestFilter(cfg, time.Hour) +} + +func recordWithEth2(t *testing.T, eth2 []byte) *enr.Record { + t.Helper() + + key, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("generate key: %v", err) + } + rec := enr.New() + if err := rec.Set("ip", net.IPv4(1, 2, 3, 4)); err != nil { + t.Fatalf("set ip: %v", err) + } + if eth2 != nil { + if err := rec.Set("eth2", eth2); err != nil { + t.Fatalf("set eth2: %v", err) + } + } + rec.SetSeq(1) + if err := rec.Sign(key); err != nil { + t.Fatalf("sign: %v", err) + } + return rec +} + +// Matches is the per-packet classification entry point, so it must decide +// without moving any counter — those numbers report admissions, and packet +// traffic dwarfs admissions by orders of magnitude. +func TestMatchesRecordsNoStats(t *testing.T) { + filter := admitTestFilter(t) + current := filter.GetCurrentForkDigest() + + cases := []struct { + name string + eth2 []byte + want bool + }{ + {"no eth2", nil, false}, + {"malformed", []byte{0x01, 0x02}, false}, + {"current digest", EncodeETH2Field(current, [4]byte{0x01, 0x00, 0x00, 0x00}, ^uint64(0)), true}, + {"unknown digest", EncodeETH2Field(ForkDigest{0xde, 0xad, 0xbe, 0xef}, [4]byte{}, 0), false}, + } + + for _, tc := range cases { + rec := recordWithEth2(t, tc.eth2) + if got := filter.Matches(rec); got != tc.want { + t.Errorf("Matches(%s) = %v, want %v", tc.name, got, tc.want) + } + } + + stats := filter.GetStats() + if stats.TotalChecks != 0 || stats.AcceptedCurrent != 0 || stats.AcceptedOld != 0 || + stats.AcceptedHistorical != 0 || stats.RejectedInvalid != 0 { + t.Fatalf("Matches moved counters: %+v", stats) + } +} + +// Admit is the only counting entry point, and TotalChecks must always equal the +// sum of the buckets — the web UI renders them in one table, so a reader must +// never see rows that do not add up. +func TestAdmitRecordsOneBucketPerCall(t *testing.T) { + filter := admitTestFilter(t) + current := filter.GetCurrentForkDigest() + + cases := []struct { + name string + eth2 []byte + wantAccept bool + wantChecks int + wantCurrent int + wantInvalid int + }{ + {"no eth2 is uncounted", nil, false, 0, 0, 0}, + {"current digest", EncodeETH2Field(current, [4]byte{0x01, 0x00, 0x00, 0x00}, ^uint64(0)), true, 1, 1, 0}, + {"unknown digest", EncodeETH2Field(ForkDigest{0xde, 0xad, 0xbe, 0xef}, [4]byte{}, 0), false, 2, 1, 1}, + {"malformed", []byte{0x01, 0x02}, false, 3, 1, 2}, + } + + for _, tc := range cases { + rec := recordWithEth2(t, tc.eth2) + if got := filter.Admit(rec); got != tc.wantAccept { + t.Errorf("Admit(%s) = %v, want %v", tc.name, got, tc.wantAccept) + } + + stats := filter.GetStats() + if stats.TotalChecks != tc.wantChecks { + t.Errorf("after %s: TotalChecks = %d, want %d", tc.name, stats.TotalChecks, tc.wantChecks) + } + if stats.AcceptedCurrent != tc.wantCurrent { + t.Errorf("after %s: AcceptedCurrent = %d, want %d", tc.name, stats.AcceptedCurrent, tc.wantCurrent) + } + if stats.RejectedInvalid != tc.wantInvalid { + t.Errorf("after %s: RejectedInvalid = %d, want %d", tc.name, stats.RejectedInvalid, tc.wantInvalid) + } + + sum := stats.AcceptedCurrent + stats.AcceptedOld + stats.AcceptedHistorical + stats.RejectedInvalid + if stats.TotalChecks != sum { + t.Errorf("after %s: TotalChecks = %d but buckets sum to %d", tc.name, stats.TotalChecks, sum) + } + } +} + +// Update mutates oldForkDigests while packets are being filtered, so the digest +// lookups must happen under the lock. Publishing the map reference and indexing +// it afterwards is a concurrent map read/write, which aborts the process. +func TestAdmitConcurrentWithUpdate(t *testing.T) { + filter := admitTestFilter(t) + + rec := recordWithEth2(t, EncodeETH2Field(ForkDigest{0x11, 0x22, 0x33, 0x44}, [4]byte{}, 0)) + + var wg sync.WaitGroup + stop := make(chan struct{}) + + // Update only writes oldForkDigests on a fork activation or a grace expiry, + // so seed an already-expired entry each round to make its cleanup loop + // delete — the same map write, just at test frequency. + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case <-stop: + return + default: + filter.mu.Lock() + filter.oldForkDigests[ForkDigest{0x11, 0x22, 0x33, 0x44}] = time.Now().Add(-2 * time.Hour) + filter.mu.Unlock() + filter.Update() + } + } + }() + + for i := 0; i < 8; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + for j := 0; j < 200; j++ { + if i%2 == 0 { + filter.Admit(rec) + } else { + filter.Matches(rec) + } + } + }(i) + } + + // The Update loop runs until the readers finish, then is joined separately. + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + time.Sleep(200 * time.Millisecond) + close(stop) + <-done + + stats := filter.GetStats() + sum := stats.AcceptedCurrent + stats.AcceptedOld + stats.AcceptedHistorical + stats.RejectedInvalid + if stats.TotalChecks != sum { + t.Fatalf("TotalChecks = %d but buckets sum to %d under concurrency", stats.TotalChecks, sum) + } +} diff --git a/bootnode/clconfig/filter.go b/bootnode/clconfig/filter.go index ba07897..8768b2c 100644 --- a/bootnode/clconfig/filter.go +++ b/bootnode/clconfig/filter.go @@ -65,15 +65,8 @@ type Logger interface { // - config: CL configuration for computing fork digests // - gracePeriod: How long to accept old fork digests (0 = default 60 minutes) // -// Example: -// -// config, _ := LoadConfig("config.yaml") -// filter := NewForkDigestFilter(config, 60*time.Minute) -// -// // Use as admission filter -// service, _ := discv5.New(&discv5.Config{ -// AdmissionFilter: filter.Filter(), -// }) +// Call Admit from admission paths and Matches from per-packet classification; +// only Admit moves the stats counters. func NewForkDigestFilter(config *Config, gracePeriod time.Duration) *ForkDigestFilter { if gracePeriod <= 0 { gracePeriod = DefaultGracePeriod @@ -105,109 +98,135 @@ func (f *ForkDigestFilter) SetLogger(logger Logger) { f.logger = logger } -// Filter returns an ENR admission filter function. -// -// This filter accepts ALL historically valid fork digests: -// - Current fork digest -// - Old fork digests (within grace period) -// - Any historically valid fork digest from the network -// -// Nodes with old digests are accepted into the routing table and will be -// pinged, which triggers ENR updates. Use ResponseFilter() to exclude them -// from FINDNODE responses. -// -// Example: +// clOutcome is the result of evaluating one record's fork digest. Each value +// maps to exactly one stats bucket, except outcomeNotCL which is deliberately +// uncounted. +type clOutcome int + +const ( + // outcomeNotCL is a record with no eth2 entry: an execution node, not a + // broken consensus node, so it must not move the CL counters. + outcomeNotCL clOutcome = iota + outcomeUndecodable + outcomeUnparsable + outcomeCurrent + outcomeOldInGrace + outcomeHistorical + outcomeUnknownDigest +) + +func (o clOutcome) accepted() bool { + return o == outcomeCurrent || o == outcomeOldInGrace || o == outcomeHistorical +} + +// classify evaluates a record's fork digest, touching neither stats nor logs. // -// filter := NewForkDigestFilter(config, 60*time.Minute) -// service, _ := discv5.New(&discv5.Config{ -// AdmissionFilter: filter.Filter(), -// ResponseFilter: filter.ResponseFilter(), -// }) -func (f *ForkDigestFilter) Filter(record *enr.Record) bool { - // No eth2 entry means an execution node, not an invalid consensus node: - // reject without moving counters (mirrors RecordELAdmission's eth gate). - // A present but undecodable entry is a broken consensus node: that counts. +// The digest lookups run inside one read-lock: Update mutates oldForkDigests, so +// publishing that map outside the lock and indexing it later is a concurrent +// map read/write, which aborts the process rather than returning an error. +func (f *ForkDigestFilter) classify(record *enr.Record) (clOutcome, ForkDigest, error) { + if record == nil { + return outcomeNotCL, ForkDigest{}, nil + } + var eth2Data []byte if err := record.Get("eth2", ð2Data); err != nil { if !record.Has("eth2") { - return false + return outcomeNotCL, ForkDigest{}, nil } - f.mu.Lock() - f.totalChecks++ - f.rejectedInvalid++ - if f.logger != nil { - f.logger.Debugf("Rejected node: undecodable eth2 field - %v", err) - } - f.mu.Unlock() - return false + return outcomeUndecodable, ForkDigest{}, err } - f.mu.Lock() - f.totalChecks++ - f.mu.Unlock() - - // Parse fork digest (first 4 bytes only) forkDigest, err := ParseETH2Field(eth2Data) if err != nil { - // Invalid eth2 field, reject - f.mu.Lock() - f.rejectedInvalid++ - if f.logger != nil { - f.logger.Debugf("Rejected node: invalid eth2 field - %v", err) - } - f.mu.Unlock() - return false + return outcomeUnparsable, ForkDigest{}, err } f.mu.RLock() - currentDigest := f.currentForkDigest - oldDigests := f.oldForkDigests - gracePeriod := f.gracePeriod - historicalDigests := f.historicalDigests - f.mu.RUnlock() + defer f.mu.RUnlock() - // Check if matches current fork digest - if forkDigest == currentDigest { - f.mu.Lock() - f.acceptedCurrent++ - f.mu.Unlock() - return true + if forkDigest == f.currentForkDigest { + return outcomeCurrent, forkDigest, nil } - // Check if matches old fork digest within grace period - if activationTime, exists := oldDigests[forkDigest]; exists { - age := time.Since(activationTime) - if age <= gracePeriod { - f.mu.Lock() - f.acceptedOld++ - f.mu.Unlock() - return true + if activationTime, exists := f.oldForkDigests[forkDigest]; exists { + if time.Since(activationTime) <= f.gracePeriod { + return outcomeOldInGrace, forkDigest, nil } - // Grace period expired but still historically valid - fall through + // Grace period expired but the digest may still be historically valid. } - // Check if it's any historically valid fork digest - // These nodes will be added to the table and pinged (triggering ENR updates) - // but may be excluded from FINDNODE responses via ResponseFilter - if historicalDigests[forkDigest] { - f.mu.Lock() - f.acceptedHistorical++ - if f.logger != nil { - f.logger.Debugf("Accepted node with historical fork digest: %s (current: %s)", forkDigest.String(), currentDigest.String()) - } - f.mu.Unlock() - return true + if f.historicalDigests[forkDigest] { + return outcomeHistorical, forkDigest, nil + } + + return outcomeUnknownDigest, forkDigest, nil +} + +// Matches reports whether a record's fork digest is acceptable. +// +// It is pure: no counter moves and nothing is logged. Use it for per-packet +// layer classification, which happens far too often to be a stats signal. +func (f *ForkDigestFilter) Matches(record *enr.Record) bool { + outcome, _, _ := f.classify(record) + return outcome.accepted() +} + +// Admit is Matches plus stats: it is the only entry point that moves the +// counters. Call it from admission paths only, never from layer classification +// (the same contract as elconfig.ForkFilter.RecordAdmission). +// +// This filter accepts ALL historically valid fork digests: the current digest, +// old digests within the grace period, and any digest from network history. +// Nodes with old digests are accepted into the routing table and pinged, which +// triggers ENR updates. +func (f *ForkDigestFilter) Admit(record *enr.Record) bool { + outcome, forkDigest, err := f.classify(record) + f.recordOutcome(outcome, forkDigest, err) + return outcome.accepted() +} + +// recordOutcome folds one admission outcome into the stats and emits the +// matching debug line, under a single lock so TotalChecks and the buckets always +// agree for any GetStats observer. +func (f *ForkDigestFilter) recordOutcome(outcome clOutcome, forkDigest ForkDigest, err error) { + if outcome == outcomeNotCL { + return } - // Unknown fork digest, reject f.mu.Lock() - f.rejectedInvalid++ - if f.logger != nil { - f.logger.Debugf("Rejected node: unknown fork digest %s (current: %s, %d historical digests known)", - forkDigest.String(), currentDigest.String(), len(historicalDigests)) + defer f.mu.Unlock() + + f.totalChecks++ + + switch outcome { + case outcomeUndecodable: + f.rejectedInvalid++ + if f.logger != nil { + f.logger.Debugf("Rejected node: undecodable eth2 field - %v", err) + } + case outcomeUnparsable: + f.rejectedInvalid++ + if f.logger != nil { + f.logger.Debugf("Rejected node: invalid eth2 field - %v", err) + } + case outcomeCurrent: + f.acceptedCurrent++ + case outcomeOldInGrace: + f.acceptedOld++ + case outcomeHistorical: + f.acceptedHistorical++ + if f.logger != nil { + f.logger.Debugf("Accepted node with historical fork digest: %s (current: %s)", forkDigest.String(), f.currentForkDigest.String()) + } + case outcomeUnknownDigest: + f.rejectedInvalid++ + if f.logger != nil { + f.logger.Debugf("Rejected node: unknown fork digest %s (current: %s, %d historical digests known)", + forkDigest.String(), f.currentForkDigest.String(), len(f.historicalDigests)) + } + case outcomeNotCL: } - f.mu.Unlock() - return false } // Update updates the fork digest based on the current epoch. @@ -335,18 +354,7 @@ func (f *ForkDigestFilter) ComputeEth2Field() []byte { func (f *ForkDigestFilter) nextForkInfo() ([4]byte, uint64) { const farFutureEpoch = ^uint64(0) - genesisTime := f.config.GetGenesisTime() - secondsPerSlot := f.config.SecondsPerSlot - if secondsPerSlot == 0 { - secondsPerSlot = 12 - } - - currentEpoch := uint64(0) - if genesisTime > 0 { - slotsPerEpoch := f.config.GetSlotsPerEpoch() - currentEpoch = uint64(GetCurrentEpoch(genesisTime, uint64(time.Now().Unix()), secondsPerSlot, slotsPerEpoch)) - } - + currentEpoch, _ := f.config.currentEpochNow() currentForkVersion := f.config.GetForkVersionAtEpoch(currentEpoch) for _, fork := range f.config.getForks() { @@ -369,23 +377,10 @@ func (f *ForkDigestFilter) nextForkInfo() ([4]byte, uint64) { // GetCurrentFork returns the name of the current fork. func (f *ForkDigestFilter) GetCurrentFork() string { - // Get genesis time - genesisTime := f.config.GetGenesisTime() - if genesisTime == 0 { - // No genesis time, fallback to "Unknown" + currentEpoch, ok := f.config.currentEpochNow() + if !ok { return "Unknown" } - - // Calculate current epoch - currentTime := uint64(time.Now().Unix()) - slotsPerEpoch := f.config.GetSlotsPerEpoch() - secondsPerSlot := f.config.SecondsPerSlot - if secondsPerSlot == 0 { - secondsPerSlot = 12 - } - currentEpoch := uint64(GetCurrentEpoch(genesisTime, currentTime, secondsPerSlot, slotsPerEpoch)) - - // Get fork name for current epoch return f.config.GetForkNameAtEpoch(currentEpoch) } @@ -481,15 +476,6 @@ func (f *ForkDigestFilter) GetRejectedInvalid() int { return f.rejectedInvalid } -// GetAcceptedHistorical returns the count of nodes accepted on a historical -// fork digest (valid chain, not the current or grace-period fork). -func (f *ForkDigestFilter) GetAcceptedHistorical() int { - f.mu.RLock() - defer f.mu.RUnlock() - - return f.acceptedHistorical -} - // GetTotalChecks returns the total number of filter checks performed. func (f *ForkDigestFilter) GetTotalChecks() int { f.mu.RLock() diff --git a/bootnode/clconfig/filter_test.go b/bootnode/clconfig/filter_test.go index dbc9035..4ad1a13 100644 --- a/bootnode/clconfig/filter_test.go +++ b/bootnode/clconfig/filter_test.go @@ -75,11 +75,11 @@ func TestNextForkInfoFallsBackToFarFuture(t *testing.T) { } } -// TestFilterSkipsRecordsWithoutEth2: a record with no eth2 entry is an +// TestAdmitSkipsRecordsWithoutEth2: a record with no eth2 entry is an // execution node, not an invalid consensus node, so it must not move any -// counter (mirrors the EL side's RecordELAdmission gate). A malformed eth2 +// counter (mirrors the EL side's AdmitELNode gate). A malformed eth2 // entry still counts as invalid. -func TestFilterSkipsRecordsWithoutEth2(t *testing.T) { +func TestAdmitSkipsRecordsWithoutEth2(t *testing.T) { cfg := &Config{ SecondsPerSlot: 12, customSlotsPerEpoch: 32, @@ -104,7 +104,7 @@ func TestFilterSkipsRecordsWithoutEth2(t *testing.T) { t.Fatalf("sign: %v", err) } - if filter.Filter(noEth2) { + if filter.Admit(noEth2) { t.Fatal("record without eth2 passed the CL filter") } stats := filter.GetStats() @@ -122,7 +122,7 @@ func TestFilterSkipsRecordsWithoutEth2(t *testing.T) { t.Fatalf("sign: %v", err) } - if filter.Filter(malformed) { + if filter.Admit(malformed) { t.Fatal("malformed eth2 passed the CL filter") } stats = filter.GetStats() @@ -140,7 +140,7 @@ func TestFilterSkipsRecordsWithoutEth2(t *testing.T) { t.Fatalf("sign: %v", err) } - if filter.Filter(undecodable) { + if filter.Admit(undecodable) { t.Fatal("undecodable eth2 passed the CL filter") } stats = filter.GetStats() diff --git a/bootnode/clfilter_test.go b/bootnode/clfilter_test.go new file mode 100644 index 0000000..2c5b9a3 --- /dev/null +++ b/bootnode/clfilter_test.go @@ -0,0 +1,117 @@ +package bootnode + +import ( + "context" + "net" + "testing" + "time" + + "github.com/ethpandaops/bootnodoor/bootnode/clconfig" + "github.com/ethpandaops/bootnodoor/db" + v5node "github.com/ethpandaops/bootnodoor/discv5/node" + "github.com/ethpandaops/bootnodoor/enr" + "github.com/ethpandaops/bootnodoor/nodes" +) + +// newCLTestService mirrors newServeAllTestService but wires the CL layer, so the +// CL fork-digest filter and its counters are reachable. +func newCLTestService(t *testing.T) *Service { + t.Helper() + + logger := quietLogger() + database := db.NewDatabase(&db.SqliteDatabaseConfig{File: ":memory:", MaxOpenConns: 5, MaxIdleConns: 2}, logger) + if err := database.Init(); err != nil { + t.Fatalf("db init: %v", err) + } + t.Cleanup(func() { database.Close() }) + if err := database.ApplyEmbeddedDbSchema(-2); err != nil { + t.Fatalf("db schema: %v", err) + } + + cfg := &Config{ + Logger: logger, + Database: database, + CLConfig: &clconfig.Config{}, + } + + key := mustKey(t) + ln, err := createLocalNode(cfg, key, net.ParseIP("1.2.3.4"), nil, 9000, nil) + if err != nil { + t.Fatalf("createLocalNode: %v", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + s := &Service{config: cfg, ctx: ctx, enrManager: NewENRManager(cfg, key, ln, false, true)} + s.clNodeDB = nodes.NewNodeDB(ctx, database, db.LayerCL, logger) + s.clTable, err = s.createTable(ln.ID(), s.clNodeDB, "CL") + if err != nil { + t.Fatalf("createTable: %v", err) + } + + return s +} + +// clNodeOnCurrentDigest builds a v5 node whose eth2 entry carries the digest the +// filter currently accepts, so it exercises the accepted path. +func clNodeOnCurrentDigest(t *testing.T, s *Service) *v5node.Node { + t.Helper() + + digest := s.enrManager.GetCLFilter().GetCurrentForkDigest() + eth2 := clconfig.EncodeETH2Field(digest, [4]byte{}, ^uint64(0)) + + key := mustKey(t) + rec := enr.New() + if err := rec.Set("ip", net.IPv4(9, 9, 9, 9)); err != nil { + t.Fatalf("set ip: %v", err) + } + if err := rec.Set("udp", uint16(9000)); err != nil { + t.Fatalf("set udp: %v", err) + } + if err := rec.Set("eth2", eth2); err != nil { + t.Fatalf("set eth2: %v", err) + } + rec.SetSeq(1) + if err := rec.Sign(key); err != nil { + t.Fatalf("sign: %v", err) + } + + n, err := v5node.New(rec) + if err != nil { + t.Fatalf("v5node.New: %v", err) + } + return n +} + +// onNodeSeen runs once per decoded discv5 message, so it must classify without +// counting: otherwise the Fork Filter card reports packet traffic rather than +// admission decisions. +func TestOnNodeSeen_LeavesCLFilterCountersUntouched(t *testing.T) { + s := newCLTestService(t) + n := clNodeOnCurrentDigest(t, s) + + for i := 0; i < 3; i++ { + s.onNodeSeen(n, time.Now()) + } + + if got := s.enrManager.GetCLFilter().GetStats().TotalChecks; got != 0 { + t.Fatalf("TotalChecks = %d after 3 onNodeSeen calls, want 0", got) + } +} + +// The counterpart guard: admission must still count, or the fix would just zero +// the UI permanently while the classification tests passed. +func TestCheckAndAddNode_RecordsCLAdmissionOnce(t *testing.T) { + s := newCLTestService(t) + n := clNodeOnCurrentDigest(t, s) + + if !s.checkAndAddNode(n) { + t.Fatal("current-digest node was not admitted to the CL table") + } + + stats := s.enrManager.GetCLFilter().GetStats() + if stats.TotalChecks != 1 || stats.AcceptedCurrent != 1 { + t.Fatalf("stats = %+v after one admission, want 1 check / 1 accepted-current", stats) + } +} diff --git a/bootnode/enr.go b/bootnode/enr.go index f6a566e..ebd33d1 100644 --- a/bootnode/enr.go +++ b/bootnode/enr.go @@ -74,7 +74,8 @@ func StaticHead() (block, timestamp uint64) { return math.MaxUint64 - 1, uint64(time.Now().Unix()) } -// UpdateENR updates the local ENR with current eth and eth2 fields. +// UpdateENR updates the local ENR with current eth and eth2 fields, reporting +// whether the record actually changed. // // It is a no-op when the computed fields already match the published record, // so periodic callers do not churn the sequence number (peers re-fetch a @@ -84,98 +85,103 @@ func StaticHead() (block, timestamp uint64) { // - On startup // - After fork transitions // - When head changes significantly (for EL fork ID Next field) -func (m *ENRManager) UpdateENR(currentBlock, currentTime uint64) error { +func (m *ENRManager) UpdateENR(currentBlock, currentTime uint64) (bool, error) { record := m.localNode.Record() - // Clone the current ENR to preserve all fields - newRecord, err := record.Clone() - if err != nil { - return fmt.Errorf("failed to clone ENR: %w", err) - } - - changed := false - // A bootnode serves no TCP, so never advertise tcp/tcp6 — including any // inherited from an ENR persisted by an older, TCP-advertising version. - newRecord.Delete("tcp") - newRecord.Delete("tcp6") + changed := record.Has("tcp") || record.Has("tcp6") - if m.servesEL && m.config.HasEL() { - forkID := m.elFilter.GetCurrentForkID(currentBlock, currentTime) - // Set eth field as a list of fork IDs - ENR.Set() will handle RLP encoding - // The eth field format is [[Hash, Next]] - a list containing fork IDs - ethField := []struct { - Hash []byte - Next uint64 - }{ - { - Hash: forkID.Hash[:], - Next: forkID.Next, - }, - } - newRecord.Set("eth", ethField) + serveEL := m.servesEL && m.config.HasEL() + serveCL := m.servesCL && m.config.HasCL() + var forkID elconfig.ForkID + switch { + case serveEL: + forkID = m.elFilter.GetCurrentForkID(currentBlock, currentTime) if current, ok := record.Eth(); !ok || len(current) == 0 || current[0].ForkID != forkID.Hash || current[0].NextForkEpoch != forkID.Next { changed = true m.config.Logger.WithField("forkID", forkID.String()).Debug("updated ENR with eth field") } - } else if record.Has("eth") { + case record.Has("eth"): // Drop any stale eth field (e.g. inherited from a reused shared ENR). - newRecord.Delete("eth") changed = true } - if m.servesCL && m.config.HasCL() { - eth2Field := m.clFilter.ComputeEth2Field() - newRecord.Set("eth2", eth2Field) + var eth2Field []byte + switch { + case serveCL: + eth2Field = m.clFilter.ComputeEth2Field() var currentEth2 []byte if err := record.Get("eth2", ¤tEth2); err != nil || !bytes.Equal(currentEth2, eth2Field) { changed = true - // eth2Field is []byte, extract first 4 bytes as fork digest for logging var forkDigest [4]byte if len(eth2Field) >= 4 { copy(forkDigest[:], eth2Field[0:4]) } m.config.Logger.WithField("forkDigest", fmt.Sprintf("%#x", forkDigest)).Debug("updated ENR with eth2 field") } - } else if record.Has("eth2") { - newRecord.Delete("eth2") + case record.Has("eth2"): changed = true } - if record.Has("tcp") || record.Has("tcp6") { - changed = true + if !changed { + return false, nil } - if !changed { - return nil + newRecord, err := record.Clone() + if err != nil { + return false, fmt.Errorf("failed to clone ENR: %w", err) + } + + newRecord.Delete("tcp") + newRecord.Delete("tcp6") + + if serveEL { + // The eth field format is [[Hash, Next]] - a list containing fork IDs. + newRecord.Set("eth", []struct { + Hash []byte + Next uint64 + }{ + { + Hash: forkID.Hash[:], + Next: forkID.Next, + }, + }) + } else { + newRecord.Delete("eth") + } + + if serveCL { + newRecord.Set("eth2", eth2Field) + } else { + newRecord.Delete("eth2") } - // Increment sequence number newRecord.SetSeq(record.Seq() + 1) - // Re-sign the record if err := newRecord.Sign(m.key); err != nil { - return fmt.Errorf("failed to sign ENR: %w", err) + return false, fmt.Errorf("failed to sign ENR: %w", err) } - // Update local node's ENR if !m.localNode.UpdateENR(newRecord) { - return fmt.Errorf("failed to update local node ENR (sequence number may be stale)") + return false, fmt.Errorf("failed to update local node ENR (sequence number may be stale)") } m.config.Logger.WithField("seq", newRecord.Seq()).Info("updated local ENR with eth/eth2 fields") - return nil + return true, nil } -// FilterELNode checks if an EL node's fork ID is valid. +// ClassifyELNode reports whether a record is an execution node on a compatible +// fork, along with the fork ID it advertised. // -// Returns true if the node should be accepted, false otherwise. -func (m *ENRManager) FilterELNode(record *enr.Record) (bool, elconfig.ForkID) { - if !m.config.HasEL() { +// It is pure: no counter moves. Use it for per-packet layer classification, and +// AdmitELNode when the result decides admission. +func (m *ENRManager) ClassifyELNode(record *enr.Record) (bool, elconfig.ForkID) { + if !m.config.HasEL() || record == nil { return false, elconfig.ForkID{} } @@ -211,16 +217,40 @@ func (m *ENRManager) FilterELNode(record *enr.Record) (bool, elconfig.ForkID) { return m.elFilter.Filter(forkID), forkID } -// FilterCLNode checks if a CL node's fork digest is valid. +// AdmitELNode is ClassifyELNode plus stats. Call it from admission paths only. +// +// Records with no eth entry are consensus nodes, not wrong-fork execution nodes, +// and are not counted (see services.AdmissionRejectedLayer). +func (m *ENRManager) AdmitELNode(record *enr.Record) (bool, elconfig.ForkID) { + accepted, forkID := m.ClassifyELNode(record) + + if m.elFilter != nil && record != nil && record.Has("eth") { + m.elFilter.RecordAdmission(accepted, forkID) + } + + return accepted, forkID +} + +// ClassifyCLNode reports whether a record is a consensus node on an accepted +// fork digest. // -// Returns true if the node should be accepted, false otherwise. -func (m *ENRManager) FilterCLNode(record *enr.Record) bool { +// It is pure: no counter moves. Use it for per-packet layer classification, and +// AdmitCLNode when the result decides admission. +func (m *ENRManager) ClassifyCLNode(record *enr.Record) bool { + if !m.config.HasCL() { + return false + } + + return m.clFilter.Matches(record) +} + +// AdmitCLNode is ClassifyCLNode plus stats. Call it from admission paths only. +func (m *ENRManager) AdmitCLNode(record *enr.Record) bool { if !m.config.HasCL() { return false } - // Use existing fork digest filter - return m.clFilter.Filter(record) + return m.clFilter.Admit(record) } // GetELFilter returns the EL fork filter (may be nil). @@ -228,16 +258,6 @@ func (m *ENRManager) GetELFilter() *elconfig.ForkFilter { return m.elFilter } -// RecordELAdmission records an EL admission decision on the filter stats. -// Records without an eth entry are consensus nodes, not wrong-fork execution -// nodes, and are not counted (see services.AdmissionRejectedLayer). -func (m *ENRManager) RecordELAdmission(record *enr.Record, accepted bool, forkID elconfig.ForkID) { - if m.elFilter == nil || record == nil || !record.Has("eth") { - return - } - m.elFilter.RecordAdmission(accepted, forkID) -} - // GetCLFilter returns the CL fork digest filter (may be nil). func (m *ENRManager) GetCLFilter() *clconfig.ForkDigestFilter { return m.clFilter diff --git a/bootnode/service.go b/bootnode/service.go index 3c84890..1382162 100644 --- a/bootnode/service.go +++ b/bootnode/service.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net" + "slices" "sync" "time" @@ -163,10 +164,14 @@ func New(cfg *Config) (*Service, error) { id.enrManager = NewENRManager(cfg, id.key, localNode, id.servesEL, id.servesCL) headBlock, headTime := StaticHead() - if uerr := id.enrManager.UpdateENR(headBlock, headTime); uerr != nil { + changed, uerr := id.enrManager.UpdateENR(headBlock, headTime) + switch { + case uerr != nil: cfg.Logger.WithError(uerr).Warn("failed to update ENR with eth/eth2 fields") - } else if serr := s.storeENR(id.storeKey, localNode.Record()); serr != nil { - cfg.Logger.WithError(serr).Warn("failed to store updated ENR") + case changed: + if serr := s.storeENR(id.storeKey, localNode.Record()); serr != nil { + cfg.Logger.WithError(serr).Warn("failed to store updated ENR") + } } } @@ -603,25 +608,35 @@ func (s *Service) refreshForkENR() { clFilter.Update() } - beforeSeq := id.localNode.Record().Seq() - if err := id.enrManager.UpdateENR(headBlock, headTime); err != nil { + changed, err := id.enrManager.UpdateENR(headBlock, headTime) + if err != nil { s.config.Logger.WithError(err).Error("failed to refresh fork fields in ENR") continue } - if id.localNode.Record().Seq() == beforeSeq { + if !changed { continue } - if err := s.storeENR(id.storeKey, id.localNode.Record()); err != nil { - s.config.Logger.WithError(err).Warn("failed to store refreshed ENR") - } - if id.servesEL && s.discv4Service != nil { - s.discv4Service.SetLocalENR(id.localNode.Record()) - } + s.publishENR(id) s.config.Logger.WithField("seq", id.localNode.Record().Seq()).Info("fork transition: re-published ENR fork fields") } } +// publishENR persists an identity's current record and pushes it to every +// service that caches a copy. The discv4 handler answers ENRRESPONSE from its +// own copy, so skipping it would keep serving a stale record. Callers must +// hold s.mu. +func (s *Service) publishENR(id *identity) { + record := id.localNode.Record() + + if err := s.storeENR(id.storeKey, record); err != nil { + s.config.Logger.WithError(err).Warn("failed to store updated ENR") + } + if id.servesEL && s.discv4Service != nil { + s.discv4Service.SetLocalENR(record) + } +} + // performTableMaintenance performs routing table maintenance. func (s *Service) performTableMaintenance() { if s.elTable != nil { @@ -843,8 +858,7 @@ func (s *Service) connectELBootnodeENR(record *enr.Record) { // Filter by fork ID before adding. Serve-all must not drop a configured seed: // rejecting the only seed leaves the table empty, so discovery never starts. if !s.config.ServeAll && s.enrManager != nil { - isEL, forkID := s.enrManager.FilterELNode(record) - s.enrManager.RecordELAdmission(record, isEL, forkID) + isEL, forkID := s.enrManager.AdmitELNode(record) if !isEL { s.config.Logger.WithFields(logrus.Fields{ "nodeID": fmt.Sprintf("%x", v5.ID().Bytes()[:8]), @@ -878,11 +892,9 @@ func (s *Service) connectELBootnodeEnode(enodeURL *enode.Enode) { // Never dial ourselves: our own enode in the bootnode list would otherwise // race our handshake challenges against our own identity. - for _, local := range s.localIDs() { - if local == nodeID { - s.config.Logger.WithField("enode", enodeURL).Debug("skipping bootnode: it is our own identity") - return - } + if slices.Contains(s.localIDs(), [32]byte(nodeID)) { + s.config.Logger.WithField("enode", enodeURL).Debug("skipping bootnode: it is our own identity") + return } // Request ENR from the node @@ -898,8 +910,7 @@ func (s *Service) connectELBootnodeEnode(enodeURL *enode.Enode) { // Filter by fork ID before adding if !s.config.ServeAll && s.enrManager != nil { - isEL, forkID := s.enrManager.FilterELNode(enrRecord) - s.enrManager.RecordELAdmission(enrRecord, isEL, forkID) + isEL, forkID := s.enrManager.AdmitELNode(enrRecord) if !isEL { s.config.Logger.WithFields(logrus.Fields{ "nodeID": fmt.Sprintf("%x", nodeID[:8]), @@ -940,7 +951,7 @@ func (s *Service) connectCLBootnodes() { nodeID := v5.ID() // Filter by fork digest before adding - if !s.config.ServeAll && s.enrManager != nil && !s.enrManager.FilterCLNode(record) { + if !s.config.ServeAll && s.enrManager != nil && !s.enrManager.AdmitCLNode(record) { s.config.Logger.WithField("nodeID", fmt.Sprintf("%x", nodeID[:8])).Warn("CL bootnode ENR has invalid fork digest, skipping") continue } @@ -952,14 +963,14 @@ func (s *Service) connectCLBootnodes() { } // addBootnodeToTable admits a configured bootnode to a routing table and -// persists it, reporting whether it was admitted. -func (s *Service) addBootnodeToTable(table *nodes.FlatTable, nodeDB *nodes.NodeDB, n *nodes.Node, logger logrus.FieldLogger) bool { +// persists it. +func (s *Service) addBootnodeToTable(table *nodes.FlatTable, nodeDB *nodes.NodeDB, n *nodes.Node, logger logrus.FieldLogger) { if table == nil { - return false + return } if !table.Add(n) { logger.Debug("bootnode not admitted to table, not persisting") - return false + return } logger.Info("added bootnode to table") @@ -969,8 +980,6 @@ func (s *Service) addBootnodeToTable(table *nodes.FlatTable, nodeDB *nodes.NodeD logger.WithError(err).Debug("failed to queue bootnode for database update") } } - - return true } // loadStoredENR loads the stored ENR from database. @@ -1016,9 +1025,9 @@ func (s *Service) onNodeSeen(n *v5node.Node, timestamp time.Time) { isEL = s.elTable != nil isCL = s.clTable != nil } else { - isEL, _ = s.enrManager.FilterELNode(n.Record()) + isEL, _ = s.enrManager.ClassifyELNode(n.Record()) if !isEL { - isCL = s.enrManager.FilterCLNode(n.Record()) + isCL = s.enrManager.ClassifyCLNode(n.Record()) } } @@ -1054,8 +1063,8 @@ func (s *Service) onFindNodeV5(id *identity, msg *v5protocol.FindNode, sourceNod // Serve-all skips this: every requester gets nodes from every served layer. if !s.config.ServeAll && id.servesEL && id.servesCL && sourceNode != nil && s.enrManager != nil { sourceRecord := sourceNode.Record() - serveEL, _ = s.enrManager.FilterELNode(sourceRecord) - serveCL = s.enrManager.FilterCLNode(sourceRecord) + serveEL, _ = s.enrManager.ClassifyELNode(sourceRecord) + serveCL = s.enrManager.ClassifyCLNode(sourceRecord) } if serveEL && s.elTable != nil { @@ -1065,12 +1074,6 @@ func (s *Service) onFindNodeV5(id *identity, msg *v5protocol.FindNode, sourceNod allNodes = append(allNodes, s.clTable.GetNodesByDistance(localID, msg.Distances, 8)...) } - // A node can sit in both tables (any dual-stack peer, and every peer under - // serve-all), so serving both layers would return it twice. - if serveEL && serveCL { - allNodes = dedupeByID(allNodes) - } - // Filter nodes based on protocol support (only return v5-capable nodes) // and apply LAN-aware filtering filteredNodes := s.filterNodesForRequester(allNodes, requester, true) @@ -1207,24 +1210,19 @@ func (s *Service) requestENRV4(n *v4node.Node) { // table. func (s *Service) admitELLookupNode(n *nodes.Node) services.AdmissionResult { if !s.config.ServeAll && n.Record() != nil && s.enrManager != nil { - isEL, forkID := s.enrManager.FilterELNode(n.Record()) - s.enrManager.RecordELAdmission(n.Record(), isEL, forkID) + isEL, forkID := s.enrManager.AdmitELNode(n.Record()) if !isEL { // A record with no eth entry is a consensus node, not an // execution node on the wrong fork. if !n.Record().Has("eth") { - if err := s.config.Database.StoreBadNode(n.IDBytes(), db.LayerEL, "not_el"); err != nil { - s.config.Logger.WithError(err).Debug("failed to store bad node") - } + s.markBadNode(n, db.LayerEL, "not_el") return services.AdmissionRejectedLayer } s.config.Logger.WithFields(logrus.Fields{ "peerID": n.PeerID(), "eth": forkID.String(), }).Debug("EL lookup admission rejected: incompatible fork id") - if err := s.config.Database.StoreBadNode(n.IDBytes(), db.LayerEL, "invalid_fork_id"); err != nil { - s.config.Logger.WithError(err).Debug("failed to store bad node") - } + s.markBadNode(n, db.LayerEL, "invalid_fork_id") return services.AdmissionRejectedFilter } } @@ -1233,39 +1231,41 @@ func (s *Service) admitELLookupNode(n *nodes.Node) services.AdmissionResult { s.probeV5Support(n) } - if !s.elTable.Add(n) { - return services.AdmissionRejectedPool - } - if err := s.config.Database.RemoveBadNode(n.IDBytes(), db.LayerEL); err != nil { - s.config.Logger.WithError(err).Debug("failed to remove from bad nodes") - } - return services.AdmissionAccepted + return s.admitToTable(n, s.elTable, db.LayerEL) } // admitCLLookupNode decides admission of a lookup-discovered node to the CL // table. func (s *Service) admitCLLookupNode(n *nodes.Node) services.AdmissionResult { if !s.config.ServeAll && n.Record() != nil && s.enrManager != nil { - if !s.enrManager.FilterCLNode(n.Record()) { + if !s.enrManager.AdmitCLNode(n.Record()) { // No eth2 entry means an execution node, not a consensus // node on the wrong digest; keep the two distinguishable. if !n.Record().Has("eth2") { - if err := s.config.Database.StoreBadNode(n.IDBytes(), db.LayerCL, "not_cl"); err != nil { - s.config.Logger.WithError(err).Debug("failed to store bad node") - } + s.markBadNode(n, db.LayerCL, "not_cl") return services.AdmissionRejectedLayer } - if err := s.config.Database.StoreBadNode(n.IDBytes(), db.LayerCL, "invalid_fork_digest"); err != nil { - s.config.Logger.WithError(err).Debug("failed to store bad node") - } + s.markBadNode(n, db.LayerCL, "invalid_fork_digest") return services.AdmissionRejectedFilter } } - if !s.clTable.Add(n) { + return s.admitToTable(n, s.clTable, db.LayerCL) +} + +// markBadNode records a rejected node so it is not retried on restart. +func (s *Service) markBadNode(n *nodes.Node, layer db.NodeLayer, reason string) { + if err := s.config.Database.StoreBadNode(n.IDBytes(), layer, reason); err != nil { + s.config.Logger.WithError(err).Debug("failed to store bad node") + } +} + +// admitToTable pools an accepted node and clears any prior bad-node record. +func (s *Service) admitToTable(n *nodes.Node, table *nodes.FlatTable, layer db.NodeLayer) services.AdmissionResult { + if !table.Add(n) { return services.AdmissionRejectedPool } - if err := s.config.Database.RemoveBadNode(n.IDBytes(), db.LayerCL); err != nil { + if err := s.config.Database.RemoveBadNode(n.IDBytes(), layer); err != nil { s.config.Logger.WithError(err).Debug("failed to remove from bad nodes") } return services.AdmissionAccepted @@ -1335,8 +1335,7 @@ func (s *Service) checkAndAddNodeV4(n *v4node.Node) bool { // Filter the node using ENR manager (EL-only for discv4) if !s.config.ServeAll && s.enrManager != nil { - filter, forkID := s.enrManager.FilterELNode(n.ENR()) - s.enrManager.RecordELAdmission(n.ENR(), filter, forkID) + filter, forkID := s.enrManager.AdmitELNode(n.ENR()) if !filter { s.config.Logger.WithFields(logrus.Fields{ "nodeID": fmt.Sprintf("%x", n.IDBytes()[:8]), @@ -1377,10 +1376,8 @@ func (s *Service) checkAndAddNode(n *v5node.Node) bool { isEL = s.elTable != nil isCL = s.clTable != nil } else { - var elForkID elconfig.ForkID - isEL, elForkID = s.enrManager.FilterELNode(n.Record()) - isCL = s.enrManager.FilterCLNode(n.Record()) - s.enrManager.RecordELAdmission(n.Record(), isEL, elForkID) + isEL, _ = s.enrManager.AdmitELNode(n.Record()) + isCL = s.enrManager.AdmitCLNode(n.Record()) } // Add to appropriate table(s) @@ -1401,23 +1398,10 @@ func (s *Service) checkAndAddNode(n *v5node.Node) bool { return added } -// dedupeByID drops repeat node IDs, keeping the first occurrence. -func dedupeByID(nodeList []*nodes.Node) []*nodes.Node { - seen := make(map[[32]byte]struct{}, len(nodeList)) - out := nodeList[:0] - for _, n := range nodeList { - id := n.ID() - if _, dup := seen[id]; dup { - continue - } - seen[id] = struct{}{} - out = append(out, n) - } - - return out -} - -// filterNodesForRequester applies LAN-aware and protocol filtering. +// filterNodesForRequester applies LAN-aware and protocol filtering. It is the +// single funnel for both protocols' FINDNODE responses, so it also enforces +// that a response never repeats a node ID — a node can sit in both tables (any +// dual-stack peer, and every peer under serve-all). func (s *Service) filterNodesForRequester(nodeList []*nodes.Node, requester *net.UDPAddr, needsV5 bool) []*nodes.Node { requesterIsLAN := v5node.IsLANAddress(requester.IP) @@ -1445,6 +1429,11 @@ func (s *Service) filterNodesForRequester(nodeList []*nodes.Node, requester *net continue } + id := n.ID() + if slices.ContainsFunc(filtered, func(kept *nodes.Node) bool { return kept.ID() == id }) { + continue + } + filtered = append(filtered, n) } @@ -1613,8 +1602,9 @@ func (s *Service) onPongReceived(remoteID []byte, sourceIP net.IP, reportedIP ne port = s.primaryIdentity().enrPort } - reporterIDStr := fmt.Sprintf("%x", remoteID[:8]) - s.ipDiscovery.ReportIP(reportedIP, port, reporterIDStr, sourceIP) + // The full ID, not a prefix: this keys the distinct-reporter threshold, so a + // truncated key would let two peers count as one. + s.ipDiscovery.ReportIP(reportedIP, port, fmt.Sprintf("%x", remoteID), sourceIP) } // updateENRWithDiscoveredIP updates every identity's ENR with the discovered IP. @@ -1660,13 +1650,6 @@ func (s *Service) updateENRWithDiscoveredIP(ip net.IP, port uint16, isIPv6 bool) "isIPv6": isIPv6, }).Info("IP discovery: consensus reached, updated ENR") - if err := s.storeENR(id.storeKey, id.localNode.Record()); err != nil { - s.config.Logger.WithError(err).Warn("failed to store updated ENR") - } - - // Keep the discv4 service's ENR in sync (EL identity only). - if id.servesEL && s.discv4Service != nil { - s.discv4Service.SetLocalENR(id.localNode.Record()) - } + s.publishENR(id) } } diff --git a/bootnode/service_test.go b/bootnode/service_test.go index 0a47731..ead5e8d 100644 --- a/bootnode/service_test.go +++ b/bootnode/service_test.go @@ -54,7 +54,7 @@ func TestUpdateENR_ELOnlyDropsInheritedEth2(t *testing.T) { t.Fatalf("createLocalNode: %v", err) } - if err := NewENRManager(cfg, key, ln, true, false).UpdateENR(0, 0); err != nil { + if _, err := NewENRManager(cfg, key, ln, true, false).UpdateENR(0, 0); err != nil { t.Fatalf("UpdateENR: %v", err) } @@ -79,7 +79,7 @@ func TestUpdateENR_DropsUnservedFields(t *testing.T) { t.Fatalf("createLocalNode: %v", err) } - if err := NewENRManager(cfg, key, ln, false, false).UpdateENR(0, 0); err != nil { + if _, err := NewENRManager(cfg, key, ln, false, false).UpdateENR(0, 0); err != nil { t.Fatalf("UpdateENR: %v", err) } @@ -95,7 +95,7 @@ func TestUpdateENR_DropsUnservedFields(t *testing.T) { // A record without an eth entry is a consensus node, not a wrong-fork // execution node, so it must not move the EL admission counters. -func TestRecordELAdmission_SkipsRecordsWithoutEth(t *testing.T) { +func TestAdmitELNode_SkipsRecordsWithoutEth(t *testing.T) { cfg := &Config{Logger: quietLogger(), ELConfig: &elconfig.ChainConfig{}, ELGenesisHash: [32]byte{1, 2, 3}, ELGenesisTime: 1000} key := mustKey(t) ln, err := createLocalNode(cfg, key, net.ParseIP("1.2.3.4"), nil, 9000, nil) @@ -105,17 +105,23 @@ func TestRecordELAdmission_SkipsRecordsWithoutEth(t *testing.T) { m := NewENRManager(cfg, key, ln, true, false) clOnly := storedENRWith(t, key, map[string][]byte{"eth2": {0xaa, 0xbb, 0xcc, 0xdd}}) - m.RecordELAdmission(clOnly, false, elconfig.ForkID{}) + m.AdmitELNode(clOnly) if got := m.GetELFilter().GetStats().TotalChecks; got != 0 { t.Fatalf("TotalChecks = %d after CL-only record, want 0", got) } elRec := storedENRWith(t, key, map[string][]byte{"eth": {0x01, 0x02, 0x03, 0x04}}) - m.RecordELAdmission(elRec, false, elconfig.ForkID{}) + m.AdmitELNode(elRec) stats := m.GetELFilter().GetStats() if stats.TotalChecks != 1 || stats.Rejected != 1 { t.Fatalf("stats = %+v after eth record, want 1 check / 1 rejection", stats) } + + // The pure path must stay silent on the same records. + m.ClassifyELNode(elRec) + if got := m.GetELFilter().GetStats().TotalChecks; got != 1 { + t.Fatalf("TotalChecks = %d after ClassifyELNode, want 1 (unchanged)", got) + } } // newTestService builds a minimal Service with the given identities and an @@ -578,7 +584,7 @@ func TestUpdateENR_PublishesCurrentEraForkID(t *testing.T) { mgr := NewENRManager(cfg, key, ln, true, false) headBlock, headTime := StaticHead() - if err := mgr.UpdateENR(headBlock, headTime); err != nil { + if _, err := mgr.UpdateENR(headBlock, headTime); err != nil { t.Fatalf("UpdateENR: %v", err) } @@ -612,13 +618,13 @@ func TestUpdateENR_NoSeqBumpWhenUnchanged(t *testing.T) { mgr := NewENRManager(cfg, key, ln, true, false) headBlock, headTime := StaticHead() - if err := mgr.UpdateENR(headBlock, headTime); err != nil { + if _, err := mgr.UpdateENR(headBlock, headTime); err != nil { t.Fatalf("first UpdateENR: %v", err) } seq := ln.Record().Seq() for i := 0; i < 3; i++ { - if err := mgr.UpdateENR(StaticHead()); err != nil { + if _, err := mgr.UpdateENR(StaticHead()); err != nil { t.Fatalf("repeat UpdateENR: %v", err) } } @@ -725,7 +731,7 @@ func TestServeAll_LeavesAdmissionCountersUntouched(t *testing.T) { } // A node present in both tables must be served once, not once per table. -func TestDedupeByID(t *testing.T) { +func TestFilterNodesForRequesterDedupes(t *testing.T) { s := newServeAllTestService(t, true) rec := storedENRWith(t, mustKey(t), nil) @@ -738,12 +744,13 @@ func TestDedupeByID(t *testing.T) { b := nodes.NewFromV5(v5, s.elNodeDB) other := nodes.NewFromV5(mustV5Node(t), s.elNodeDB) - got := dedupeByID([]*nodes.Node{a, b, other, a}) + requester := &net.UDPAddr{IP: net.ParseIP("8.8.8.8"), Port: 30303} + got := s.filterNodesForRequester([]*nodes.Node{a, b, other, a}, requester, true) if len(got) != 2 { - t.Fatalf("dedupeByID returned %d nodes, want 2", len(got)) + t.Fatalf("filterNodesForRequester returned %d nodes, want 2", len(got)) } if got[0].ID() != a.ID() || got[1].ID() != other.ID() { - t.Errorf("dedupeByID did not keep first occurrences in order") + t.Errorf("filterNodesForRequester did not keep first occurrences in order") } } diff --git a/discv4/node/node.go b/discv4/node/node.go index 4761b9b..1262115 100644 --- a/discv4/node/node.go +++ b/discv4/node/node.go @@ -55,6 +55,14 @@ type Node struct { bondExpiration time.Time consecutiveTimeout uint32 // Bond-specific consecutive timeout counter + // bondedIPs maps a proven remote IP to when its bond expires. Keyed per IP + // because a bond proves reachability at one address only: addr is rewritten + // from whatever source last sent us a packet, so serving requests on the + // strength of a bond earned elsewhere lets a spoofed source reflect our + // replies at a third party. Ports are excluded so a NAT remap does not + // silently drop a peer mid-bond. + bondedIPs map[string]time.Time + // Statistics (shared with generic node wrapper) stats *stats.SharedStats @@ -270,6 +278,21 @@ func (n *Node) IsBonded() bool { return true } +// IsBondedFrom reports whether this node proved reachability at addr's IP and +// that proof is still valid. Inbound request handlers must use this rather than +// IsBonded, so a bond earned at one address cannot serve replies to another. +func (n *Node) IsBondedFrom(addr *net.UDPAddr) bool { + if addr == nil || addr.IP == nil { + return false + } + + n.bondMu.RLock() + defer n.bondMu.RUnlock() + + expiry, ok := n.bondedIPs[addr.IP.String()] + return ok && time.Now().Before(expiry) +} + // MarkPingSent records that we sent a PING to this node. func (n *Node) MarkPingSent() { now := time.Now() @@ -305,8 +328,11 @@ func (n *Node) MarkPongSent() { // MarkPongReceived records that we received a PONG from this node. // -// This establishes or renews the bond. -func (n *Node) MarkPongReceived(bondDuration time.Duration) { +// This establishes or renews the bond. provenAddr is the address the answered +// PING was sent to, not the PONG's source: the source is attacker-chosen on a +// spoofed packet, so binding the bond to it would prove nothing. Pass nil only +// where no endpoint was proven. +func (n *Node) MarkPongReceived(bondDuration time.Duration, provenAddr *net.UDPAddr) { now := time.Now() n.bondMu.Lock() @@ -314,6 +340,17 @@ func (n *Node) MarkPongReceived(bondDuration time.Duration) { n.bondStatus = BondStatusBonded n.bondExpiration = now.Add(bondDuration) n.consecutiveTimeout = 0 + if provenAddr != nil && provenAddr.IP != nil { + if n.bondedIPs == nil { + n.bondedIPs = make(map[string]time.Time) + } + for ip, expiry := range n.bondedIPs { + if now.After(expiry) { + delete(n.bondedIPs, ip) + } + } + n.bondedIPs[provenAddr.IP.String()] = now.Add(bondDuration) + } n.bondMu.Unlock() n.statsRef().ResetFailureCount() diff --git a/discv4/protocol/endpoint_proof_test.go b/discv4/protocol/endpoint_proof_test.go new file mode 100644 index 0000000..88918a2 --- /dev/null +++ b/discv4/protocol/endpoint_proof_test.go @@ -0,0 +1,248 @@ +package protocol + +import ( + "context" + "net" + "sync" + "testing" + "time" + + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethpandaops/bootnodoor/discv4/node" +) + +// recordingTransport captures the destinations we send to, so a test can assert +// that no reply was reflected at a spoofed address. +type recordingTransport struct { + mu sync.Mutex + sent []string +} + +func (r *recordingTransport) SendTo(_ []byte, to *net.UDPAddr) error { + r.mu.Lock() + defer r.mu.Unlock() + r.sent = append(r.sent, to.String()) + return nil +} + +func (r *recordingTransport) Send(_ []byte, to *net.UDPAddr, _ *net.UDPAddr) error { + return r.SendTo(nil, to) +} + +func (r *recordingTransport) sentTo(addr *net.UDPAddr) bool { + r.mu.Lock() + defer r.mu.Unlock() + for _, s := range r.sent { + if s == addr.String() { + return true + } + } + return false +} + +func proofHandler(t *testing.T) (*Handler, *recordingTransport, context.CancelFunc) { + t.Helper() + key, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("generate key: %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + tr := &recordingTransport{} + return NewHandler(ctx, HandlerConfig{PrivateKey: key, LocalAddr: testAddr()}, tr), tr, cancel +} + +// bondAt drives a full PING/PONG exchange so the node ends up bonded at addr, +// the way production does: register the PING we sent, then answer it. +func bondAt(t *testing.T, h *Handler, n *node.Node, addr *net.UDPAddr) { + t.Helper() + + n.SetAddr(addr) + hash := []byte("ping-hash-" + addr.String()) + if _, err := h.addPendingRequest(hash, n, PingPacket); err != nil { + t.Fatalf("addPendingRequest: %v", err) + } + pong := &Pong{ReplyTok: hash, Expiration: MakeExpiration(20 * time.Second)} + if err := h.handlePong(n, addr, pong); err != nil { + t.Fatalf("handlePong: %v", err) + } + if !n.IsBondedFrom(addr) { + t.Fatalf("node not bonded at %s after a matched PONG", addr) + } +} + +// A bond proves reachability at one address only. Serving FINDNODE from any +// other source lets an attacker who bonded legitimately spoof a victim's source +// and have us reflect the much larger NEIGHBORS at that victim. +func TestFindnodeFromUnbondedAddressRejected(t *testing.T) { + h, tr, cancel := proofHandler(t) + defer cancel() + + n, _ := makeKeyedNode(t, 30303) + attacker := &net.UDPAddr{IP: net.IPv4(1, 2, 3, 4), Port: 30303} + bondAt(t, h, n, attacker) + + victim := &net.UDPAddr{IP: net.IPv4(203, 0, 113, 9), Port: 30303} + err := h.handleFindnode(n, victim, testAddr(), &Findnode{Expiration: MakeExpiration(20 * time.Second)}) + if err == nil { + t.Fatal("FINDNODE from an unbonded source address was served") + } + if tr.sentTo(victim) { + t.Fatal("reflected a reply at the spoofed victim address") + } + if h.GetStats().UnbondedFindnode == 0 { + t.Error("unbondedFindnode counter did not move") + } +} + +// The legitimate case must still work, or the gate has simply broken discovery. +func TestFindnodeFromBondedAddressServed(t *testing.T) { + h, _, cancel := proofHandler(t) + defer cancel() + + n, _ := makeKeyedNode(t, 30303) + addr := &net.UDPAddr{IP: net.IPv4(1, 2, 3, 4), Port: 30303} + bondAt(t, h, n, addr) + + if err := h.handleFindnode(n, addr, testAddr(), &Findnode{Expiration: MakeExpiration(20 * time.Second)}); err != nil { + t.Fatalf("FINDNODE from the bonded address was refused: %v", err) + } +} + +// One node ID can legitimately bond over both address families, so a per-IP bond +// must not let the second exchange invalidate the first. +func TestDualStackPeerKeepsBothBonds(t *testing.T) { + h, _, cancel := proofHandler(t) + defer cancel() + + n, _ := makeKeyedNode(t, 30303) + v4 := &net.UDPAddr{IP: net.IPv4(1, 2, 3, 4), Port: 30303} + v6 := &net.UDPAddr{IP: net.ParseIP("2001:db8::1"), Port: 30303} + + bondAt(t, h, n, v4) + bondAt(t, h, n, v6) + + if !n.IsBondedFrom(v4) { + t.Error("IPv4 bond was lost when the IPv6 bond was established") + } + if !n.IsBondedFrom(v6) { + t.Error("IPv6 bond was not established") + } +} + +// Receiving a PING proves nothing about the source: we pong whatever address the +// packet claimed, so bonding here would bond a spoofed victim. +func TestInboundPingDoesNotBond(t *testing.T) { + h, _, cancel := proofHandler(t) + defer cancel() + + n, _ := makeKeyedNode(t, 30303) + addr := &net.UDPAddr{IP: net.IPv4(5, 6, 7, 8), Port: 30303} + n.SetAddr(addr) + + ping := &Ping{Version: 4, Expiration: MakeExpiration(20 * time.Second)} + if err := h.handlePing(n, addr, testAddr(), ping, []byte("hash")); err != nil { + t.Fatalf("handlePing: %v", err) + } + + if n.IsBondedFrom(addr) { + t.Fatal("an inbound PING alone established a bond") + } + if n.LastSeen().IsZero() { + t.Error("MarkPingReceived did not take effect") + } +} + +// A PONG must not be matched by a token belonging to a different request type: +// peers know the hashes of the packets we send them. +func TestPongMatchingRejectsNonPingRequest(t *testing.T) { + h, _, cancel := proofHandler(t) + defer cancel() + + n, _ := makeKeyedNode(t, 30303) + addr := n.Addr() + + called := 0 + h.config.OnPongReceived = func(*node.Node, net.IP, uint16) { called++ } + + hash := []byte("enr-request-hash") + if _, err := h.addPendingRequest(hash, n, ENRRequestPacket); err != nil { + t.Fatalf("addPendingRequest: %v", err) + } + + pong := &Pong{ + ReplyTok: hash, + To: NewEndpoint(&net.UDPAddr{IP: net.IPv4(9, 9, 9, 9), Port: 30303}, 0), + Expiration: MakeExpiration(20 * time.Second), + } + if err := h.handlePong(n, addr, pong); err != nil { + t.Fatalf("handlePong: %v", err) + } + + if n.IsBondedFrom(addr) { + t.Error("a PONG matching an ENRREQUEST token established a bond") + } + if called != 0 { + t.Errorf("OnPongReceived fired %d times for a non-PING match", called) + } +} + +// The match is consumed once, so a replayed PONG cannot cast repeated +// external-IP votes off a single PING. +func TestReplayedPongAppliesSideEffectsOnce(t *testing.T) { + h, _, cancel := proofHandler(t) + defer cancel() + + n, _ := makeKeyedNode(t, 30303) + addr := n.Addr() + + called := 0 + h.config.OnPongReceived = func(*node.Node, net.IP, uint16) { called++ } + + hash := []byte("ping-hash") + if _, err := h.addPendingRequest(hash, n, PingPacket); err != nil { + t.Fatalf("addPendingRequest: %v", err) + } + + pong := &Pong{ + ReplyTok: hash, + To: NewEndpoint(&net.UDPAddr{IP: net.IPv4(9, 9, 9, 9), Port: 30303}, 0), + Expiration: MakeExpiration(20 * time.Second), + } + for i := 0; i < 3; i++ { + if err := h.handlePong(n, addr, pong); err != nil { + t.Fatalf("handlePong %d: %v", i, err) + } + } + + if called != 1 { + t.Fatalf("OnPongReceived fired %d times for a replayed PONG, want 1", called) + } +} + +// A PONG whose source is not the address the PING went to proves only that +// somebody received that PING, which is what the spoofing attack relies on. +func TestPongFromWrongSourceRejected(t *testing.T) { + h, _, cancel := proofHandler(t) + defer cancel() + + n, _ := makeKeyedNode(t, 30303) + sentTo := n.Addr() + + hash := []byte("ping-hash") + if _, err := h.addPendingRequest(hash, n, PingPacket); err != nil { + t.Fatalf("addPendingRequest: %v", err) + } + + victim := &net.UDPAddr{IP: net.IPv4(203, 0, 113, 9), Port: 30303} + pong := &Pong{ReplyTok: hash, Expiration: MakeExpiration(20 * time.Second)} + if err := h.handlePong(n, victim, pong); err != nil { + t.Fatalf("handlePong: %v", err) + } + + if n.IsBondedFrom(victim) { + t.Fatal("a PONG spoofed from a victim address bonded that address") + } + if n.IsBondedFrom(sentTo) { + t.Fatal("a PONG from the wrong source bonded the PING destination") + } +} diff --git a/discv4/protocol/handler.go b/discv4/protocol/handler.go index 2cf19a0..cef3465 100644 --- a/discv4/protocol/handler.go +++ b/discv4/protocol/handler.go @@ -5,6 +5,7 @@ import ( "crypto/ecdsa" "fmt" "net" + "slices" "sync" "time" @@ -134,6 +135,12 @@ type PendingRequest struct { // ToNode is the destination node ToNode *node.Node + // DestIP is the IP the request was sent to, snapshotted at send time. + // ToNode.Addr() is unusable for verifying a response's origin because + // getOrCreateNode rewrites it from every inbound packet, including the + // spoofed one a response check is meant to catch. + DestIP net.IP + // PacketType is the type of request PacketType byte @@ -332,9 +339,11 @@ func (h *Handler) handlePing(fromNode *node.Node, from *net.UDPAddr, localAddr * return err } - // Mark node as bonded: they pinged us, we ponged them. - // This allows THEM to query US with FINDNODE immediately. - fromNode.MarkPongReceived(h.config.BondExpiration) + // Receiving a PING grants no bond: we ponged whatever address the packet + // claimed, which proves nothing if that source was spoofed. Bonding here + // would let an attacker bond a victim's address and then have us reflect + // NEIGHBORS at it. The bond is established by the reciprocal PING below, + // when its PONG comes back from the address we sent it to. // IMPORTANT: For bidirectional bonding (required by strict clients like reth for ENRRequest), // we also need to establish that WE can reach THEM, not just that they can reach us. @@ -375,19 +384,23 @@ func (h *Handler) handlePong(fromNode *node.Node, from *net.UDPAddr, pong *Pong) return ErrExpired } - // Mark pong received (establishes bond) - fromNode.MarkPongReceived(h.config.BondExpiration) + // Nothing below may run for a PONG we did not solicit from this address: it + // establishes a bond, casts a vote in the external-IP election that rewrites + // our published ENR, and can trigger outbound ENR traffic. + req := h.consumePendingPing(pong.ReplyTok, fromNode.ID(), from) + if req == nil { + return nil + } + + // Bind the bond to the address we proved, not the packet's source. + fromNode.MarkPongReceived(h.config.BondExpiration, &net.UDPAddr{IP: req.DestIP, Port: from.Port}) - // Call OnPongReceived callback with the IP and port reported in the PONG - // The To field in PONG contains our address as seen by the remote peer + // The To field in PONG contains our address as seen by the remote peer. if h.config.OnPongReceived != nil && pong.To.IP != nil && pong.To.UDP > 0 { h.config.OnPongReceived(fromNode, pong.To.IP, pong.To.UDP) } - // Match to pending requests - for _, req := range h.getPendingRequests(pong.ReplyTok, fromNode.ID()) { - h.deliverResponse(req, pong) - } + h.deliverResponse(req, pong) // Check if remote node has newer ENR if pong.ENRSeq > 0 && fromNode.ENR() != nil { @@ -414,8 +427,9 @@ func (h *Handler) handleFindnode(fromNode *node.Node, from *net.UDPAddr, localAd return ErrExpired } - // Check if node is bonded - if !fromNode.IsBonded() { + // Bonded at this source address specifically: a bond earned elsewhere would + // let a spoofed source have us reflect NEIGHBORS at a third party. + if !fromNode.IsBondedFrom(from) { h.incrementUnbondedFindnode() logrus.WithField("node_id", fmt.Sprintf("%x", fromNode.IDBytes()[:8])). Debug("Rejected FINDNODE from unbonded node") @@ -449,8 +463,6 @@ func (h *Handler) handleNeighbors(fromNode *node.Node, from *net.UDPAddr, neighb return ErrExpired } - h.incrementFindnodeResponsesRecv() - // Only accept NEIGHBORS in response to a FINDNODE we actually sent to this // node. Dropping unsolicited NEIGHBORS prevents a peer we never queried from // making us accumulate node records without bound. @@ -459,6 +471,10 @@ func (h *Handler) handleNeighbors(fromNode *node.Node, from *net.UDPAddr, neighb return nil } + // Counted after the gate: this reports responses to our queries, so counting + // unsolicited packets here would let any peer inflate it. + h.incrementFindnodeResponsesRecv() + // Accumulate the response, keyed by the matched request's hash so each // FINDNODE gets exactly one entry and a fresh request never collides with // a delivered one. Room is reserved before decoding, so records past the @@ -549,8 +565,8 @@ func (h *Handler) handleENRRequest(fromNode *node.Node, from *net.UDPAddr, local // This prevents amplification attacks and matches reth's behavior. // Only respond to ENRRequest if we've established a bidirectional bond: // - We sent them a PING - // - They sent us a PONG - if !fromNode.IsBonded() { + // - They sent us a PONG from this address + if !fromNode.IsBondedFrom(from) { logrus.WithFields(logrus.Fields{ "from": from.String(), "node_id": fmt.Sprintf("%x", fromNode.IDBytes()[:8]), @@ -943,9 +959,15 @@ func requestKey(hash []byte, id node.ID) string { // peer with one already in flight is rejected: NEIGHBORS carries no reply // token, so two in-flight FINDNODEs to one peer cannot be told apart. func (h *Handler) addPendingRequest(hash []byte, toNode *node.Node, packetType byte) (*PendingRequest, error) { + var destIP net.IP + if addr := toNode.Addr(); addr != nil && addr.IP != nil { + destIP = append(net.IP(nil), addr.IP...) + } + req := &PendingRequest{ RequestHash: hash, ToNode: toNode, + DestIP: destIP, PacketType: packetType, CreatedAt: time.Now(), Timeout: time.Now().Add(h.config.RequestTimeout), @@ -973,6 +995,47 @@ func (h *Handler) getPendingRequests(replyTok []byte, id node.ID) []*PendingRequ return append([]*PendingRequest(nil), h.requests[requestKey(replyTok, id)]...) } +// consumePendingPing removes and returns the pending PING this PONG answers, or +// nil if there is none. +// +// Three properties beyond "a token matched" are required before a PONG may +// mutate state, and all three are enforced here so no caller can forget one: +// +// - from must be the IP the PING was sent to. The token alone proves only that +// somebody received that PING; an attacker who receives it at their own +// address can replay it with a victim's source and bond the victim. +// - the request must be a PING. Peers know the hashes of packets we sent them, +// so an ENRREQUEST or FINDNODE hash would otherwise match as a reply token. +// - the entry is deleted here, under the same lock, so a replayed PONG finds +// nothing and the side effects run at most once per PING. +func (h *Handler) consumePendingPing(replyTok []byte, id node.ID, from *net.UDPAddr) *PendingRequest { + if from == nil || from.IP == nil { + return nil + } + + key := requestKey(replyTok, id) + + h.requestsMu.Lock() + defer h.requestsMu.Unlock() + + reqs := h.requests[key] + for i, req := range reqs { + if req.PacketType != PingPacket || req.DestIP == nil || !req.DestIP.Equal(from.IP) { + continue + } + + reqs = slices.Delete(reqs, i, i+1) + if len(reqs) == 0 { + delete(h.requests, key) + } else { + h.requests[key] = reqs + } + return req + } + + return nil +} + // findPendingFindnode returns the pending FINDNODE request awaiting a response // from the given node, or nil if none exists. func (h *Handler) findPendingFindnode(id node.ID) *PendingRequest { @@ -1004,11 +1067,8 @@ func (h *Handler) removePendingRequest(req *PendingRequest) { defer h.requestsMu.Unlock() reqs := h.requests[key] - for i, r := range reqs { - if r == req { - reqs = append(reqs[:i], reqs[i+1:]...) - break - } + if i := slices.Index(reqs, req); i >= 0 { + reqs = slices.Delete(reqs, i, i+1) } if len(reqs) == 0 { delete(h.requests, key) @@ -1055,12 +1115,7 @@ func (h *Handler) cleanup() { // Clean up expired requests h.requestsMu.Lock() for key, reqs := range h.requests { - kept := reqs[:0] - for _, req := range reqs { - if !now.After(req.Timeout) { - kept = append(kept, req) - } - } + kept := slices.DeleteFunc(reqs, func(req *PendingRequest) bool { return now.After(req.Timeout) }) if len(kept) == 0 { delete(h.requests, key) } else { @@ -1080,16 +1135,37 @@ func (h *Handler) cleanup() { // Evict stale, unbonded nodes so the map stays bounded. Bonded nodes are // kept until their bond expires, after which IsBonded reports false and they - // become eligible here. + // become eligible here. Scanning under the read lock keeps a full-map sweep + // from stalling every inbound packet in getOrCreateNode. + stale := h.staleNodes(now) + if len(stale) == 0 { + return + } + h.nodesMu.Lock() - for id, n := range h.nodes { - if !n.IsBonded() && now.Sub(n.LastSeen()) > h.config.NodeTTL { + for _, id := range stale { + // Re-check: a node may have been seen again since the scan. + if n, ok := h.nodes[id]; ok && !n.IsBonded() && now.Sub(n.LastSeen()) > h.config.NodeTTL { delete(h.nodes, id) } } h.nodesMu.Unlock() } +// staleNodes returns the IDs of unbonded nodes past their TTL. +func (h *Handler) staleNodes(now time.Time) []node.ID { + h.nodesMu.RLock() + defer h.nodesMu.RUnlock() + + var stale []node.ID + for id, n := range h.nodes { + if !n.IsBonded() && now.Sub(n.LastSeen()) > h.config.NodeTTL { + stale = append(stale, id) + } + } + return stale +} + // Statistics func (h *Handler) incrementPacketsReceived() { diff --git a/discv4/protocol/handler_test.go b/discv4/protocol/handler_test.go index d21528f..5a86262 100644 --- a/discv4/protocol/handler_test.go +++ b/discv4/protocol/handler_test.go @@ -60,7 +60,7 @@ func TestCleanupEvictsStaleUnbondedNodes(t *testing.T) { pubBonded, idBonded := makeNodeID(t) bonded := h.getOrCreateNode(idBonded, pubBonded, testAddr()) - bonded.MarkPongReceived(time.Hour) // establish a live bond + bonded.MarkPongReceived(time.Hour, testAddr()) // establish a live bond time.Sleep(40 * time.Millisecond) // age both past NodeTTL diff --git a/discv4/protocol/pending_neighbors_test.go b/discv4/protocol/pending_neighbors_test.go index e8853b1..5b5d91f 100644 --- a/discv4/protocol/pending_neighbors_test.go +++ b/discv4/protocol/pending_neighbors_test.go @@ -213,7 +213,7 @@ func TestFindnodeRemovesCompletedRequest(t *testing.T) { h := NewHandler(ctx, HandlerConfig{PrivateKey: key}, stubTransport{}) to := makeDiscv4Node(t) - to.MarkPongReceived(time.Hour) + to.MarkPongReceived(time.Hour, to.Addr()) target := EncodePubkey(&key.PublicKey) type result struct { diff --git a/discv5/protocol/handler.go b/discv5/protocol/handler.go index 4405f2a..4b38006 100644 --- a/discv5/protocol/handler.go +++ b/discv5/protocol/handler.go @@ -977,8 +977,12 @@ func (h *Handler) handlePong(msg *Pong, remoteID node.ID, from *net.UDPAddr, rem "nodeID": remoteID, }).Debug("handler: received PONG") - // Match with pending request - h.requests.MatchResponse(msg.RequestID, remoteID, msg) + // An unsolicited PONG must not reach the side effects below: OnPongReceived + // casts a vote in the external-IP election that rewrites our published ENR, + // and the ENR branch triggers outbound traffic. + if !h.requests.MatchResponse(msg.RequestID, remoteID, msg) { + return nil + } // Call OnPongReceived callback with the source IP and the IP/port reported in the PONG // The IP and Port fields in PONG contain our address as seen by the remote peer diff --git a/discv5/protocol/request.go b/discv5/protocol/request.go index 9bf787b..582ec90 100644 --- a/discv5/protocol/request.go +++ b/discv5/protocol/request.go @@ -129,6 +129,27 @@ func (rt *RequestTracker) AddRequest(requestID []byte, n *node.Node, msg Message return req.ResponseChan } +// respondsTo reports whether resp is the response type request expects. +// +// An unknown request type matches nothing: a new request/response pair must be +// registered here deliberately rather than defaulting to accepting any reply. +func respondsTo(request, resp Message) bool { + if request == nil || resp == nil { + return false + } + + switch request.Type() { + case PingMsg: + return resp.Type() == PongMsg + case FindNodeMsg: + return resp.Type() == NodesMsg + case TalkReqMsg: + return resp.Type() == TalkRespMsg + default: + return false + } +} + // MatchResponse matches a response to a pending request. // // Returns true if the request was matched and notified. @@ -147,6 +168,14 @@ func (rt *RequestTracker) MatchResponse(requestID []byte, nodeID node.ID, msg Me return false } + // The response must be the kind this request asked for. Request IDs are ours + // but the peer learns them, so without this a PONG can match a pending + // FINDNODE: it would both fire the PONG side effects and consume the entry + // below, silently stranding the lookup that was waiting on it. + if !respondsTo(req.Message, msg) { + return false + } + // Handle multi-packet NODES responses if nodesMsg, ok := msg.(*Nodes); ok && nodesMsg.Total > 1 { // Initialize accumulator on first packet diff --git a/discv5/protocol/request_match_test.go b/discv5/protocol/request_match_test.go new file mode 100644 index 0000000..71606a3 --- /dev/null +++ b/discv5/protocol/request_match_test.go @@ -0,0 +1,57 @@ +package protocol + +import ( + "testing" + "time" + + "github.com/ethpandaops/bootnodoor/discv5/node" +) + +// Request IDs are ours but the peer learns them, so a response must also be the +// kind the request asked for. Otherwise a PONG matches a pending FINDNODE: it +// fires the PONG side effects and consumes the entry, stranding the lookup. +func TestMatchResponseRejectsWrongResponseType(t *testing.T) { + rt := NewRequestTracker(time.Second) + + n, err := node.New(signedRecord(t, generateKey(t), 1, nil)) + if err != nil { + t.Fatalf("node.New: %v", err) + } + + requestID := []byte{0x01, 0x02, 0x03, 0x04} + ch := rt.AddRequest(requestID, n, &FindNode{RequestID: requestID, Distances: []uint{1}}) + + pong := &Pong{RequestID: requestID, IP: []byte{9, 9, 9, 9}, Port: 30303} + if rt.MatchResponse(requestID, n.ID(), pong) { + t.Fatal("a PONG matched a pending FINDNODE") + } + + select { + case resp := <-ch: + t.Fatalf("pending FINDNODE was resolved by a PONG: %+v", resp) + default: + } + + nodes := &Nodes{RequestID: requestID, Total: 1} + if !rt.MatchResponse(requestID, n.ID(), nodes) { + t.Fatal("the matching NODES response was rejected") + } +} + +// The PING/PONG pair must still match, or gating handlePong on this would drop +// every legitimate PONG. +func TestMatchResponseAcceptsPongForPing(t *testing.T) { + rt := NewRequestTracker(time.Second) + + n, err := node.New(signedRecord(t, generateKey(t), 1, nil)) + if err != nil { + t.Fatalf("node.New: %v", err) + } + + requestID := []byte{0x0a, 0x0b} + rt.AddRequest(requestID, n, &Ping{RequestID: requestID}) + + if !rt.MatchResponse(requestID, n.ID(), &Pong{RequestID: requestID}) { + t.Fatal("a PONG did not match its pending PING") + } +} diff --git a/services/ipdiscovery.go b/services/ipdiscovery.go index eefc8fd..c6ab92e 100644 --- a/services/ipdiscovery.go +++ b/services/ipdiscovery.go @@ -75,12 +75,14 @@ type IPDiscovery struct { // ipReport tracks reports for a specific IP:Port combination type ipReport struct { - ip net.IP - port uint16 - count int - firstSeen time.Time - lastSeen time.Time - reporterIDs []string // Track which peers reported this (for debugging) + ip net.IP + port uint16 + count int + firstSeen time.Time + lastSeen time.Time + // Counted by identity as well as by source IP: source addresses are + // spoofable, so one node ID must not satisfy the thresholds on its own. + reporterIDs map[string]int // Track distinct reporter node IDs -> count reporterIPs map[string]int // Track distinct reporter IPs -> count } @@ -199,7 +201,7 @@ func (ipd *IPDiscovery) ReportIP(ip net.IP, port uint16, reporterID string, repo ip: ip, port: port, firstSeen: now, - reporterIDs: make([]string, 0), + reporterIDs: make(map[string]int), reporterIPs: make(map[string]int), } reports[addrKey] = report @@ -208,7 +210,7 @@ func (ipd *IPDiscovery) ReportIP(ip net.IP, port uint16, reporterID string, repo // Update report report.count++ report.lastSeen = now - report.reporterIDs = append(report.reporterIDs, reporterID) + report.reporterIDs[reporterID]++ // Track reporter IP reporterIPStr := reporterIP.String() @@ -328,9 +330,11 @@ func (ipd *IPDiscovery) checkConsensusForFamilyLocked(isIPv6 bool) { if maxRecentAddr != "" && maxRecentAddr != currentAddrKey && maxRecentReport != nil { recentMajority := float64(maxRecentCount) / float64(totalRecentCount) distinctIPCount := len(maxRecentReport.reporterIPs) + distinctIDCount := len(maxRecentReport.reporterIDs) - // Enforce distinct IP threshold for address changes too - if recentMajority >= ipd.majorityThreshold && distinctIPCount >= ipd.minDistinctIPs { + // Enforce both distinct thresholds for address changes too + if recentMajority >= ipd.majorityThreshold && + distinctIPCount >= ipd.minDistinctIPs && distinctIDCount >= ipd.minDistinctIPs { // Address change detected! ipd.logger.WithFields(logrus.Fields{ "family": familyName, @@ -381,13 +385,15 @@ func (ipd *IPDiscovery) checkConsensusForFamilyLocked(isIPv6 bool) { // Check distinct IP count distinctIPCount := len(maxReport.reporterIPs) - if distinctIPCount < ipd.minDistinctIPs { + distinctIDCount := len(maxReport.reporterIDs) + if distinctIPCount < ipd.minDistinctIPs || distinctIDCount < ipd.minDistinctIPs { ipd.logger.WithFields(logrus.Fields{ "family": familyName, "addr": fmt.Sprintf("%s:%d", maxReport.ip.String(), maxReport.port), "distinctIPs": distinctIPCount, + "distinctIDs": distinctIDCount, "minDistinct": ipd.minDistinctIPs, - }).Debug("IP discovery: insufficient distinct reporter IPs") + }).Debug("IP discovery: insufficient distinct reporters") return } diff --git a/services/ipdiscovery_test.go b/services/ipdiscovery_test.go new file mode 100644 index 0000000..cda440f --- /dev/null +++ b/services/ipdiscovery_test.go @@ -0,0 +1,70 @@ +package services + +import ( + "fmt" + "net" + "testing" + "time" + + "github.com/sirupsen/logrus" +) + +func quietIPDiscovery(t *testing.T) (*IPDiscovery, <-chan string) { + t.Helper() + + logger := logrus.New() + logger.SetLevel(logrus.PanicLevel) + + reached := make(chan string, 4) + ipd := NewIPDiscovery(IPDiscoveryConfig{ + MinReports: 5, + MinDistinctIPs: 3, + Logger: logger, + OnConsensusReached: func(ip net.IP, port uint16, isIPv6 bool) { + reached <- fmt.Sprintf("%s:%d", ip.String(), port) + }, + }) + return ipd, reached +} + +// A single peer must not reach consensus on its own, however many source +// addresses it appears to report from: UDP source IPs are spoofable, so +// distinct-IP alone is not a measure of independent opinions. +func TestConsensusRequiresDistinctReporters(t *testing.T) { + ipd, reached := quietIPDiscovery(t) + + external := net.ParseIP("203.0.113.7") + for i := 0; i < 8; i++ { + reporterIP := net.IPv4(198, 51, 100, byte(1+i%4)) + ipd.ReportIP(external, 30303, "same-reporter-node-id-0000000000", reporterIP) + } + + // The callback is dispatched with `go`, so a non-blocking receive here would + // pass even when consensus fired. + select { + case addr := <-reached: + t.Fatalf("one reporter reached consensus on %s across spoofed source IPs", addr) + case <-time.After(500 * time.Millisecond): + } +} + +// The same report volume from genuinely distinct peers must still reach +// consensus, so the new gate does not simply disable IP discovery. +func TestConsensusReachedWithDistinctReporters(t *testing.T) { + ipd, reached := quietIPDiscovery(t) + + external := net.ParseIP("203.0.113.7") + for i := 0; i < 6; i++ { + reporterIP := net.IPv4(198, 51, 100, byte(1+i)) + ipd.ReportIP(external, 30303, fmt.Sprintf("reporter-node-id-%026d", i), reporterIP) + } + + select { + case addr := <-reached: + if addr != "203.0.113.7:30303" { + t.Fatalf("consensus on %s, want 203.0.113.7:30303", addr) + } + case <-time.After(2 * time.Second): + t.Fatal("distinct reporters did not reach consensus") + } +} From 6dd2f08c2bc60f5ad0c905a360b001e62f62a665 Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Wed, 29 Jul 2026 10:40:17 -0500 Subject: [PATCH 09/49] fix(discv4): bind response matching to the address actually queried Follow-up from review of c3d9e16. The endpoint proof was incomplete: the recorded destination could differ from where the packet went, and the proven address did not reach the code that consumes it. - Ping/Findnode/RequestENR read Node.Addr() separately for the To endpoint, the pending request and the send. Concurrent receive workers rewrite that address between those reads, so DestIP could name an endpoint the request never went to: legitimate PONGs rejected, or a bond granted for an address that never received a PING. Each sender now captures the address once and addPendingRequest takes it explicitly, so the recorded and actual destinations cannot diverge. - OnPongReceived now carries the proven address. The bootnode was re-reading from.Addr() for the IP-discovery source, which any later spoofed packet from the same identity rewrites, letting an unproven address count toward the distinct-source threshold. - ENRRESPONSE and NEIGHBORS apply the same destination check as PONG. ENRRESPONSE additionally requires an ENRREQUEST: it matched on packet hash alone, so a peer could answer with the hash of some other packet we sent it and resolve the wrong waiter. pendingFindnodeLocked stays address-agnostic; it enforces one in-flight FINDNODE per peer, which is not an endpoint question. --- bootnode/service.go | 5 +- discv4/protocol/endpoint_proof_test.go | 53 +++++++++++-- discv4/protocol/handler.go | 91 +++++++++++++++++------ discv4/protocol/pending_neighbors_test.go | 12 +-- discv4/protocol/pending_request_test.go | 20 ++--- discv4/protocol/response_delivery_test.go | 5 +- 6 files changed, 137 insertions(+), 49 deletions(-) diff --git a/bootnode/service.go b/bootnode/service.go index 1382162..55f741d 100644 --- a/bootnode/service.go +++ b/bootnode/service.go @@ -340,9 +340,8 @@ func (s *Service) initDiscv4(id *identity) error { discv4Config.OnNodeSeen = func(n *v4node.Node, timestamp time.Time) { s.onNodeSeenV4(n, timestamp) } - discv4Config.OnPongReceived = func(from *v4node.Node, ip net.IP, port uint16) { - sourceIP := from.Addr().IP - s.onPongReceived(from.IDBytes(), sourceIP, ip, port) + discv4Config.OnPongReceived = func(from *v4node.Node, provenAddr *net.UDPAddr, ip net.IP, port uint16) { + s.onPongReceived(from.IDBytes(), provenAddr.IP, ip, port) } // OnENRRequest: discv4 service handles this internally using LocalENR from config // No callback needed - it will automatically respond with the ENR diff --git a/discv4/protocol/endpoint_proof_test.go b/discv4/protocol/endpoint_proof_test.go index 88918a2..38ff8fb 100644 --- a/discv4/protocol/endpoint_proof_test.go +++ b/discv4/protocol/endpoint_proof_test.go @@ -58,7 +58,7 @@ func bondAt(t *testing.T, h *Handler, n *node.Node, addr *net.UDPAddr) { n.SetAddr(addr) hash := []byte("ping-hash-" + addr.String()) - if _, err := h.addPendingRequest(hash, n, PingPacket); err != nil { + if _, err := h.addPendingRequest(hash, n, PingPacket, n.Addr()); err != nil { t.Fatalf("addPendingRequest: %v", err) } pong := &Pong{ReplyTok: hash, Expiration: MakeExpiration(20 * time.Second)} @@ -162,10 +162,10 @@ func TestPongMatchingRejectsNonPingRequest(t *testing.T) { addr := n.Addr() called := 0 - h.config.OnPongReceived = func(*node.Node, net.IP, uint16) { called++ } + h.config.OnPongReceived = func(*node.Node, *net.UDPAddr, net.IP, uint16) { called++ } hash := []byte("enr-request-hash") - if _, err := h.addPendingRequest(hash, n, ENRRequestPacket); err != nil { + if _, err := h.addPendingRequest(hash, n, ENRRequestPacket, n.Addr()); err != nil { t.Fatalf("addPendingRequest: %v", err) } @@ -196,10 +196,10 @@ func TestReplayedPongAppliesSideEffectsOnce(t *testing.T) { addr := n.Addr() called := 0 - h.config.OnPongReceived = func(*node.Node, net.IP, uint16) { called++ } + h.config.OnPongReceived = func(*node.Node, *net.UDPAddr, net.IP, uint16) { called++ } hash := []byte("ping-hash") - if _, err := h.addPendingRequest(hash, n, PingPacket); err != nil { + if _, err := h.addPendingRequest(hash, n, PingPacket, n.Addr()); err != nil { t.Fatalf("addPendingRequest: %v", err) } @@ -219,6 +219,47 @@ func TestReplayedPongAppliesSideEffectsOnce(t *testing.T) { } } +// The proven address must reach the IP-discovery callback directly. Reading it +// back off the node would hand over whatever address the last inbound packet +// set, which is attacker-controlled and undoes the endpoint proof. +func TestPongCallbackReceivesProvenAddress(t *testing.T) { + h, _, cancel := proofHandler(t) + defer cancel() + + n, _ := makeKeyedNode(t, 30303) + sentTo := n.Addr() + + var got *net.UDPAddr + h.config.OnPongReceived = func(_ *node.Node, provenAddr *net.UDPAddr, _ net.IP, _ uint16) { + got = provenAddr + } + + hash := []byte("ping-hash") + if _, err := h.addPendingRequest(hash, n, PingPacket, sentTo); err != nil { + t.Fatalf("addPendingRequest: %v", err) + } + + // A concurrent packet from the same identity rewrites the node's address + // before the PONG is processed, exactly as getOrCreateNode does. + n.SetAddr(&net.UDPAddr{IP: net.IPv4(203, 0, 113, 9), Port: 30303}) + + pong := &Pong{ + ReplyTok: hash, + To: NewEndpoint(&net.UDPAddr{IP: net.IPv4(9, 9, 9, 9), Port: 30303}, 0), + Expiration: MakeExpiration(20 * time.Second), + } + if err := h.handlePong(n, sentTo, pong); err != nil { + t.Fatalf("handlePong: %v", err) + } + + if got == nil { + t.Fatal("OnPongReceived never fired for a solicited PONG") + } + if !got.IP.Equal(sentTo.IP) { + t.Fatalf("callback got %s, want the proven %s", got.IP, sentTo.IP) + } +} + // A PONG whose source is not the address the PING went to proves only that // somebody received that PING, which is what the spoofing attack relies on. func TestPongFromWrongSourceRejected(t *testing.T) { @@ -229,7 +270,7 @@ func TestPongFromWrongSourceRejected(t *testing.T) { sentTo := n.Addr() hash := []byte("ping-hash") - if _, err := h.addPendingRequest(hash, n, PingPacket); err != nil { + if _, err := h.addPendingRequest(hash, n, PingPacket, n.Addr()); err != nil { t.Fatalf("addPendingRequest: %v", err) } diff --git a/discv4/protocol/handler.go b/discv4/protocol/handler.go index cef3465..8c86d10 100644 --- a/discv4/protocol/handler.go +++ b/discv4/protocol/handler.go @@ -38,7 +38,12 @@ type OnNodeSeenCallback func(n *node.Node, timestamp time.Time) // OnPongReceivedCallback is called when a PONG response is received. // The ip and port parameters contain our external address as seen by the remote peer. -type OnPongReceivedCallback func(from *node.Node, ip net.IP, port uint16) +// +// provenAddr is the address the answered PING was sent to. Callers must use it +// rather than from.Addr(), which any later inbound packet rewrites, including a +// spoofed one; attributing a report to that address would undo the endpoint +// proof this callback is gated on. +type OnPongReceivedCallback func(from *node.Node, provenAddr *net.UDPAddr, ip net.IP, port uint16) // Handler handles incoming and outgoing discv4 protocol messages. // @@ -393,11 +398,12 @@ func (h *Handler) handlePong(fromNode *node.Node, from *net.UDPAddr, pong *Pong) } // Bind the bond to the address we proved, not the packet's source. - fromNode.MarkPongReceived(h.config.BondExpiration, &net.UDPAddr{IP: req.DestIP, Port: from.Port}) + provenAddr := &net.UDPAddr{IP: req.DestIP, Port: from.Port} + fromNode.MarkPongReceived(h.config.BondExpiration, provenAddr) // The To field in PONG contains our address as seen by the remote peer. if h.config.OnPongReceived != nil && pong.To.IP != nil && pong.To.UDP > 0 { - h.config.OnPongReceived(fromNode, pong.To.IP, pong.To.UDP) + h.config.OnPongReceived(fromNode, provenAddr, pong.To.IP, pong.To.UDP) } h.deliverResponse(req, pong) @@ -464,9 +470,11 @@ func (h *Handler) handleNeighbors(fromNode *node.Node, from *net.UDPAddr, neighb } // Only accept NEIGHBORS in response to a FINDNODE we actually sent to this - // node. Dropping unsolicited NEIGHBORS prevents a peer we never queried from - // making us accumulate node records without bound. - matchedReq := h.findPendingFindnode(fromNode.ID()) + // address. Dropping unsolicited NEIGHBORS prevents a peer we never queried + // from making us accumulate node records without bound; requiring the source + // to be the address queried stops a peer answering from a spoofed one. + // NEIGHBORS carries no reply token, so the match is by node ID plus endpoint. + matchedReq := h.findPendingFindnode(fromNode.ID(), from) if matchedReq == nil { return nil } @@ -593,10 +601,12 @@ func (h *Handler) handleENRResponse(fromNode *node.Node, from *net.UDPAddr, resp "enr_seq": resp.Record.Seq(), }).Debug("Received ENRRESPONSE") - // Only a response to a request we actually sent to this peer may touch any - // state: ENRRESPONSE carries no expiration, so an unsolicited replay could - // otherwise roll the node back to an older record. - reqs := h.getPendingRequests(resp.ReplyTok, fromNode.ID()) + // Only a response to an ENRREQUEST we actually sent to this address may touch + // any state: ENRRESPONSE carries no expiration, so an unsolicited replay could + // otherwise roll the node back to an older record. The type and destination + // must both match, or a peer could answer with the hash of some other packet + // we sent it and resolve the wrong waiter. + reqs := h.pendingRequestsFrom(resp.ReplyTok, fromNode.ID(), from, ENRRequestPacket) if len(reqs) == 0 { return nil } @@ -628,13 +638,17 @@ func (h *Handler) handleENRResponse(fromNode *node.Node, from *net.UDPAddr, resp // Ping sends a PING request to a node. func (h *Handler) Ping(n *node.Node) (*Pong, error) { + // Read the address once: inbound packets rewrite it, and the endpoint proof + // requires the recorded destination to be the one we actually sent to. + destAddr := n.Addr() + // Build PING message ping := &Ping{ Version: 4, // A bootnode serves no RLPx, so it advertises tcp-port 0; the recipient's // tcp is not the sender's to set (spec: to = [ip, udp-port, 0]). From: NewEndpoint(h.config.LocalAddr, 0), - To: NewEndpoint(n.Addr(), 0), + To: NewEndpoint(destAddr, 0), Expiration: MakeExpiration(h.config.ExpirationWindow), } @@ -650,14 +664,14 @@ func (h *Handler) Ping(n *node.Node) (*Pong, error) { } // Register pending request; removal is deferred so every exit path clears it. - req, err := h.addPendingRequest(hash, n, PingPacket) + req, err := h.addPendingRequest(hash, n, PingPacket, destAddr) if err != nil { return nil, err } defer h.removePendingRequest(req) // Send packet - if err := h.transport.SendTo(packet, n.Addr()); err != nil { + if err := h.transport.SendTo(packet, destAddr); err != nil { return nil, err } @@ -713,14 +727,15 @@ func (h *Handler) Findnode(n *node.Node, target []byte) ([]*node.Node, error) { // Register pending request. Removal is deferred so every exit path clears // it: a completed request left in the map keeps matching later NEIGHBORS // from that node and reopens collection windows until cleanup runs. - req, err := h.addPendingRequest(hash, n, FindnodePacket) + destAddr := n.Addr() + req, err := h.addPendingRequest(hash, n, FindnodePacket, destAddr) if err != nil { return nil, err } defer h.removePendingRequest(req) // Send packet - if err := h.transport.SendTo(packet, n.Addr()); err != nil { + if err := h.transport.SendTo(packet, destAddr); err != nil { return nil, err } @@ -767,14 +782,15 @@ func (h *Handler) RequestENR(n *node.Node) (*enr.Record, error) { } // Register pending request; removal is deferred so every exit path clears it. - pendingReq, err := h.addPendingRequest(hash, n, ENRRequestPacket) + destAddr := n.Addr() + pendingReq, err := h.addPendingRequest(hash, n, ENRRequestPacket, destAddr) if err != nil { return nil, err } defer h.removePendingRequest(pendingReq) // Send packet - if err := h.transport.SendTo(packet, n.Addr()); err != nil { + if err := h.transport.SendTo(packet, destAddr); err != nil { return nil, err } @@ -958,10 +974,13 @@ func requestKey(hash []byte, id node.ID) string { // addPendingRequest registers a new pending request. A second FINDNODE to a // peer with one already in flight is rejected: NEIGHBORS carries no reply // token, so two in-flight FINDNODEs to one peer cannot be told apart. -func (h *Handler) addPendingRequest(hash []byte, toNode *node.Node, packetType byte) (*PendingRequest, error) { +// destAddr must be the address the caller sends the packet to, captured once: +// toNode.Addr() is rewritten by concurrent inbound packets, so reading it here +// can record an endpoint the request never went to. +func (h *Handler) addPendingRequest(hash []byte, toNode *node.Node, packetType byte, destAddr *net.UDPAddr) (*PendingRequest, error) { var destIP net.IP - if addr := toNode.Addr(); addr != nil && addr.IP != nil { - destIP = append(net.IP(nil), addr.IP...) + if destAddr != nil && destAddr.IP != nil { + destIP = append(net.IP(nil), destAddr.IP...) } req := &PendingRequest{ @@ -995,6 +1014,25 @@ func (h *Handler) getPendingRequests(replyTok []byte, id node.ID) []*PendingRequ return append([]*PendingRequest(nil), h.requests[requestKey(replyTok, id)]...) } +// pendingRequestsFrom returns the pending requests of the given type that match +// this reply token and were sent to this address. +func (h *Handler) pendingRequestsFrom(replyTok []byte, id node.ID, from *net.UDPAddr, packetType byte) []*PendingRequest { + if from == nil || from.IP == nil { + return nil + } + + h.requestsMu.RLock() + defer h.requestsMu.RUnlock() + + var out []*PendingRequest + for _, req := range h.requests[requestKey(replyTok, id)] { + if req.PacketType == packetType && req.DestIP != nil && req.DestIP.Equal(from.IP) { + out = append(out, req) + } + } + return out +} + // consumePendingPing removes and returns the pending PING this PONG answers, or // nil if there is none. // @@ -1038,10 +1076,19 @@ func (h *Handler) consumePendingPing(replyTok []byte, id node.ID, from *net.UDPA // findPendingFindnode returns the pending FINDNODE request awaiting a response // from the given node, or nil if none exists. -func (h *Handler) findPendingFindnode(id node.ID) *PendingRequest { +func (h *Handler) findPendingFindnode(id node.ID, from *net.UDPAddr) *PendingRequest { + if from == nil || from.IP == nil { + return nil + } + h.requestsMu.RLock() defer h.requestsMu.RUnlock() - return h.pendingFindnodeLocked(id) + + req := h.pendingFindnodeLocked(id) + if req == nil || req.DestIP == nil || !req.DestIP.Equal(from.IP) { + return nil + } + return req } func (h *Handler) pendingFindnodeLocked(id node.ID) *PendingRequest { diff --git a/discv4/protocol/pending_neighbors_test.go b/discv4/protocol/pending_neighbors_test.go index 5b5d91f..6104d21 100644 --- a/discv4/protocol/pending_neighbors_test.go +++ b/discv4/protocol/pending_neighbors_test.go @@ -70,7 +70,7 @@ func TestNeighborsAccumulationCapped(t *testing.T) { defer cancel() from := makeDiscv4Node(t) - if _, err := h.addPendingRequest([]byte("req"), from, FindnodePacket); err != nil { + if _, err := h.addPendingRequest([]byte("req"), from, FindnodePacket, from.Addr()); err != nil { t.Fatalf("addPendingRequest: %v", err) } @@ -104,7 +104,7 @@ func TestNeighborsDeliveredToWaiter(t *testing.T) { defer cancel() from := makeDiscv4Node(t) - req, err := h.addPendingRequest([]byte("req"), from, FindnodePacket) + req, err := h.addPendingRequest([]byte("req"), from, FindnodePacket, from.Addr()) if err != nil { t.Fatalf("addPendingRequest: %v", err) } @@ -157,7 +157,7 @@ func TestNeighborsCapAppliesBeforeNodePersistence(t *testing.T) { defer cancel() from := makeDiscv4Node(t) - if _, err := h.addPendingRequest([]byte("req"), from, FindnodePacket); err != nil { + if _, err := h.addPendingRequest([]byte("req"), from, FindnodePacket, from.Addr()); err != nil { t.Fatalf("addPendingRequest: %v", err) } @@ -227,7 +227,7 @@ func TestFindnodeRemovesCompletedRequest(t *testing.T) { }() deadline := time.Now().Add(2 * time.Second) - for h.findPendingFindnode(to.ID()) == nil { + for h.findPendingFindnode(to.ID(), to.Addr()) == nil { if time.Now().After(deadline) { t.Fatal("pending FINDNODE never registered") } @@ -257,7 +257,7 @@ func TestNeighborsPersistenceCapExactUnderConcurrency(t *testing.T) { defer cancel() from := makeDiscv4Node(t) - if _, err := h.addPendingRequest([]byte("req"), from, FindnodePacket); err != nil { + if _, err := h.addPendingRequest([]byte("req"), from, FindnodePacket, from.Addr()); err != nil { t.Fatalf("addPendingRequest: %v", err) } @@ -294,7 +294,7 @@ func TestNeighborsAfterDeliveryPersistNothing(t *testing.T) { defer cancel() from := makeDiscv4Node(t) - req, err := h.addPendingRequest([]byte("req"), from, FindnodePacket) + req, err := h.addPendingRequest([]byte("req"), from, FindnodePacket, from.Addr()) if err != nil { t.Fatalf("addPendingRequest: %v", err) } diff --git a/discv4/protocol/pending_request_test.go b/discv4/protocol/pending_request_test.go index 2eababe..7f57876 100644 --- a/discv4/protocol/pending_request_test.go +++ b/discv4/protocol/pending_request_test.go @@ -69,11 +69,11 @@ func TestIdenticalRequestsToDifferentPeersDoNotAlias(t *testing.T) { nodeB, keyB := makeKeyedNode(t, 30302) hash := []byte("same-second-packet") - reqA, err := h.addPendingRequest(hash, nodeA, ENRRequestPacket) + reqA, err := h.addPendingRequest(hash, nodeA, ENRRequestPacket, nodeA.Addr()) if err != nil { t.Fatalf("addPendingRequest A: %v", err) } - reqB, err := h.addPendingRequest(hash, nodeB, ENRRequestPacket) + reqB, err := h.addPendingRequest(hash, nodeB, ENRRequestPacket, nodeB.Addr()) if err != nil { t.Fatalf("addPendingRequest B: %v", err) } @@ -106,11 +106,11 @@ func TestSamePeerDuplicateRequestsBothComplete(t *testing.T) { n, key := makeKeyedNode(t, 30301) hash := []byte("same-second-packet") - req1, err := h.addPendingRequest(hash, n, ENRRequestPacket) + req1, err := h.addPendingRequest(hash, n, ENRRequestPacket, n.Addr()) if err != nil { t.Fatalf("addPendingRequest 1: %v", err) } - req2, err := h.addPendingRequest(hash, n, ENRRequestPacket) + req2, err := h.addPendingRequest(hash, n, ENRRequestPacket, n.Addr()) if err != nil { t.Fatalf("addPendingRequest 2: %v", err) } @@ -160,7 +160,7 @@ func TestStaleENRResponseNotInstalled(t *testing.T) { n.SetENR(newer) hash := []byte("pending") - req, err := h.addPendingRequest(hash, n, ENRRequestPacket) + req, err := h.addPendingRequest(hash, n, ENRRequestPacket, n.Addr()) if err != nil { t.Fatalf("addPendingRequest: %v", err) } @@ -187,7 +187,7 @@ func TestMismatchedIdentityENRResponseDropped(t *testing.T) { _, otherKey := makeKeyedNode(t, 30302) hash := []byte("pending") - req, err := h.addPendingRequest(hash, n, ENRRequestPacket) + req, err := h.addPendingRequest(hash, n, ENRRequestPacket, n.Addr()) if err != nil { t.Fatalf("addPendingRequest: %v", err) } @@ -213,11 +213,11 @@ func TestIdenticalFindnodeToDifferentPeersSeparateAccumulators(t *testing.T) { nodeB, _ := makeKeyedNode(t, 30302) hash := []byte("same-target-same-second") - reqA, err := h.addPendingRequest(hash, nodeA, FindnodePacket) + reqA, err := h.addPendingRequest(hash, nodeA, FindnodePacket, nodeA.Addr()) if err != nil { t.Fatalf("addPendingRequest A: %v", err) } - reqB, err := h.addPendingRequest(hash, nodeB, FindnodePacket) + reqB, err := h.addPendingRequest(hash, nodeB, FindnodePacket, nodeB.Addr()) if err != nil { t.Fatalf("addPendingRequest B: %v", err) } @@ -252,10 +252,10 @@ func TestSecondFindnodeToSamePeerRejected(t *testing.T) { defer cancel() n, _ := makeKeyedNode(t, 30301) - if _, err := h.addPendingRequest([]byte("hash-1"), n, FindnodePacket); err != nil { + if _, err := h.addPendingRequest([]byte("hash-1"), n, FindnodePacket, n.Addr()); err != nil { t.Fatalf("first findnode: %v", err) } - if _, err := h.addPendingRequest([]byte("hash-2"), n, FindnodePacket); err == nil { + if _, err := h.addPendingRequest([]byte("hash-2"), n, FindnodePacket, n.Addr()); err == nil { t.Fatal("second in-flight findnode to the same peer was accepted") } } diff --git a/discv4/protocol/response_delivery_test.go b/discv4/protocol/response_delivery_test.go index 1e85de8..4bd37d8 100644 --- a/discv4/protocol/response_delivery_test.go +++ b/discv4/protocol/response_delivery_test.go @@ -21,7 +21,8 @@ func TestDeliverResponseNeverBlocks(t *testing.T) { h, cancel := newTestHandler(t) defer cancel() - req, err := h.addPendingRequest([]byte("reqhash"), makeDiscv4Node(t), PingPacket) + dest := makeDiscv4Node(t) + req, err := h.addPendingRequest([]byte("reqhash"), dest, PingPacket, dest.Addr()) if err != nil { t.Fatalf("addPendingRequest: %v", err) } @@ -67,7 +68,7 @@ func TestDuplicateResponsesWaiterGetsOneNoLeak(t *testing.T) { hash := []byte("reqhash") to := makeDiscv4Node(t) - req, err := h.addPendingRequest(hash, to, PingPacket) + req, err := h.addPendingRequest(hash, to, PingPacket, to.Addr()) if err != nil { t.Fatalf("addPendingRequest: %v", err) } From 917fd8b59078169c63ca136a9c5ba577aba9050e Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Wed, 29 Jul 2026 11:57:37 -0500 Subject: [PATCH 10/49] fix(bootnode): honour EnableIPDiscovery and never override an explicit ENR IP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit EnableIPDiscovery defaulted to false and was read in exactly one place — reconcileStoredENR's startup ip6-stripping branch. Nothing in the runtime path consulted it: New built the IPDiscovery service unconditionally, both OnPongReceived callbacks were wired unconditionally, and there was no CLI flag at all. So the feature ran for everyone while claiming to be off. updateENRWithDiscoveredIP also checked neither the flag nor ENRIPProvided, so a bootnode started with an explicit --enr-ip had that address (and, on a shared socket, its port) overwritten in the live ENR and republished once peer reports reached consensus. reconcileStoredENR restored it on restart, making the symptom a flip-flop rather than a permanent change. - Default EnableIPDiscovery to true so current behaviour is preserved, and add --enable-ip-discovery to turn it off. Skipping construction is the whole enforcement: onPongReceived already returns early on a nil service. - Refuse to move an address the operator set explicitly, per the contract Config.ENRIPProvided already documents and only startup honoured. Behaviour change: with the default now true, reconcileStoredENR's !EnableIPDiscovery branch stops firing, so a stored ip6 is retained at startup instead of stripped. That is coherent — with discovery on, let it correct the address rather than discard it — but it is a startup change, not only a flag default. --- bootnode/config.go | 6 ++-- bootnode/ipdiscovery_gate_test.go | 48 +++++++++++++++++++++++++++++++ bootnode/service.go | 32 +++++++++++++++------ cmd/bootnodoor/main.go | 9 ++++-- 4 files changed, 81 insertions(+), 14 deletions(-) create mode 100644 bootnode/ipdiscovery_gate_test.go diff --git a/bootnode/config.go b/bootnode/config.go index 71904be..ef947a8 100644 --- a/bootnode/config.go +++ b/bootnode/config.go @@ -126,7 +126,9 @@ type Config struct { // Discovery configuration - // EnableIPDiscovery enables automatic IP discovery from PONG responses (default: false) + // EnableIPDiscovery enables automatic IP discovery from PONG responses + // (default: true). An explicitly configured ENRIP/ENRIP6 is never overridden + // by discovery regardless of this setting. EnableIPDiscovery bool // GracePeriod is the grace period for accepting old fork digests (default: 60 minutes) @@ -157,7 +159,7 @@ func DefaultConfig() *Config { EnableDiscv5: true, SessionLifetime: 12 * time.Hour, MaxSessions: 1024, - EnableIPDiscovery: false, + EnableIPDiscovery: true, GracePeriod: 60 * time.Minute, } } diff --git a/bootnode/ipdiscovery_gate_test.go b/bootnode/ipdiscovery_gate_test.go new file mode 100644 index 0000000..5c2efd4 --- /dev/null +++ b/bootnode/ipdiscovery_gate_test.go @@ -0,0 +1,48 @@ +package bootnode + +import ( + "net" + "testing" +) + +// An explicitly configured ENR address is authoritative, so peer reports must not +// move it while running. reconcileStoredENR honours this at startup; without the +// same check at runtime a configured address is overwritten and only restored on +// the next restart. +func TestUpdateENRWithDiscoveredIP_KeepsExplicitAddress(t *testing.T) { + el := &identity{key: mustKey(t), servesEL: true, bindPort: 9000, enrPort: 9000, storeKey: "local_enr"} + s := newTestService(t, []*identity{el}) + s.config.ENRIPProvided = true + + before := el.localNode.Record().IP() + if before == nil { + t.Fatal("test identity has no ENR IP to protect") + } + + s.updateENRWithDiscoveredIP(net.ParseIP("9.9.9.9"), 31000, false) + + if got := el.localNode.Record().IP(); !got.Equal(before) { + t.Fatalf("configured ENR IP was overwritten by discovery: %v -> %v", before, got) + } +} + +// The same path must still self-correct when the address was auto-detected, or +// the guard above would disable IP discovery entirely. +func TestUpdateENRWithDiscoveredIP_UpdatesAutoDetectedAddress(t *testing.T) { + el := &identity{key: mustKey(t), servesEL: true, bindPort: 9000, enrPort: 9000, storeKey: "local_enr"} + s := newTestService(t, []*identity{el}) + + s.updateENRWithDiscoveredIP(net.ParseIP("9.9.9.9"), 31000, false) + + if got := el.localNode.Record().IP(); !got.Equal(net.ParseIP("9.9.9.9")) { + t.Fatalf("auto-detected ENR IP = %v, want the discovered 9.9.9.9", got) + } +} + +// The default was declared false while nothing in the runtime path read it, so +// discovery ran unconditionally. Pin the default that the runtime gate now honours. +func TestDefaultConfigEnablesIPDiscovery(t *testing.T) { + if !DefaultConfig().EnableIPDiscovery { + t.Fatal("EnableIPDiscovery default is false; the runtime gate would disable discovery for everyone") + } +} diff --git a/bootnode/service.go b/bootnode/service.go index 55f741d..872e0b7 100644 --- a/bootnode/service.go +++ b/bootnode/service.go @@ -180,16 +180,19 @@ func New(cfg *Config) (*Service, error) { s.localNode = primary.localNode s.enrManager = primary.enrManager - // Create IP discovery service - ipDiscoveryCfg := services.IPDiscoveryConfig{ - MinReports: 5, // Require 5 reports - MinDistinctIPs: 3, // From at least 3 distinct IPs - Logger: cfg.Logger, - OnConsensusReached: func(ip net.IP, port uint16, isIPv6 bool) { - s.updateENRWithDiscoveredIP(ip, port, isIPv6) - }, + // Create IP discovery service. Leaving it nil when disabled is the whole + // enforcement: onPongReceived already returns early on a nil service, so no + // peer report can reach consensus and rewrite the ENR. + if cfg.EnableIPDiscovery { + s.ipDiscovery = services.NewIPDiscovery(services.IPDiscoveryConfig{ + MinReports: 5, // Require 5 reports + MinDistinctIPs: 3, // From at least 3 distinct IPs + Logger: cfg.Logger, + OnConsensusReached: func(ip net.IP, port uint16, isIPv6 bool) { + s.updateENRWithDiscoveredIP(ip, port, isIPv6) + }, + }) } - s.ipDiscovery = services.NewIPDiscovery(ipDiscoveryCfg) // Create node databases for enabled layers var err error @@ -1608,6 +1611,17 @@ func (s *Service) onPongReceived(remoteID []byte, sourceIP net.IP, reportedIP ne // updateENRWithDiscoveredIP updates every identity's ENR with the discovered IP. func (s *Service) updateENRWithDiscoveredIP(ip net.IP, port uint16, isIPv6 bool) { + // An explicitly configured address is authoritative (see Config.ENRIPProvided), + // so peer reports must not move it. reconcileStoredENR already honours this at + // startup; without the same check here a configured address is overwritten + // while running and only restored on restart. + if isIPv6 && s.config.ENRIP6Provided { + return + } + if !isIPv6 && s.config.ENRIPProvided { + return + } + s.mu.Lock() defer s.mu.Unlock() diff --git a/cmd/bootnodoor/main.go b/cmd/bootnodoor/main.go index 75a1d11..c5f7e80 100644 --- a/cmd/bootnodoor/main.go +++ b/cmd/bootnodoor/main.go @@ -59,9 +59,10 @@ var ( clEnrPort int // ENR configuration - enrIP string - enrIP6 string - enrPort int + enrIP string + enrIP6 string + enrPort int + enableIPDiscovery bool // Logging logLevel string @@ -139,6 +140,7 @@ func init() { rootCmd.Flags().StringVar(&enrIP, "enr-ip", "", "IPv4 address to advertise in ENR (auto-detected if not specified)") rootCmd.Flags().StringVar(&enrIP6, "enr-ip6", "", "IPv6 address to advertise in ENR (optional)") rootCmd.Flags().IntVar(&enrPort, "enr-port", 0, "UDP port to advertise in ENR (0 = use bind-port)") + rootCmd.Flags().BoolVar(&enableIPDiscovery, "enable-ip-discovery", true, "Learn the external address from peer PONG reports (never overrides --enr-ip/--enr-ip6)") // Logging rootCmd.Flags().StringVar(&logLevel, "log-level", "info", "Log level (debug, info, warn, error)") @@ -521,6 +523,7 @@ func runBootnode(cmd *cobra.Command, args []string) error { config.ENRIP6 = enrIPv6 config.ENRIPProvided = enrIP != "" config.ENRIP6Provided = enrIP6 != "" + config.EnableIPDiscovery = enableIPDiscovery config.ENRPort = enrUDPPort config.EnableDiscv4 = enableDiscv4 config.EnableDiscv5 = enableDiscv5 From fa30b6892deb4db4fee2e87600803bd93437491e Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Wed, 29 Jul 2026 12:04:46 -0500 Subject: [PATCH 11/49] fix(discv5): stop unauthenticated packets moving, deleting or replacing sessions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit handleOrdinaryPacket located the session by the unauthenticated src-id header, migrated its address before decrypting, and on decrypt failure deleted it and challenged the packet source. An attacker who knew only a peer's public node ID could therefore migrate then destroy that peer's session from any address, and repeat it — permanent session denial, with in-flight requests dying on timeout. Four routes closed: - Address migration now happens only after DecryptMessage succeeds. AES-GCM over the header proves possession of the session key, and the source address is not in the AAD, so a NAT-rebound peer still migrates from its new address. - Decrypt failure keeps the session. Recovery is a handshake and Cache.Put replaces the entry by node ID when it lands, so deleting bought nothing and was the whole DoS. The challenge still goes to the packet source: a peer that genuinely lost its keys sends a random packet, which by definition fails to decrypt, and its source is the only address it can be reached at. Answering at the session address would be a reflection primitive against the real peer. - The GetByAddr fallback is gone. Sessions are keyed by sender node ID and every creation site uses the correct ID, so it never helped a correct peer — but it reached the same deletion without knowing the victim's node ID at all, by spoofing its IP:port with a random src-id, and was an O(n) scan over up to 1000 sessions driven by unauthenticated traffic. - A forged WHOAREYOU no longer replaces a live session. Not deleting was insufficient: with a request in flight, recovery derives fresh keys and Put overwrites the session before the remote proves anything, leaving a session present but unreadable by the real peer. Sessions now remember the nonces of recent ordinary packets they sent (bounded, 16) and a challenge quoting a nonce we never sent is dropped. Also: Session.RemoteAddr becomes remoteAddr behind an Addr() accessor, fixing an unlocked read racing UpdateAddr; and Session.String() no longer holds the read lock across Age()/IdleTime(), which take it again. --- discv5/protocol/handler.go | 93 ++++++---- discv5/protocol/session_proof_test.go | 239 ++++++++++++++++++++++++++ discv5/session/cache.go | 4 +- discv5/session/session.go | 66 ++++++- 4 files changed, 356 insertions(+), 46 deletions(-) create mode 100644 discv5/protocol/session_proof_test.go diff --git a/discv5/protocol/handler.go b/discv5/protocol/handler.go index 4b38006..7ff155b 100644 --- a/discv5/protocol/handler.go +++ b/discv5/protocol/handler.go @@ -431,27 +431,12 @@ func (h *Handler) handleOrdinaryPacket(packet *Packet, from *net.UDPAddr, localA var srcNodeID node.ID copy(srcNodeID[:], packet.SrcID) - // Look up session by node ID first (most efficient and handles IP changes) + // Sessions are keyed by the sender's node ID and every creation site uses the + // correct ID, so this always hits when a session exists. There is deliberately + // no address fallback: srcID is unauthenticated, so matching a session by + // source address alone let anyone who could guess a peer's IP:port reach the + // failure path below and tear that peer's session down. sess := h.config.Sessions.Get(srcNodeID) - - // If session exists, verify and update address if needed - if sess != nil { - // Check if the address has changed - if sess.RemoteAddr.String() != from.String() { - h.config.Logger.WithFields(logrus.Fields{ - "nodeID": srcNodeID.String()[:16], - "oldAddr": sess.RemoteAddr, - "newAddr": from, - }).Info("handler: node address changed, updating session") - - // Update the session's remote address - sess.UpdateAddr(from) - } - } else { - // No session by node ID, try lookup by address (slower fallback) - sess = h.config.Sessions.GetByAddr(from) - } - if sess == nil { // No session exists, send WHOAREYOU challenge h.config.Logger.WithFields(logrus.Fields{ @@ -470,22 +455,36 @@ func (h *Handler) handleOrdinaryPacket(packet *Packet, from *net.UDPAddr, localA packet.Message, ) if err != nil { - // Decryption failed - session is corrupted/expired - // Delete the session immediately to force a new handshake - h.config.Sessions.Delete(sess.RemoteID) + // Keep the session. Anyone can send an undecryptable packet naming this + // node ID, so deleting here let an attacker who knows only a peer's public + // ID destroy that peer's session at will. A peer that genuinely lost its + // keys recovers via the handshake, and Cache.Put replaces this entry by + // node ID when it lands, so deletion buys nothing. + // + // The challenge still goes to the packet source: the legitimate "restarted + // and lost my keys" case is a random packet, which by definition fails to + // decrypt, and the source is the only address such a peer is reachable at. + // Answering at sess.Addr() instead would be a reflection primitive. h.config.Logger.WithFields(logrus.Fields{ "nodeID": sess.RemoteID.String()[:16], "addr": from, "error": err, - }).Debug("handler: decryption failed, deleted session and sending WHOAREYOU") + }).Debug("handler: decryption failed, keeping session and sending WHOAREYOU") - // Extract dest node ID from packet srcID and send WHOAREYOU - if len(packet.SrcID) != 32 { - return fmt.Errorf("no source node ID in packet") - } - var destNodeID node.ID - copy(destNodeID[:], packet.SrcID) - return h.sendWHOAREYOU(from, destNodeID, packet.Header.Nonce, localAddr) + return h.sendWHOAREYOU(from, srcNodeID, packet.Header.Nonce, localAddr) + } + + // Only now is the sender proven: AES-GCM over the header authenticates + // possession of the session key, and the source address is not part of the + // AAD, so a NAT-rebound peer decrypts fine from its new address. + if sess.Addr().String() != from.String() { + h.config.Logger.WithFields(logrus.Fields{ + "nodeID": srcNodeID.String()[:16], + "oldAddr": sess.Addr(), + "newAddr": from, + }).Info("handler: node address changed, updating session") + + sess.UpdateAddr(from) } // Decode message from plaintext @@ -556,16 +555,29 @@ func (h *Handler) handleWHOAREYOUPacket(packet *Packet, from *net.UDPAddr, local return fmt.Errorf("no pending handshake for %s", from) } + // Answering this challenge derives fresh keys and replaces the session, so + // a WHOAREYOU nobody authenticated must not reach that path: otherwise + // anyone able to reach us from this address could swap a working session + // for keys the real peer never agreed to. Only a peer that actually + // received one of our packets can quote its nonce. + if !sess.SentNonce(packet.Header.Nonce) { + h.config.Logger.WithFields(logrus.Fields{ + "nodeID": sess.RemoteID.String()[:16], + "addr": from, + }).Debug("handler: ignoring WHOAREYOU referencing a nonce we never sent") + return fmt.Errorf("unsolicited WHOAREYOU from %s", from) + } + // Look up pending request for this node to get the message to replay pendingReq := h.requests.GetPendingRequestForNode(sess.RemoteID) if pendingReq == nil { - // No pending request either - just delete stale session + // Keep the session: the handshake replaces it by node ID if the peer + // really did lose its keys, so deleting here only helps an attacker. h.config.Logger.WithFields(logrus.Fields{ "nodeID": sess.RemoteID.String()[:16], "addr": from, "age": sess.Age(), - }).Info("handler: received unexpected WHOAREYOU with no pending request, deleting stale session") - h.config.Sessions.Delete(sess.RemoteID) + }).Info("handler: received unexpected WHOAREYOU with no pending request") return fmt.Errorf("no pending handshake or request for %s", from) } @@ -587,8 +599,9 @@ func (h *Handler) handleWHOAREYOUPacket(packet *Packet, from *net.UDPAddr, local h.pendingHandshakes[pendingKey] = pending h.mu.Unlock() - // Delete the stale session - we'll create a new one during handshake - h.config.Sessions.Delete(sess.RemoteID) + // The stale session is not deleted here: sendHandshakePacket's Put replaces + // it by node ID once the new keys exist, so deleting first only widens the + // window in which the peer has no session at all. // Continue processing the WHOAREYOU with our pending handshake // After handshake completes, the message will be sent with the SAME request ID, @@ -1230,6 +1243,10 @@ func (h *Handler) SendMessage(msg Message, remoteID node.ID, to *net.UDPAddr, re if err != nil { return fmt.Errorf("failed to encode ordinary packet: %w", err) } + + // Remembered so a WHOAREYOU quoting this nonce can be told apart from a + // forged one; answering a forged challenge would replace the session keys. + sess.RecordSentNonce(nonce) } // Send via UDP transport @@ -1355,6 +1372,10 @@ func (h *Handler) SendMessageFrom(msg Message, remoteID node.ID, to *net.UDPAddr if err != nil { return fmt.Errorf("failed to encode ordinary packet: %w", err) } + + // Remembered so a WHOAREYOU quoting this nonce can be told apart from a + // forged one; answering a forged challenge would replace the session keys. + sess.RecordSentNonce(nonce) } // Send via UDP transport from the specified local address diff --git a/discv5/protocol/session_proof_test.go b/discv5/protocol/session_proof_test.go new file mode 100644 index 0000000..680d048 --- /dev/null +++ b/discv5/protocol/session_proof_test.go @@ -0,0 +1,239 @@ +package protocol + +import ( + "context" + "net" + "sync" + "testing" + "time" + + "github.com/ethpandaops/bootnodoor/discv5/node" + "github.com/ethpandaops/bootnodoor/discv5/session" + "github.com/sirupsen/logrus" +) + +type sentPacket struct { + data []byte + to *net.UDPAddr +} + +// recordingTransport captures where packets went, so a test can assert a +// challenge was not reflected at the victim's address. +type recordingTransport struct { + mu sync.Mutex + sent []sentPacket +} + +func (r *recordingTransport) SendTo(data []byte, to *net.UDPAddr) error { + r.mu.Lock() + defer r.mu.Unlock() + r.sent = append(r.sent, sentPacket{data: data, to: to}) + return nil +} + +func (r *recordingTransport) Send(data []byte, to *net.UDPAddr, _ *net.UDPAddr) error { + return r.SendTo(data, to) +} + +func (r *recordingTransport) destinations() []string { + r.mu.Lock() + defer r.mu.Unlock() + out := make([]string, 0, len(r.sent)) + for _, p := range r.sent { + out = append(out, p.to.String()) + } + return out +} + +func sessionHandler(t *testing.T) (*Handler, *recordingTransport, *session.Cache, context.CancelFunc) { + t.Helper() + + logger := logrus.New() + logger.SetLevel(logrus.PanicLevel) + + localKey := generateKey(t) + localNode, err := node.New(signedRecord(t, localKey, 1, nil)) + if err != nil { + t.Fatalf("node.New: %v", err) + } + + cache := session.NewCache(16, time.Hour, logger) + ctx, cancel := context.WithCancel(context.Background()) + h := NewHandler(ctx, HandlerConfig{ + LocalNode: localNode, + Sessions: cache, + PrivateKey: localKey, + Logger: logger, + }) + + tr := &recordingTransport{} + h.SetTransport(tr) + return h, tr, cache, cancel +} + +// victimSession installs a session for a peer reachable at addr, returning the +// peer's node ID and the keys the session was built with. +func victimSession(t *testing.T, cache *session.Cache, addr *net.UDPAddr) (*node.Node, *session.Session) { + t.Helper() + + peerKey := generateKey(t) + peerNode, err := node.New(signedRecord(t, peerKey, 1, nil)) + if err != nil { + t.Fatalf("node.New: %v", err) + } + + keys := &session.SessionKeys{ + InitiatorKey: []byte("0123456789abcdef"), + RecipientKey: []byte("fedcba9876543210"), + } + sess := session.NewSession(peerNode.ID(), addr, keys, false, time.Hour) + sess.SetNode(peerNode) + cache.Put(sess) + + return peerNode, sess +} + +// garbageOrdinaryPacket builds a syntactically valid ordinary packet that names +// srcID but whose ciphertext cannot decrypt against any real session key. +func garbageOrdinaryPacket(t *testing.T, srcID, destID node.ID) []byte { + t.Helper() + + nonce := make([]byte, 12) + for i := range nonce { + nonce[i] = byte(i + 1) + } + + authdata := srcID[:] + maskingIV, _, err := BuildOrdinaryHeaderData(srcID, nonce, authdata) + if err != nil { + t.Fatalf("BuildOrdinaryHeaderData: %v", err) + } + + data, err := EncodeOrdinaryPacket(srcID, destID, maskingIV, nonce, authdata, []byte("not-a-valid-ciphertext")) + if err != nil { + t.Fatalf("EncodeOrdinaryPacket: %v", err) + } + return data +} + +// A node ID is public information from an ENR, so an attacker who knows only that +// must not be able to move a peer's session to an address of their choosing. +func TestSpoofedOrdinaryPacketDoesNotMigrateSession(t *testing.T) { + h, _, cache, cancel := sessionHandler(t) + defer cancel() + + victimAddr := &net.UDPAddr{IP: net.IPv4(198, 51, 100, 5), Port: 30303} + victim, sess := victimSession(t, cache, victimAddr) + victimID := victim.ID() + + attackerAddr := &net.UDPAddr{IP: net.IPv4(203, 0, 113, 9), Port: 40404} + data := garbageOrdinaryPacket(t, victimID, h.config.LocalNode.ID()) + _ = h.HandleIncomingPacket(data, attackerAddr, victimAddr) + + if got := sess.Addr().String(); got != victimAddr.String() { + t.Fatalf("session migrated to %s on an unauthenticated packet, want %s", got, victimAddr) + } +} + +// Deleting on decrypt failure let anyone holding a peer's public node ID destroy +// that peer's session, repeatably. The session must survive. +func TestDecryptFailureKeepsSession(t *testing.T) { + h, tr, cache, cancel := sessionHandler(t) + defer cancel() + + victimAddr := &net.UDPAddr{IP: net.IPv4(198, 51, 100, 5), Port: 30303} + victim, _ := victimSession(t, cache, victimAddr) + victimID := victim.ID() + + attackerAddr := &net.UDPAddr{IP: net.IPv4(203, 0, 113, 9), Port: 40404} + data := garbageOrdinaryPacket(t, victimID, h.config.LocalNode.ID()) + + for i := 0; i < 5; i++ { + _ = h.HandleIncomingPacket(data, attackerAddr, victimAddr) + if cache.Get(victimID) == nil { + t.Fatalf("session destroyed by unauthenticated packet %d", i+1) + } + } + + // The challenge answers the packet source, which is the only address a peer + // that genuinely lost its keys could be reached at. + for _, dst := range tr.destinations() { + if dst == victimAddr.String() { + t.Fatal("challenge was reflected at the victim's address") + } + } +} + +// WHOAREYOU is unauthenticated and answering one replaces the session keys, so a +// challenge quoting a nonce we never sent must be ignored. +func TestSpoofedWhoareyouDoesNotReplaceSession(t *testing.T) { + h, _, cache, cancel := sessionHandler(t) + defer cancel() + + victimAddr := &net.UDPAddr{IP: net.IPv4(198, 51, 100, 5), Port: 30303} + victim, sess := victimSession(t, cache, victimAddr) + victimID := victim.ID() + + keysBefore := sess.EncryptionKey() + + nonce := make([]byte, 12) + for i := range nonce { + nonce[i] = 0xAA + } + data, _, err := EncodeWHOAREYOUPacket(h.config.LocalNode.ID(), nonce, &WHOAREYOUChallenge{ + IDNonce: make([]byte, 16), + ENRSeq: 0, + }) + if err != nil { + t.Fatalf("EncodeWHOAREYOUPacket: %v", err) + } + + _ = h.HandleIncomingPacket(data, victimAddr, victimAddr) + + after := cache.Get(victimID) + if after == nil { + t.Fatal("spoofed WHOAREYOU destroyed the session") + } + if string(after.EncryptionKey()) != string(keysBefore) { + t.Fatal("spoofed WHOAREYOU replaced the session keys") + } +} + +// With a request in flight the forged challenge reaches handshake recovery, which +// derives new keys and replaces the session by node ID. Asserting a session still +// exists is not enough here — it exists but the real peer cannot read it. +func TestSpoofedWhoareyouWithPendingRequestKeepsKeys(t *testing.T) { + h, _, cache, cancel := sessionHandler(t) + defer cancel() + + victimAddr := &net.UDPAddr{IP: net.IPv4(198, 51, 100, 5), Port: 30303} + victim, sess := victimSession(t, cache, victimAddr) + victimID := victim.ID() + + keysBefore := string(sess.EncryptionKey()) + + requestID := []byte{0x01, 0x02, 0x03, 0x04} + h.requests.AddRequest(requestID, victim, &Ping{RequestID: requestID}) + + nonce := make([]byte, 12) + for i := range nonce { + nonce[i] = 0xBB + } + data, _, err := EncodeWHOAREYOUPacket(h.config.LocalNode.ID(), nonce, &WHOAREYOUChallenge{ + IDNonce: make([]byte, 16), + ENRSeq: 0, + }) + if err != nil { + t.Fatalf("EncodeWHOAREYOUPacket: %v", err) + } + + _ = h.HandleIncomingPacket(data, victimAddr, victimAddr) + + after := cache.Get(victimID) + if after == nil { + t.Fatal("spoofed WHOAREYOU destroyed the session") + } + if string(after.EncryptionKey()) != keysBefore { + t.Fatal("spoofed WHOAREYOU replaced the session keys via handshake recovery") + } +} diff --git a/discv5/session/cache.go b/discv5/session/cache.go index 3536f74..1ffd44f 100644 --- a/discv5/session/cache.go +++ b/discv5/session/cache.go @@ -118,7 +118,7 @@ func (c *Cache) Put(session *Session) { // Store the session c.sessions[session.RemoteID] = session - c.logger.WithField("nodeID", session.RemoteID).WithField("addr", session.RemoteAddr).WithField("lifetime", c.sessionLifetime).Trace("cached new session") + c.logger.WithField("nodeID", session.RemoteID).WithField("addr", session.Addr()).WithField("lifetime", c.sessionLifetime).Trace("cached new session") } // Delete removes a session from the cache. @@ -216,7 +216,7 @@ func (c *Cache) GetByAddr(addr *net.UDPAddr) *Session { defer c.mu.RUnlock() for _, session := range c.sessions { - if session.RemoteAddr.String() == addr.String() { + if session.Addr().String() == addr.String() { if !session.IsExpired() { session.Touch() return session diff --git a/discv5/session/session.go b/discv5/session/session.go index 2cd1d20..d4246fd 100644 --- a/discv5/session/session.go +++ b/discv5/session/session.go @@ -2,6 +2,7 @@ package session import ( "net" + "slices" "sync" "time" @@ -20,8 +21,10 @@ type Session struct { // RemoteID is the node ID of the remote peer RemoteID node.ID - // RemoteAddr is the network address of the remote peer - RemoteAddr *net.UDPAddr + // remoteAddr is the network address of the remote peer. Unexported because it + // is mutated by UpdateAddr under mu while packets are handled concurrently; + // read it through Addr(). + remoteAddr *net.UDPAddr // Node is the full node information (ENR, etc.) // This allows protocol operations to access node data without a separate table lookup @@ -42,6 +45,9 @@ type Session struct { // LastUsed is the last time this session was used LastUsed time.Time + // sentNonces holds the nonces of recent ordinary packets we sent, oldest first. + sentNonces []string + // mu protects mutable fields mu sync.RWMutex } @@ -69,7 +75,7 @@ func NewSession( return &Session{ RemoteID: remoteID, - RemoteAddr: remoteAddr, + remoteAddr: remoteAddr, Keys: keys, IsInitiator: isInitiator, CreatedAt: now, @@ -109,14 +115,59 @@ func (s *Session) SetNode(n *node.Node) { s.Node = n } +// maxSentNonces bounds the remembered nonces. A WHOAREYOU answers a packet we +// sent moments ago, so only the most recent few can legitimately be referenced. +const maxSentNonces = 16 + +// RecordSentNonce remembers the nonce of an ordinary packet we sent on this +// session, so a WHOAREYOU claiming to answer it can be verified. +func (s *Session) RecordSentNonce(nonce []byte) { + if len(nonce) == 0 { + return + } + + s.mu.Lock() + defer s.mu.Unlock() + + s.sentNonces = append(s.sentNonces, string(nonce)) + if len(s.sentNonces) > maxSentNonces { + s.sentNonces = s.sentNonces[len(s.sentNonces)-maxSentNonces:] + } +} + +// SentNonce reports whether nonce belongs to a packet we sent on this session. +// +// WHOAREYOU is unauthenticated, and answering one replaces this session's keys, +// so a forged challenge must not be able to reach that path. Only a peer that +// actually received one of our packets can quote its nonce back. +func (s *Session) SentNonce(nonce []byte) bool { + if len(nonce) == 0 { + return false + } + + s.mu.RLock() + defer s.mu.RUnlock() + + return slices.Contains(s.sentNonces, string(nonce)) +} + +// Addr returns the remote address for this session. +func (s *Session) Addr() *net.UDPAddr { + s.mu.RLock() + defer s.mu.RUnlock() + + return s.remoteAddr +} + // UpdateAddr updates the remote address for this session. // -// This is called when we detect that a node has moved to a different IP address. +// Only call this once the sender is authenticated: an unauthenticated packet +// naming this node ID must not be able to steer where the session points. func (s *Session) UpdateAddr(addr *net.UDPAddr) { s.mu.Lock() defer s.mu.Unlock() - s.RemoteAddr = addr + s.remoteAddr = addr } // GetNode returns the node reference for this session. @@ -181,14 +232,13 @@ func (s *Session) TimeUntilExpiry() time.Duration { // String returns a human-readable representation of the session. func (s *Session) String() string { - s.mu.RLock() - defer s.mu.RUnlock() - role := "recipient" if s.IsInitiator { role = "initiator" } + // Age and IdleTime take the read lock themselves; holding it here as well + // would be a recursive RLock, which deadlocks if a writer queues in between. return "Session{" + "RemoteID: " + s.RemoteID.String() + ", Role: " + role + From 01b9fbc83b06f9627834ba36094e69a983a9258f Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Wed, 29 Jul 2026 12:09:08 -0500 Subject: [PATCH 12/49] fix(discv5): bind requests to their destination and gate the IP vote on it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PendingRequest recorded no address, and all three AddRequest callers re-read n.Addr() for the send, so the recorded request and the transmitted destination came from two separate reads of state that a newer ENR can move — including one supplied by the peer being verified. This is the divergence already fixed on the discv4 side in 6dd2f08. AddRequest now takes the destination explicitly and each caller captures n.Addr() once for both the record and the send. MatchResponse returns the matched request instead of a bool so callers can reach DestAddr. handlePong gates the OnPongReceived IP-discovery vote on the PONG's source matching that destination by IP. A session peer can send a correctly typed, correctly encrypted PONG from a forged source, and that source is what feeds the distinct-reporter threshold. Response delivery stays source-agnostic so NAT rebinding and mobile peers keep working — only the vote needs the endpoint proven, which is why the check sits at the side effect and not in the match. handleNodes and handleTalkResp continue to ignore the result; their delivery semantics are unchanged. --- discv5/protocol/handler.go | 27 ++++++--- discv5/protocol/request.go | 28 ++++++--- discv5/protocol/request_match_test.go | 38 +++++++++++-- discv5/protocol/session_proof_test.go | 81 ++++++++++++++++++++++++++- discv5/service.go | 6 +- 5 files changed, 156 insertions(+), 24 deletions(-) diff --git a/discv5/protocol/handler.go b/discv5/protocol/handler.go index 7ff155b..5ec81e5 100644 --- a/discv5/protocol/handler.go +++ b/discv5/protocol/handler.go @@ -993,13 +993,18 @@ func (h *Handler) handlePong(msg *Pong, remoteID node.ID, from *net.UDPAddr, rem // An unsolicited PONG must not reach the side effects below: OnPongReceived // casts a vote in the external-IP election that rewrites our published ENR, // and the ENR branch triggers outbound traffic. - if !h.requests.MatchResponse(msg.RequestID, remoteID, msg) { + req := h.requests.MatchResponse(msg.RequestID, remoteID, msg) + if req == nil { return nil } - // Call OnPongReceived callback with the source IP and the IP/port reported in the PONG - // The IP and Port fields in PONG contain our address as seen by the remote peer - if h.config.OnPongReceived != nil && len(msg.IP) > 0 && msg.Port > 0 { + // The IP-discovery vote additionally requires the PONG to come from the address + // we pinged. A session peer can send an authenticated PONG from a forged source, + // and from.IP is what feeds the distinct-reporter threshold. Delivery above + // stays source-agnostic so NAT rebinding and mobile peers keep working; only + // this vote needs the endpoint proven. + if h.config.OnPongReceived != nil && len(msg.IP) > 0 && msg.Port > 0 && + req.DestAddr != nil && req.DestAddr.IP.Equal(from.IP) { reportedIP := net.IP(msg.IP) h.config.OnPongReceived(remoteID, from.IP, reportedIP, msg.Port) } @@ -1652,11 +1657,15 @@ func (h *Handler) SendPing(n *node.Node) (<-chan *Response, error) { ENRSeq: h.config.LocalNode.Record().Seq(), } + // One read of the address for both the record and the send: a newer ENR can + // move it in between, and the endpoint proof needs them to agree. + destAddr := n.Addr() + // Register pending request (store message and node for replay if session becomes stale) - respChan := h.requests.AddRequest(requestID, n, ping) + respChan := h.requests.AddRequest(requestID, n, ping, destAddr) // Send PING (pass node object so it's available for handshake if needed) - if err := h.SendMessage(ping, n.ID(), n.Addr(), n); err != nil { + if err := h.SendMessage(ping, n.ID(), destAddr, n); err != nil { h.config.Logger.WithFields(logrus.Fields{ "to": n.Addr(), "nodeID": n.ID(), @@ -1825,11 +1834,13 @@ func (h *Handler) SendFindNode(n *node.Node, distances []uint) (<-chan *Response Distances: distances, } + destAddr := n.Addr() + // Register pending request (store message and node for replay if session becomes stale) - respChan := h.requests.AddRequest(requestID, n, findNode) + respChan := h.requests.AddRequest(requestID, n, findNode, destAddr) // Send FINDNODE (pass node object so it's available for handshake if needed) - if err := h.SendMessage(findNode, n.ID(), n.Addr(), n); err != nil { + if err := h.SendMessage(findNode, n.ID(), destAddr, n); err != nil { h.config.Logger.WithFields(logrus.Fields{ "to": n.Addr(), "nodeID": n.ID(), diff --git a/discv5/protocol/request.go b/discv5/protocol/request.go index 582ec90..cc12224 100644 --- a/discv5/protocol/request.go +++ b/discv5/protocol/request.go @@ -1,6 +1,7 @@ package protocol import ( + "net" "sync" "time" @@ -41,6 +42,11 @@ type PendingRequest struct { // Message is the original message that was sent (for replay after re-handshake) Message Message + // DestAddr is the address this request was sent to, captured at send time. + // Node.Addr() cannot verify a response's origin because a newer ENR moves it, + // including one supplied by the peer being verified. + DestAddr *net.UDPAddr + // Timeout is when the request expires Timeout time.Time @@ -101,8 +107,12 @@ func NewRequestTracker(timeout time.Duration) *RequestTracker { // // The message and node parameters are stored for replay if the session becomes stale. // +// destAddr must be the address the caller sends this request to, captured once: +// n.Addr() is derived from the ENR and moves when a newer record arrives, so +// reading it again at send time can record an endpoint the request never went to. +// // Returns a channel that will receive the response or timeout. -func (rt *RequestTracker) AddRequest(requestID []byte, n *node.Node, msg Message) <-chan *Response { +func (rt *RequestTracker) AddRequest(requestID []byte, n *node.Node, msg Message, destAddr *net.UDPAddr) <-chan *Response { rt.mu.Lock() defer rt.mu.Unlock() @@ -113,6 +123,7 @@ func (rt *RequestTracker) AddRequest(requestID []byte, n *node.Node, msg Message NodeID: n.ID(), Node: n, Message: msg, + DestAddr: destAddr, Timeout: now.Add(rt.timeout), ResponseChan: make(chan *Response, 1), Retries: 0, @@ -152,20 +163,21 @@ func respondsTo(request, resp Message) bool { // MatchResponse matches a response to a pending request. // -// Returns true if the request was matched and notified. -func (rt *RequestTracker) MatchResponse(requestID []byte, nodeID node.ID, msg Message) bool { +// Returns the matched request, or nil if there was none. Callers that gate a side +// effect on where the request was sent need DestAddr from the result. +func (rt *RequestTracker) MatchResponse(requestID []byte, nodeID node.ID, msg Message) *PendingRequest { rt.mu.Lock() defer rt.mu.Unlock() key := string(requestID) req, exists := rt.requests[key] if !exists { - return false + return nil } // Verify node ID matches if req.NodeID != nodeID { - return false + return nil } // The response must be the kind this request asked for. Request IDs are ours @@ -173,7 +185,7 @@ func (rt *RequestTracker) MatchResponse(requestID []byte, nodeID node.ID, msg Me // FINDNODE: it would both fire the PONG side effects and consume the entry // below, silently stranding the lookup that was waiting on it. if !respondsTo(req.Message, msg) { - return false + return nil } // Handle multi-packet NODES responses @@ -195,7 +207,7 @@ func (rt *RequestTracker) MatchResponse(requestID []byte, nodeID node.ID, msg Me // If we haven't received all packets yet, keep waiting if req.ReceivedCount < req.ExpectedTotal { - return true + return req } // All packets received, send accumulated response @@ -218,7 +230,7 @@ func (rt *RequestTracker) MatchResponse(requestID []byte, nodeID node.ID, msg Me delete(rt.requests, key) close(req.ResponseChan) - return true + return req } // handleTimeout handles request timeout. diff --git a/discv5/protocol/request_match_test.go b/discv5/protocol/request_match_test.go index 71606a3..ddcc1cd 100644 --- a/discv5/protocol/request_match_test.go +++ b/discv5/protocol/request_match_test.go @@ -1,6 +1,7 @@ package protocol import ( + "net" "testing" "time" @@ -18,11 +19,12 @@ func TestMatchResponseRejectsWrongResponseType(t *testing.T) { t.Fatalf("node.New: %v", err) } + peerAddr := &net.UDPAddr{IP: net.IPv4(203, 0, 113, 1), Port: 30303} requestID := []byte{0x01, 0x02, 0x03, 0x04} - ch := rt.AddRequest(requestID, n, &FindNode{RequestID: requestID, Distances: []uint{1}}) + ch := rt.AddRequest(requestID, n, &FindNode{RequestID: requestID, Distances: []uint{1}}, peerAddr) pong := &Pong{RequestID: requestID, IP: []byte{9, 9, 9, 9}, Port: 30303} - if rt.MatchResponse(requestID, n.ID(), pong) { + if rt.MatchResponse(requestID, n.ID(), pong) != nil { t.Fatal("a PONG matched a pending FINDNODE") } @@ -33,11 +35,36 @@ func TestMatchResponseRejectsWrongResponseType(t *testing.T) { } nodes := &Nodes{RequestID: requestID, Total: 1} - if !rt.MatchResponse(requestID, n.ID(), nodes) { + if rt.MatchResponse(requestID, n.ID(), nodes) == nil { t.Fatal("the matching NODES response was rejected") } } +// MatchResponse must hand back the destination so handlePong can tell a PONG that +// came from the address we pinged apart from one with a forged source. Sessions +// are keyed by node ID and accept packets from a changed address, so the source +// alone proves nothing about where the request went. +func TestMatchResponseReportsRequestDestination(t *testing.T) { + rt := NewRequestTracker(time.Second) + + n, err := node.New(signedRecord(t, generateKey(t), 1, nil)) + if err != nil { + t.Fatalf("node.New: %v", err) + } + + peerAddr := &net.UDPAddr{IP: net.IPv4(198, 51, 100, 7), Port: 30303} + requestID := []byte{0x11, 0x22} + rt.AddRequest(requestID, n, &Ping{RequestID: requestID}, peerAddr) + + req := rt.MatchResponse(requestID, n.ID(), &Pong{RequestID: requestID}) + if req == nil { + t.Fatal("a PONG did not match its pending PING") + } + if req.DestAddr == nil || !req.DestAddr.IP.Equal(peerAddr.IP) { + t.Fatalf("DestAddr = %v, want the address the PING was sent to (%v)", req.DestAddr, peerAddr.IP) + } +} + // The PING/PONG pair must still match, or gating handlePong on this would drop // every legitimate PONG. func TestMatchResponseAcceptsPongForPing(t *testing.T) { @@ -48,10 +75,11 @@ func TestMatchResponseAcceptsPongForPing(t *testing.T) { t.Fatalf("node.New: %v", err) } + peerAddr := &net.UDPAddr{IP: net.IPv4(203, 0, 113, 1), Port: 30303} requestID := []byte{0x0a, 0x0b} - rt.AddRequest(requestID, n, &Ping{RequestID: requestID}) + rt.AddRequest(requestID, n, &Ping{RequestID: requestID}, peerAddr) - if !rt.MatchResponse(requestID, n.ID(), &Pong{RequestID: requestID}) { + if rt.MatchResponse(requestID, n.ID(), &Pong{RequestID: requestID}) == nil { t.Fatal("a PONG did not match its pending PING") } } diff --git a/discv5/protocol/session_proof_test.go b/discv5/protocol/session_proof_test.go index 680d048..53081e1 100644 --- a/discv5/protocol/session_proof_test.go +++ b/discv5/protocol/session_proof_test.go @@ -116,6 +116,85 @@ func garbageOrdinaryPacket(t *testing.T, srcID, destID node.ID) []byte { return data } +// authenticPacket encrypts msg with the session key so it decrypts successfully, +// letting a test vary only the source address. +func authenticPacket(t *testing.T, sess *session.Session, srcID, destID node.ID, msg Message) []byte { + t.Helper() + + msgBytes, err := msg.Encode() + if err != nil { + t.Fatalf("msg.Encode: %v", err) + } + plaintext := make([]byte, 1+len(msgBytes)) + plaintext[0] = msg.Type() + copy(plaintext[1:], msgBytes) + + nonce := make([]byte, 12) + for i := range nonce { + nonce[i] = byte(i + 9) + } + + authdata := srcID[:] + maskingIV, headerData, err := BuildOrdinaryHeaderData(srcID, nonce, authdata) + if err != nil { + t.Fatalf("BuildOrdinaryHeaderData: %v", err) + } + + ciphertext, err := session.EncryptMessage(sess.DecryptionKey(), nonce, headerData, plaintext) + if err != nil { + t.Fatalf("EncryptMessage: %v", err) + } + + data, err := EncodeOrdinaryPacket(srcID, destID, maskingIV, nonce, authdata, ciphertext) + if err != nil { + t.Fatalf("EncodeOrdinaryPacket: %v", err) + } + return data +} + +// A session peer can send a correctly encrypted PONG from a forged source, and +// that source is what feeds the distinct-reporter threshold in IP discovery. The +// vote must require the PONG to come from the address the PING was sent to, while +// delivery stays source-agnostic so NAT rebinding still works. +func TestPongVoteRequiresRequestDestination(t *testing.T) { + for _, tc := range []struct { + name string + fromAddr *net.UDPAddr + wantVote bool + }{ + {"from the pinged address", &net.UDPAddr{IP: net.IPv4(198, 51, 100, 5), Port: 30303}, true}, + {"from a forged source", &net.UDPAddr{IP: net.IPv4(203, 0, 113, 9), Port: 40404}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + h, _, cache, cancel := sessionHandler(t) + defer cancel() + + pingedAddr := &net.UDPAddr{IP: net.IPv4(198, 51, 100, 5), Port: 30303} + peer, sess := victimSession(t, cache, pingedAddr) + + votes := 0 + h.config.OnPongReceived = func(node.ID, net.IP, net.IP, uint16) { votes++ } + + requestID := []byte{0x07, 0x08} + h.requests.AddRequest(requestID, peer, &Ping{RequestID: requestID}, pingedAddr) + + pong := &Pong{RequestID: requestID, IP: []byte{9, 9, 9, 9}, Port: 30303} + data := authenticPacket(t, sess, peer.ID(), h.config.LocalNode.ID(), pong) + + if err := h.HandleIncomingPacket(data, tc.fromAddr, pingedAddr); err != nil { + t.Fatalf("HandleIncomingPacket: %v", err) + } + + if tc.wantVote && votes != 1 { + t.Fatalf("votes = %d for a PONG from the pinged address, want 1", votes) + } + if !tc.wantVote && votes != 0 { + t.Fatalf("votes = %d for a PONG from a forged source, want 0", votes) + } + }) + } +} + // A node ID is public information from an ENR, so an attacker who knows only that // must not be able to move a peer's session to an address of their choosing. func TestSpoofedOrdinaryPacketDoesNotMigrateSession(t *testing.T) { @@ -213,7 +292,7 @@ func TestSpoofedWhoareyouWithPendingRequestKeepsKeys(t *testing.T) { keysBefore := string(sess.EncryptionKey()) requestID := []byte{0x01, 0x02, 0x03, 0x04} - h.requests.AddRequest(requestID, victim, &Ping{RequestID: requestID}) + h.requests.AddRequest(requestID, victim, &Ping{RequestID: requestID}, victimAddr) nonce := make([]byte, 12) for i := range nonce { diff --git a/discv5/service.go b/discv5/service.go index 0b64d1f..9e4ec15 100644 --- a/discv5/service.go +++ b/discv5/service.go @@ -367,10 +367,12 @@ func (s *Service) TalkReq(n *node.Node, protocolName string, request []byte) ([] Request: request, } + destAddr := n.Addr() + // Register pending request and send - respChan := s.handler.Requests().AddRequest(requestID, n, talkReq) + respChan := s.handler.Requests().AddRequest(requestID, n, talkReq, destAddr) - if err := s.handler.SendMessage(talkReq, n.ID(), n.Addr(), n); err != nil { + if err := s.handler.SendMessage(talkReq, n.ID(), destAddr, n); err != nil { s.handler.Requests().CancelRequest(requestID) return nil, fmt.Errorf("failed to send talkreq: %w", err) } From 3b1fd31b70e7ffec79789d37552c08f7f9f5eb04 Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Wed, 29 Jul 2026 12:13:37 -0500 Subject: [PATCH 13/49] fix(discv4): promote a node's address only on a proven endpoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit getOrCreateNode rewrote a known node's canonical address from every inbound packet's source, before any handler checked expiration or solicitation. That address is what every sender reads and what sendNeighbors republishes, and nodes.NewFromV4 stores the handler's own object — so a peer could name a third party in NEIGHBORS at an address of its choosing and we would republish it to every FINDNODE querier. HandlePacket also refreshed liveness and fired OnNodeSeen before validation, and OnNodeSeen can emit PING/ENRREQUEST traffic. - lookupOrCreateNode uses the address only when creating an unknown node, and is now read-locked on the hit path. handleNeighbors uses it too, so a claimed record can no longer move a third party's address. - promoteAddr installs the address, called only from handlePong for the destination a matched PING was sent to. It promotes req.ToNode as well as fromNode: ping/lookup build v4 nodes ad hoc, so the object a sender read can differ from the one the handler resolved. - noteSeen/noteProven split the pre-dispatch work. noteSeen runs in every handler right after the expiration check — liveness is identity-scoped and the signature authenticates the identity, and withholding it would evict a peer that is actively signing packets but whose bond has lapsed, which then returns with no proven addresses at all. noteProven adds OnNodeSeen and sits behind each handler's proof gate. handlePing's reciprocal PING now targets the source it just ponged rather than the canonical address. Without this the split blackholes any peer that moves: it would be pinged only at its old address, never answer, never bond, and be refused service permanently. This is not a new lever — PINGs are signed, so an attacker can only ping as themselves, and a spoofed source yields one PONG plus one rate-limited PING at the victim, neither larger than the trigger. Scope: this migrates the handler's node, not the routing table's copy. nodes.Node stores addr as a snapshot taken at admission and has no production SetAddr caller, so table-driven pings keep using the frozen value exactly as before. Fixing that needs nodes/iplimit.go re-keyed, since it counts per IP at Add time; tracked separately. Adds discv4/protocol/handle_packet_test.go — the first coverage HandlePacket has ever had, including the moved-peer regression test. --- discv4/protocol/handle_packet_test.go | 200 ++++++++++++++++++++++ discv4/protocol/handler.go | 136 ++++++++++++--- discv4/protocol/handler_test.go | 8 +- discv4/protocol/pending_neighbors_test.go | 2 +- 4 files changed, 313 insertions(+), 33 deletions(-) create mode 100644 discv4/protocol/handle_packet_test.go diff --git a/discv4/protocol/handle_packet_test.go b/discv4/protocol/handle_packet_test.go new file mode 100644 index 0000000..d621241 --- /dev/null +++ b/discv4/protocol/handle_packet_test.go @@ -0,0 +1,200 @@ +package protocol + +import ( + "crypto/ecdsa" + "net" + "testing" + "time" + + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethpandaops/bootnodoor/discv4/node" +) + +// encodeFrom builds a real signed packet from a peer, so tests can drive +// HandlePacket end to end rather than calling handlers directly. +func encodeFrom(t *testing.T, key *ecdsa.PrivateKey, msg Packet) ([]byte, []byte) { + t.Helper() + data, hash, err := Encode(key, msg) + if err != nil { + t.Fatalf("Encode: %v", err) + } + return data, hash +} + +func packetHandler(t *testing.T) (*Handler, *recordingTransport, func()) { + h, tr, cancel := proofHandler(t) + return h, tr, cancel +} + +// A peer's claimed source address must not become the node's canonical address: +// every sender reads it and sendNeighbors republishes it, so an unauthenticated +// packet could otherwise steer our traffic and poison what we tell others. +func TestHandlePacketDoesNotMoveCanonicalAddress(t *testing.T) { + h, _, cancel := packetHandler(t) + defer cancel() + + peerKey, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("generate key: %v", err) + } + knownAddr := &net.UDPAddr{IP: net.IPv4(198, 51, 100, 5), Port: 30303} + n := node.New(&peerKey.PublicKey, knownAddr) + + h.nodesMu.Lock() + h.nodes[n.ID()] = n + h.nodesMu.Unlock() + + spoofed := &net.UDPAddr{IP: net.IPv4(203, 0, 113, 9), Port: 40404} + data, _ := encodeFrom(t, peerKey, &Findnode{ + Target: EncodePubkey(&peerKey.PublicKey), + Expiration: MakeExpiration(20 * time.Second), + }) + _ = h.HandlePacket(data, spoofed, testAddr()) + + if got := n.Addr().String(); got != knownAddr.String() { + t.Fatalf("canonical address moved to %s on an unauthenticated packet, want %s", got, knownAddr) + } +} + +// An expired packet must not refresh liveness or fire OnNodeSeen. The node has to +// pre-exist, because creating one stamps LastSeen. +func TestHandlePacketExpiredTouchesNothing(t *testing.T) { + h, _, cancel := packetHandler(t) + defer cancel() + + seen := 0 + h.config.OnNodeSeen = func(*node.Node, time.Time) { seen++ } + + peerKey, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("generate key: %v", err) + } + addr := &net.UDPAddr{IP: net.IPv4(198, 51, 100, 5), Port: 30303} + n := node.New(&peerKey.PublicKey, addr) + + h.nodesMu.Lock() + h.nodes[n.ID()] = n + h.nodesMu.Unlock() + + before := n.LastSeen() + time.Sleep(5 * time.Millisecond) + + data, _ := encodeFrom(t, peerKey, &Findnode{ + Target: EncodePubkey(&peerKey.PublicKey), + Expiration: uint64(time.Now().Add(-time.Minute).Unix()), + }) + _ = h.HandlePacket(data, addr, testAddr()) + + if !n.LastSeen().Equal(before) { + t.Error("expired packet refreshed liveness") + } + if seen != 0 { + t.Errorf("expired packet fired OnNodeSeen %d times", seen) + } +} + +// An unbonded FINDNODE is refused, so it must not admit the node either — that is +// the callback which can spawn outbound traffic toward an unproven address. +func TestHandlePacketUnbondedFindnodeDoesNotAdmit(t *testing.T) { + h, _, cancel := packetHandler(t) + defer cancel() + + seen := 0 + h.config.OnNodeSeen = func(*node.Node, time.Time) { seen++ } + + peerKey, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("generate key: %v", err) + } + addr := &net.UDPAddr{IP: net.IPv4(198, 51, 100, 5), Port: 30303} + + data, _ := encodeFrom(t, peerKey, &Findnode{ + Target: EncodePubkey(&peerKey.PublicKey), + Expiration: MakeExpiration(20 * time.Second), + }) + _ = h.HandlePacket(data, addr, testAddr()) + + if seen != 0 { + t.Fatalf("unbonded FINDNODE fired OnNodeSeen %d times, want 0", seen) + } +} + +// The blackhole regression test. Removing the address rewrite means a peer that +// moves is only reachable if the reciprocal PING goes to the source we just +// ponged; otherwise it is pinged at its old address forever, never bonds, and is +// refused service permanently. +func TestHandlePacketMovedPeerRebondsAtNewAddress(t *testing.T) { + h, tr, cancel := packetHandler(t) + defer cancel() + + peerKey, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("generate key: %v", err) + } + oldAddr := &net.UDPAddr{IP: net.IPv4(198, 51, 100, 5), Port: 30303} + newAddr := &net.UDPAddr{IP: net.IPv4(198, 51, 100, 77), Port: 30303} + n := node.New(&peerKey.PublicKey, oldAddr) + + h.nodesMu.Lock() + h.nodes[n.ID()] = n + h.nodesMu.Unlock() + + // The peer pings us from its new address. + data, _ := encodeFrom(t, peerKey, &Ping{ + Version: 4, + From: NewEndpoint(newAddr, 0), + To: NewEndpoint(testAddr(), 0), + Expiration: MakeExpiration(20 * time.Second), + }) + if err := h.HandlePacket(data, newAddr, testAddr()); err != nil { + t.Fatalf("HandlePacket(ping): %v", err) + } + + // The reciprocal PING is sent from a goroutine. + deadline := time.Now().Add(2 * time.Second) + var pingHash []byte + for time.Now().Before(deadline) { + if req := h.findPendingPingTo(n.ID(), newAddr); req != nil { + pingHash = req.RequestHash + break + } + time.Sleep(5 * time.Millisecond) + } + if pingHash == nil { + t.Fatalf("no PING was sent to the peer's new address; destinations: %v", tr.sent) + } + + // Its PONG from the new address proves the endpoint. + pongData, _ := encodeFrom(t, peerKey, &Pong{ + To: NewEndpoint(testAddr(), 0), + ReplyTok: pingHash, + Expiration: MakeExpiration(20 * time.Second), + }) + if err := h.HandlePacket(pongData, newAddr, testAddr()); err != nil { + t.Fatalf("HandlePacket(pong): %v", err) + } + + if !n.IsBondedFrom(newAddr) { + t.Error("peer did not bond at its new address") + } + if got := n.Addr().String(); got != newAddr.String() { + t.Errorf("canonical address = %s after a proven PONG, want %s", got, newAddr) + } +} + +// findPendingPingTo reports the pending PING sent to addr, for tests that need the +// reply token of a PING the handler emitted itself. +func (h *Handler) findPendingPingTo(id node.ID, addr *net.UDPAddr) *PendingRequest { + h.requestsMu.RLock() + defer h.requestsMu.RUnlock() + + for _, reqs := range h.requests { + for _, req := range reqs { + if req.PacketType == PingPacket && req.ToNode != nil && req.ToNode.ID() == id && + req.DestIP != nil && req.DestIP.Equal(addr.IP) { + return req + } + } + } + return nil +} diff --git a/discv4/protocol/handler.go b/discv4/protocol/handler.go index 8c86d10..1e13a83 100644 --- a/discv4/protocol/handler.go +++ b/discv4/protocol/handler.go @@ -142,8 +142,8 @@ type PendingRequest struct { // DestIP is the IP the request was sent to, snapshotted at send time. // ToNode.Addr() is unusable for verifying a response's origin because - // getOrCreateNode rewrites it from every inbound packet, including the - // spoofed one a response check is meant to catch. + // lookupOrCreateNode deliberately never rewrites it, but the node object can + // still be re-addressed by a proven promotion between send and response. DestIP net.IP // PacketType is the type of request @@ -282,17 +282,11 @@ func (h *Handler) HandlePacket(data []byte, from *net.UDPAddr, localAddr *net.UD fromNodeID := node.PubkeyToID(pubkey) - // Get or create node - fromNode := h.getOrCreateNode(fromNodeID, pubkey, from) - - // Update last seen - fromNode.UpdateLastSeen() - fromNode.IncrementPacketsReceived() - - // Call OnNodeSeen callback - if h.config.OnNodeSeen != nil { - h.config.OnNodeSeen(fromNode, time.Now()) - } + // Look up the node without promoting this packet's source to its canonical + // address, and without touching liveness or firing OnNodeSeen: none of that is + // warranted before the handler has checked expiration and solicitation. Each + // handler states its own gate and calls noteSeen/noteProven itself. + fromNode := h.lookupOrCreateNode(fromNodeID, pubkey, from) // Dispatch by packet type switch p := packet.(type) { @@ -329,6 +323,15 @@ func (h *Handler) handlePing(fromNode *node.Node, from *net.UDPAddr, localAddr * return ErrExpired } + h.noteSeen(fromNode) + + // Admission and its outbound traffic need a proven source; an already-bonded + // peer has one. Otherwise the reciprocal PING below proves it a moment later + // and its PONG runs noteProven then. + if fromNode.IsBondedFrom(from) { + h.noteProven(fromNode) + } + // Mark ping received fromNode.MarkPingReceived() @@ -361,9 +364,11 @@ func (h *Handler) handlePing(fromNode *node.Node, from *net.UDPAddr, localAddr * // Only spawn goroutine if we're actually going to ping (don't create unnecessary goroutines) if timeSinceLastPing > 100*time.Millisecond { - // Send PING back in goroutine to establish bidirectional bond + // Ping the source we just ponged, not the canonical address: a peer that + // moved is only reachable at its new address, and its PONG from there is + // what proves the new endpoint. go func() { - if _, err := h.Ping(fromNode); err != nil { + if _, err := h.pingTo(fromNode, from); err != nil { logrus.WithFields(logrus.Fields{ "node_id": fmt.Sprintf("%x", fromNode.IDBytes()[:8]), "error": err, @@ -389,6 +394,8 @@ func (h *Handler) handlePong(fromNode *node.Node, from *net.UDPAddr, pong *Pong) return ErrExpired } + h.noteSeen(fromNode) + // Nothing below may run for a PONG we did not solicit from this address: it // establishes a bond, casts a vote in the external-IP election that rewrites // our published ENR, and can trigger outbound ENR traffic. @@ -399,7 +406,10 @@ func (h *Handler) handlePong(fromNode *node.Node, from *net.UDPAddr, pong *Pong) // Bind the bond to the address we proved, not the packet's source. provenAddr := &net.UDPAddr{IP: req.DestIP, Port: from.Port} + h.promoteAddr(fromNode, provenAddr) + h.promoteAddr(req.ToNode, provenAddr) fromNode.MarkPongReceived(h.config.BondExpiration, provenAddr) + h.noteProven(fromNode) // The To field in PONG contains our address as seen by the remote peer. if h.config.OnPongReceived != nil && pong.To.IP != nil && pong.To.UDP > 0 { @@ -433,6 +443,8 @@ func (h *Handler) handleFindnode(fromNode *node.Node, from *net.UDPAddr, localAd return ErrExpired } + h.noteSeen(fromNode) + // Bonded at this source address specifically: a bond earned elsewhere would // let a spoofed source have us reflect NEIGHBORS at a third party. if !fromNode.IsBondedFrom(from) { @@ -442,6 +454,7 @@ func (h *Handler) handleFindnode(fromNode *node.Node, from *net.UDPAddr, localAd return fmt.Errorf("node not bonded") } + h.noteProven(fromNode) h.incrementFindnodeRequestsRecv() // Call callback to get nodes @@ -469,6 +482,8 @@ func (h *Handler) handleNeighbors(fromNode *node.Node, from *net.UDPAddr, neighb return ErrExpired } + h.noteSeen(fromNode) + // Only accept NEIGHBORS in response to a FINDNODE we actually sent to this // address. Dropping unsolicited NEIGHBORS prevents a peer we never queried // from making us accumulate node records without bound; requiring the source @@ -481,6 +496,7 @@ func (h *Handler) handleNeighbors(fromNode *node.Node, from *net.UDPAddr, neighb // Counted after the gate: this reports responses to our queries, so counting // unsolicited packets here would let any peer inflate it. + h.noteProven(fromNode) h.incrementFindnodeResponsesRecv() // Accumulate the response, keyed by the matched request's hash so each @@ -522,7 +538,7 @@ func (h *Handler) handleNeighbors(fromNode *node.Node, from *net.UDPAddr, neighb } nodeID := node.PubkeyToID(pubkey) - nodes = append(nodes, h.getOrCreateNode(nodeID, pubkey, addr)) + nodes = append(nodes, h.lookupOrCreateNode(nodeID, pubkey, addr)) } h.pendingNeighborsMu.Lock() @@ -601,6 +617,8 @@ func (h *Handler) handleENRResponse(fromNode *node.Node, from *net.UDPAddr, resp "enr_seq": resp.Record.Seq(), }).Debug("Received ENRRESPONSE") + h.noteSeen(fromNode) + // Only a response to an ENRREQUEST we actually sent to this address may touch // any state: ENRRESPONSE carries no expiration, so an unsolicited replay could // otherwise roll the node back to an older record. The type and destination @@ -627,6 +645,10 @@ func (h *Handler) handleENRResponse(fromNode *node.Node, from *net.UDPAddr, resp fromNode.UpdateENR(resp.Record) + // After UpdateENR, so OnNodeSeen sees the record and admits the node instead of + // requesting an ENR it already has. + h.noteProven(fromNode) + for _, req := range reqs { h.deliverResponse(req, resp.Record) } @@ -636,12 +658,18 @@ func (h *Handler) handleENRResponse(fromNode *node.Node, from *net.UDPAddr, resp // Sending Methods -// Ping sends a PING request to a node. +// Ping sends a PING request to a node at its canonical address. func (h *Handler) Ping(n *node.Node) (*Pong, error) { - // Read the address once: inbound packets rewrite it, and the endpoint proof - // requires the recorded destination to be the one we actually sent to. - destAddr := n.Addr() + return h.pingTo(n, n.Addr()) +} +// pingTo sends a PING to an explicit destination. +// +// handlePing uses it to ping back the source it just ponged, rather than the +// node's canonical address. That is what lets a peer which moved re-prove its new +// endpoint: without it, a moved peer would be pinged only at its old address, never +// answer, and so never bond or be served again. +func (h *Handler) pingTo(n *node.Node, destAddr *net.UDPAddr) (*Pong, error) { // Build PING message ping := &Ping{ Version: 4, @@ -912,17 +940,27 @@ func (h *Handler) sendENRResponse(to *node.Node, addr *net.UDPAddr, localAddr *n // Node Management -// getOrCreateNode gets an existing node or creates a new one. -func (h *Handler) getOrCreateNode(id node.ID, pubkey *ecdsa.PublicKey, addr *net.UDPAddr) *node.Node { +// lookupOrCreateNode returns the tracked node for id, creating one at addr if the +// id is unknown. +// +// addr is used ONLY when creating: an existing node's canonical address is never +// rewritten here, because addr is either an unauthenticated packet source or an +// address a peer claimed in a NEIGHBORS record. Every sender reads that address, +// and sendNeighbors republishes it, so letting either source set it would steer +// our outbound traffic and let a peer poison what we publish about a third party. +// Only promoteAddr, on a proven endpoint, may move it. +func (h *Handler) lookupOrCreateNode(id node.ID, pubkey *ecdsa.PublicKey, addr *net.UDPAddr) *node.Node { + h.nodesMu.RLock() + n, exists := h.nodes[id] + h.nodesMu.RUnlock() + if exists { + return n + } + h.nodesMu.Lock() defer h.nodesMu.Unlock() - n, exists := h.nodes[id] - if exists { - // Update address if changed - if n.Addr().String() != addr.String() { - n.SetAddr(addr) - } + if n, exists := h.nodes[id]; exists { return n } @@ -944,6 +982,48 @@ func (h *Handler) getOrCreateNode(id node.ID, pubkey *ecdsa.PublicKey, addr *net return n } +// promoteAddr installs a proven endpoint as n's canonical address. +// +// Only handlePong may call this, and only for the address a matched PING was sent +// to — see lookupOrCreateNode for why nothing else may move it. +func (h *Handler) promoteAddr(n *node.Node, proven *net.UDPAddr) { + if n == nil || proven == nil || proven.IP == nil { + return + } + if n.Addr().String() == proven.String() { + return + } + + n.SetAddr(proven) + + logrus.WithFields(logrus.Fields{ + "node_id": fmt.Sprintf("%x", n.IDBytes()[:8]), + "addr": proven.String(), + }).Debug("promoted proven endpoint to canonical address") +} + +// noteSeen refreshes identity-scoped liveness. +// +// Safe for any non-expired packet: the signature authenticates the identity, so a +// peer can only refresh its own liveness. Withholding it until the source is +// proven would evict a peer that is actively signing packets but whose bond has +// lapsed, and it would come back with no proven addresses at all. +func (h *Handler) noteSeen(n *node.Node) { + n.UpdateLastSeen() + n.IncrementPacketsReceived() +} + +// noteProven is noteSeen plus OnNodeSeen, which admits the node to the routing +// table and can spawn outbound PING/ENRREQUEST traffic toward it. It requires a +// proven or solicited source. +func (h *Handler) noteProven(n *node.Node) { + h.noteSeen(n) + + if h.config.OnNodeSeen != nil { + h.config.OnNodeSeen(n, time.Now()) + } +} + // GetNode returns a node by ID. func (h *Handler) GetNode(id node.ID) *node.Node { h.nodesMu.RLock() diff --git a/discv4/protocol/handler_test.go b/discv4/protocol/handler_test.go index 5a86262..bc90d8a 100644 --- a/discv4/protocol/handler_test.go +++ b/discv4/protocol/handler_test.go @@ -39,7 +39,7 @@ func TestGetOrCreateNodeRespectsMaxNodes(t *testing.T) { for i := 0; i < maxNodes*5; i++ { pub, id := makeNodeID(t) - h.getOrCreateNode(id, pub, testAddr()) + h.lookupOrCreateNode(id, pub, testAddr()) } if got := len(h.AllNodes()); got != maxNodes { @@ -56,10 +56,10 @@ func TestCleanupEvictsStaleUnbondedNodes(t *testing.T) { h := NewHandler(ctx, HandlerConfig{MaxNodes: 1000, NodeTTL: 20 * time.Millisecond}, nil) pubStale, idStale := makeNodeID(t) - h.getOrCreateNode(idStale, pubStale, testAddr()) + h.lookupOrCreateNode(idStale, pubStale, testAddr()) pubBonded, idBonded := makeNodeID(t) - bonded := h.getOrCreateNode(idBonded, pubBonded, testAddr()) + bonded := h.lookupOrCreateNode(idBonded, pubBonded, testAddr()) bonded.MarkPongReceived(time.Hour, testAddr()) // establish a live bond time.Sleep(40 * time.Millisecond) // age both past NodeTTL @@ -84,7 +84,7 @@ func TestCleanupReclaimsFloodedNodes(t *testing.T) { for i := 0; i < 500; i++ { pub, id := makeNodeID(t) - h.getOrCreateNode(id, pub, testAddr()) + h.lookupOrCreateNode(id, pub, testAddr()) } if got := len(h.AllNodes()); got != 500 { t.Fatalf("setup: expected 500 tracked nodes, got %d", got) diff --git a/discv4/protocol/pending_neighbors_test.go b/discv4/protocol/pending_neighbors_test.go index 6104d21..b2a556c 100644 --- a/discv4/protocol/pending_neighbors_test.go +++ b/discv4/protocol/pending_neighbors_test.go @@ -185,7 +185,7 @@ func TestFreshNodeSurvivesCleanup(t *testing.T) { t.Fatalf("generate key: %v", err) } id := node.PubkeyToID(&key.PublicKey) - h.getOrCreateNode(id, &key.PublicKey, &net.UDPAddr{IP: net.IPv4(9, 9, 9, 9), Port: 30303}) + h.lookupOrCreateNode(id, &key.PublicKey, &net.UDPAddr{IP: net.IPv4(9, 9, 9, 9), Port: 30303}) h.cleanup() From ee7fd2cb3afffb70f285967d32e244b83e6b36c6 Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Wed, 29 Jul 2026 12:28:36 -0500 Subject: [PATCH 14/49] fix(discv5,discv4): address review of the endpoint-proof commits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Cache.Put evicted whenever the map was full, without checking whether the key already existed. fa30b68 made that reachable: handshake recovery now retains the stale session instead of deleting it first, so replacing it at capacity evicted an unrelated live peer and left the map one short. Only evict when the Put actually grows the map. Adds the first tests for discv5/session. - handleENRRequest called neither noteSeen nor noteProven, so accepted ENRREQUEST traffic stopped refreshing LastSeen and never re-admitted the peer, letting an actively communicating node age out. It now matches every other handler: noteSeen after the expiration check, noteProven after the bond gate. - Widen the sent-nonce window from 16 to 64. Too small silently drops a legitimate peer's restart recovery until its next packet; the window only has to cover what we can send to one peer within a request lifetime. Noted, not changed: GetPendingRequestForNode picks an arbitrary pending request for the peer, so with concurrent requests in flight recovery can replay one the challenge did not refer to. That predates these commits — the nonce check gates whether recovery runs at all, not which request it carries. --- discv4/protocol/handler.go | 4 +++ discv5/session/cache.go | 7 ++-- discv5/session/cache_test.go | 70 ++++++++++++++++++++++++++++++++++++ discv5/session/session.go | 7 ++-- 4 files changed, 84 insertions(+), 4 deletions(-) create mode 100644 discv5/session/cache_test.go diff --git a/discv4/protocol/handler.go b/discv4/protocol/handler.go index 1e13a83..87fbaa1 100644 --- a/discv4/protocol/handler.go +++ b/discv4/protocol/handler.go @@ -585,6 +585,8 @@ func (h *Handler) handleENRRequest(fromNode *node.Node, from *net.UDPAddr, local return ErrExpired } + h.noteSeen(fromNode) + // IMPORTANT: Check if node is bonded (bidirectional bond required) // This prevents amplification attacks and matches reth's behavior. // Only respond to ENRRequest if we've established a bidirectional bond: @@ -598,6 +600,8 @@ func (h *Handler) handleENRRequest(fromNode *node.Node, from *net.UDPAddr, local return fmt.Errorf("node not bonded") } + h.noteProven(fromNode) + // Call callback if h.config.OnENRRequest != nil { if err := h.config.OnENRRequest(fromNode); err != nil { diff --git a/discv5/session/cache.go b/discv5/session/cache.go index 1ffd44f..792e938 100644 --- a/discv5/session/cache.go +++ b/discv5/session/cache.go @@ -109,8 +109,11 @@ func (c *Cache) Put(session *Session) { c.mu.Lock() defer c.mu.Unlock() - // Check if we need to evict - if len(c.sessions) >= c.maxSessions { + // Replacing an existing key frees no slot, so only evict when this Put grows + // the map. Handshake recovery replaces a retained session by node ID; without + // this check it would evict an unrelated live peer and leave the map short. + _, replacing := c.sessions[session.RemoteID] + if !replacing && len(c.sessions) >= c.maxSessions { // Find and remove the least recently used session c.evictLRU() } diff --git a/discv5/session/cache_test.go b/discv5/session/cache_test.go new file mode 100644 index 0000000..c6854d5 --- /dev/null +++ b/discv5/session/cache_test.go @@ -0,0 +1,70 @@ +package session + +import ( + "net" + "testing" + "time" + + "github.com/ethpandaops/bootnodoor/discv5/node" + "github.com/sirupsen/logrus" +) + +func quietCache(t *testing.T, maxSessions int) *Cache { + t.Helper() + logger := logrus.New() + logger.SetLevel(logrus.PanicLevel) + return NewCache(maxSessions, time.Hour, logger) +} + +func testSession(id byte, addr *net.UDPAddr) *Session { + var nodeID node.ID + nodeID[0] = id + keys := &SessionKeys{ + InitiatorKey: []byte("0123456789abcdef"), + RecipientKey: []byte("fedcba9876543210"), + } + return NewSession(nodeID, addr, keys, false, time.Hour) +} + +// Replacing an existing session frees no slot, so it must not evict anyone. +// Handshake recovery replaces a retained session by node ID; evicting on that +// path would drop an unrelated live peer and leave the cache below capacity. +func TestPutReplacingDoesNotEvict(t *testing.T) { + cache := quietCache(t, 2) + addr := &net.UDPAddr{IP: net.IPv4(198, 51, 100, 1), Port: 30303} + + first := testSession(1, addr) + second := testSession(2, addr) + cache.Put(first) + cache.Put(second) + + if cache.Count() != 2 { + t.Fatalf("Count = %d after filling, want 2", cache.Count()) + } + + // Recovery touches the stale entry, then replaces it with fresh keys. + cache.Get(first.RemoteID) + cache.Put(testSession(1, addr)) + + if cache.Count() != 2 { + t.Errorf("Count = %d after replacing an existing session, want 2", cache.Count()) + } + if cache.Get(second.RemoteID) == nil { + t.Error("replacing one session evicted an unrelated peer") + } +} + +// Adding a genuinely new session at capacity must still evict, or the cache +// would grow without bound. +func TestPutNewAtCapacityEvicts(t *testing.T) { + cache := quietCache(t, 2) + addr := &net.UDPAddr{IP: net.IPv4(198, 51, 100, 1), Port: 30303} + + cache.Put(testSession(1, addr)) + cache.Put(testSession(2, addr)) + cache.Put(testSession(3, addr)) + + if cache.Count() > 2 { + t.Fatalf("Count = %d, want the cache bounded at 2", cache.Count()) + } +} diff --git a/discv5/session/session.go b/discv5/session/session.go index d4246fd..2c7d686 100644 --- a/discv5/session/session.go +++ b/discv5/session/session.go @@ -116,8 +116,11 @@ func (s *Session) SetNode(n *node.Node) { } // maxSentNonces bounds the remembered nonces. A WHOAREYOU answers a packet we -// sent moments ago, so only the most recent few can legitimately be referenced. -const maxSentNonces = 16 +// sent moments ago, so the window only has to cover the traffic we can send to +// one peer within a request lifetime; sized well above that, because being too +// small silently drops a legitimate peer's restart recovery until its next +// packet, while being generous costs a few hundred bytes per session. +const maxSentNonces = 64 // RecordSentNonce remembers the nonce of an ordinary packet we sent on this // session, so a WHOAREYOU claiming to answer it can be verified. From 73aff5664fe52eca2da5f57c43a5c9d5bd1c01b8 Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Wed, 29 Jul 2026 12:49:25 -0500 Subject: [PATCH 15/49] refactor: cleanups from the review of the endpoint-proof work - discv4 noteProven called noteSeen, but every call site already did, so IncrementPacketsReceived double-counted on most packets. It now only fires the callback, which is the decision it actually owns. - discv5 SendMessage and SendMessageFrom were 122 near-identical lines, and the RecordSentNonce fix had to be pasted into both. SendTo(data, to) is literally Send(data, to, nil), so SendMessage delegates. ~120 lines gone and the sent- nonce invariant has one home. - Delete discv5/ipdiscovery.go: an unreferenced, diverged copy of services/ipdiscovery.go, missing the distinct-reporter hardening and taking no reporter IP at all. The spoofable version of a function just hardened. - Delete getPendingRequests: no production caller, and the one lookup helper with no type or destination binding, i.e. the shape the rest of this work exists to remove. The two tests use pendingRequestsFrom. - Delete four callerless ForkDigestFilter accessors and an unreachable outcomeNotCL switch arm; drop the packetHandler alias for proofHandler; make nodes.ENR delegate to Record rather than repeat it. Efficiency, both measured on the paths they sit on: - enr.EncodeRLPBytes took the write lock to read its cache, costing 8.6x under contention at 10 cores. A bootnode serves the same records to every requester, so those are exactly the contended ones. Read path now takes RLock. - discv5 compared session and packet addresses via String() on every authenticated packet: 77ns/6 allocs against a 271ns decrypt. Now compares IP and port directly, 1.9ns/0 allocs. Also trims comments: the "unauthenticated source must not become canonical" invariant was written out four times; it now lives on lookupOrCreateNode with the others pointing at it. --- bootnode/clconfig/filter.go | 33 -- discv4/protocol/handle_packet_test.go | 13 +- discv4/protocol/handler.go | 37 +- discv4/protocol/pending_request_test.go | 2 +- discv4/protocol/response_delivery_test.go | 2 +- discv5/ipdiscovery.go | 479 ---------------------- discv5/protocol/handler.go | 123 +----- discv5/session/session.go | 2 +- enr/encoding.go | 10 + nodes/node.go | 6 +- 10 files changed, 33 insertions(+), 674 deletions(-) delete mode 100644 discv5/ipdiscovery.go diff --git a/bootnode/clconfig/filter.go b/bootnode/clconfig/filter.go index 8768b2c..0e2ae69 100644 --- a/bootnode/clconfig/filter.go +++ b/bootnode/clconfig/filter.go @@ -225,7 +225,6 @@ func (f *ForkDigestFilter) recordOutcome(outcome clOutcome, forkDigest ForkDiges f.logger.Debugf("Rejected node: unknown fork digest %s (current: %s, %d historical digests known)", forkDigest.String(), f.currentForkDigest.String(), len(f.historicalDigests)) } - case outcomeNotCL: } } @@ -452,38 +451,6 @@ func (f *ForkDigestFilter) GetOldDigests() map[string]time.Duration { return result } -// GetAcceptedCurrent returns the count of nodes accepted with current fork digest. -func (f *ForkDigestFilter) GetAcceptedCurrent() int { - f.mu.RLock() - defer f.mu.RUnlock() - - return f.acceptedCurrent -} - -// GetAcceptedOld returns the count of nodes accepted with old fork digests. -func (f *ForkDigestFilter) GetAcceptedOld() int { - f.mu.RLock() - defer f.mu.RUnlock() - - return f.acceptedOld -} - -// GetRejectedInvalid returns the count of nodes rejected due to invalid fork digest. -func (f *ForkDigestFilter) GetRejectedInvalid() int { - f.mu.RLock() - defer f.mu.RUnlock() - - return f.rejectedInvalid -} - -// GetTotalChecks returns the total number of filter checks performed. -func (f *ForkDigestFilter) GetTotalChecks() int { - f.mu.RLock() - defer f.mu.RUnlock() - - return f.totalChecks -} - // GetPreviousForkDigest returns the previous fork digest as a hex string. func (f *ForkDigestFilter) GetPreviousForkDigest() string { return f.config.GetPreviousForkDigest().String() diff --git a/discv4/protocol/handle_packet_test.go b/discv4/protocol/handle_packet_test.go index d621241..0418c84 100644 --- a/discv4/protocol/handle_packet_test.go +++ b/discv4/protocol/handle_packet_test.go @@ -21,16 +21,11 @@ func encodeFrom(t *testing.T, key *ecdsa.PrivateKey, msg Packet) ([]byte, []byte return data, hash } -func packetHandler(t *testing.T) (*Handler, *recordingTransport, func()) { - h, tr, cancel := proofHandler(t) - return h, tr, cancel -} - // A peer's claimed source address must not become the node's canonical address: // every sender reads it and sendNeighbors republishes it, so an unauthenticated // packet could otherwise steer our traffic and poison what we tell others. func TestHandlePacketDoesNotMoveCanonicalAddress(t *testing.T) { - h, _, cancel := packetHandler(t) + h, _, cancel := proofHandler(t) defer cancel() peerKey, err := crypto.GenerateKey() @@ -59,7 +54,7 @@ func TestHandlePacketDoesNotMoveCanonicalAddress(t *testing.T) { // An expired packet must not refresh liveness or fire OnNodeSeen. The node has to // pre-exist, because creating one stamps LastSeen. func TestHandlePacketExpiredTouchesNothing(t *testing.T) { - h, _, cancel := packetHandler(t) + h, _, cancel := proofHandler(t) defer cancel() seen := 0 @@ -96,7 +91,7 @@ func TestHandlePacketExpiredTouchesNothing(t *testing.T) { // An unbonded FINDNODE is refused, so it must not admit the node either — that is // the callback which can spawn outbound traffic toward an unproven address. func TestHandlePacketUnbondedFindnodeDoesNotAdmit(t *testing.T) { - h, _, cancel := packetHandler(t) + h, _, cancel := proofHandler(t) defer cancel() seen := 0 @@ -124,7 +119,7 @@ func TestHandlePacketUnbondedFindnodeDoesNotAdmit(t *testing.T) { // ponged; otherwise it is pinged at its old address forever, never bonds, and is // refused service permanently. func TestHandlePacketMovedPeerRebondsAtNewAddress(t *testing.T) { - h, tr, cancel := packetHandler(t) + h, tr, cancel := proofHandler(t) defer cancel() peerKey, err := crypto.GenerateKey() diff --git a/discv4/protocol/handler.go b/discv4/protocol/handler.go index 87fbaa1..1e1be30 100644 --- a/discv4/protocol/handler.go +++ b/discv4/protocol/handler.go @@ -140,10 +140,9 @@ type PendingRequest struct { // ToNode is the destination node ToNode *node.Node - // DestIP is the IP the request was sent to, snapshotted at send time. - // ToNode.Addr() is unusable for verifying a response's origin because - // lookupOrCreateNode deliberately never rewrites it, but the node object can - // still be re-addressed by a proven promotion between send and response. + // DestIP is the IP the request was sent to, snapshotted at send time; see + // lookupOrCreateNode. ToNode.Addr() cannot serve here because promoteAddr can + // move it between send and response. DestIP net.IP // PacketType is the type of request @@ -362,7 +361,6 @@ func (h *Handler) handlePing(fromNode *node.Node, from *net.UDPAddr, localAddr * lastPingSent := fromNode.LastPingSent() timeSinceLastPing := time.Since(lastPingSent) - // Only spawn goroutine if we're actually going to ping (don't create unnecessary goroutines) if timeSinceLastPing > 100*time.Millisecond { // Ping the source we just ponged, not the canonical address: a peer that // moved is only reachable at its new address, and its PONG from there is @@ -986,10 +984,9 @@ func (h *Handler) lookupOrCreateNode(id node.ID, pubkey *ecdsa.PublicKey, addr * return n } -// promoteAddr installs a proven endpoint as n's canonical address. -// -// Only handlePong may call this, and only for the address a matched PING was sent -// to — see lookupOrCreateNode for why nothing else may move it. +// promoteAddr installs a proven endpoint as n's canonical address. Only +// handlePong may call it, for the address a matched PING was sent to; see +// lookupOrCreateNode for why nothing else may move it. func (h *Handler) promoteAddr(n *node.Node, proven *net.UDPAddr) { if n == nil || proven == nil || proven.IP == nil { return @@ -1017,12 +1014,11 @@ func (h *Handler) noteSeen(n *node.Node) { n.IncrementPacketsReceived() } -// noteProven is noteSeen plus OnNodeSeen, which admits the node to the routing -// table and can spawn outbound PING/ENRREQUEST traffic toward it. It requires a -// proven or solicited source. +// noteProven fires OnNodeSeen, which admits the node to the routing table and can +// spawn outbound PING/ENRREQUEST traffic toward it. It requires a proven or +// solicited source, so it sits behind each handler's gate while noteSeen runs +// ahead of it. func (h *Handler) noteProven(n *node.Node) { - h.noteSeen(n) - if h.config.OnNodeSeen != nil { h.config.OnNodeSeen(n, time.Now()) } @@ -1058,9 +1054,8 @@ func requestKey(hash []byte, id node.ID) string { // addPendingRequest registers a new pending request. A second FINDNODE to a // peer with one already in flight is rejected: NEIGHBORS carries no reply // token, so two in-flight FINDNODEs to one peer cannot be told apart. -// destAddr must be the address the caller sends the packet to, captured once: -// toNode.Addr() is rewritten by concurrent inbound packets, so reading it here -// can record an endpoint the request never went to. +// destAddr must be the address the caller sends the packet to, captured once, +// so the recorded and actual destinations cannot diverge. func (h *Handler) addPendingRequest(hash []byte, toNode *node.Node, packetType byte, destAddr *net.UDPAddr) (*PendingRequest, error) { var destIP net.IP if destAddr != nil && destAddr.IP != nil { @@ -1090,14 +1085,6 @@ func (h *Handler) addPendingRequest(hash []byte, toNode *node.Node, packetType b return req, nil } -// getPendingRequests returns the pending requests matching a reply token and -// its sender, so a response can only resolve requests sent to that peer. -func (h *Handler) getPendingRequests(replyTok []byte, id node.ID) []*PendingRequest { - h.requestsMu.RLock() - defer h.requestsMu.RUnlock() - return append([]*PendingRequest(nil), h.requests[requestKey(replyTok, id)]...) -} - // pendingRequestsFrom returns the pending requests of the given type that match // this reply token and were sent to this address. func (h *Handler) pendingRequestsFrom(replyTok []byte, id node.ID, from *net.UDPAddr, packetType byte) []*PendingRequest { diff --git a/discv4/protocol/pending_request_test.go b/discv4/protocol/pending_request_test.go index 7f57876..bf09332 100644 --- a/discv4/protocol/pending_request_test.go +++ b/discv4/protocol/pending_request_test.go @@ -116,7 +116,7 @@ func TestSamePeerDuplicateRequestsBothComplete(t *testing.T) { } h.removePendingRequest(req1) - if got := len(h.getPendingRequests(hash, n.ID())); got != 1 { + if got := len(h.pendingRequestsFrom(hash, n.ID(), n.Addr(), ENRRequestPacket)); got != 1 { t.Fatalf("after removing one duplicate, %d pending remain, want 1", got) } diff --git a/discv4/protocol/response_delivery_test.go b/discv4/protocol/response_delivery_test.go index 4bd37d8..1b1d651 100644 --- a/discv4/protocol/response_delivery_test.go +++ b/discv4/protocol/response_delivery_test.go @@ -89,7 +89,7 @@ func TestDuplicateResponsesWaiterGetsOneNoLeak(t *testing.T) { for i := 0; i < dups; i++ { go func() { defer wg.Done() - for _, r := range h.getPendingRequests(hash, to.ID()) { + for _, r := range h.pendingRequestsFrom(hash, to.ID(), to.Addr(), PingPacket) { h.deliverResponse(r, "pong") } }() diff --git a/discv5/ipdiscovery.go b/discv5/ipdiscovery.go deleted file mode 100644 index bf6bb17..0000000 --- a/discv5/ipdiscovery.go +++ /dev/null @@ -1,479 +0,0 @@ -package discv5 - -import ( - "fmt" - "net" - "sync" - "time" - - "github.com/sirupsen/logrus" -) - -// DefaultMinReports is the minimum number of PONG responses needed before considering IP valid -const DefaultMinReports = 5 - -// DefaultMajorityThreshold is the percentage threshold for IP consensus (0.0-1.0) -const DefaultMajorityThreshold = 0.75 - -// DefaultReportExpiry is how long to keep IP reports before expiring them -const DefaultReportExpiry = 30 * time.Minute - -// DefaultRecentWindow is the time window to consider reports "recent" for IP change detection -const DefaultRecentWindow = 5 * time.Minute - -// IPDiscovery tracks external IP addresses and ports reported by peers via PONG messages. -// -// It implements a consensus mechanism to detect the node's public IP address and port: -// - Collects IP:Port from PONG responses (shows our address as seen by remote peer) -// - Tracks IPv4 and IPv6 independently (separate consensus for each) -// - Requires minimum number of reports before considering an address valid -// - Requires majority threshold (e.g., 75%) for consensus -// - Expires old reports to handle IP/port changes -type IPDiscovery struct { - // mu protects the internal state - mu sync.RWMutex - - // ipv4Reports maps "IP:Port" string to report info for IPv4 - ipv4Reports map[string]*ipReport - - // ipv6Reports maps "IP:Port" string to report info for IPv6 - ipv6Reports map[string]*ipReport - - // currentConsensusIPv4 is the IPv4 address that reached consensus - currentConsensusIPv4 net.IP - - // currentConsensusIPv4Port is the IPv4 port that reached consensus - currentConsensusIPv4Port uint16 - - // currentConsensusIPv6 is the IPv6 address that reached consensus - currentConsensusIPv6 net.IP - - // currentConsensusIPv6Port is the IPv6 port that reached consensus - currentConsensusIPv6Port uint16 - - // config - minReports int // Minimum reports needed - majorityThreshold float64 // Threshold for majority (0.0-1.0) - reportExpiry time.Duration // How long to keep reports - recentWindow time.Duration // Time window for recent reports - onConsensusReached func(ip net.IP, port uint16, isIPv6 bool) // Callback when consensus is reached - logger logrus.FieldLogger - - // stats - totalReportsIPv4 int - totalReportsIPv6 int - consensusReachedIPv4 bool - consensusReachedIPv6 bool -} - -// ipReport tracks reports for a specific IP:Port combination -type ipReport struct { - ip net.IP - port uint16 - count int - firstSeen time.Time - lastSeen time.Time - reporterIDs []string // Track which peers reported this (for debugging) -} - -// IPDiscoveryConfig contains configuration for IP discovery -type IPDiscoveryConfig struct { - // MinReports is the minimum number of PONG responses needed (default: 3) - MinReports int - - // MajorityThreshold is the percentage needed for consensus (default: 0.75) - MajorityThreshold float64 - - // ReportExpiry is how long to keep reports (default: 30 minutes) - ReportExpiry time.Duration - - // RecentWindow is the time window to consider reports "recent" (default: 5 minutes) - // Used for detecting IP changes - recent reports get priority - RecentWindow time.Duration - - // OnConsensusReached is called when IP:Port consensus is reached or changes - // isIPv6 indicates whether this is an IPv6 address (true) or IPv4 (false) - OnConsensusReached func(ip net.IP, port uint16, isIPv6 bool) - - // Logger for debug messages - Logger logrus.FieldLogger -} - -// NewIPDiscovery creates a new IP discovery service. -func NewIPDiscovery(cfg IPDiscoveryConfig) *IPDiscovery { - if cfg.MinReports <= 0 { - cfg.MinReports = DefaultMinReports - } - if cfg.MajorityThreshold <= 0 || cfg.MajorityThreshold > 1.0 { - cfg.MajorityThreshold = DefaultMajorityThreshold - } - if cfg.ReportExpiry <= 0 { - cfg.ReportExpiry = DefaultReportExpiry - } - if cfg.RecentWindow <= 0 { - cfg.RecentWindow = DefaultRecentWindow - } - if cfg.Logger == nil { - cfg.Logger = logrus.New() - } - - return &IPDiscovery{ - ipv4Reports: make(map[string]*ipReport), - ipv6Reports: make(map[string]*ipReport), - minReports: cfg.MinReports, - majorityThreshold: cfg.MajorityThreshold, - reportExpiry: cfg.ReportExpiry, - recentWindow: cfg.RecentWindow, - onConsensusReached: cfg.OnConsensusReached, - logger: cfg.Logger, - } -} - -// ReportIP records an IP address and port from a PONG response. -// -// Parameters: -// - ip: The IP address as reported by the remote peer -// - port: The port as reported by the remote peer -// - reporterID: The node ID of the peer that sent the PONG (for tracking) -func (ipd *IPDiscovery) ReportIP(ip net.IP, port uint16, reporterID string) { - if ip == nil || ip.IsLoopback() || ip.IsUnspecified() { - // Ignore invalid IPs - return - } - - if port == 0 { - // Ignore invalid ports - return - } - - // Determine if IPv4 or IPv6 - isIPv6 := ip.To4() == nil - - ipd.mu.Lock() - defer ipd.mu.Unlock() - - // Clean up expired reports first - ipd.cleanupExpiredLocked() - - // Use "IP:Port" as the key - addrKey := fmt.Sprintf("%s:%d", ip.String(), port) - now := time.Now() - - // Select appropriate reports map - var reports map[string]*ipReport - var totalReports *int - if isIPv6 { - reports = ipd.ipv6Reports - totalReports = &ipd.totalReportsIPv6 - } else { - reports = ipd.ipv4Reports - totalReports = &ipd.totalReportsIPv4 - } - - // Get or create report for this IP:Port - report, exists := reports[addrKey] - if !exists { - report = &ipReport{ - ip: ip, - port: port, - firstSeen: now, - reporterIDs: make([]string, 0), - } - reports[addrKey] = report - } - - // Update report - report.count++ - report.lastSeen = now - report.reporterIDs = append(report.reporterIDs, reporterID) - *totalReports++ - - ipd.logger.WithFields(logrus.Fields{ - "addr": addrKey, - "ipv6": isIPv6, - "count": report.count, - "reporter": reporterID[:16], - "totalReports": *totalReports, - }).Debug("IP discovery: received address report") - - // Check for consensus (check both IPv4 and IPv6) - ipd.checkConsensusLocked() -} - -// checkConsensusLocked checks if an IP:Port has reached consensus for both IPv4 and IPv6. -// Must be called with lock held. -// -// This function handles both initial consensus and address changes: -// - For initial consensus: requires minimum reports and majority threshold -// - For address changes: prioritizes recent reports to detect when IP or port has changed -func (ipd *IPDiscovery) checkConsensusLocked() { - // Check IPv4 consensus - ipd.checkConsensusForFamilyLocked(false) - - // Check IPv6 consensus - ipd.checkConsensusForFamilyLocked(true) -} - -// checkConsensusForFamilyLocked checks consensus for a specific address family (IPv4 or IPv6). -// Must be called with lock held. -func (ipd *IPDiscovery) checkConsensusForFamilyLocked(isIPv6 bool) { - now := time.Now() - - // Select appropriate maps and state - var reports map[string]*ipReport - var currentConsensusIP *net.IP - var currentConsensusPort *uint16 - var consensusReached *bool - var totalReports *int - familyName := "IPv4" - - if isIPv6 { - reports = ipd.ipv6Reports - currentConsensusIP = &ipd.currentConsensusIPv6 - currentConsensusPort = &ipd.currentConsensusIPv6Port - consensusReached = &ipd.consensusReachedIPv6 - totalReports = &ipd.totalReportsIPv6 - familyName = "IPv6" - } else { - reports = ipd.ipv4Reports - currentConsensusIP = &ipd.currentConsensusIPv4 - currentConsensusPort = &ipd.currentConsensusIPv4Port - consensusReached = &ipd.consensusReachedIPv4 - totalReports = &ipd.totalReportsIPv4 - } - - // Separate recent reports from all reports - recentReports := make(map[string]int) - allReports := make(map[string]int) - - for addrKey, report := range reports { - allReports[addrKey] = report.count - - // Count reports within the recent window - if now.Sub(report.lastSeen) <= ipd.recentWindow { - recentReports[addrKey] = report.count - } - } - - // Calculate totals - totalCount := 0 - for _, count := range allReports { - totalCount += count - } - - totalRecentCount := 0 - for _, count := range recentReports { - totalRecentCount += count - } - - // Need minimum reports before considering consensus - if totalCount < ipd.minReports { - return - } - - // Current consensus address key - currentAddrKey := "" - if *currentConsensusIP != nil && *currentConsensusPort != 0 { - currentAddrKey = fmt.Sprintf("%s:%d", (*currentConsensusIP).String(), *currentConsensusPort) - } - - // If we already have consensus, check recent reports for address changes - if *consensusReached && totalRecentCount >= ipd.minReports { - // Find address with most recent reports - var maxRecentAddr string - maxRecentCount := 0 - for addrKey, count := range recentReports { - if count > maxRecentCount { - maxRecentCount = count - maxRecentAddr = addrKey - } - } - - // Check if recent reports show consensus on a DIFFERENT address - if maxRecentAddr != "" && maxRecentAddr != currentAddrKey { - recentMajority := float64(maxRecentCount) / float64(totalRecentCount) - - if recentMajority >= ipd.majorityThreshold { - // Address change detected! - newReport := reports[maxRecentAddr] - - ipd.logger.WithFields(logrus.Fields{ - "family": familyName, - "oldAddr": currentAddrKey, - "newAddr": maxRecentAddr, - "recentCount": maxRecentCount, - "recentTotal": totalRecentCount, - "recentMajority": recentMajority, - }).Warn("IP discovery: address change detected") - - // Clear old reports to prevent flip-flopping - for k := range reports { - delete(reports, k) - } - - // Re-add only the report for the new address - if newReport != nil { - reports[maxRecentAddr] = newReport - } - - *currentConsensusIP = newReport.ip - *currentConsensusPort = newReport.port - *totalReports = maxRecentCount - - // Call callback for address change - if ipd.onConsensusReached != nil { - ip := newReport.ip - port := newReport.port - go ipd.onConsensusReached(ip, port, isIPv6) - } - return - } - } - } - - // Check for initial consensus or stable consensus on all reports - var maxReport *ipReport - maxCount := 0 - for _, report := range reports { - if report.count > maxCount { - maxCount = report.count - maxReport = report - } - } - - if maxReport == nil { - return - } - - // Check if it meets majority threshold - majority := float64(maxReport.count) / float64(totalCount) - if majority >= ipd.majorityThreshold { - // Consensus reached! - addrChanged := !*consensusReached || - *currentConsensusIP == nil || - !maxReport.ip.Equal(*currentConsensusIP) || - maxReport.port != *currentConsensusPort - - if addrChanged { - ipd.logger.WithFields(logrus.Fields{ - "family": familyName, - "addr": fmt.Sprintf("%s:%d", maxReport.ip.String(), maxReport.port), - "count": maxReport.count, - "total": totalCount, - "majority": majority, - "threshold": ipd.majorityThreshold, - }).Info("IP discovery: consensus reached") - - *currentConsensusIP = maxReport.ip - *currentConsensusPort = maxReport.port - *consensusReached = true - - // Call callback if provided - if ipd.onConsensusReached != nil { - // Call in goroutine to avoid blocking - ip := maxReport.ip - port := maxReport.port - go ipd.onConsensusReached(ip, port, isIPv6) - } - } - } -} - -// cleanupExpiredLocked removes reports older than reportExpiry. -// Must be called with lock held. -func (ipd *IPDiscovery) cleanupExpiredLocked() { - now := time.Now() - - // Clean up IPv4 reports - for addrKey, report := range ipd.ipv4Reports { - if now.Sub(report.lastSeen) > ipd.reportExpiry { - delete(ipd.ipv4Reports, addrKey) - ipd.logger.WithField("addr", addrKey).Debug("IP discovery: expired old IPv4 report") - } - } - - // Clean up IPv6 reports - for addrKey, report := range ipd.ipv6Reports { - if now.Sub(report.lastSeen) > ipd.reportExpiry { - delete(ipd.ipv6Reports, addrKey) - ipd.logger.WithField("addr", addrKey).Debug("IP discovery: expired old IPv6 report") - } - } -} - -// GetConsensusIP returns the current consensus IPv4 address, or nil if no consensus. -// For IPv6, this returns nil. Use GetStats() for complete information. -func (ipd *IPDiscovery) GetConsensusIP() net.IP { - ipd.mu.RLock() - defer ipd.mu.RUnlock() - return ipd.currentConsensusIPv4 -} - -// GetStats returns statistics about IP discovery. -type IPDiscoveryStats struct { - TotalReportsIPv4 int - TotalReportsIPv6 int - UniqueIPv4Addrs int - UniqueIPv6Addrs int - ConsensusReachedIPv4 bool - ConsensusReachedIPv6 bool - ConsensusIPv4Addr string // "IP:Port" format - ConsensusIPv6Addr string // "IP:Port" format - IPv4Reports map[string]int // "IP:Port" -> count - IPv6Reports map[string]int // "IP:Port" -> count -} - -// GetStats returns current statistics. -func (ipd *IPDiscovery) GetStats() IPDiscoveryStats { - ipd.mu.RLock() - defer ipd.mu.RUnlock() - - stats := IPDiscoveryStats{ - TotalReportsIPv4: ipd.totalReportsIPv4, - TotalReportsIPv6: ipd.totalReportsIPv6, - UniqueIPv4Addrs: len(ipd.ipv4Reports), - UniqueIPv6Addrs: len(ipd.ipv6Reports), - ConsensusReachedIPv4: ipd.consensusReachedIPv4, - ConsensusReachedIPv6: ipd.consensusReachedIPv6, - IPv4Reports: make(map[string]int), - IPv6Reports: make(map[string]int), - } - - if ipd.currentConsensusIPv4 != nil && ipd.currentConsensusIPv4Port > 0 { - stats.ConsensusIPv4Addr = fmt.Sprintf("%s:%d", ipd.currentConsensusIPv4.String(), ipd.currentConsensusIPv4Port) - } - - if ipd.currentConsensusIPv6 != nil && ipd.currentConsensusIPv6Port > 0 { - stats.ConsensusIPv6Addr = fmt.Sprintf("%s:%d", ipd.currentConsensusIPv6.String(), ipd.currentConsensusIPv6Port) - } - - for addrKey, report := range ipd.ipv4Reports { - stats.IPv4Reports[addrKey] = report.count - } - - for addrKey, report := range ipd.ipv6Reports { - stats.IPv6Reports[addrKey] = report.count - } - - return stats -} - -// Reset clears all reports and resets consensus state. -// This can be used when the node's network changes. -func (ipd *IPDiscovery) Reset() { - ipd.mu.Lock() - defer ipd.mu.Unlock() - - ipd.ipv4Reports = make(map[string]*ipReport) - ipd.ipv6Reports = make(map[string]*ipReport) - ipd.currentConsensusIPv4 = nil - ipd.currentConsensusIPv4Port = 0 - ipd.currentConsensusIPv6 = nil - ipd.currentConsensusIPv6Port = 0 - ipd.consensusReachedIPv4 = false - ipd.consensusReachedIPv6 = false - ipd.totalReportsIPv4 = 0 - ipd.totalReportsIPv6 = 0 - - ipd.logger.Info("IP discovery: reset all reports") -} diff --git a/discv5/protocol/handler.go b/discv5/protocol/handler.go index 5ec81e5..0854068 100644 --- a/discv5/protocol/handler.go +++ b/discv5/protocol/handler.go @@ -477,7 +477,7 @@ func (h *Handler) handleOrdinaryPacket(packet *Packet, from *net.UDPAddr, localA // Only now is the sender proven: AES-GCM over the header authenticates // possession of the session key, and the source address is not part of the // AAD, so a NAT-rebound peer decrypts fine from its new address. - if sess.Addr().String() != from.String() { + if cur := sess.Addr(); cur == nil || cur.Port != from.Port || !cur.IP.Equal(from.IP) { h.config.Logger.WithFields(logrus.Fields{ "nodeID": srcNodeID.String()[:16], "oldAddr": sess.Addr(), @@ -1159,126 +1159,7 @@ func (h *Handler) handleTalkResp(msg *TalkResp, remoteID node.ID, from *net.UDPA // to send arbitrary messages through the protocol handler. // remoteNode is optional - if provided, it will be stored in pending handshakes for WHOAREYOU responses. func (h *Handler) SendMessage(msg Message, remoteID node.ID, to *net.UDPAddr, remoteNode *node.Node) error { - // Look up session - sess := h.config.Sessions.Get(remoteID) - - var packetBytes []byte - var err error - - if sess == nil { - // No session - send random packet to trigger WHOAREYOU from receiver - - // Store pending message for handshake completion - // Include the node object if we have it (needed for handshake) - handshakeKey := makeHandshakeKey(remoteID, to) - now := time.Now() - pending := &PendingHandshake{ - Message: msg, - ToNode: remoteNode, - ToAddr: to, - ToNodeID: remoteID, - CreatedAt: now, - LastRetry: now, - RetryCount: 0, - MaxRetries: 3, // Retry up to 3 times before giving up - } - - h.mu.Lock() - accepted := h.addPendingHandshake(handshakeKey, pending) - h.mu.Unlock() - - if !accepted { - return fmt.Errorf("pending handshake limit reached") - } - - // Log if we don't have node info for potential handshake - if remoteNode == nil { - h.config.Logger.WithField("remoteID", remoteID).Debug("handler: sending random packet without node info, may fail handshake if WHOAREYOU received") - } - - // Encode random packet (go-ethereum style) - // This will be 91 bytes: IV(16) + header(23) + authdata(32) + random(20) - packetBytes, err = EncodeRandomPacket(h.config.LocalNode.ID(), remoteID) - if err != nil { - return fmt.Errorf("failed to encode random packet: %w", err) - } - } else { - // Have session - encrypt and send normally - - // Encode message plaintext: message-type (1 byte) + RLP-encoded message - msgBytes, err := msg.Encode() - if err != nil { - return fmt.Errorf("failed to encode message: %w", err) - } - - // Build plaintext: message type + message data - plaintext := make([]byte, 1+len(msgBytes)) - plaintext[0] = msg.Type() - copy(plaintext[1:], msgBytes) - - // Generate nonce - nonce, err := crypto.GenerateRandomBytes(12) - if err != nil { - return fmt.Errorf("failed to generate nonce: %w", err) - } - - // Get local node ID - localNodeID := h.config.LocalNode.ID() - - // Authdata for ordinary message with session: srcID (32 bytes) - authdata := localNodeID[:] - - // Build unmasked header data for GCM authentication - // This returns: maskingIV, unmasked headerData (IV || header || authdata) - maskingIV, headerData, err := BuildOrdinaryHeaderData(localNodeID, nonce, authdata) - if err != nil { - return fmt.Errorf("failed to build header data: %w", err) - } - - // Encrypt message using session key - // GCM uses unmasked headerData as additional authenticated data - ciphertext, err := session.EncryptMessage(sess.EncryptionKey(), nonce, headerData, plaintext) - if err != nil { - return fmt.Errorf("failed to encrypt message: %w", err) - } - - // Now encode the full packet with the encrypted message - // This uses the same maskingIV to ensure consistency - packetBytes, err = EncodeOrdinaryPacket(localNodeID, remoteID, maskingIV, nonce, authdata, ciphertext) - if err != nil { - return fmt.Errorf("failed to encode ordinary packet: %w", err) - } - - // Remembered so a WHOAREYOU quoting this nonce can be told apart from a - // forged one; answering a forged challenge would replace the session keys. - sess.RecordSentNonce(nonce) - } - - // Send via UDP transport - h.mu.RLock() - transport := h.transport - h.mu.RUnlock() - - if transport == nil { - return fmt.Errorf("transport not initialized") - } - - if err := transport.SendTo(packetBytes, to); err != nil { - return fmt.Errorf("failed to send packet: %w", err) - } - - h.mu.Lock() - h.packetsSent++ - h.mu.Unlock() - - h.config.Logger.WithFields(logrus.Fields{ - "type": msg.Type(), - "to": to, - "nodeID": remoteID, - "size": len(packetBytes), - }).Trace("sent message") - - return nil + return h.SendMessageFrom(msg, remoteID, to, remoteNode, nil) } // SendMessageFrom sends a message to a remote node from a specific local address. diff --git a/discv5/session/session.go b/discv5/session/session.go index 2c7d686..a73b93a 100644 --- a/discv5/session/session.go +++ b/discv5/session/session.go @@ -134,7 +134,7 @@ func (s *Session) RecordSentNonce(nonce []byte) { s.sentNonces = append(s.sentNonces, string(nonce)) if len(s.sentNonces) > maxSentNonces { - s.sentNonces = s.sentNonces[len(s.sentNonces)-maxSentNonces:] + s.sentNonces = s.sentNonces[1:] } } diff --git a/enr/encoding.go b/enr/encoding.go index 95627c8..3da1854 100644 --- a/enr/encoding.go +++ b/enr/encoding.go @@ -19,6 +19,16 @@ import ( // // Returns ErrRecordTooLarge if the encoded record exceeds 300 bytes. func (r *Record) EncodeRLPBytes() ([]byte, error) { + // A bootnode serves the same closest-node set to every requester, so the hot + // records are re-encoded concurrently; taking the write lock for a cache read + // serialises that on one mutex. + r.mu.RLock() + cached := r.raw + r.mu.RUnlock() + if len(cached) > 0 { + return cached, nil + } + r.mu.Lock() defer r.mu.Unlock() diff --git a/nodes/node.go b/nodes/node.go index 59a88d7..5ce07ea 100644 --- a/nodes/node.go +++ b/nodes/node.go @@ -139,11 +139,9 @@ func (n *Node) PublicKey() *ecdsa.PublicKey { return n.pubKey } -// ENR returns the node's ENR record. +// ENR returns the node's ENR record. Alias for Record. func (n *Node) ENR() *enr.Record { - n.mu.RLock() - defer n.mu.RUnlock() - return n.enr + return n.Record() } // Addr returns the node's UDP address. From 59403555fbb4d14adc6463bb945441c8057b0708 Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Wed, 29 Jul 2026 12:57:00 -0500 Subject: [PATCH 16/49] fix(discv5): keep Zone in the session address comparison 73aff56 replaced sess.Addr().String() != from.String() with an explicit IP and port comparison to avoid two allocations per authenticated packet, but dropped UDPAddr.Zone. Cache.GetByAddr still matches on the full address string, so for scoped IPv6 the two could disagree: a peer that changed interface would not migrate here, then miss GetByAddr on the WHOAREYOU path and fail to recover. Comparing Zone restores exact equivalence with the old String() form and is still allocation-free. --- discv5/protocol/handler.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/discv5/protocol/handler.go b/discv5/protocol/handler.go index 0854068..0c1311f 100644 --- a/discv5/protocol/handler.go +++ b/discv5/protocol/handler.go @@ -477,7 +477,10 @@ func (h *Handler) handleOrdinaryPacket(packet *Packet, from *net.UDPAddr, localA // Only now is the sender proven: AES-GCM over the header authenticates // possession of the session key, and the source address is not part of the // AAD, so a NAT-rebound peer decrypts fine from its new address. - if cur := sess.Addr(); cur == nil || cur.Port != from.Port || !cur.IP.Equal(from.IP) { + // Zone is part of the comparison because Cache.GetByAddr matches on the full + // address string; ignoring it here would let the two disagree for scoped IPv6 + // and strand a peer that changed interface. + if cur := sess.Addr(); cur == nil || cur.Port != from.Port || cur.Zone != from.Zone || !cur.IP.Equal(from.IP) { h.config.Logger.WithFields(logrus.Fields{ "nodeID": srcNodeID.String()[:16], "oldAddr": sess.Addr(), From 165af25e3ceea5c1a40685ac361a3729d8e81692 Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Wed, 29 Jul 2026 13:00:55 -0500 Subject: [PATCH 17/49] fix(bootnode): refresh last-seen on both layers for a dual-layer peer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit checkAndAddNode admits EL and CL independently, and each admission builds its own nodes.Node wrapping the same v5 node — two objects, two last-seen fields. onNodeSeen classified EL-xor-CL, so for a record carrying both eth and eth2 only the EL copy was ever refreshed. The CL copy's last-seen stayed at the zero time from admission onwards, making it look infinitely old to the CL table's age-based sweep while the peer was actively talking to us. The xor existed to keep the counting filter off the per-packet path. Classify is pure since the Classify/Admit split, so both layers can be evaluated with no counter effect and ~19ns for the second call. Test asserts both tables' last-seen advance; with the xor restored the CL side reads 0001-01-01. --- bootnode/duallayer_test.go | 128 +++++++++++++++++++++++++++++++++++++ bootnode/service.go | 10 ++- 2 files changed, 132 insertions(+), 6 deletions(-) create mode 100644 bootnode/duallayer_test.go diff --git a/bootnode/duallayer_test.go b/bootnode/duallayer_test.go new file mode 100644 index 0000000..4e661a2 --- /dev/null +++ b/bootnode/duallayer_test.go @@ -0,0 +1,128 @@ +package bootnode + +import ( + "context" + "net" + "testing" + "time" + + "github.com/ethpandaops/bootnodoor/bootnode/clconfig" + "github.com/ethpandaops/bootnodoor/bootnode/elconfig" + "github.com/ethpandaops/bootnodoor/db" + v5node "github.com/ethpandaops/bootnodoor/discv5/node" + "github.com/ethpandaops/bootnodoor/enr" + "github.com/ethpandaops/bootnodoor/nodes" +) + +// newDualLayerService wires both tables so a record carrying eth and eth2 is +// admitted to each. +func newDualLayerService(t *testing.T) *Service { + t.Helper() + + logger := quietLogger() + database := db.NewDatabase(&db.SqliteDatabaseConfig{File: ":memory:", MaxOpenConns: 5, MaxIdleConns: 2}, logger) + if err := database.Init(); err != nil { + t.Fatalf("db init: %v", err) + } + t.Cleanup(func() { database.Close() }) + if err := database.ApplyEmbeddedDbSchema(-2); err != nil { + t.Fatalf("db schema: %v", err) + } + + cfg := &Config{ + Logger: logger, + Database: database, + ELConfig: &elconfig.ChainConfig{}, + ELGenesisHash: [32]byte{1, 2, 3}, + ELGenesisTime: 1000, + CLConfig: &clconfig.Config{}, + } + + key := mustKey(t) + ln, err := createLocalNode(cfg, key, net.ParseIP("1.2.3.4"), nil, 9000, nil) + if err != nil { + t.Fatalf("createLocalNode: %v", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + s := &Service{config: cfg, ctx: ctx, enrManager: NewENRManager(cfg, key, ln, true, true)} + s.elNodeDB = nodes.NewNodeDB(ctx, database, db.LayerEL, logger) + s.clNodeDB = nodes.NewNodeDB(ctx, database, db.LayerCL, logger) + if s.elTable, err = s.createTable(ln.ID(), s.elNodeDB, "EL"); err != nil { + t.Fatalf("createTable EL: %v", err) + } + if s.clTable, err = s.createTable(ln.ID(), s.clNodeDB, "CL"); err != nil { + t.Fatalf("createTable CL: %v", err) + } + return s +} + +// dualLayerNode builds a v5 node advertising the fork id and fork digest this +// service currently accepts, so it is admitted to both tables. +func dualLayerNode(t *testing.T, s *Service) *v5node.Node { + t.Helper() + + forkID := s.enrManager.GetELFilter().GetCurrentForkID(StaticHead()) + digest := s.enrManager.GetCLFilter().GetCurrentForkDigest() + + key := mustKey(t) + rec := enr.New() + if err := rec.Set("ip", net.IPv4(9, 9, 9, 9)); err != nil { + t.Fatalf("set ip: %v", err) + } + if err := rec.Set("udp", uint16(9000)); err != nil { + t.Fatalf("set udp: %v", err) + } + if err := rec.Set("eth", []struct { + Hash []byte + Next uint64 + }{{Hash: forkID.Hash[:], Next: forkID.Next}}); err != nil { + t.Fatalf("set eth: %v", err) + } + if err := rec.Set("eth2", clconfig.EncodeETH2Field(digest, [4]byte{}, ^uint64(0))); err != nil { + t.Fatalf("set eth2: %v", err) + } + rec.SetSeq(1) + if err := rec.Sign(key); err != nil { + t.Fatalf("sign: %v", err) + } + + n, err := v5node.New(rec) + if err != nil { + t.Fatalf("v5node.New: %v", err) + } + return n +} + +// A dual-layer peer is admitted to both tables as two separate node objects with +// their own last-seen. Refreshing only one on inbound traffic leaves the other +// ageing out while the peer is actively talking to us. +func TestOnNodeSeenRefreshesBothLayers(t *testing.T) { + s := newDualLayerService(t) + n := dualLayerNode(t, s) + + if !s.checkAndAddNode(n) { + t.Fatal("dual-layer node was not admitted") + } + + el := s.elTable.Get(n.ID()) + cl := s.clTable.Get(n.ID()) + if el == nil || cl == nil { + t.Fatalf("node not in both tables: el=%v cl=%v", el != nil, cl != nil) + } + if el == cl { + t.Skip("tables share one node object; this test only means something when they differ") + } + + refreshed := time.Now().Add(time.Hour) + s.onNodeSeen(n, refreshed) + + if got := s.elTable.Get(n.ID()).LastSeen(); !got.Equal(refreshed) { + t.Errorf("EL last-seen = %v, want the refreshed %v", got, refreshed) + } + if got := s.clTable.Get(n.ID()).LastSeen(); !got.Equal(refreshed) { + t.Errorf("CL last-seen = %v, want the refreshed %v", got, refreshed) + } +} diff --git a/bootnode/service.go b/bootnode/service.go index 872e0b7..df7465b 100644 --- a/bootnode/service.go +++ b/bootnode/service.go @@ -1019,18 +1019,16 @@ func (s *Service) onNodeSeen(n *v5node.Node, timestamp time.Time) { if s.enrManager != nil { nodeID := n.ID() - // Serve-all pools a node into every table, so refresh last-seen wherever it - // actually lives rather than by classification. Classified mode is - // EL-xor-CL, so the CL filter only runs when the EL one declines. + // Both layers, independently, as checkAndAddNode admits them: a dual-layer + // record becomes two node objects with their own last-seen, so refreshing + // only one lets the other age out while the peer is actively talking. var isEL, isCL bool if s.config.ServeAll { isEL = s.elTable != nil isCL = s.clTable != nil } else { isEL, _ = s.enrManager.ClassifyELNode(n.Record()) - if !isEL { - isCL = s.enrManager.ClassifyCLNode(n.Record()) - } + isCL = s.enrManager.ClassifyCLNode(n.Record()) } if isEL && s.elTable != nil && s.elNodeDB != nil { From a3a31fa39af1075de6ccfb9551ad0896a5776e7b Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Wed, 29 Jul 2026 13:07:36 -0500 Subject: [PATCH 18/49] test(bootnode): exercise the real dual-layer staleness path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of 165af25 was right that the test proved less than claimed, and that the commit message overstated the bug. NewFromV5 ends in v5.SetStats, so the v5 node shares stats with whichever wrapper was constructed last — CL, since checkAndAddNode builds it second — and handleMessage calls SetLastSeen on the v5 node before OnNodeSeen. So on a freshly admitted peer the handler already refreshes the CL copy for free, and the "CL last-seen stays at the zero value" result came from the test calling onNodeSeen without the handler's preceding SetLastSeen. The staleness is real but narrower: a re-admission (an ENR refresh, say) builds a fresh wrapper, FlatTable.Add keeps the entry it already has and discards the new one, but SetStats has already repointed the v5 node at the discarded copy. From then on the table's own object is refreshed only by onNodeSeen, so classifying one layer and not the other strands the other. The test now re-admits, then applies SetLastSeen and onNodeSeen in the order the handler produces them, and seeds a known admission timestamp so a regression shows a stale time rather than a zero one. With the xor restored the CL entry reads its admission time while the refresh is an hour later. The fix in 165af25 is unchanged and still correct. --- bootnode/duallayer_test.go | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/bootnode/duallayer_test.go b/bootnode/duallayer_test.go index 4e661a2..b94a306 100644 --- a/bootnode/duallayer_test.go +++ b/bootnode/duallayer_test.go @@ -96,19 +96,23 @@ func dualLayerNode(t *testing.T, s *Service) *v5node.Node { return n } -// A dual-layer peer is admitted to both tables as two separate node objects with -// their own last-seen. Refreshing only one on inbound traffic leaves the other -// ageing out while the peer is actively talking to us. +// A dual-layer peer occupies both tables as two node objects with their own +// last-seen. The handler's SetLastSeen reaches only whichever one currently +// shares stats with the v5 node, and re-admission repoints that at a wrapper the +// tables discarded — so onNodeSeen has to refresh both itself. func TestOnNodeSeenRefreshesBothLayers(t *testing.T) { s := newDualLayerService(t) n := dualLayerNode(t, s) + // Admit with a known last-seen, so a later failure shows a stale timestamp + // rather than a zero one and cannot be mistaken for "never populated". + admitted := time.Now() + n.SetLastSeen(admitted) + if !s.checkAndAddNode(n) { t.Fatal("dual-layer node was not admitted") } - - el := s.elTable.Get(n.ID()) - cl := s.clTable.Get(n.ID()) + el, cl := s.elTable.Get(n.ID()), s.clTable.Get(n.ID()) if el == nil || cl == nil { t.Fatalf("node not in both tables: el=%v cl=%v", el != nil, cl != nil) } @@ -116,7 +120,13 @@ func TestOnNodeSeenRefreshesBothLayers(t *testing.T) { t.Skip("tables share one node object; this test only means something when they differ") } + // Re-admission, as an ENR refresh would do: repoints the v5 node's stats at a + // wrapper the tables do not hold. + s.checkAndAddNode(n) + + // One inbound packet, in the order the handler produces it. refreshed := time.Now().Add(time.Hour) + n.SetLastSeen(refreshed) s.onNodeSeen(n, refreshed) if got := s.elTable.Get(n.ID()).LastSeen(); !got.Equal(refreshed) { From f3ae9a7231437d4a1c4bf2bab046a8245ca0f0e7 Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Wed, 29 Jul 2026 15:31:13 -0500 Subject: [PATCH 19/49] fix(db): key nodes and bad_nodes on (nodeid, layer) Rows in both tables are per node and layer, but nodeid was the sole primary key, so a dual-layer peer's second write replaced its first while every read filtered on (nodeid, layer). Nodes lost a layer on restart; bad_nodes lost the suppression that stops repeated ENR requests. --- db/layer_key_test.go | 139 ++++++++++++++++++++ db/migration_down_test.go | 78 +++++++++++ db/migration_layer_key_test.go | 98 ++++++++++++++ db/nodes.go | 4 +- db/schema/20260729200000_node_layer_key.sql | 117 ++++++++++++++++ 5 files changed, 434 insertions(+), 2 deletions(-) create mode 100644 db/layer_key_test.go create mode 100644 db/migration_down_test.go create mode 100644 db/migration_layer_key_test.go create mode 100644 db/schema/20260729200000_node_layer_key.sql diff --git a/db/layer_key_test.go b/db/layer_key_test.go new file mode 100644 index 0000000..c18744a --- /dev/null +++ b/db/layer_key_test.go @@ -0,0 +1,139 @@ +package db + +import ( + "testing" + "time" + + "github.com/jmoiron/sqlx" + "github.com/sirupsen/logrus" +) + +func testDB(t *testing.T) *Database { + t.Helper() + + logger := logrus.New() + logger.SetLevel(logrus.ErrorLevel) + + database := NewDatabase(&SqliteDatabaseConfig{File: ":memory:"}, logger) + if err := database.Init(); err != nil { + t.Fatalf("init: %v", err) + } + t.Cleanup(func() { database.Close() }) + if err := database.ApplyEmbeddedDbSchema(-2); err != nil { + t.Fatalf("schema: %v", err) + } + return database +} + +// A dual-layer peer occupies one row per layer. With nodeid as the sole primary +// key both upserts collide, so the second layer overwrites the first's +// layer-specific fork digest while the row keeps the first layer's tag — and +// every read filters on (nodeid, layer), so one layer silently disappears. +func TestUpsertKeepsBothLayers(t *testing.T) { + database := testDB(t) + + id := []byte("0123456789abcdef0123456789abcdef") + elDigest := []byte{0xaa, 0xaa, 0xaa, 0xaa} + clDigest := []byte{0xbb, 0xbb, 0xbb, 0xbb} + + for _, tc := range []struct { + layer NodeLayer + digest []byte + }{{LayerEL, elDigest}, {LayerCL, clDigest}} { + n := &Node{ + NodeID: id, Layer: string(tc.layer), Port: 30303, Seq: 1, + ForkDigest: tc.digest, FirstSeen: time.Now().Unix(), ENR: []byte("enr"), + } + if err := database.RunDBTransaction(func(tx *sqlx.Tx) error { + return database.UpsertNode(tx, n) + }); err != nil { + t.Fatalf("upsert %s: %v", tc.layer, err) + } + } + + el, err := database.GetNode(LayerEL, id) + if err != nil { + t.Fatalf("EL row missing after the CL upsert: %v", err) + } + cl, err := database.GetNode(LayerCL, id) + if err != nil { + t.Fatalf("CL row missing after the EL upsert: %v", err) + } + if string(el.ForkDigest) != string(elDigest) { + t.Errorf("EL fork digest = %x, want %x", el.ForkDigest, elDigest) + } + if string(cl.ForkDigest) != string(clDigest) { + t.Errorf("CL fork digest = %x, want %x", cl.ForkDigest, clDigest) + } +} + +// Same collision on the ENR-update path, which is the one admission uses. +func TestUpdateNodeENRKeepsBothLayers(t *testing.T) { + database := testDB(t) + + id := []byte("fedcba9876543210fedcba9876543210") + + for _, tc := range []struct { + layer NodeLayer + digest []byte + }{{LayerEL, []byte{1, 1, 1, 1}}, {LayerCL, []byte{2, 2, 2, 2}}} { + if err := database.RunDBTransaction(func(tx *sqlx.Tx) error { + return database.UpdateNodeENR(tx, tc.layer, id, nil, nil, 30303, 1, tc.digest, []byte("enr"), true, true) + }); err != nil { + t.Fatalf("update %s: %v", tc.layer, err) + } + } + + if _, err := database.GetNode(LayerEL, id); err != nil { + t.Errorf("EL row missing: %v", err) + } + if _, err := database.GetNode(LayerCL, id); err != nil { + t.Errorf("CL row missing: %v", err) + } + got, err := database.CountAllNodes() + if err != nil { + t.Fatalf("CountAllNodes: %v", err) + } + if got != 2 { + t.Errorf("total rows across layers = %d, want 2", got) + } +} + +// markBadNode is called per layer, and INSERT OR REPLACE keyed on nodeid alone +// drops the other layer's entry — so a peer rejected on both layers stays +// suppressed on only whichever was written last, defeating the cache that +// exists to stop repeated ENR requests. +func TestBadNodeSuppressionSurvivesBothLayers(t *testing.T) { + database := testDB(t) + + id := []byte("badbadbadbadbadbadbadbadbadbadba") + + if err := database.StoreBadNode(id, LayerEL, "invalid_fork_id"); err != nil { + t.Fatalf("store EL: %v", err) + } + if err := database.StoreBadNode(id, LayerCL, "invalid_fork_digest"); err != nil { + t.Fatalf("store CL: %v", err) + } + + elBad, _, elReason, err := database.IsBadNode(id, LayerEL, time.Hour) + if err != nil { + t.Fatalf("IsBadNode EL: %v", err) + } + clBad, _, clReason, err := database.IsBadNode(id, LayerCL, time.Hour) + if err != nil { + t.Fatalf("IsBadNode CL: %v", err) + } + + if !elBad { + t.Error("EL rejection was forgotten after the CL rejection was recorded") + } + if !clBad { + t.Error("CL rejection was forgotten after the EL rejection was recorded") + } + if elBad && elReason != "invalid_fork_id" { + t.Errorf("EL reason = %q, want invalid_fork_id", elReason) + } + if clBad && clReason != "invalid_fork_digest" { + t.Errorf("CL reason = %q, want invalid_fork_digest", clReason) + } +} diff --git a/db/migration_down_test.go b/db/migration_down_test.go new file mode 100644 index 0000000..907b9fc --- /dev/null +++ b/db/migration_down_test.go @@ -0,0 +1,78 @@ +package db + +import ( + "path/filepath" + "testing" + + "github.com/jmoiron/sqlx" + "github.com/pressly/goose/v3" + "github.com/sirupsen/logrus" +) + +// The down migration collapses two rows into a nodeid primary key, so it has to +// actually run and has to pick a row deterministically rather than erroring on +// the constraint. +func TestLayerKeyDownMigrationCollapsesDeterministically(t *testing.T) { + file := filepath.Join(t.TempDir(), "down.db") + + logger := logrus.New() + logger.SetLevel(logrus.ErrorLevel) + + database := NewDatabase(&SqliteDatabaseConfig{File: file}, logger) + if err := database.Init(); err != nil { + t.Fatalf("init: %v", err) + } + defer database.Close() + if err := database.ApplyEmbeddedDbSchema(-2); err != nil { + t.Fatalf("schema: %v", err) + } + + dualID := []byte("dddddddddddddddddddddddddddddddd") + clOnlyID := []byte("11111111111111111111111111111111") + + for _, tc := range []struct { + id []byte + layer NodeLayer + }{{dualID, LayerEL}, {dualID, LayerCL}, {clOnlyID, LayerCL}} { + n := &Node{ + NodeID: tc.id, Layer: string(tc.layer), Port: 30303, Seq: 1, + ForkDigest: []byte{1, 2, 3, 4}, FirstSeen: 1000, ENR: []byte("enr-" + string(tc.layer)), + } + if err := database.RunDBTransaction(func(tx *sqlx.Tx) error { + return database.UpsertNode(tx, n) + }); err != nil { + t.Fatalf("seed %s: %v", tc.layer, err) + } + } + + goose.SetLogger(&gooseLogger{logger: logger}) + goose.SetBaseFS(embedSchema) + if err := goose.SetDialect("sqlite3"); err != nil { + t.Fatalf("dialect: %v", err) + } + if err := goose.Down(database.writerDb.DB, "schema"); err != nil { + t.Fatalf("down migration failed to run: %v", err) + } + + var rows []struct { + NodeID []byte `db:"nodeid"` + Layer string `db:"layer"` + } + if err := database.ReaderDb.Select(&rows, "SELECT nodeid, layer FROM nodes ORDER BY layer"); err != nil { + t.Fatalf("select after down: %v", err) + } + if len(rows) != 2 { + t.Fatalf("rows after collapse = %d, want 2", len(rows)) + } + + byID := map[string]string{} + for _, r := range rows { + byID[string(r.NodeID)] = r.Layer + } + if got := byID[string(dualID)]; got != string(LayerEL) { + t.Errorf("dual-layer node collapsed to layer %q, want el", got) + } + if got := byID[string(clOnlyID)]; got != string(LayerCL) { + t.Errorf("cl-only node collapsed to layer %q, want cl", got) + } +} diff --git a/db/migration_layer_key_test.go b/db/migration_layer_key_test.go new file mode 100644 index 0000000..0f54c43 --- /dev/null +++ b/db/migration_layer_key_test.go @@ -0,0 +1,98 @@ +package db + +import ( + "path/filepath" + "testing" + + "github.com/jmoiron/sqlx" + "github.com/sirupsen/logrus" +) + +const schemaBeforeLayerKey = 20251106015541 + +func openAt(t *testing.T, file string, version int64) *Database { + t.Helper() + + logger := logrus.New() + logger.SetLevel(logrus.ErrorLevel) + + database := NewDatabase(&SqliteDatabaseConfig{File: file}, logger) + if err := database.Init(); err != nil { + t.Fatalf("init: %v", err) + } + if err := database.ApplyEmbeddedDbSchema(version); err != nil { + t.Fatalf("schema %d: %v", version, err) + } + return database +} + +// The up migration rebuilds both tables, so it must carry existing rows across +// rather than silently starting empty. +func TestLayerKeyMigrationPreservesExistingRows(t *testing.T) { + file := filepath.Join(t.TempDir(), "nodes.db") + + elID := []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + clID := []byte("cccccccccccccccccccccccccccccccc") + + // Seeded with raw SQL: the Go upserts now target the composite key and + // cannot write the pre-migration schema. + old := openAt(t, file, schemaBeforeLayerKey) + if err := old.RunDBTransaction(func(tx *sqlx.Tx) error { + for _, tc := range []struct { + id []byte + layer NodeLayer + }{{elID, LayerEL}, {clID, LayerCL}} { + if _, err := tx.Exec( + `INSERT INTO nodes (nodeid, layer, port, seq, fork_digest, first_seen, enr, has_v4, has_v5) + VALUES (?, ?, 30303, 7, ?, 1000, ?, 1, 1)`, + tc.id, string(tc.layer), []byte{9, 9, 9, 9}, []byte("enr-"+string(tc.layer))); err != nil { + return err + } + } + _, err := tx.Exec( + `INSERT INTO bad_nodes (nodeid, layer, rejected_at, reason) VALUES (?, ?, ?, ?)`, + elID, string(LayerEL), 1000, "invalid_fork_id") + return err + }); err != nil { + t.Fatalf("seed: %v", err) + } + old.Close() + + migrated := openAt(t, file, -2) + defer migrated.Close() + + el, err := migrated.GetNode(LayerEL, elID) + if err != nil { + t.Fatalf("EL row lost by the migration: %v", err) + } + if el.Seq != 7 || string(el.ENR) != "enr-el" { + t.Errorf("EL row mangled: seq=%d enr=%q", el.Seq, el.ENR) + } + if _, err := migrated.GetNode(LayerCL, clID); err != nil { + t.Errorf("CL row lost by the migration: %v", err) + } + if isBad, _, reason, err := migrated.IsBadNode(elID, LayerEL, 0); err != nil { + t.Errorf("IsBadNode: %v", err) + } else if !isBad || reason != "invalid_fork_id" { + t.Errorf("bad node lost by the migration: isBad=%v reason=%q", isBad, reason) + } + + // The point of the migration: both layers now coexist for one ID. + for _, layer := range []NodeLayer{LayerEL, LayerCL} { + n := &Node{ + NodeID: elID, Layer: string(layer), Port: 30303, Seq: 8, + ForkDigest: []byte{1, 2, 3, 4}, FirstSeen: 1000, ENR: []byte("enr"), + } + if err := migrated.RunDBTransaction(func(tx *sqlx.Tx) error { + return migrated.UpsertNode(tx, n) + }); err != nil { + t.Fatalf("post-migration upsert %s: %v", layer, err) + } + } + if _, err := migrated.GetNode(LayerEL, elID); err != nil { + t.Errorf("EL row missing after dual-layer upsert: %v", err) + } + if _, err := migrated.GetNode(LayerCL, elID); err != nil { + t.Errorf("CL row missing after dual-layer upsert: %v", err) + } +} diff --git a/db/nodes.go b/db/nodes.go index e15a1a5..52a8c66 100644 --- a/db/nodes.go +++ b/db/nodes.go @@ -134,7 +134,7 @@ func (d *Database) UpsertNode(tx *sqlx.Tx, node *Node) error { _, err := tx.Exec(` INSERT INTO nodes (nodeid, layer, ip, ipv6, port, seq, fork_digest, first_seen, last_seen, last_active, enr, has_v4, has_v5, success_count, failure_count, avg_rtt) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16) - ON CONFLICT(nodeid) DO UPDATE SET + ON CONFLICT(nodeid, layer) DO UPDATE SET ip = excluded.ip, ipv6 = excluded.ipv6, port = excluded.port, @@ -159,7 +159,7 @@ func (d *Database) UpdateNodeENR(tx *sqlx.Tx, layer NodeLayer, nodeID []byte, ip _, err := tx.Exec(` INSERT INTO nodes (nodeid, layer, ip, ipv6, port, seq, fork_digest, first_seen, last_seen, last_active, enr, has_v4, has_v5, success_count, failure_count, avg_rtt) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, NULL, NULL, $9, $10, $11, 0, 0, 0) - ON CONFLICT(nodeid) DO UPDATE SET + ON CONFLICT(nodeid, layer) DO UPDATE SET ip = excluded.ip, ipv6 = excluded.ipv6, port = excluded.port, diff --git a/db/schema/20260729200000_node_layer_key.sql b/db/schema/20260729200000_node_layer_key.sql new file mode 100644 index 0000000..546f9a6 --- /dev/null +++ b/db/schema/20260729200000_node_layer_key.sql @@ -0,0 +1,117 @@ +-- +goose Up +-- +goose StatementBegin + +-- Reads filter on (nodeid, layer), so keying on nodeid alone let a dual-layer +-- peer's second write replace its first. SQLite cannot alter a primary key, +-- hence the rebuild. Rows already collapsed are carried over as-is; the lost +-- layer returns only on rediscovery. + +CREATE TABLE "nodes_new" ( + "nodeid" BLOB NOT NULL, + "layer" TEXT NOT NULL, + "ip" BLOB, + "ipv6" BLOB, + "port" INTEGER, + "seq" INTEGER, + "fork_digest" BLOB, + "first_seen" INTEGER, + "last_seen" INTEGER, + "last_active" INTEGER, + "enr" BLOB, + "has_v4" INTEGER DEFAULT 0, + "has_v5" INTEGER DEFAULT 1, + "success_count" INTEGER DEFAULT 0, + "failure_count" INTEGER DEFAULT 0, + "avg_rtt" INTEGER DEFAULT 0, + PRIMARY KEY ("nodeid", "layer") +); + +INSERT INTO "nodes_new" SELECT + nodeid, layer, ip, ipv6, port, seq, fork_digest, first_seen, last_seen, + last_active, enr, has_v4, has_v5, success_count, failure_count, avg_rtt +FROM "nodes"; + +DROP TABLE "nodes"; +ALTER TABLE "nodes_new" RENAME TO "nodes"; + +CREATE INDEX IF NOT EXISTS "idx_nodes_layer" ON "nodes" ("layer"); +CREATE INDEX IF NOT EXISTS "idx_nodes_last_active" ON "nodes" ("last_active" DESC); +CREATE INDEX IF NOT EXISTS "idx_nodes_fork_digest" ON "nodes" ("fork_digest"); +CREATE INDEX IF NOT EXISTS "idx_nodes_layer_last_active" ON "nodes" ("layer", "last_active" DESC); + +CREATE TABLE "bad_nodes_new" ( + "nodeid" BLOB NOT NULL, + "layer" TEXT NOT NULL, + "rejected_at" INTEGER NOT NULL, + "reason" TEXT, + PRIMARY KEY ("nodeid", "layer") +); + +INSERT INTO "bad_nodes_new" SELECT nodeid, layer, rejected_at, reason FROM "bad_nodes"; + +DROP TABLE "bad_nodes"; +ALTER TABLE "bad_nodes_new" RENAME TO "bad_nodes"; + +CREATE INDEX IF NOT EXISTS "idx_bad_nodes_layer" ON "bad_nodes" ("layer"); +CREATE INDEX IF NOT EXISTS "idx_bad_nodes_rejected_at" ON "bad_nodes" ("rejected_at"); + +-- +goose StatementEnd +-- +goose Down +-- +goose StatementBegin + +-- Lossy by necessity: two rows cannot both fit a nodeid primary key. Keeping +-- the EL row makes the collapse deterministic rather than insertion-ordered. + +CREATE TABLE "nodes_old" ( + "nodeid" BLOB PRIMARY KEY, + "layer" TEXT NOT NULL, + "ip" BLOB, + "ipv6" BLOB, + "port" INTEGER, + "seq" INTEGER, + "fork_digest" BLOB, + "first_seen" INTEGER, + "last_seen" INTEGER, + "last_active" INTEGER, + "enr" BLOB, + "has_v4" INTEGER DEFAULT 0, + "has_v5" INTEGER DEFAULT 1, + "success_count" INTEGER DEFAULT 0, + "failure_count" INTEGER DEFAULT 0, + "avg_rtt" INTEGER DEFAULT 0 +); + +INSERT INTO "nodes_old" SELECT + nodeid, layer, ip, ipv6, port, seq, fork_digest, first_seen, last_seen, + last_active, enr, has_v4, has_v5, success_count, failure_count, avg_rtt +FROM "nodes" +WHERE layer = 'el' + OR nodeid NOT IN (SELECT nodeid FROM "nodes" WHERE layer = 'el'); + +DROP TABLE "nodes"; +ALTER TABLE "nodes_old" RENAME TO "nodes"; + +CREATE INDEX IF NOT EXISTS "idx_nodes_layer" ON "nodes" ("layer"); +CREATE INDEX IF NOT EXISTS "idx_nodes_last_active" ON "nodes" ("last_active" DESC); +CREATE INDEX IF NOT EXISTS "idx_nodes_fork_digest" ON "nodes" ("fork_digest"); +CREATE INDEX IF NOT EXISTS "idx_nodes_layer_last_active" ON "nodes" ("layer", "last_active" DESC); + +CREATE TABLE "bad_nodes_old" ( + "nodeid" BLOB PRIMARY KEY, + "layer" TEXT NOT NULL, + "rejected_at" INTEGER NOT NULL, + "reason" TEXT +); + +INSERT INTO "bad_nodes_old" SELECT nodeid, layer, rejected_at, reason +FROM "bad_nodes" +WHERE layer = 'el' + OR nodeid NOT IN (SELECT nodeid FROM "bad_nodes" WHERE layer = 'el'); + +DROP TABLE "bad_nodes"; +ALTER TABLE "bad_nodes_old" RENAME TO "bad_nodes"; + +CREATE INDEX IF NOT EXISTS "idx_bad_nodes_layer" ON "bad_nodes" ("layer"); +CREATE INDEX IF NOT EXISTS "idx_bad_nodes_rejected_at" ON "bad_nodes" ("rejected_at"); + +-- +goose StatementEnd From 97adbbe5c126c7f7d13abb3b9ccdb9dbef9d4e5f Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Wed, 29 Jul 2026 15:35:31 -0500 Subject: [PATCH 20/49] fix(nodes): persist admitted nodes and drain the write queue on close FlatTable.Add marked nodes dirty but never enqueued them, so organically discovered nodes lived only in memory and were lost on restart. The queue consumer also abandoned its channel backlog on context cancellation, and Stop cancels before closing, so Close now refuses new work and flushes what is left. The full upsert wrote last_active as NULL while its branch cleared the DirtyLastActive that admission had just set, leaving configured bootnodes sorting as the most inactive rows. --- nodes/admission_persist_test.go | 161 ++++++++++++++++++++++++++++++++ nodes/flattable.go | 6 ++ nodes/nodedb.go | 43 ++++++++- 3 files changed, 207 insertions(+), 3 deletions(-) create mode 100644 nodes/admission_persist_test.go diff --git a/nodes/admission_persist_test.go b/nodes/admission_persist_test.go new file mode 100644 index 0000000..7dc4aa8 --- /dev/null +++ b/nodes/admission_persist_test.go @@ -0,0 +1,161 @@ +package nodes + +import ( + "context" + "net" + "path/filepath" + "testing" + "time" + + "github.com/ethpandaops/bootnodoor/db" + "github.com/sirupsen/logrus" +) + +func persistTestDB(t *testing.T, file string) *db.Database { + t.Helper() + + logger := logrus.New() + logger.SetLevel(logrus.ErrorLevel) + + database := db.NewDatabase(&db.SqliteDatabaseConfig{File: file}, logger) + if err := database.Init(); err != nil { + t.Fatalf("init: %v", err) + } + if err := database.ApplyEmbeddedDbSchema(-2); err != nil { + t.Fatalf("schema: %v", err) + } + return database +} + +func quietTableLogger() logrus.FieldLogger { + logger := logrus.New() + logger.SetLevel(logrus.ErrorLevel) + return logger +} + +func newPersistTable(t *testing.T, ndb *NodeDB, logger logrus.FieldLogger) *FlatTable { + t.Helper() + + table, err := NewFlatTable(FlatTableConfig{DB: ndb, MaxActiveNodes: 10, Logger: logger}) + if err != nil { + t.Fatalf("new table: %v", err) + } + return table +} + +// Admission puts a node in the active pool and marks it dirty, but nothing ever +// enqueued it, so organically discovered nodes were never written at all. +func TestAdmissionPersistsNode(t *testing.T) { + database := persistTestDB(t, filepath.Join(t.TempDir(), "admit.db")) + defer database.Close() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + logger := quietTableLogger() + ndb := NewNodeDB(ctx, database, db.LayerCL, logger) + table := newPersistTable(t, ndb, logger) + + n := NewFromV5(makeV5At(t, net.IPv4(10, 0, 0, 7)), ndb) + if !table.Add(n) { + t.Fatal("node was not admitted") + } + + deadline := time.Now().Add(5 * time.Second) + for ndb.Count() == 0 { + if time.Now().After(deadline) { + t.Fatal("admitted node was never persisted") + } + time.Sleep(20 * time.Millisecond) + } +} + +// A node admitted immediately before shutdown must not be lost: the consumer +// abandons the channel backlog on ctx cancellation, so Close has to drain it. +func TestAdmissionSurvivesImmediateClose(t *testing.T) { + file := filepath.Join(t.TempDir(), "close.db") + database := persistTestDB(t, file) + + ctx, cancel := context.WithCancel(context.Background()) + logger := quietTableLogger() + ndb := NewNodeDB(ctx, database, db.LayerCL, logger) + table := newPersistTable(t, ndb, logger) + + n := NewFromV5(makeV5At(t, net.IPv4(10, 0, 0, 9)), ndb) + if !table.Add(n) { + t.Fatal("node was not admitted") + } + + cancel() + ndb.Close() + database.Close() + + reopened := persistTestDB(t, file) + defer reopened.Close() + + count, err := reopened.CountNodes(db.LayerCL) + if err != nil { + t.Fatalf("count: %v", err) + } + if count != 1 { + t.Errorf("persisted nodes after immediate close = %d, want 1", count) + } +} + +// QueueUpdate must refuse work once Close has begun rather than accept it into a +// queue nobody will drain. +func TestQueueUpdateRejectedAfterClose(t *testing.T) { + database := persistTestDB(t, filepath.Join(t.TempDir(), "gate.db")) + defer database.Close() + + ctx, cancel := context.WithCancel(context.Background()) + logger := quietTableLogger() + ndb := NewNodeDB(ctx, database, db.LayerCL, logger) + + cancel() + ndb.Close() + + n := NewFromV5(makeV5At(t, net.IPv4(10, 0, 0, 11)), ndb) + n.MarkDirty(DirtyFull) + if err := ndb.QueueUpdate(n); err == nil { + t.Error("QueueUpdate accepted a node after Close; it will never be written") + } +} + +// The DirtyFull branch clears every remaining flag after upserting, so the +// upsert itself has to carry last_active or the DirtyLastActive set during +// admission is discarded and the row sorts as the most inactive. +func TestFullUpsertPersistsLastActive(t *testing.T) { + database := persistTestDB(t, filepath.Join(t.TempDir(), "active.db")) + defer database.Close() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + logger := quietTableLogger() + ndb := NewNodeDB(ctx, database, db.LayerCL, logger) + + n := NewFromV5(makeV5At(t, net.IPv4(10, 0, 0, 21)), ndb) + n.SetLastActive(time.Now()) + n.MarkDirty(DirtyFull) + if err := ndb.QueueUpdate(n); err != nil { + t.Fatalf("queue: %v", err) + } + + deadline := time.Now().Add(5 * time.Second) + for ndb.Count() == 0 { + if time.Now().After(deadline) { + t.Fatal("node was never persisted") + } + time.Sleep(20 * time.Millisecond) + } + + id := n.IDBytes() + stored, err := database.GetNode(db.LayerCL, id[:]) + if err != nil { + t.Fatalf("load: %v", err) + } + if !stored.LastActive.Valid || stored.LastActive.Int64 == 0 { + t.Error("last_active was written as NULL by the full upsert") + } +} diff --git a/nodes/flattable.go b/nodes/flattable.go index caf22c7..b1338a7 100644 --- a/nodes/flattable.go +++ b/nodes/flattable.go @@ -267,6 +267,9 @@ func (t *FlatTable) Add(n *Node) bool { // Queue ENR update (preserves stats) existing.MarkDirty(DirtyENR) + if err := t.db.QueueUpdate(existing); err != nil { + t.logger.WithError(err).WithField("peerID", existing.PeerID()).Debug("failed to queue ENR update") + } if t.nodeChangedCallback != nil { t.nodeChangedCallback(existing) @@ -332,6 +335,9 @@ func (t *FlatTable) Add(n *Node) bool { // Queue ENR update to DB and mark as active n.MarkDirty(DirtyENR) n.SetLastActive(time.Now()) + if err := t.db.QueueUpdate(n); err != nil { + t.logger.WithError(err).WithField("peerID", n.PeerID()).Debug("failed to queue admitted node") + } if t.nodeChangedCallback != nil { t.nodeChangedCallback(n) diff --git a/nodes/nodedb.go b/nodes/nodedb.go index 3fc452e..c546731 100644 --- a/nodes/nodedb.go +++ b/nodes/nodedb.go @@ -28,6 +28,7 @@ type NodeDB struct { updateQueue chan *Node updateQueueSet map[[32]byte]*Node // Tracks pending updates by nodeID updateQueueLock sync.Mutex + closing bool // Set under updateQueueLock so no write is accepted after Close starts draining // Stats tracking stats NodeDBStats @@ -78,6 +79,10 @@ func (ndb *NodeDB) QueueUpdate(n *Node) error { ndb.updateQueueLock.Lock() defer ndb.updateQueueLock.Unlock() + if ndb.closing { + return fmt.Errorf("node db is closing") + } + // Check if there's already a pending update for this node if _, ok := ndb.updateQueueSet[nodeID]; ok { // Node already queued - dirty flags will accumulate automatically @@ -128,7 +133,7 @@ func (ndb *NodeDB) processUpdateQueue() { for { select { case <-ndb.ctx.Done(): - // Process remaining batch + batch = ndb.drainQueue(batch) if len(batch) > 0 { ndb.batchUpdate(batch) } @@ -154,6 +159,18 @@ func (ndb *NodeDB) processUpdateQueue() { } } +// drainQueue moves everything currently queued into batch without blocking. +func (ndb *NodeDB) drainQueue(batch []*Node) []*Node { + for { + select { + case node := <-ndb.updateQueue: + batch = append(batch, node) + default: + return batch + } + } +} + // batchUpdate performs a batch update of nodes. func (ndb *NodeDB) batchUpdate(nodes []*Node) { if len(nodes) == 0 { @@ -349,6 +366,14 @@ func (ndb *NodeDB) upsertNodeTx(tx *sqlx.Tx, n *Node) error { lastSeen.Int64 = stats.LastSeen.Unix() } + // The DirtyFull branch clears every other flag once this upsert runs, so a + // DirtyLastActive set alongside it would otherwise be dropped. + lastActive := sql.NullInt64{} + if t := n.LastActive(); !t.IsZero() { + lastActive.Valid = true + lastActive.Int64 = t.Unix() + } + // Extract fork digest based on layer var forkDigest []byte if ndb.layer == db.LayerEL { @@ -383,7 +408,7 @@ func (ndb *NodeDB) upsertNodeTx(tx *sqlx.Tx, n *Node) error { ForkDigest: forkDigest, FirstSeen: firstSeen, LastSeen: lastSeen, - LastActive: sql.NullInt64{}, // Updated separately + LastActive: lastActive, ENR: enrBytes, HasV4: n.HasV4(), HasV5: n.HasV5(), @@ -541,9 +566,21 @@ func (ndb *NodeDB) LoadRandom(limit int) ([]*Node, error) { } // Close stops the update queue processor and waits for pending updates. +// +// The processor exits on context cancellation, and Stop cancels before calling +// here, so a producer can still enqueue after the processor is gone. Refusing +// new work first and flushing afterwards is what makes that write-or-reject +// rather than a silent drop. func (ndb *NodeDB) Close() { - // Wait for queue processor to finish + ndb.updateQueueLock.Lock() + ndb.closing = true + ndb.updateQueueLock.Unlock() + ndb.wg.Wait() + + if batch := ndb.drainQueue(nil); len(batch) > 0 { + ndb.batchUpdate(batch) + } } // GetStats returns current database statistics. From 3535cd10a30e5e9e5249f4ec1a534a4e8f58b8e4 Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Wed, 29 Jul 2026 15:38:38 -0500 Subject: [PATCH 21/49] fix(nodes): count active and inactive as sets, not a subtraction TotalNodes came from the database and ActiveNodes from memory, so the six consumers subtracting them reported more active than total and a negative inactive count whenever a write had not landed yet. GetStats now derives the union and the difference directly. --- db/nodes.go | 8 +++ nodes/flattable.go | 17 ++++- nodes/nodedb.go | 20 ++++++ nodes/stats_population_test.go | 117 +++++++++++++++++++++++++++++++++ nodes/types.go | 1 + webui/handlers/cl-nodes.go | 4 +- webui/handlers/el-nodes.go | 4 +- webui/handlers/overview.go | 4 +- 8 files changed, 168 insertions(+), 7 deletions(-) create mode 100644 nodes/stats_population_test.go diff --git a/db/nodes.go b/db/nodes.go index 52a8c66..6661b49 100644 --- a/db/nodes.go +++ b/db/nodes.go @@ -107,6 +107,14 @@ func (d *Database) CountNodes(layer NodeLayer) (int, error) { return count, err } +// GetNodeIDs returns the node IDs persisted for a specific layer. +func (d *Database) GetNodeIDs(layer NodeLayer) ([][]byte, error) { + d.trackQuery() + var ids [][]byte + err := d.ReaderDb.Select(&ids, "SELECT nodeid FROM nodes WHERE layer = $1", string(layer)) + return ids, err +} + // CountAllNodes returns the total number of nodes (all layers). func (d *Database) CountAllNodes() (int, error) { d.trackQuery() diff --git a/nodes/flattable.go b/nodes/flattable.go index b1338a7..18d7dbb 100644 --- a/nodes/flattable.go +++ b/nodes/flattable.go @@ -733,16 +733,31 @@ func (t *FlatTable) ActiveSize() int { } // GetStats returns statistics about the table. +// +// Active comes from memory and persisted from the database, so the two are +// counted as sets rather than subtracted: an admission whose write has not +// landed yet would otherwise report more active than total. func (t *FlatTable) GetStats() TableStats { + persisted := t.db.PersistedIDs() + t.mu.RLock() defer t.mu.RUnlock() activeCount := len(t.activeNodes) - totalCount := t.db.Count() + + inactiveCount := 0 + totalCount := activeCount + for _, id := range persisted { + if _, active := t.activeNodes[id]; !active { + inactiveCount++ + totalCount++ + } + } return TableStats{ TotalNodes: totalCount, ActiveNodes: activeCount, + InactiveNodes: inactiveCount, AdmissionRejections: t.admissionRejections, IPLimitRejections: t.ipLimitRejections, DeadNodesRemoved: t.deadNodesRemoved, diff --git a/nodes/nodedb.go b/nodes/nodedb.go index c546731..1ad1c41 100644 --- a/nodes/nodedb.go +++ b/nodes/nodedb.go @@ -609,6 +609,26 @@ func (ndb *NodeDB) List() []*Node { return nodes } +// PersistedIDs returns the node IDs persisted for this layer. +func (ndb *NodeDB) PersistedIDs() [][32]byte { + rows, err := ndb.db.GetNodeIDs(ndb.layer) + if err != nil { + ndb.logger.WithError(err).Warn("failed to list persisted node ids") + return nil + } + + ids := make([][32]byte, 0, len(rows)) + for _, raw := range rows { + if len(raw) != 32 { + continue + } + var id [32]byte + copy(id[:], raw) + ids = append(ids, id) + } + return ids +} + // Count returns the total number of nodes in the database. func (ndb *NodeDB) Count() int { count, err := ndb.db.CountNodes(ndb.layer) diff --git a/nodes/stats_population_test.go b/nodes/stats_population_test.go new file mode 100644 index 0000000..2d8606e --- /dev/null +++ b/nodes/stats_population_test.go @@ -0,0 +1,117 @@ +package nodes + +import ( + "context" + "net" + "path/filepath" + "testing" + "time" + + "github.com/ethpandaops/bootnodoor/db" +) + +// TotalNodes came from the database and ActiveNodes from memory, so consumers +// subtracting them could report more active than total and a negative inactive +// count. The three populations are asserted exactly, because the inequality +// alone is also satisfied by reporting total == active and inactive == 0. +func TestGetStatsCountsPopulationsExactly(t *testing.T) { + database := persistTestDB(t, filepath.Join(t.TempDir(), "pop.db")) + defer database.Close() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + logger := quietTableLogger() + ndb := NewNodeDB(ctx, database, db.LayerCL, logger) + table := newPersistTable(t, ndb, logger) + + // Persisted but never admitted: two rows written straight through the queue. + for i := 0; i < 2; i++ { + n := NewFromV5(makeV5At(t, net.IPv4(10, 1, 0, byte(i+1))), ndb) + n.MarkDirty(DirtyFull) + if err := ndb.QueueUpdate(n); err != nil { + t.Fatalf("queue: %v", err) + } + } + waitForPersisted(t, ndb, 2) + + // Admitted, and therefore also persisted: overlapping population. + admitted := make([]*Node, 0, 3) + for i := 0; i < 3; i++ { + n := NewFromV5(makeV5At(t, net.IPv4(10, 2, 0, byte(i+1))), ndb) + if !table.Add(n) { + t.Fatalf("node %d not admitted", i) + } + admitted = append(admitted, n) + } + waitForPersisted(t, ndb, 5) + + stats := table.GetStats() + if stats.ActiveNodes != 3 { + t.Errorf("ActiveNodes = %d, want 3", stats.ActiveNodes) + } + if stats.TotalNodes != 5 { + t.Errorf("TotalNodes = %d, want 5 (union of persisted and active)", stats.TotalNodes) + } + if stats.InactiveNodes != 2 { + t.Errorf("InactiveNodes = %d, want 2 (persisted but not active)", stats.InactiveNodes) + } + + // Demotion drops a node from the active pool while its row remains. + table.mu.Lock() + delete(table.activeNodes, admitted[0].ID()) + table.mu.Unlock() + + stats = table.GetStats() + if stats.ActiveNodes != 2 { + t.Errorf("after demotion ActiveNodes = %d, want 2", stats.ActiveNodes) + } + if stats.TotalNodes != 5 { + t.Errorf("after demotion TotalNodes = %d, want 5", stats.TotalNodes) + } + if stats.InactiveNodes != 3 { + t.Errorf("after demotion InactiveNodes = %d, want 3", stats.InactiveNodes) + } +} + +// An active node whose write has not landed yet must not make active exceed +// total, which is what produced the negative count in the devnet run. +func TestGetStatsHoldsInvariantBeforePersistence(t *testing.T) { + database := persistTestDB(t, filepath.Join(t.TempDir(), "lag.db")) + defer database.Close() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + logger := quietTableLogger() + ndb := NewNodeDB(ctx, database, db.LayerCL, logger) + table := newPersistTable(t, ndb, logger) + + for i := 0; i < 4; i++ { + n := NewFromV5(makeV5At(t, net.IPv4(10, 3, 0, byte(i+1))), ndb) + table.mu.Lock() + table.activeNodes[n.ID()] = n + table.ipLimiter.Add(n) + table.mu.Unlock() + } + + stats := table.GetStats() + if stats.ActiveNodes > stats.TotalNodes { + t.Errorf("ActiveNodes %d > TotalNodes %d", stats.ActiveNodes, stats.TotalNodes) + } + if stats.InactiveNodes < 0 { + t.Errorf("InactiveNodes = %d, want >= 0", stats.InactiveNodes) + } +} + +func waitForPersisted(t *testing.T, ndb *NodeDB, want int) { + t.Helper() + + deadline := time.Now().Add(5 * time.Second) + for ndb.Count() < want { + if time.Now().After(deadline) { + t.Fatalf("only %d of %d nodes persisted", ndb.Count(), want) + } + time.Sleep(20 * time.Millisecond) + } +} diff --git a/nodes/types.go b/nodes/types.go index dda838a..5b84bc3 100644 --- a/nodes/types.go +++ b/nodes/types.go @@ -36,6 +36,7 @@ type NodeChangedCallback func(*Node) type TableStats struct { TotalNodes int ActiveNodes int + InactiveNodes int AdmissionRejections int IPLimitRejections int DeadNodesRemoved int diff --git a/webui/handlers/cl-nodes.go b/webui/handlers/cl-nodes.go index a44b6f1..b638fba 100644 --- a/webui/handlers/cl-nodes.go +++ b/webui/handlers/cl-nodes.go @@ -103,7 +103,7 @@ func (fh *FrontendHandler) CLNodes(w http.ResponseWriter, r *http.Request) { pageData := CLNodesPageData{ TotalNodes: stats.TotalNodes, ActiveNodes: stats.ActiveNodes, - InactiveNodes: stats.TotalNodes - stats.ActiveNodes, + InactiveNodes: stats.InactiveNodes, AliveNodes: aliveCount, DeadNodes: deadCount, CurrentForkDigest: currentForkDigest, @@ -180,7 +180,7 @@ func (fh *FrontendHandler) CLNodesJSON(w http.ResponseWriter, r *http.Request) { pageData := CLNodesPageData{ TotalNodes: stats.TotalNodes, ActiveNodes: stats.ActiveNodes, - InactiveNodes: stats.TotalNodes - stats.ActiveNodes, + InactiveNodes: stats.InactiveNodes, AliveNodes: aliveCount, DeadNodes: deadCount, CurrentForkDigest: currentForkDigest, diff --git a/webui/handlers/el-nodes.go b/webui/handlers/el-nodes.go index 11eb7b1..bcd340a 100644 --- a/webui/handlers/el-nodes.go +++ b/webui/handlers/el-nodes.go @@ -120,7 +120,7 @@ func (fh *FrontendHandler) ELNodes(w http.ResponseWriter, r *http.Request) { pageData := ELNodesPageData{ TotalNodes: stats.TotalNodes, ActiveNodes: stats.ActiveNodes, - InactiveNodes: stats.TotalNodes - stats.ActiveNodes, + InactiveNodes: stats.InactiveNodes, AliveNodes: aliveCount, DeadNodes: deadCount, CurrentForkDigest: currentForkDigest, @@ -209,7 +209,7 @@ func (fh *FrontendHandler) ELNodesJSON(w http.ResponseWriter, r *http.Request) { pageData := ELNodesPageData{ TotalNodes: stats.TotalNodes, ActiveNodes: stats.ActiveNodes, - InactiveNodes: stats.TotalNodes - stats.ActiveNodes, + InactiveNodes: stats.InactiveNodes, AliveNodes: aliveCount, DeadNodes: deadCount, CurrentForkDigest: currentForkDigest, diff --git a/webui/handlers/overview.go b/webui/handlers/overview.go index 6564dea..2f837ba 100644 --- a/webui/handlers/overview.go +++ b/webui/handlers/overview.go @@ -333,7 +333,7 @@ func (fh *FrontendHandler) getOverviewPageData() (*OverviewPageData, error) { // Get EL table stats if available if elTable := fh.bootnodeService.ELTable(); elTable != nil { elStats := elTable.GetStats() - elInactiveNodes := elStats.TotalNodes - elStats.ActiveNodes + elInactiveNodes := elStats.InactiveNodes pageData.ELActiveNodes = elStats.ActiveNodes pageData.ELTotalNodes = elStats.TotalNodes pageData.ELTableStats = TableStats{ @@ -350,7 +350,7 @@ func (fh *FrontendHandler) getOverviewPageData() (*OverviewPageData, error) { // Get CL table stats if available if clTable := fh.bootnodeService.CLTable(); clTable != nil { clStats := clTable.GetStats() - clInactiveNodes := clStats.TotalNodes - clStats.ActiveNodes + clInactiveNodes := clStats.InactiveNodes pageData.CLActiveNodes = clStats.ActiveNodes pageData.CLTotalNodes = clStats.TotalNodes pageData.CLTableStats = TableStats{ From b5b78dcb8df5a43fedd96f609c1934d5f3b7a15b Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Wed, 29 Jul 2026 15:42:59 -0500 Subject: [PATCH 22/49] fix(transport): distinguish other-protocol packets from unrecognised ones discv5 registers first and rejected anything it could not decode, so every ordinary discv4 packet on the shared socket was counted invalid before the transport re-dispatched it: 17749 of 21140 received in a devnet run. Only the dispatcher knows the final outcome, so it counts fallthrough and unhandled separately and the UI reports both. --- bootnode/stats.go | 2 + transport/dispatch_metrics_test.go | 68 ++++++++++++++++++++++++++ transport/metrics.go | 52 ++++++++++++++------ transport/udp.go | 10 +++- webui/handlers/overview.go | 19 ++++--- webui/templates/overview/overview.html | 5 ++ 6 files changed, 132 insertions(+), 24 deletions(-) create mode 100644 transport/dispatch_metrics_test.go diff --git a/bootnode/stats.go b/bootnode/stats.go index 12bfe5e..5b4d46d 100644 --- a/bootnode/stats.go +++ b/bootnode/stats.go @@ -97,6 +97,8 @@ func (s *Service) GetStats() Stats { out.Packets.SendErrors += m.SendErrors out.Packets.ReceiveErrors += m.ReceiveErrors out.Packets.RateLimited += m.RateLimited + out.Packets.PacketsFellThrough += m.PacketsFellThrough + out.Packets.PacketsUnhandled += m.PacketsUnhandled } } diff --git a/transport/dispatch_metrics_test.go b/transport/dispatch_metrics_test.go new file mode 100644 index 0000000..fa3bc9f --- /dev/null +++ b/transport/dispatch_metrics_test.go @@ -0,0 +1,68 @@ +package transport + +import ( + "net" + "testing" + + "github.com/sirupsen/logrus" +) + +func dispatchTestTransport(t *testing.T, handlers ...PacketHandler) *UDPTransport { + t.Helper() + + logger := logrus.New() + logger.SetLevel(logrus.ErrorLevel) + + return &UDPTransport{logger: logger, metrics: NewMetrics(), handlers: handlers} +} + +// discv5 registers first and rejects anything it cannot decode, so a normal +// discv4 packet is only recognised on the second attempt. Counting the first +// handler's rejection as "invalid" made 84% of ordinary traffic look invalid, so +// the distinction has to be made here, where the final outcome is known. +func TestDispatchDistinguishesFallthroughFromUnhandled(t *testing.T) { + accept := func(_ []byte, _ *net.UDPAddr, _ *net.UDPAddr) bool { return true } + reject := func(_ []byte, _ *net.UDPAddr, _ *net.UDPAddr) bool { return false } + + from := &net.UDPAddr{IP: net.IPv4(10, 0, 0, 1), Port: 30303} + local := &net.UDPAddr{IP: net.IPv4(10, 0, 0, 2), Port: 9000} + + t.Run("first handler accepts", func(t *testing.T) { + tr := dispatchTestTransport(t, accept, reject) + tr.dispatchPacket([]byte("packet"), from, local) + + got := tr.Metrics().Snapshot() + if got.PacketsFellThrough != 0 { + t.Errorf("PacketsFellThrough = %d, want 0", got.PacketsFellThrough) + } + if got.PacketsUnhandled != 0 { + t.Errorf("PacketsUnhandled = %d, want 0", got.PacketsUnhandled) + } + }) + + t.Run("second handler accepts", func(t *testing.T) { + tr := dispatchTestTransport(t, reject, accept) + tr.dispatchPacket([]byte("packet"), from, local) + + got := tr.Metrics().Snapshot() + if got.PacketsFellThrough != 1 { + t.Errorf("PacketsFellThrough = %d, want 1", got.PacketsFellThrough) + } + if got.PacketsUnhandled != 0 { + t.Errorf("PacketsUnhandled = %d, want 0", got.PacketsUnhandled) + } + }) + + t.Run("nobody accepts", func(t *testing.T) { + tr := dispatchTestTransport(t, reject, reject) + tr.dispatchPacket([]byte("packet"), from, local) + + got := tr.Metrics().Snapshot() + if got.PacketsFellThrough != 0 { + t.Errorf("PacketsFellThrough = %d, want 0", got.PacketsFellThrough) + } + if got.PacketsUnhandled != 1 { + t.Errorf("PacketsUnhandled = %d, want 1", got.PacketsUnhandled) + } + }) +} diff --git a/transport/metrics.go b/transport/metrics.go index 2b35e5e..0bfa6fe 100644 --- a/transport/metrics.go +++ b/transport/metrics.go @@ -21,6 +21,22 @@ type Metrics struct { sendErrors atomic.Uint64 receiveErrors atomic.Uint64 rateLimited atomic.Uint64 + + // Dispatch outcomes. A packet the first handler declines but a later one + // accepts is normal traffic for the other protocol on a shared socket; only + // a packet no handler accepts is unrecognised. + packetsFellThrough atomic.Uint64 + packetsUnhandled atomic.Uint64 +} + +// RecordFellThrough records a packet accepted by a handler other than the first. +func (m *Metrics) RecordFellThrough() { + m.packetsFellThrough.Add(1) +} + +// RecordUnhandled records a packet no handler accepted. +func (m *Metrics) RecordUnhandled() { + m.packetsUnhandled.Add(1) } // NewMetrics creates a new metrics tracker. @@ -62,14 +78,16 @@ func (m *Metrics) IncrementDropped() { // Snapshot returns a snapshot of the current metrics. type MetricsSnapshot struct { - PacketsSent uint64 - PacketsReceived uint64 - PacketsDropped uint64 - BytesSent uint64 - BytesReceived uint64 - SendErrors uint64 - ReceiveErrors uint64 - RateLimited uint64 + PacketsSent uint64 + PacketsReceived uint64 + PacketsDropped uint64 + BytesSent uint64 + BytesReceived uint64 + SendErrors uint64 + ReceiveErrors uint64 + RateLimited uint64 + PacketsFellThrough uint64 + PacketsUnhandled uint64 } // Snapshot returns a snapshot of the current metrics. @@ -81,14 +99,16 @@ type MetricsSnapshot struct { // snapshot.PacketsSent, snapshot.PacketsReceived) func (m *Metrics) Snapshot() MetricsSnapshot { return MetricsSnapshot{ - PacketsSent: m.packetsSent.Load(), - PacketsReceived: m.packetsReceived.Load(), - PacketsDropped: m.packetsDropped.Load(), - BytesSent: m.bytesSent.Load(), - BytesReceived: m.bytesReceived.Load(), - SendErrors: m.sendErrors.Load(), - ReceiveErrors: m.receiveErrors.Load(), - RateLimited: m.rateLimited.Load(), + PacketsSent: m.packetsSent.Load(), + PacketsReceived: m.packetsReceived.Load(), + PacketsDropped: m.packetsDropped.Load(), + BytesSent: m.bytesSent.Load(), + BytesReceived: m.bytesReceived.Load(), + SendErrors: m.sendErrors.Load(), + ReceiveErrors: m.receiveErrors.Load(), + RateLimited: m.rateLimited.Load(), + PacketsFellThrough: m.packetsFellThrough.Load(), + PacketsUnhandled: m.packetsUnhandled.Load(), } } diff --git a/transport/udp.go b/transport/udp.go index afbf34d..464d207 100644 --- a/transport/udp.go +++ b/transport/udp.go @@ -391,13 +391,19 @@ func (t *UDPTransport) dispatchPacket(data []byte, from *net.UDPAddr, localAddr t.handlersMu.RUnlock() // Try each handler in order - for _, handler := range handlers { + for i, handler := range handlers { if handler(data, from, localAddr) { - // Handler accepted the packet + if i > 0 && t.metrics != nil { + t.metrics.RecordFellThrough() + } return } } + if t.metrics != nil { + t.metrics.RecordUnhandled() + } + // No handler recognized the packet t.logger.WithFields(logrus.Fields{ "from": from, diff --git a/webui/handlers/overview.go b/webui/handlers/overview.go index 2f837ba..1f2ce6e 100644 --- a/webui/handlers/overview.go +++ b/webui/handlers/overview.go @@ -105,11 +105,12 @@ type OverviewPageData struct { PendingChallenges int // Handler stats - PacketsReceived int - PacketsSent int - InvalidPackets int - FilteredResponses int - FindNodeReceived int + PacketsReceived int + PacketsSent int + InvalidPackets int + WrongProtocolPackets int + FilteredResponses int + FindNodeReceived int // CL fork digest filter stats FilterAcceptedCurrent int @@ -503,7 +504,13 @@ func (fh *FrontendHandler) getOverviewPageData() (*OverviewPageData, error) { // discv5-specific views stay on the handler counters. pageData.PacketsReceived = int(stats.Packets.PacketsReceived) pageData.PacketsSent = int(stats.Packets.PacketsSent) - pageData.InvalidPackets = stats.Discv5.InvalidPackets + int(stats.Discv4.InvalidPackets) + + // A packet the first handler declines but a later one accepts is ordinary + // traffic for the other protocol on the shared socket, so only the packets + // nothing accepted are unrecognised. Summing the per-handler "invalid" + // counters instead reported most normal discv4 load as invalid. + pageData.WrongProtocolPackets = int(stats.Packets.PacketsFellThrough) + pageData.InvalidPackets = int(stats.Packets.PacketsUnhandled) pageData.FilteredResponses = stats.Discv5.FilteredResponses pageData.FindNodeReceived = stats.Discv5.FindNodeReceived + int(stats.Discv4.FindnodeRequestsRecv) diff --git a/webui/templates/overview/overview.html b/webui/templates/overview/overview.html index 7df7972..c0656bc 100644 --- a/webui/templates/overview/overview.html +++ b/webui/templates/overview/overview.html @@ -463,6 +463,10 @@
Packet Statistics
FINDNODE Received {{ .FindNodeReceived }} + + Other Protocol + {{ .WrongProtocolPackets }} + Invalid Packets {{ .InvalidPackets }} @@ -892,6 +896,7 @@
Old Fork Digests (Grace Period)
updateValue('[data-stat="packets-sent"]', data.PacketsSent); updateValue('[data-stat="findnode-received"]', data.FindNodeReceived); updateValue('[data-stat="invalid-packets"]', data.InvalidPackets); + updateValue('[data-stat="wrong-protocol-packets"]', data.WrongProtocolPackets); updateValue('[data-stat="filtered-responses"]', data.FilteredResponses); /* Fork info (if present) */ From f510adf6ecba3edc97acb8f6be20b3d6190c4ae9 Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Wed, 29 Jul 2026 15:48:47 -0500 Subject: [PATCH 23/49] fix(discv4): coalesce the PONG-driven ENR refresh A PONG advertising a newer sequence spawned an unguarded RequestENR, which always PINGs and sleeps 500ms before the ENRREQUEST. Its PONG re-entered handlePong with the cached sequence still stale, so refreshes multiplied at RTT speed: a scripted peer saw 25433 PINGs and 18662 ENRREQUESTs in two seconds, matching a devnet capture of ~4000 cycles across two peers during a fork. The claim is taken at the trigger, not inside RequestENR, because a goroutine descheduled past the winner's release would otherwise become a new winner. A sequence observed mid-refresh still earns one more round; failures retry twice with backoff. RequestENR itself is unchanged, so the lookup path is unaffected. Also makes the service-level guard atomic: Load-then-Store let two callers through, and delete-on-completion could drop a later claimant's entry. --- bootnode/service.go | 35 ++-- discv4/protocol/enr_refresh_test.go | 284 ++++++++++++++++++++++++++++ discv4/protocol/handler.go | 119 +++++++++++- 3 files changed, 421 insertions(+), 17 deletions(-) create mode 100644 discv4/protocol/enr_refresh_test.go diff --git a/bootnode/service.go b/bootnode/service.go index df7465b..fd0c142 100644 --- a/bootnode/service.go +++ b/bootnode/service.go @@ -801,8 +801,9 @@ func (s *Service) cleanupStaleENRRequests() { s.pendingENRRequestsV4.Range(func(key, value interface{}) bool { if timestamp, ok := value.(time.Time); ok { - if now.Sub(timestamp) > staleThreshold { - s.pendingENRRequestsV4.Delete(key) + // Delete only the entry we just judged stale: a fresh claim may have + // replaced it between the Range read and here. + if now.Sub(timestamp) > staleThreshold && s.pendingENRRequestsV4.CompareAndDelete(key, value) { cleanedCount++ } } @@ -1160,24 +1161,28 @@ func (s *Service) requestENRV4(n *v4node.Node) { nodeID := n.ID() now := time.Now() - // Check if we already have a recent pending ENR request for this node - if val, exists := s.pendingENRRequestsV4.Load(nodeID); exists { - if timestamp, ok := val.(time.Time); ok { - // If request is less than 30 seconds old, skip (still pending) - if time.Since(timestamp) < 30*time.Second { - return - } - // Request is stale (>30s), replace it + // Claim the slot atomically: a Load followed by a Store lets two callers both + // through, and takeover of an entry older than 30s has to stay possible, so a + // bare LoadOrStore is not enough either. + for { + val, loaded := s.pendingENRRequestsV4.LoadOrStore(nodeID, now) + if !loaded { + break + } + timestamp, ok := val.(time.Time) + if ok && time.Since(timestamp) < 30*time.Second { + return + } + if s.pendingENRRequestsV4.CompareAndSwap(nodeID, val, now) { + break } } - // Mark as pending with current timestamp - s.pendingENRRequestsV4.Store(nodeID, now) - // Run in goroutine to avoid blocking packet handling go func() { - // Remove from pending when done - defer s.pendingENRRequestsV4.Delete(nodeID) + // Release only our own claim: an unconditional delete would drop the entry + // of whoever took over after our 30s window expired. + defer s.pendingENRRequestsV4.CompareAndDelete(nodeID, now) // IMPORTANT: Some clients (like reth) require bidirectional bonding before responding to ENRRequest. // Bidirectional bonding means: diff --git a/discv4/protocol/enr_refresh_test.go b/discv4/protocol/enr_refresh_test.go new file mode 100644 index 0000000..559ca85 --- /dev/null +++ b/discv4/protocol/enr_refresh_test.go @@ -0,0 +1,284 @@ +package protocol + +import ( + "context" + "crypto/ecdsa" + "net" + "sync" + "testing" + "time" + + "github.com/ethpandaops/bootnodoor/discv4/node" +) + +// scriptedPeer decodes what the handler sends and answers it the way a real peer +// would. recordingTransport cannot be used here: it discards the packet bytes, +// and a PONG-triggered refresh emits nothing unless its PINGs are answered. +type scriptedPeer struct { + t *testing.T + h *Handler + key *ecdsa.PrivateKey + + mu sync.Mutex + pings int + enrReqs int + enrSeq uint64 + stopped bool + pongAddr *net.UDPAddr +} + +func (p *scriptedPeer) SendTo(data []byte, to *net.UDPAddr) error { + packet, _, hash, err := Decode(data) + if err != nil { + return nil + } + + p.mu.Lock() + if p.stopped { + p.mu.Unlock() + return nil + } + seq := p.enrSeq + switch packet.(type) { + case *Ping: + p.pings++ + case *ENRRequest: + p.enrReqs++ + } + p.mu.Unlock() + + var reply []byte + switch packet.(type) { + case *Ping: + reply, _ = EncodePacket(p.key, &Pong{ + To: Endpoint{IP: net.IPv4(127, 0, 0, 1), UDP: 30303}, + ReplyTok: hash, + Expiration: MakeExpiration(20 * time.Second), + ENRSeq: seq, + }) + case *ENRRequest: + reply, _ = EncodePacket(p.key, &ENRResponse{ + ReplyTok: hash, + Record: signedV4Record(p.t, p.key, seq), + }) + } + + if reply != nil { + go func() { + if err := p.h.HandlePacket(reply, p.pongAddr, nil); err != nil { + p.t.Logf("reply not accepted: %v", err) + } + }() + } + return nil +} + +func (p *scriptedPeer) Send(data []byte, to *net.UDPAddr, _ *net.UDPAddr) error { + return p.SendTo(data, to) +} + +func (p *scriptedPeer) counts() (int, int) { + p.mu.Lock() + defer p.mu.Unlock() + return p.pings, p.enrReqs +} + +func (p *scriptedPeer) stop() { + p.mu.Lock() + p.stopped = true + p.mu.Unlock() +} + +// A PONG advertising a sequence above the cached record starts an ENR refresh. +// That refresh PINGs, and its PONG re-enters handlePong with the cached sequence +// still stale, so an unguarded trigger spawns another refresh at RTT speed — +// thousands of PING/ENRREQUEST pairs against one peer in the devnet capture. +func TestPongDrivenENRRefreshRunsOnce(t *testing.T) { + peer := &scriptedPeer{t: t, enrSeq: 5, pongAddr: &net.UDPAddr{IP: net.IPv4(1, 2, 3, 4), Port: 30303}} + + h, cancel := newScriptedHandler(t, peer) + defer cancel() + peer.h = h + + n, key := makeKeyedNode(t, 30303) + peer.key = key + if !n.UpdateENR(signedV4Record(t, key, 1)) { + t.Fatal("seed record was not installed") + } + h.nodesMu.Lock() + h.nodes[n.ID()] = n + h.nodesMu.Unlock() + + req, err := h.addPendingRequest([]byte("seed-ping-hash"), n, PingPacket, n.Addr()) + if err != nil { + t.Fatalf("addPendingRequest: %v", err) + } + + pong := &Pong{ + To: Endpoint{IP: net.IPv4(127, 0, 0, 1), UDP: 30303}, + ReplyTok: req.RequestHash, + Expiration: MakeExpiration(20 * time.Second), + ENRSeq: 5, + } + data, err := EncodePacket(key, pong) + if err != nil { + t.Fatalf("encode pong: %v", err) + } + if err := h.HandlePacket(data, n.Addr(), nil); err != nil { + t.Fatalf("handle pong: %v", err) + } + + time.Sleep(2 * time.Second) + peer.stop() + pings, enrReqs := peer.counts() + + if enrReqs > 1 { + t.Errorf("ENRREQUESTs sent = %d, want at most 1 (refresh not coalesced)", enrReqs) + } + if pings > 2 { + t.Errorf("PINGs sent = %d, want at most 2 (refresh not coalesced)", pings) + } + t.Logf("pings=%d enrRequests=%d", pings, enrReqs) +} + +func newScriptedHandler(t *testing.T, peer *scriptedPeer) (*Handler, func()) { + t.Helper() + + ctx, cancel := context.WithCancel(context.Background()) + h := NewHandler(ctx, HandlerConfig{ + PrivateKey: mustHandlerKey(t), + LocalAddr: &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 30304}, + BondExpiration: time.Hour, + NodeTTL: time.Hour, + ExpirationWindow: 20 * time.Second, + }, peer) + return h, cancel +} + +func mustHandlerKey(t *testing.T) *ecdsa.PrivateKey { + t.Helper() + _, key := makeKeyedNode(t, 1) + return key +} + +func (p *scriptedPeer) setSeq(seq uint64) { + p.mu.Lock() + p.enrSeq = seq + p.mu.Unlock() +} + +func deliverPong(t *testing.T, h *Handler, n *node.Node, key *ecdsa.PrivateKey, token []byte, seq uint64) { + t.Helper() + + req, err := h.addPendingRequest(token, n, PingPacket, n.Addr()) + if err != nil { + t.Fatalf("addPendingRequest: %v", err) + } + data, err := EncodePacket(key, &Pong{ + To: Endpoint{IP: net.IPv4(127, 0, 0, 1), UDP: 30303}, + ReplyTok: req.RequestHash, + Expiration: MakeExpiration(20 * time.Second), + ENRSeq: seq, + }) + if err != nil { + t.Fatalf("encode pong: %v", err) + } + if err := h.HandlePacket(data, n.Addr(), nil); err != nil { + t.Fatalf("handle pong: %v", err) + } +} + +// A sequence advertised after the running attempt started is real new data, so it +// must produce exactly one more refresh — coalescing must not swallow it. +func TestENRRefreshRearmsForBumpDuringRefresh(t *testing.T) { + peer := &scriptedPeer{t: t, enrSeq: 5, pongAddr: &net.UDPAddr{IP: net.IPv4(1, 2, 3, 4), Port: 30303}} + + h, cancel := newScriptedHandler(t, peer) + defer cancel() + peer.h = h + + n, key := makeKeyedNode(t, 30303) + peer.key = key + if !n.UpdateENR(signedV4Record(t, key, 1)) { + t.Fatal("seed record was not installed") + } + h.nodesMu.Lock() + h.nodes[n.ID()] = n + h.nodesMu.Unlock() + + deliverPong(t, h, n, key, []byte("first-ping-hash"), 5) + + // Ping sleeps 500ms before the ENRREQUEST, so this lands mid-refresh. + time.Sleep(100 * time.Millisecond) + peer.setSeq(6) + deliverPong(t, h, n, key, []byte("second-ping-hash"), 6) + + time.Sleep(3 * time.Second) + peer.stop() + _, enrReqs := peer.counts() + + if enrReqs != 2 { + t.Errorf("ENRREQUESTs sent = %d, want 2 (one per observed bump)", enrReqs) + } +} + +// Eviction removes the refresh state; a refresh completing afterwards must not +// resurrect an entry, or the map grows for every peer that ever left. +func TestENRRefreshStateClearedOnEviction(t *testing.T) { + peer := &scriptedPeer{t: t, enrSeq: 5, pongAddr: &net.UDPAddr{IP: net.IPv4(1, 2, 3, 4), Port: 30303}} + + h, cancel := newScriptedHandler(t, peer) + defer cancel() + peer.h = h + + n, key := makeKeyedNode(t, 30303) + peer.key = key + if !n.UpdateENR(signedV4Record(t, key, 1)) { + t.Fatal("seed record was not installed") + } + + h.startENRRefresh(n, 5) + + h.enrRefreshMu.Lock() + delete(h.enrRefresh, n.ID()) + h.enrRefreshMu.Unlock() + + time.Sleep(2 * time.Second) + peer.stop() + + h.enrRefreshMu.Lock() + _, present := h.enrRefresh[n.ID()] + h.enrRefreshMu.Unlock() + + if present { + t.Error("refresh state was recreated after eviction") + } +} + +// Concurrent triggers must not race on the refresh map. +func TestENRRefreshConcurrentTriggers(t *testing.T) { + peer := &scriptedPeer{t: t, enrSeq: 5, pongAddr: &net.UDPAddr{IP: net.IPv4(1, 2, 3, 4), Port: 30303}} + + h, cancel := newScriptedHandler(t, peer) + defer cancel() + peer.h = h + + n, key := makeKeyedNode(t, 30303) + peer.key = key + if !n.UpdateENR(signedV4Record(t, key, 1)) { + t.Fatal("seed record was not installed") + } + + var wg sync.WaitGroup + for i := 0; i < 50; i++ { + wg.Add(1) + go func(seq uint64) { + defer wg.Done() + h.startENRRefresh(n, seq) + }(uint64(5 + i%3)) + } + wg.Wait() + + time.Sleep(1500 * time.Millisecond) + peer.stop() +} diff --git a/discv4/protocol/handler.go b/discv4/protocol/handler.go index 1e1be30..6488625 100644 --- a/discv4/protocol/handler.go +++ b/discv4/protocol/handler.go @@ -67,6 +67,12 @@ type Handler struct { nodesMu sync.RWMutex nodes map[node.ID]*node.Node + // In-flight PONG-driven ENR refreshes, keyed by node ID. The refresh cannot + // update the cached sequence before its own PING is answered, so without this + // every PONG on the way re-triggers it. + enrRefreshMu sync.Mutex + enrRefresh map[node.ID]*enrRefreshState + // Pending requests, keyed by packet hash + destination node ID: the hash // alone aliases across peers (deterministic signatures, 1s Expiration // granularity), and identical requests to one peer share a key's slice. @@ -236,6 +242,7 @@ func NewHandler(ctx context.Context, config HandlerConfig, transport Transport) ctx: ctx, transport: transport, nodes: make(map[node.ID]*node.Node), + enrRefresh: make(map[node.ID]*enrRefreshState), requests: make(map[string][]*PendingRequest), pendingNeighbors: make(map[string]*PendingNeighborsResponse), localENR: config.LocalENR, @@ -419,14 +426,114 @@ func (h *Handler) handlePong(fromNode *node.Node, from *net.UDPAddr, pong *Pong) // Check if remote node has newer ENR if pong.ENRSeq > 0 && fromNode.ENR() != nil { if pong.ENRSeq > fromNode.ENR().Seq() { - // Request updated ENR - go h.RequestENR(fromNode) + h.startENRRefresh(fromNode, pong.ENRSeq) } } return nil } +// maxENRRefreshRetries bounds retries after a failed refresh so a peer that +// never answers cannot keep one running. +const maxENRRefreshRetries = 2 + +// enrRefreshState tracks one peer's automatic ENR refresh. +type enrRefreshState struct { + inFlight bool + + // targetSeq is what the running attempt is fetching; highestSeenSeq is the + // largest advertised since. Only highestSeenSeq > targetSeq means a genuinely + // newer record appeared mid-refresh and another round is warranted. Comparing + // against the installed record instead would also retry after a failed or + // stale response, which never terminates. + targetSeq uint64 + highestSeenSeq uint64 + + retries int +} + +// startENRRefresh claims the refresh for a peer and runs at most one at a time. +// The claim is taken here rather than inside RequestENR because a goroutine +// descheduled past the winner's release would otherwise become a new winner — +// under exactly the load this is meant to prevent. +func (h *Handler) startENRRefresh(n *node.Node, advertisedSeq uint64) { + id := n.ID() + + h.enrRefreshMu.Lock() + state := h.enrRefresh[id] + if state == nil { + state = &enrRefreshState{} + h.enrRefresh[id] = state + } + if advertisedSeq > state.highestSeenSeq { + state.highestSeenSeq = advertisedSeq + } + if state.inFlight { + h.enrRefreshMu.Unlock() + return + } + state.inFlight = true + state.targetSeq = state.highestSeenSeq + state.retries = 0 + h.enrRefreshMu.Unlock() + + go h.runENRRefresh(n) +} + +// runENRRefresh fetches a peer's record, repeating only for a sequence observed +// after the current attempt started or a bounded number of failures. +func (h *Handler) runENRRefresh(n *node.Node) { + id := n.ID() + + for { + _, err := h.RequestENR(n) + + h.enrRefreshMu.Lock() + state := h.enrRefresh[id] + if state == nil { + h.enrRefreshMu.Unlock() + return + } + + if state.highestSeenSeq > state.targetSeq { + state.targetSeq = state.highestSeenSeq + state.retries = 0 + h.enrRefreshMu.Unlock() + continue + } + + if err != nil && state.retries < maxENRRefreshRetries { + state.retries++ + backoff := time.Duration(state.retries) * h.config.ExpirationWindow + h.enrRefreshMu.Unlock() + + select { + case <-time.After(backoff): + case <-h.ctx.Done(): + h.releaseENRRefresh(id) + return + } + continue + } + + state.inFlight = false + state.retries = 0 + h.enrRefreshMu.Unlock() + return + } +} + +// releaseENRRefresh clears the in-flight claim without recreating a state entry +// that eviction has already removed. +func (h *Handler) releaseENRRefresh(id node.ID) { + h.enrRefreshMu.Lock() + if state := h.enrRefresh[id]; state != nil { + state.inFlight = false + state.retries = 0 + } + h.enrRefreshMu.Unlock() +} + // handleFindnode processes a FINDNODE request. func (h *Handler) handleFindnode(fromNode *node.Node, from *net.UDPAddr, localAddr *net.UDPAddr, findnode *Findnode) error { logrus.WithFields(logrus.Fields{ @@ -1261,13 +1368,21 @@ func (h *Handler) cleanup() { } h.nodesMu.Lock() + evicted := make([]node.ID, 0, len(stale)) for _, id := range stale { // Re-check: a node may have been seen again since the scan. if n, ok := h.nodes[id]; ok && !n.IsBonded() && now.Sub(n.LastSeen()) > h.config.NodeTTL { delete(h.nodes, id) + evicted = append(evicted, id) } } h.nodesMu.Unlock() + + h.enrRefreshMu.Lock() + for _, id := range evicted { + delete(h.enrRefresh, id) + } + h.enrRefreshMu.Unlock() } // staleNodes returns the IDs of unbonded nodes past their TTL. From aea31616b16bc52062881daef5ba6449d9ec0b33 Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Wed, 29 Jul 2026 15:52:32 -0500 Subject: [PATCH 24/49] fix(bootnode): arm the fork ENR refresh at the next boundary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The refresh ran on a free-running one-minute ticker, so the advertised eth/eth2 fields kept the previous fork for up to a full period after activation — 8s, 20s, 22s and 51s across six devnet transitions. It now waits for the next scheduled boundary, with a one-minute reconciliation tick as the backstop for a missing schedule, a clock jump, or a boundary passed during startup. Boundaries come from the raw fork epochs rather than GetAllForkDigestInfos, which deduplicates by digest: the eth2 next-fork tuple changes at a boundary even when the current digest does not. --- bootnode/clconfig/config.go | 23 ++++++++ bootnode/fork_boundary_test.go | 89 +++++++++++++++++++++++++++++++ bootnode/service.go | 95 +++++++++++++++++++++++++++++++++- 3 files changed, 206 insertions(+), 1 deletion(-) create mode 100644 bootnode/fork_boundary_test.go diff --git a/bootnode/clconfig/config.go b/bootnode/clconfig/config.go index 6b4754d..691b1dd 100644 --- a/bootnode/clconfig/config.go +++ b/bootnode/clconfig/config.go @@ -77,6 +77,29 @@ func (c *Config) getForks() []forkDefinition { return c.forks } +// ForkEpochs returns every scheduled fork epoch in ascending order, including +// BPO entries and excluding far-future placeholders. +// +// Distinct from GetAllForkDigestInfos, which deduplicates by digest: the eth2 +// next-fork tuple changes at a boundary even when the current digest does not, +// so a caller scheduling work per boundary needs the raw epochs. +func (c *Config) ForkEpochs() []uint64 { + forks := c.getForks() + + epochs := make([]uint64, 0, len(forks)) + for i := range forks { + epoch := forks[i].epoch + if epoch == math.MaxUint64 { + continue + } + if len(epochs) > 0 && epochs[len(epochs)-1] == epoch { + continue + } + epochs = append(epochs, epoch) + } + return epochs +} + // GetForkEpoch returns the epoch for a given fork name. // Returns nil if the fork is not defined. func (c *Config) GetForkEpoch(forkName string) *uint64 { diff --git a/bootnode/fork_boundary_test.go b/bootnode/fork_boundary_test.go new file mode 100644 index 0000000..057f7a3 --- /dev/null +++ b/bootnode/fork_boundary_test.go @@ -0,0 +1,89 @@ +package bootnode + +import ( + "os" + "path/filepath" + "strconv" + "testing" + "time" + + "github.com/ethpandaops/bootnodoor/bootnode/clconfig" +) + +func boundaryService(t *testing.T, genesis uint64, electraEpoch string) *Service { + t.Helper() + + yaml := "PRESET_BASE: mainnet\n" + + "MIN_GENESIS_TIME: " + strconv.FormatUint(genesis, 10) + "\n" + + "GENESIS_DELAY: 0\n" + + "SECONDS_PER_SLOT: 12\n" + + "SLOTS_PER_EPOCH: 32\n" + + "GENESIS_FORK_VERSION: 0x10000000\n" + + "ALTAIR_FORK_VERSION: 0x20000000\nALTAIR_FORK_EPOCH: 0\n" + + "BELLATRIX_FORK_VERSION: 0x30000000\nBELLATRIX_FORK_EPOCH: 0\n" + + "CAPELLA_FORK_VERSION: 0x40000000\nCAPELLA_FORK_EPOCH: 0\n" + + "DENEB_FORK_VERSION: 0x50000000\nDENEB_FORK_EPOCH: 0\n" + + "ELECTRA_FORK_VERSION: 0x60000000\nELECTRA_FORK_EPOCH: " + electraEpoch + "\n" + + path := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(path, []byte(yaml), 0o600); err != nil { + t.Fatalf("write config: %v", err) + } + cl, err := clconfig.LoadConfig(path) + if err != nil { + t.Fatalf("load config: %v", err) + } + return &Service{config: &Config{CLConfig: cl}} +} + +// A free-running ticker left the record advertising the previous fork for up to +// its full period, so the wait has to track the next scheduled boundary. +func TestNextForkBoundaryTracksSchedule(t *testing.T) { + now := time.Now() + s := boundaryService(t, uint64(now.Unix()), "1") + + next, ok := s.nextForkBoundary(now) + if !ok { + t.Fatal("no boundary found for an epoch-1 fork") + } + wantAt := now.Add(384 * time.Second) + if diff := next.Sub(wantAt); diff > 2*time.Second || diff < -2*time.Second { + t.Errorf("boundary = %v, want ~%v", next, wantAt) + } +} + +// Boundaries already passed must not yield a negative or immediate timer. +func TestNextForkBoundaryAllPassed(t *testing.T) { + now := time.Now() + s := boundaryService(t, uint64(now.Add(-10*time.Hour).Unix()), "1") + + if _, ok := s.nextForkBoundary(now); ok { + t.Error("a past boundary was reported as upcoming") + } + if delay := s.nextForkRefreshDelay(); delay != maxForkRefreshDelay { + t.Errorf("delay = %v, want the backstop %v", delay, maxForkRefreshDelay) + } +} + +// With no genesis data no epoch has a wall clock, so the backstop must carry the +// refresh instead of the timer firing continuously. +func TestNextForkBoundaryWithoutGenesis(t *testing.T) { + s := &Service{config: &Config{CLConfig: &clconfig.Config{SecondsPerSlot: 12}}} + + if _, ok := s.nextForkBoundary(time.Now()); ok { + t.Error("a boundary was reported with no genesis time") + } + if delay := s.nextForkRefreshDelay(); delay != maxForkRefreshDelay { + t.Errorf("delay = %v, want the backstop %v", delay, maxForkRefreshDelay) + } +} + +// A distant boundary is capped so the reconciliation backstop still runs. +func TestNextForkRefreshDelayCapped(t *testing.T) { + now := time.Now() + s := boundaryService(t, uint64(now.Unix()), "100") + + if delay := s.nextForkRefreshDelay(); delay != maxForkRefreshDelay { + t.Errorf("delay = %v, want the cap %v", delay, maxForkRefreshDelay) + } +} diff --git a/bootnode/service.go b/bootnode/service.go index fd0c142..b0a712b 100644 --- a/bootnode/service.go +++ b/bootnode/service.go @@ -3,6 +3,7 @@ package bootnode import ( "context" "fmt" + "math" "net" "slices" "sync" @@ -538,7 +539,14 @@ func (s *Service) maintenanceLoop() { supportCheck := time.NewTicker(30 * time.Minute) // Check protocol support every 30 minutes badNodesCleanup := time.NewTicker(24 * time.Hour) // Cleanup bad nodes once per day enrRequestCleanup := time.NewTicker(1 * time.Minute) // Cleanup stale ENR requests every minute - forkRefresh := time.NewTicker(1 * time.Minute) // Re-publish eth/eth2 when a fork activates + + // Fork fields are re-published on a timer armed for the next boundary rather + // than polled: peers see the transition immediately and re-request our record, + // so a free-running tick left us advertising the previous fork for up to its + // full period. forkReconcile is the backstop for a missing schedule, a clock + // jump, or a boundary that passed while we were starting. + forkRefresh := time.NewTimer(s.nextForkRefreshDelay()) + forkReconcile := time.NewTicker(1 * time.Minute) defer tableMaintenance.Stop() defer alivenessCheck.Stop() @@ -547,6 +555,7 @@ func (s *Service) maintenanceLoop() { defer badNodesCleanup.Stop() defer enrRequestCleanup.Stop() defer forkRefresh.Stop() + defer forkReconcile.Stop() for { select { @@ -573,8 +582,92 @@ func (s *Service) maintenanceLoop() { case <-forkRefresh.C: s.refreshForkENR() + forkRefresh.Reset(s.nextForkRefreshDelay()) + + case <-forkReconcile.C: + // Refresh before recomputing: if a boundary was missed, this is what + // corrects the record, and the delay must be measured from now. + s.refreshForkENR() + if !forkRefresh.Stop() { + select { + case <-forkRefresh.C: + default: + } + } + forkRefresh.Reset(s.nextForkRefreshDelay()) + } + } +} + +// forkRefreshLead re-publishes just before a boundary so the record is already +// correct when peers act on the transition. +const forkRefreshLead = 500 * time.Millisecond + +// maxForkRefreshDelay caps the wait so a schedule that yields no future boundary +// still reaches refreshForkENR at the old cadence. +const maxForkRefreshDelay = time.Minute + +// nextForkRefreshDelay returns how long until the next scheduled fork boundary. +func (s *Service) nextForkRefreshDelay() time.Duration { + now := time.Now() + + next, ok := s.nextForkBoundary(now) + if !ok { + return maxForkRefreshDelay + } + + delay := next.Sub(now) + forkRefreshLead + if delay < time.Millisecond { + delay = time.Millisecond + } + if delay > maxForkRefreshDelay { + delay = maxForkRefreshDelay + } + return delay +} + +// nextForkBoundary returns the earliest CL or EL fork activation after now. +func (s *Service) nextForkBoundary(now time.Time) (time.Time, bool) { + var next time.Time + found := false + + consider := func(t time.Time) { + if !t.After(now) { + return + } + if !found || t.Before(next) { + next = t + found = true } } + + if cfg := s.config.CLConfig; cfg != nil { + genesis := cfg.GetGenesisTime() + slotsPerEpoch := cfg.GetSlotsPerEpoch() + secondsPerSlot := cfg.SecondsPerSlot + if genesis > 0 && slotsPerEpoch > 0 && secondsPerSlot > 0 { + for _, epoch := range cfg.ForkEpochs() { + // Overflow guard: a placeholder epoch would wrap the product. + if epoch > math.MaxUint64/(slotsPerEpoch*secondsPerSlot) { + continue + } + consider(time.Unix(int64(genesis+epoch*slotsPerEpoch*secondsPerSlot), 0)) + } + } + } + + if s.enrManager != nil { + if filter := s.enrManager.GetELFilter(); filter != nil { + for _, fork := range filter.GetAllForkIDsWithNames() { + // Block-numbered forks have no wall clock; only timestamps do. + if fork.IsTime && fork.Activation <= math.MaxInt64 { + consider(time.Unix(int64(fork.Activation), 0)) + } + } + } + } + + return next, found } // localIDs returns the node IDs of every discovery identity, so discovery can From ac12e1cd6d7a3c34da7dd5fd7260333fc95d9465 Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Wed, 29 Jul 2026 16:05:33 -0500 Subject: [PATCH 25/49] fix: address review findings in the devnet fix set - batchUpdate cleared every dirty flag after writing, discarding any flag marked while the write was in flight: that caller saw the node already queued and did not enqueue it again. Clear only the observed flags and requeue the remainder. - The full upsert's conflict clause never assigned last_active, so an existing row kept a stale or NULL timestamp. COALESCE so a caller without one cannot blank a stored value. - A peer advertising a higher sequence in every PONG could re-arm the ENR refresh indefinitely, holding a goroutine and its traffic until shutdown. Bound the rounds one claim may run; a later PONG opens a fresh claim. - The fork overflow guard divided by a product that could itself wrap to zero, panicking the maintenance goroutine. Check each multiplication first. - Default the refresh backoff when ExpirationWindow is unset, so a retry is delayed rather than immediate. --- bootnode/service.go | 24 ++++++++++-- db/layer_key_test.go | 51 +++++++++++++++++++++++++ db/nodes.go | 2 + discv4/protocol/enr_refresh_test.go | 59 +++++++++++++++++++++++++---- discv4/protocol/handler.go | 23 +++++++++-- nodes/admission_persist_test.go | 28 ++++++++++++++ nodes/node.go | 10 +++++ nodes/nodedb.go | 24 ++++++++++-- 8 files changed, 204 insertions(+), 17 deletions(-) diff --git a/bootnode/service.go b/bootnode/service.go index b0a712b..3a79365 100644 --- a/bootnode/service.go +++ b/bootnode/service.go @@ -626,6 +626,24 @@ func (s *Service) nextForkRefreshDelay() time.Duration { return delay } +// forkOffsetSeconds returns seconds from genesis to an epoch, reporting false if +// any step would overflow. Each multiplication is checked before it happens: a +// placeholder epoch or an absurd slot length would otherwise wrap, and dividing +// by an already-wrapped product panics. +func forkOffsetSeconds(epoch, slotsPerEpoch, secondsPerSlot uint64) (uint64, bool) { + if slotsPerEpoch == 0 || secondsPerSlot == 0 { + return 0, false + } + if slotsPerEpoch > math.MaxUint64/secondsPerSlot { + return 0, false + } + epochSeconds := slotsPerEpoch * secondsPerSlot + if epoch > math.MaxUint64/epochSeconds { + return 0, false + } + return epoch * epochSeconds, true +} + // nextForkBoundary returns the earliest CL or EL fork activation after now. func (s *Service) nextForkBoundary(now time.Time) (time.Time, bool) { var next time.Time @@ -647,11 +665,11 @@ func (s *Service) nextForkBoundary(now time.Time) (time.Time, bool) { secondsPerSlot := cfg.SecondsPerSlot if genesis > 0 && slotsPerEpoch > 0 && secondsPerSlot > 0 { for _, epoch := range cfg.ForkEpochs() { - // Overflow guard: a placeholder epoch would wrap the product. - if epoch > math.MaxUint64/(slotsPerEpoch*secondsPerSlot) { + offset, ok := forkOffsetSeconds(epoch, slotsPerEpoch, secondsPerSlot) + if !ok || genesis > math.MaxInt64-offset { continue } - consider(time.Unix(int64(genesis+epoch*slotsPerEpoch*secondsPerSlot), 0)) + consider(time.Unix(int64(genesis+offset), 0)) } } } diff --git a/db/layer_key_test.go b/db/layer_key_test.go index c18744a..de5de68 100644 --- a/db/layer_key_test.go +++ b/db/layer_key_test.go @@ -1,6 +1,7 @@ package db import ( + "database/sql" "testing" "time" @@ -137,3 +138,53 @@ func TestBadNodeSuppressionSurvivesBothLayers(t *testing.T) { t.Errorf("CL reason = %q, want invalid_fork_digest", clReason) } } + +// The full upsert supplies last_active, but its conflict clause has to assign it +// too: an already-persisted row would otherwise keep a stale or NULL timestamp +// while the DirtyFull branch cleared the flag that would have fixed it. +func TestUpsertUpdatesLastActiveOnExistingRow(t *testing.T) { + database := testDB(t) + + id := []byte("aaaabbbbccccddddeeeeffff00001111") + + if err := database.RunDBTransaction(func(tx *sqlx.Tx) error { + return database.UpdateNodeENR(tx, LayerEL, id, nil, nil, 30303, 1, []byte{1, 2, 3, 4}, []byte("enr"), true, true) + }); err != nil { + t.Fatalf("seed row: %v", err) + } + + active := time.Now().Unix() + n := &Node{ + NodeID: id, Layer: string(LayerEL), Port: 30303, Seq: 2, + ForkDigest: []byte{1, 2, 3, 4}, FirstSeen: 1000, ENR: []byte("enr2"), + LastActive: sql.NullInt64{Valid: true, Int64: active}, + } + if err := database.RunDBTransaction(func(tx *sqlx.Tx) error { + return database.UpsertNode(tx, n) + }); err != nil { + t.Fatalf("upsert: %v", err) + } + + stored, err := database.GetNode(LayerEL, id) + if err != nil { + t.Fatalf("load: %v", err) + } + if !stored.LastActive.Valid || stored.LastActive.Int64 != active { + t.Errorf("last_active = %v, want %d", stored.LastActive, active) + } + + // A caller with no timestamp must not blank the stored one. + n.LastActive = sql.NullInt64{} + if err := database.RunDBTransaction(func(tx *sqlx.Tx) error { + return database.UpsertNode(tx, n) + }); err != nil { + t.Fatalf("second upsert: %v", err) + } + stored, err = database.GetNode(LayerEL, id) + if err != nil { + t.Fatalf("reload: %v", err) + } + if !stored.LastActive.Valid || stored.LastActive.Int64 != active { + t.Errorf("last_active was blanked by an upsert without a timestamp: %v", stored.LastActive) + } +} diff --git a/db/nodes.go b/db/nodes.go index 6661b49..3619215 100644 --- a/db/nodes.go +++ b/db/nodes.go @@ -149,6 +149,8 @@ func (d *Database) UpsertNode(tx *sqlx.Tx, node *Node) error { seq = excluded.seq, fork_digest = excluded.fork_digest, last_seen = excluded.last_seen, + -- COALESCE so a caller without a timestamp cannot blank a stored one. + last_active = COALESCE(excluded.last_active, nodes.last_active), enr = excluded.enr, has_v4 = excluded.has_v4, has_v5 = excluded.has_v5, diff --git a/discv4/protocol/enr_refresh_test.go b/discv4/protocol/enr_refresh_test.go index 559ca85..aad98ff 100644 --- a/discv4/protocol/enr_refresh_test.go +++ b/discv4/protocol/enr_refresh_test.go @@ -19,12 +19,13 @@ type scriptedPeer struct { h *Handler key *ecdsa.PrivateKey - mu sync.Mutex - pings int - enrReqs int - enrSeq uint64 - stopped bool - pongAddr *net.UDPAddr + mu sync.Mutex + pings int + enrReqs int + enrSeq uint64 + stopped bool + bumpEachPong bool + pongAddr *net.UDPAddr } func (p *scriptedPeer) SendTo(data []byte, to *net.UDPAddr) error { @@ -38,13 +39,19 @@ func (p *scriptedPeer) SendTo(data []byte, to *net.UDPAddr) error { p.mu.Unlock() return nil } - seq := p.enrSeq switch packet.(type) { case *Ping: p.pings++ + // Bump on PING only, so every round's PONG advertises strictly more than + // the record the previous round installed. Bumping on the ENRREQUEST too + // would put the installed record ahead and the loop would not re-arm. + if p.bumpEachPong { + p.enrSeq++ + } case *ENRRequest: p.enrReqs++ } + seq := p.enrSeq p.mu.Unlock() var reply []byte @@ -282,3 +289,41 @@ func TestENRRefreshConcurrentTriggers(t *testing.T) { time.Sleep(1500 * time.Millisecond) peer.stop() } + +// A peer that advertises a higher sequence in every PONG could otherwise re-arm +// the refresh forever, holding a goroutine and generating traffic until shutdown. +func TestENRRefreshBoundedAgainstEndlessBumps(t *testing.T) { + peer := &scriptedPeer{t: t, enrSeq: 5, pongAddr: &net.UDPAddr{IP: net.IPv4(1, 2, 3, 4), Port: 30303}, bumpEachPong: true} + + h, cancel := newScriptedHandler(t, peer) + defer cancel() + peer.h = h + + n, key := makeKeyedNode(t, 30303) + peer.key = key + if !n.UpdateENR(signedV4Record(t, key, 1)) { + t.Fatal("seed record was not installed") + } + h.nodesMu.Lock() + h.nodes[n.ID()] = n + h.nodesMu.Unlock() + + deliverPong(t, h, n, key, []byte("bump-ping-hash"), 5) + + time.Sleep(4 * time.Second) + peer.stop() + _, enrReqs := peer.counts() + + // Literal, not maxENRRefreshRounds: comparing against the constant under test + // makes the assertion vacuous when the bound is raised. + if enrReqs > 4 { + t.Errorf("ENRREQUESTs sent = %d, want at most 4 rounds per claim", enrReqs) + } + + h.enrRefreshMu.Lock() + inFlight := h.enrRefresh[n.ID()].inFlight + h.enrRefreshMu.Unlock() + if inFlight { + t.Error("refresh still marked in flight after the round bound was reached") + } +} diff --git a/discv4/protocol/handler.go b/discv4/protocol/handler.go index 6488625..d18c65e 100644 --- a/discv4/protocol/handler.go +++ b/discv4/protocol/handler.go @@ -437,6 +437,13 @@ func (h *Handler) handlePong(fromNode *node.Node, from *net.UDPAddr, pong *Pong) // never answers cannot keep one running. const maxENRRefreshRetries = 2 +// maxENRRefreshRounds bounds the rounds one claim may run. Re-arming on a +// sequence observed mid-refresh is otherwise unbounded: a peer that advertises a +// higher sequence in every PONG keeps the goroutine and its PING/ENRREQUEST +// traffic alive indefinitely. Past the bound the claim ends, and a later PONG has +// to open a fresh one. +const maxENRRefreshRounds = 4 + // enrRefreshState tracks one peer's automatic ENR refresh. type enrRefreshState struct { inFlight bool @@ -450,6 +457,7 @@ type enrRefreshState struct { highestSeenSeq uint64 retries int + rounds int } // startENRRefresh claims the refresh for a peer and runs at most one at a time. @@ -475,6 +483,7 @@ func (h *Handler) startENRRefresh(n *node.Node, advertisedSeq uint64) { state.inFlight = true state.targetSeq = state.highestSeenSeq state.retries = 0 + state.rounds = 0 h.enrRefreshMu.Unlock() go h.runENRRefresh(n) @@ -495,16 +504,24 @@ func (h *Handler) runENRRefresh(n *node.Node) { return } - if state.highestSeenSeq > state.targetSeq { + state.rounds++ + + if state.highestSeenSeq > state.targetSeq && state.rounds < maxENRRefreshRounds { state.targetSeq = state.highestSeenSeq state.retries = 0 h.enrRefreshMu.Unlock() continue } - if err != nil && state.retries < maxENRRefreshRetries { + if err != nil && state.retries < maxENRRefreshRetries && state.rounds < maxENRRefreshRounds { state.retries++ - backoff := time.Duration(state.retries) * h.config.ExpirationWindow + // ExpirationWindow can be zero in a bare config; a zero backoff would + // make the retry immediate rather than delayed. + unit := h.config.ExpirationWindow + if unit <= 0 { + unit = 20 * time.Second + } + backoff := time.Duration(state.retries) * unit h.enrRefreshMu.Unlock() select { diff --git a/nodes/admission_persist_test.go b/nodes/admission_persist_test.go index 7dc4aa8..49b15d2 100644 --- a/nodes/admission_persist_test.go +++ b/nodes/admission_persist_test.go @@ -159,3 +159,31 @@ func TestFullUpsertPersistsLastActive(t *testing.T) { t.Error("last_active was written as NULL by the full upsert") } } + +// batchUpdate snapshots the dirty flags, writes, then clears. Clearing +// everything discarded any flag marked while the write was in flight, because +// that caller saw the node already queued and did not enqueue it again. +func TestClearDirtyFlagsMaskKeepsUnobservedFlags(t *testing.T) { + database := persistTestDB(t, filepath.Join(t.TempDir(), "flags.db")) + defer database.Close() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + ndb := NewNodeDB(ctx, database, db.LayerCL, quietTableLogger()) + n := NewFromV5(makeV5At(t, net.IPv4(10, 4, 0, 1)), ndb) + + n.MarkDirty(DirtyENR) + observed := n.GetDirtyFlags() + n.MarkDirty(DirtyLastActive) + + if remaining := n.ClearDirtyFlagsMask(observed); !remaining { + t.Error("ClearDirtyFlagsMask reported nothing left, so the later flag would not be requeued") + } + if got := n.GetDirtyFlags(); got&DirtyLastActive == 0 { + t.Error("a flag marked after the snapshot was cleared unwritten") + } + if got := n.GetDirtyFlags(); got&DirtyENR != 0 { + t.Error("the observed flag was not cleared") + } +} diff --git a/nodes/node.go b/nodes/node.go index 5ce07ea..fe0a598 100644 --- a/nodes/node.go +++ b/nodes/node.go @@ -513,6 +513,16 @@ func (n *Node) ClearDirtyFlags() { n.dirtyMu.Unlock() } +// ClearDirtyFlagsMask clears only the given flags and reports whether any +// remain. Writers clear what they observed rather than everything, so a flag +// marked while the batch was in flight survives to the next round. +func (n *Node) ClearDirtyFlagsMask(flags DirtyFlags) bool { + n.dirtyMu.Lock() + defer n.dirtyMu.Unlock() + n.dirtyFields &^= flags + return n.dirtyFields != 0 +} + // LastActive returns the last active timestamp. func (n *Node) LastActive() time.Time { n.mu.RLock() diff --git a/nodes/nodedb.go b/nodes/nodedb.go index 1ad1c41..6d747d1 100644 --- a/nodes/nodedb.go +++ b/nodes/nodedb.go @@ -182,7 +182,10 @@ func (ndb *NodeDB) batchUpdate(nodes []*Node) { "layer": ndb.layer, }).Debug("processing batch update") + var requeue []*Node + err := ndb.db.RunDBTransaction(func(tx *sqlx.Tx) error { + requeue = requeue[:0] for _, node := range nodes { dirtyFlags := node.GetDirtyFlags() nodeID := node.ID() @@ -199,8 +202,10 @@ func (ndb *NodeDB) batchUpdate(nodes []*Node) { ndb.logger.WithError(err).WithField("nodeID", fmt.Sprintf("%x", nodeID[:8])).Error("failed to upsert node in batch") continue } - // Full upsert covers everything, clear all dirty flags - node.ClearDirtyFlags() + // Full upsert covers everything this pass observed. + if node.ClearDirtyFlagsMask(dirtyFlags) { + requeue = append(requeue, node) + } continue } @@ -247,8 +252,11 @@ func (ndb *NodeDB) batchUpdate(nodes []*Node) { } } - // Clear dirty flags after successful update - node.ClearDirtyFlags() + // Clear only what this pass observed, so a flag marked while the + // batch was running is not erased unwritten. + if node.ClearDirtyFlagsMask(dirtyFlags) { + requeue = append(requeue, node) + } } return nil }) @@ -269,6 +277,14 @@ func (ndb *NodeDB) batchUpdate(nodes []*Node) { } ndb.updateQueueLock.Unlock() + // Re-queue anything marked dirty while this batch was in flight: those + // callers saw the node already queued and returned without enqueueing. + for _, node := range requeue { + if err := ndb.QueueUpdate(node); err != nil { + ndb.logger.WithError(err).Debug("failed to requeue node dirtied during batch") + } + } + // Track processed updates ndb.statsLock.Lock() ndb.stats.ProcessedUpdates += int64(len(nodes)) From df72c158b5c991e00a2e1d42ebc2191e8a5ccd2a Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Wed, 29 Jul 2026 16:12:52 -0500 Subject: [PATCH 26/49] fix: close the residual windows from the second review round - Clearing the observed flag bits could still drop a same-bit re-mark made while the write was in flight, since the new mark is indistinguishable from the one written. Snapshot a generation alongside the flags and skip clearing entirely when it moved. - The refresh round bound was per claim, so a peer could send one PING, take the reciprocal PING's higher-sequence PONG, and open a fresh four-round claim immediately. Exhausting the rounds now starts a cooldown. - The signed-range check subtracted an unsigned offset from MaxInt64, which wraps for an offset past MaxInt64 and let an unrepresentable timestamp through. --- bootnode/service.go | 4 ++- discv4/protocol/enr_refresh_test.go | 41 +++++++++++++++++++++++++++++ discv4/protocol/handler.go | 15 ++++++++++- nodes/admission_persist_test.go | 20 +++++++++----- nodes/node.go | 23 +++++++++++++--- nodes/nodedb.go | 6 ++--- 6 files changed, 94 insertions(+), 15 deletions(-) diff --git a/bootnode/service.go b/bootnode/service.go index 3a79365..6a24111 100644 --- a/bootnode/service.go +++ b/bootnode/service.go @@ -666,7 +666,9 @@ func (s *Service) nextForkBoundary(now time.Time) (time.Time, bool) { if genesis > 0 && slotsPerEpoch > 0 && secondsPerSlot > 0 { for _, epoch := range cfg.ForkEpochs() { offset, ok := forkOffsetSeconds(epoch, slotsPerEpoch, secondsPerSlot) - if !ok || genesis > math.MaxInt64-offset { + // Bound offset first: MaxInt64-offset is unsigned arithmetic and + // would wrap for an offset past MaxInt64, letting the guard pass. + if !ok || offset > math.MaxInt64 || genesis > math.MaxInt64-offset { continue } consider(time.Unix(int64(genesis+offset), 0)) diff --git a/discv4/protocol/enr_refresh_test.go b/discv4/protocol/enr_refresh_test.go index aad98ff..621f2a9 100644 --- a/discv4/protocol/enr_refresh_test.go +++ b/discv4/protocol/enr_refresh_test.go @@ -327,3 +327,44 @@ func TestENRRefreshBoundedAgainstEndlessBumps(t *testing.T) { t.Error("refresh still marked in flight after the round bound was reached") } } + +// The round bound is per claim, so without a cooldown a peer could open a fresh +// claim immediately and sustain the same rate one PING at a time. +func TestENRRefreshCooldownAfterRoundsExhausted(t *testing.T) { + peer := &scriptedPeer{t: t, enrSeq: 5, pongAddr: &net.UDPAddr{IP: net.IPv4(1, 2, 3, 4), Port: 30303}, bumpEachPong: true} + + h, cancel := newScriptedHandler(t, peer) + defer cancel() + peer.h = h + + n, key := makeKeyedNode(t, 30303) + peer.key = key + if !n.UpdateENR(signedV4Record(t, key, 1)) { + t.Fatal("seed record was not installed") + } + h.nodesMu.Lock() + h.nodes[n.ID()] = n + h.nodesMu.Unlock() + + deliverPong(t, h, n, key, []byte("cooldown-ping-1"), 5) + time.Sleep(3500 * time.Millisecond) + + _, afterFirst := peer.counts() + + h.enrRefreshMu.Lock() + cooling := time.Now().Before(h.enrRefresh[n.ID()].cooldownUntil) + h.enrRefreshMu.Unlock() + if !cooling { + t.Fatal("no cooldown was set after the rounds were exhausted") + } + + // A fresh trigger during the cooldown must not open another claim. + deliverPong(t, h, n, key, []byte("cooldown-ping-2"), 99) + time.Sleep(1500 * time.Millisecond) + peer.stop() + + _, afterSecond := peer.counts() + if afterSecond != afterFirst { + t.Errorf("ENRREQUESTs went %d -> %d during the cooldown, want no new claim", afterFirst, afterSecond) + } +} diff --git a/discv4/protocol/handler.go b/discv4/protocol/handler.go index d18c65e..5da041f 100644 --- a/discv4/protocol/handler.go +++ b/discv4/protocol/handler.go @@ -444,6 +444,10 @@ const maxENRRefreshRetries = 2 // to open a fresh one. const maxENRRefreshRounds = 4 +// enrRefreshCooldown is how long a peer waits for a new claim after exhausting +// one, capping sustained refresh traffic per peer regardless of what it advertises. +const enrRefreshCooldown = 30 * time.Second + // enrRefreshState tracks one peer's automatic ENR refresh. type enrRefreshState struct { inFlight bool @@ -458,6 +462,12 @@ type enrRefreshState struct { retries int rounds int + + // cooldownUntil applies after a claim exhausts its rounds. Without it the + // bound is per claim only: one inbound PING earns a reciprocal PING, whose + // higher-sequence PONG opens a fresh claim, so a peer could sustain the same + // ENRREQUEST rate with one packet per claim. + cooldownUntil time.Time } // startENRRefresh claims the refresh for a peer and runs at most one at a time. @@ -476,7 +486,7 @@ func (h *Handler) startENRRefresh(n *node.Node, advertisedSeq uint64) { if advertisedSeq > state.highestSeenSeq { state.highestSeenSeq = advertisedSeq } - if state.inFlight { + if state.inFlight || time.Now().Before(state.cooldownUntil) { h.enrRefreshMu.Unlock() return } @@ -533,6 +543,9 @@ func (h *Handler) runENRRefresh(n *node.Node) { continue } + if state.rounds >= maxENRRefreshRounds { + state.cooldownUntil = time.Now().Add(enrRefreshCooldown) + } state.inFlight = false state.retries = 0 h.enrRefreshMu.Unlock() diff --git a/nodes/admission_persist_test.go b/nodes/admission_persist_test.go index 49b15d2..f4148c6 100644 --- a/nodes/admission_persist_test.go +++ b/nodes/admission_persist_test.go @@ -163,7 +163,7 @@ func TestFullUpsertPersistsLastActive(t *testing.T) { // batchUpdate snapshots the dirty flags, writes, then clears. Clearing // everything discarded any flag marked while the write was in flight, because // that caller saw the node already queued and did not enqueue it again. -func TestClearDirtyFlagsMaskKeepsUnobservedFlags(t *testing.T) { +func TestClearDirtySnapshotKeepsUnobservedFlags(t *testing.T) { database := persistTestDB(t, filepath.Join(t.TempDir(), "flags.db")) defer database.Close() @@ -174,16 +174,24 @@ func TestClearDirtyFlagsMaskKeepsUnobservedFlags(t *testing.T) { n := NewFromV5(makeV5At(t, net.IPv4(10, 4, 0, 1)), ndb) n.MarkDirty(DirtyENR) - observed := n.GetDirtyFlags() + observed, gen := n.DirtySnapshot() n.MarkDirty(DirtyLastActive) - if remaining := n.ClearDirtyFlagsMask(observed); !remaining { - t.Error("ClearDirtyFlagsMask reported nothing left, so the later flag would not be requeued") + if remaining := n.ClearDirtySnapshot(observed, gen); !remaining { + t.Error("ClearDirtySnapshot reported nothing left, so the later mark would not be requeued") } if got := n.GetDirtyFlags(); got&DirtyLastActive == 0 { t.Error("a flag marked after the snapshot was cleared unwritten") } - if got := n.GetDirtyFlags(); got&DirtyENR != 0 { - t.Error("the observed flag was not cleared") + + // Re-marking the same bit must also survive: the generation moved, so the + // observed write cannot be assumed to cover the newer value. + observed, gen = n.DirtySnapshot() + n.MarkDirty(DirtyLastActive) + if remaining := n.ClearDirtySnapshot(observed, gen); !remaining { + t.Error("a same-bit re-mark during the write was dropped") + } + if got := n.GetDirtyFlags(); got&DirtyLastActive == 0 { + t.Error("same-bit re-mark was cleared unwritten") } } diff --git a/nodes/node.go b/nodes/node.go index fe0a598..22df325 100644 --- a/nodes/node.go +++ b/nodes/node.go @@ -64,6 +64,7 @@ type Node struct { // Dirty tracking for database updates dirtyMu sync.Mutex dirtyFields DirtyFlags + dirtyGen uint64 } // NewFromV4 creates a generic Node from a discv4 node. @@ -495,6 +496,7 @@ func (n *Node) CalculateScore(forkInfo *ForkScoringInfo) float64 { func (n *Node) MarkDirty(flags DirtyFlags) { n.dirtyMu.Lock() n.dirtyFields |= flags + n.dirtyGen++ n.dirtyMu.Unlock() } @@ -513,12 +515,25 @@ func (n *Node) ClearDirtyFlags() { n.dirtyMu.Unlock() } -// ClearDirtyFlagsMask clears only the given flags and reports whether any -// remain. Writers clear what they observed rather than everything, so a flag -// marked while the batch was in flight survives to the next round. -func (n *Node) ClearDirtyFlagsMask(flags DirtyFlags) bool { +// DirtySnapshot returns the current flags and a generation that changes on every +// subsequent MarkDirty, so a writer can tell whether anything was marked while it +// was working. +func (n *Node) DirtySnapshot() (DirtyFlags, uint64) { n.dirtyMu.Lock() defer n.dirtyMu.Unlock() + return n.dirtyFields, n.dirtyGen +} + +// ClearDirtySnapshot clears the snapshotted flags and reports whether the node is +// still dirty. Clearing is skipped entirely when the generation moved: the same +// bit may have been re-marked for a newer value, which is indistinguishable from +// the one just written, so the field would otherwise be dropped unwritten. +func (n *Node) ClearDirtySnapshot(flags DirtyFlags, gen uint64) bool { + n.dirtyMu.Lock() + defer n.dirtyMu.Unlock() + if n.dirtyGen != gen { + return true + } n.dirtyFields &^= flags return n.dirtyFields != 0 } diff --git a/nodes/nodedb.go b/nodes/nodedb.go index 6d747d1..1443eb6 100644 --- a/nodes/nodedb.go +++ b/nodes/nodedb.go @@ -187,7 +187,7 @@ func (ndb *NodeDB) batchUpdate(nodes []*Node) { err := ndb.db.RunDBTransaction(func(tx *sqlx.Tx) error { requeue = requeue[:0] for _, node := range nodes { - dirtyFlags := node.GetDirtyFlags() + dirtyFlags, dirtyGen := node.DirtySnapshot() nodeID := node.ID() ndb.logger.WithFields(logrus.Fields{ @@ -203,7 +203,7 @@ func (ndb *NodeDB) batchUpdate(nodes []*Node) { continue } // Full upsert covers everything this pass observed. - if node.ClearDirtyFlagsMask(dirtyFlags) { + if node.ClearDirtySnapshot(dirtyFlags, dirtyGen) { requeue = append(requeue, node) } continue @@ -254,7 +254,7 @@ func (ndb *NodeDB) batchUpdate(nodes []*Node) { // Clear only what this pass observed, so a flag marked while the // batch was running is not erased unwritten. - if node.ClearDirtyFlagsMask(dirtyFlags) { + if node.ClearDirtySnapshot(dirtyFlags, dirtyGen) { requeue = append(requeue, node) } } From 179cbd7336a99e3371153199bd1cab379a74d759 Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Wed, 29 Jul 2026 16:21:50 -0500 Subject: [PATCH 27/49] fix: close the third-round review findings - The queue-set entry was removed after the flags were cleared, so a caller that marked the node in between had its QueueUpdate swallowed as already-queued and then the entry deleted underneath it. Clear after removal instead. - Bounding rounds per claim did not bound the rate: a peer could advertise one increment per claim, finish in a single round, and reopen on the next PONG. Every completed claim now starts a minimum interval, with the longer cooldown reserved for an exhausted one. - A bump observed during a cooldown was remembered but never fetched unless another PONG arrived later. The cleanup tick now resumes those. --- discv4/protocol/enr_refresh_test.go | 39 ++++++++++++++++++++ discv4/protocol/handler.go | 55 +++++++++++++++++++++++++++-- nodes/nodedb.go | 33 +++++++++-------- 3 files changed, 109 insertions(+), 18 deletions(-) diff --git a/discv4/protocol/enr_refresh_test.go b/discv4/protocol/enr_refresh_test.go index 621f2a9..5d45b53 100644 --- a/discv4/protocol/enr_refresh_test.go +++ b/discv4/protocol/enr_refresh_test.go @@ -368,3 +368,42 @@ func TestENRRefreshCooldownAfterRoundsExhausted(t *testing.T) { t.Errorf("ENRREQUESTs went %d -> %d during the cooldown, want no new claim", afterFirst, afterSecond) } } + +// A bump seen during a cooldown is remembered but not fetched, so the cached +// record would stay stale unless another PONG happened to arrive later. +func TestENRRefreshResumesBumpDeferredByCooldown(t *testing.T) { + peer := &scriptedPeer{t: t, enrSeq: 5, pongAddr: &net.UDPAddr{IP: net.IPv4(1, 2, 3, 4), Port: 30303}} + + h, cancel := newScriptedHandler(t, peer) + defer cancel() + peer.h = h + + n, key := makeKeyedNode(t, 30303) + peer.key = key + if !n.UpdateENR(signedV4Record(t, key, 1)) { + t.Fatal("seed record was not installed") + } + h.nodesMu.Lock() + h.nodes[n.ID()] = n + h.nodesMu.Unlock() + + h.enrRefreshMu.Lock() + h.enrRefresh[n.ID()] = &enrRefreshState{highestSeenSeq: 9, targetSeq: 1} + h.enrRefreshMu.Unlock() + + peer.setSeq(9) + h.resumeDeferredENRRefreshes() + + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + if _, enrReqs := peer.counts(); enrReqs > 0 { + break + } + time.Sleep(50 * time.Millisecond) + } + peer.stop() + + if _, enrReqs := peer.counts(); enrReqs == 0 { + t.Error("a bump deferred by cooldown was never fetched") + } +} diff --git a/discv4/protocol/handler.go b/discv4/protocol/handler.go index 5da041f..21283b1 100644 --- a/discv4/protocol/handler.go +++ b/discv4/protocol/handler.go @@ -444,10 +444,15 @@ const maxENRRefreshRetries = 2 // to open a fresh one. const maxENRRefreshRounds = 4 -// enrRefreshCooldown is how long a peer waits for a new claim after exhausting -// one, capping sustained refresh traffic per peer regardless of what it advertises. +// enrRefreshCooldown follows a claim that exhausted its rounds. const enrRefreshCooldown = 30 * time.Second +// enrRefreshMinInterval separates consecutive claims for one peer. Bounding +// rounds alone is not enough: a peer can advertise one increment per claim, let +// it finish in a single round, and reopen on the next PONG, sustaining the same +// ENRREQUEST rate without ever exhausting a claim. +const enrRefreshMinInterval = 5 * time.Second + // enrRefreshState tracks one peer's automatic ENR refresh. type enrRefreshState struct { inFlight bool @@ -543,9 +548,11 @@ func (h *Handler) runENRRefresh(n *node.Node) { continue } + cooldown := enrRefreshMinInterval if state.rounds >= maxENRRefreshRounds { - state.cooldownUntil = time.Now().Add(enrRefreshCooldown) + cooldown = enrRefreshCooldown } + state.cooldownUntil = time.Now().Add(cooldown) state.inFlight = false state.retries = 0 h.enrRefreshMu.Unlock() @@ -553,6 +560,47 @@ func (h *Handler) runENRRefresh(n *node.Node) { } } +// resumeDeferredENRRefreshes starts refreshes for peers whose newer sequence was +// observed during a cooldown. Without this the bump is remembered but never +// fetched unless another PONG happens to arrive after the cooldown expires. +func (h *Handler) resumeDeferredENRRefreshes() { + now := time.Now() + + type pending struct { + id node.ID + seq uint64 + } + var due []pending + + h.enrRefreshMu.Lock() + for id, state := range h.enrRefresh { + if state.inFlight || now.Before(state.cooldownUntil) { + continue + } + if state.highestSeenSeq > state.targetSeq { + due = append(due, pending{id, state.highestSeenSeq}) + } + } + h.enrRefreshMu.Unlock() + + if len(due) == 0 { + return + } + + h.nodesMu.RLock() + resume := make([]*node.Node, 0, len(due)) + for _, p := range due { + if n, ok := h.nodes[p.id]; ok && n.ENR() != nil && p.seq > n.ENR().Seq() { + resume = append(resume, n) + } + } + h.nodesMu.RUnlock() + + for _, n := range resume { + h.startENRRefresh(n, 0) + } +} + // releaseENRRefresh clears the in-flight claim without recreating a state entry // that eviction has already removed. func (h *Handler) releaseENRRefresh(id node.ID) { @@ -1357,6 +1405,7 @@ func (h *Handler) cleanupLoop() { select { case <-ticker.C: h.cleanup() + h.resumeDeferredENRRefreshes() case <-h.ctx.Done(): return } diff --git a/nodes/nodedb.go b/nodes/nodedb.go index 1443eb6..b9e959e 100644 --- a/nodes/nodedb.go +++ b/nodes/nodedb.go @@ -182,10 +182,15 @@ func (ndb *NodeDB) batchUpdate(nodes []*Node) { "layer": ndb.layer, }).Debug("processing batch update") - var requeue []*Node + type written struct { + node *Node + flags DirtyFlags + gen uint64 + } + var processed []written err := ndb.db.RunDBTransaction(func(tx *sqlx.Tx) error { - requeue = requeue[:0] + processed = processed[:0] for _, node := range nodes { dirtyFlags, dirtyGen := node.DirtySnapshot() nodeID := node.ID() @@ -202,10 +207,7 @@ func (ndb *NodeDB) batchUpdate(nodes []*Node) { ndb.logger.WithError(err).WithField("nodeID", fmt.Sprintf("%x", nodeID[:8])).Error("failed to upsert node in batch") continue } - // Full upsert covers everything this pass observed. - if node.ClearDirtySnapshot(dirtyFlags, dirtyGen) { - requeue = append(requeue, node) - } + processed = append(processed, written{node, dirtyFlags, dirtyGen}) continue } @@ -252,11 +254,7 @@ func (ndb *NodeDB) batchUpdate(nodes []*Node) { } } - // Clear only what this pass observed, so a flag marked while the - // batch was running is not erased unwritten. - if node.ClearDirtySnapshot(dirtyFlags, dirtyGen) { - requeue = append(requeue, node) - } + processed = append(processed, written{node, dirtyFlags, dirtyGen}) } return nil }) @@ -277,10 +275,15 @@ func (ndb *NodeDB) batchUpdate(nodes []*Node) { } ndb.updateQueueLock.Unlock() - // Re-queue anything marked dirty while this batch was in flight: those - // callers saw the node already queued and returned without enqueueing. - for _, node := range requeue { - if err := ndb.QueueUpdate(node); err != nil { + // Clear after the set entry is gone, so a caller that marked the node while + // the write was running either enqueued itself or is requeued here. Clearing + // first would let its QueueUpdate be swallowed as already-queued and then have + // the entry deleted underneath it. + for _, p := range processed { + if !p.node.ClearDirtySnapshot(p.flags, p.gen) { + continue + } + if err := ndb.QueueUpdate(p.node); err != nil { ndb.logger.WithError(err).Debug("failed to requeue node dirtied during batch") } } From 5c4596454bb6531b4f99de89529702146c205b8c Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Wed, 29 Jul 2026 16:28:51 -0500 Subject: [PATCH 28/49] fix(nodes): never clear dirty flags for writes that did not land A failed per-field update, and a transaction that failed at commit, both still had their snapshots cleared as if persisted, so the ENR, stats or timestamp was silently lost until an unrelated change happened to mark the node again. Track per-node write failures, discard the processed set entirely when the commit fails, and requeue anything not confirmed written. --- nodes/nodedb.go | 36 +++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/nodes/nodedb.go b/nodes/nodedb.go index b9e959e..09496b2 100644 --- a/nodes/nodedb.go +++ b/nodes/nodedb.go @@ -194,6 +194,7 @@ func (ndb *NodeDB) batchUpdate(nodes []*Node) { for _, node := range nodes { dirtyFlags, dirtyGen := node.DirtySnapshot() nodeID := node.ID() + writeFailed := false ndb.logger.WithFields(logrus.Fields{ "nodeID": fmt.Sprintf("%x", nodeID[:8]), @@ -216,6 +217,7 @@ func (ndb *NodeDB) batchUpdate(nodes []*Node) { ndb.logger.WithField("nodeID", fmt.Sprintf("%x", nodeID[:8])).Debug("updating ENR") if err := ndb.updateNodeENRTx(tx, node); err != nil { ndb.logger.WithError(err).WithField("nodeID", fmt.Sprintf("%x", nodeID[:8])).Error("failed to update ENR in batch") + writeFailed = true } } @@ -224,6 +226,7 @@ func (ndb *NodeDB) batchUpdate(nodes []*Node) { ndb.logger.WithField("nodeID", fmt.Sprintf("%x", nodeID[:8])).Debug("updating stats") if err := ndb.updateNodeStatsTx(tx, node); err != nil { ndb.logger.WithError(err).WithField("nodeID", fmt.Sprintf("%x", nodeID[:8])).Error("failed to update stats in batch") + writeFailed = true } } @@ -233,6 +236,7 @@ func (ndb *NodeDB) batchUpdate(nodes []*Node) { if !lastActive.IsZero() { if err := ndb.db.UpdateNodeLastActive(tx, ndb.layer, nodeID[:], lastActive.Unix()); err != nil { ndb.logger.WithError(err).WithField("nodeID", fmt.Sprintf("%x", nodeID[:8])).Error("failed to update last_active in batch") + writeFailed = true } } } @@ -243,6 +247,7 @@ func (ndb *NodeDB) batchUpdate(nodes []*Node) { if !lastSeen.IsZero() { if err := ndb.db.UpdateNodeLastSeen(tx, ndb.layer, nodeID[:], lastSeen.Unix()); err != nil { ndb.logger.WithError(err).WithField("nodeID", fmt.Sprintf("%x", nodeID[:8])).Error("failed to update last_seen in batch") + writeFailed = true } } } @@ -251,15 +256,21 @@ func (ndb *NodeDB) batchUpdate(nodes []*Node) { if dirtyFlags&DirtyProtocol != 0 { if err := ndb.updateNodeProtocolSupportTx(tx, nodeID, node.HasV4(), node.HasV5()); err != nil { ndb.logger.WithError(err).WithField("nodeID", fmt.Sprintf("%x", nodeID[:8])).Error("failed to update protocol support in batch") + writeFailed = true } } - processed = append(processed, written{node, dirtyFlags, dirtyGen}) + if !writeFailed { + processed = append(processed, written{node, dirtyFlags, dirtyGen}) + } } return nil }) if err != nil { + // Nothing reached the database, so no snapshot may be cleared: the flags + // are all that will bring these nodes back on a later pass. + processed = processed[:0] ndb.logger.WithError(err).Error("failed to commit batch transaction") } else { ndb.logger.WithFields(logrus.Fields{ @@ -279,12 +290,27 @@ func (ndb *NodeDB) batchUpdate(nodes []*Node) { // the write was running either enqueued itself or is requeued here. Clearing // first would let its QueueUpdate be swallowed as already-queued and then have // the entry deleted underneath it. + persisted := make(map[[32]byte]bool, len(processed)) + requeue := make([]*Node, 0, len(nodes)) + for _, p := range processed { - if !p.node.ClearDirtySnapshot(p.flags, p.gen) { - continue + persisted[p.node.ID()] = true + if p.node.ClearDirtySnapshot(p.flags, p.gen) { + requeue = append(requeue, p.node) } - if err := ndb.QueueUpdate(p.node); err != nil { - ndb.logger.WithError(err).Debug("failed to requeue node dirtied during batch") + } + + // A node whose write failed keeps its flags, but it is out of the queue set + // now, so nothing would bring it back until an unrelated update marked it. + for _, node := range nodes { + if !persisted[node.ID()] { + requeue = append(requeue, node) + } + } + + for _, node := range requeue { + if err := ndb.QueueUpdate(node); err != nil { + ndb.logger.WithError(err).Debug("failed to requeue node after batch") } } From 9e32cdb5f67ba553b433d21082d35e9fa303a870 Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Wed, 29 Jul 2026 16:32:56 -0500 Subject: [PATCH 29/49] fix(nodes): back off between consecutive batch failures Requeueing unwritten nodes keeps data from being lost, but on a persistent database error the requeue refills the batch and the next pass runs immediately. batchUpdate now reports whether it committed, and the consumer sleeps proportionally to the run of failures, capped at five seconds. --- nodes/nodedb.go | 40 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/nodes/nodedb.go b/nodes/nodedb.go index 09496b2..c6df58f 100644 --- a/nodes/nodedb.go +++ b/nodes/nodedb.go @@ -130,6 +130,11 @@ func (ndb *NodeDB) processUpdateQueue() { ticker := time.NewTicker(1000 * time.Millisecond) defer ticker.Stop() + // Failures requeue their nodes so nothing is lost, which on a persistent + // database error would otherwise spin: the requeue refills the batch and the + // next pass runs immediately. Back off between consecutive failures instead. + failures := 0 + for { select { case <-ndb.ctx.Done(): @@ -144,7 +149,7 @@ func (ndb *NodeDB) processUpdateQueue() { // Process when batch reaches 50 items if len(batch) >= 50 { - ndb.batchUpdate(batch) + failures = ndb.runBatch(batch, failures) batch = batch[:0] time.Sleep(10 * time.Millisecond) // Avoid hammering DB } @@ -152,13 +157,37 @@ func (ndb *NodeDB) processUpdateQueue() { case <-ticker.C: // Process any pending items if len(batch) > 0 { - ndb.batchUpdate(batch) + failures = ndb.runBatch(batch, failures) batch = batch[:0] } } } } +// maxBatchBackoff caps the delay after repeated batch failures. +const maxBatchBackoff = 5 * time.Second + +// runBatch writes a batch and sleeps proportionally to how many consecutive +// batches have failed, returning the updated count. +func (ndb *NodeDB) runBatch(batch []*Node, failures int) int { + if ndb.batchUpdate(batch) { + return 0 + } + + failures++ + + backoff := time.Duration(failures) * 100 * time.Millisecond + if backoff > maxBatchBackoff { + backoff = maxBatchBackoff + } + + select { + case <-time.After(backoff): + case <-ndb.ctx.Done(): + } + return failures +} + // drainQueue moves everything currently queued into batch without blocking. func (ndb *NodeDB) drainQueue(batch []*Node) []*Node { for { @@ -172,9 +201,10 @@ func (ndb *NodeDB) drainQueue(batch []*Node) []*Node { } // batchUpdate performs a batch update of nodes. -func (ndb *NodeDB) batchUpdate(nodes []*Node) { +// batchUpdate writes a batch and reports whether the transaction committed. +func (ndb *NodeDB) batchUpdate(nodes []*Node) bool { if len(nodes) == 0 { - return + return true } ndb.logger.WithFields(logrus.Fields{ @@ -318,6 +348,8 @@ func (ndb *NodeDB) batchUpdate(nodes []*Node) { ndb.statsLock.Lock() ndb.stats.ProcessedUpdates += int64(len(nodes)) ndb.statsLock.Unlock() + + return err == nil } // updateNodeENRTx updates only ENR info within a transaction. From f7edae0c6ff3da6919a552cecbcccdb031392439 Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Wed, 29 Jul 2026 16:35:55 -0500 Subject: [PATCH 30/49] fix(nodes): treat a partial batch as a failure for backoff purposes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Row-level write errors leave the transaction callback returning nil, so a committed transaction was reported as success and reset the backoff even though the failed nodes had been requeued — retrying every 10ms. A batch now counts as successful only when every node in it was persisted. --- nodes/nodedb.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nodes/nodedb.go b/nodes/nodedb.go index c6df58f..e4c321c 100644 --- a/nodes/nodedb.go +++ b/nodes/nodedb.go @@ -349,7 +349,10 @@ func (ndb *NodeDB) batchUpdate(nodes []*Node) bool { ndb.stats.ProcessedUpdates += int64(len(nodes)) ndb.statsLock.Unlock() - return err == nil + // Row-level failures leave the callback returning nil, so a committed + // transaction is not proof every node landed. Reporting success on a partial + // batch would reset the backoff while the requeued nodes retry immediately. + return err == nil && len(processed) == len(nodes) } // updateNodeENRTx updates only ENR info within a transaction. From 7f7f12a94f6dbff3e923553c376f650ea10a789f Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Wed, 29 Jul 2026 18:28:24 -0500 Subject: [PATCH 31/49] fix(bootnode): poll after a fork boundary instead of skipping to the next MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit nextForkBoundary only considers activations strictly after now, so a refresh firing on the boundary before the new digest is computable found no change and then armed for the following boundary — a full backstop minute away. A devnet BPO transition took 75s that way, worse than the ticker it replaced. The refresh now polls every second for 90s after any boundary, so the lag is bounded by the poll interval rather than by where the backstop chain happened to land. --- bootnode/fork_boundary_test.go | 40 +++++++++++++++++++++++++- bootnode/service.go | 52 +++++++++++++++++++++++++++++----- 2 files changed, 84 insertions(+), 8 deletions(-) diff --git a/bootnode/fork_boundary_test.go b/bootnode/fork_boundary_test.go index 057f7a3..91754d1 100644 --- a/bootnode/fork_boundary_test.go +++ b/bootnode/fork_boundary_test.go @@ -79,11 +79,49 @@ func TestNextForkBoundaryWithoutGenesis(t *testing.T) { } // A distant boundary is capped so the reconciliation backstop still runs. +// Genesis is placed beyond the settle window, since genesis is itself a boundary +// and would otherwise put this inside the post-boundary polling period. func TestNextForkRefreshDelayCapped(t *testing.T) { now := time.Now() - s := boundaryService(t, uint64(now.Unix()), "100") + s := boundaryService(t, uint64(now.Add(-2*forkSettleWindow).Unix()), "100") if delay := s.nextForkRefreshDelay(); delay != maxForkRefreshDelay { t.Errorf("delay = %v, want the cap %v", delay, maxForkRefreshDelay) } } + +// A boundary that has just passed is skipped by nextForkBoundary, so arming for +// the following one leaves a full backstop hole: a fire landing on the boundary +// before the digest is computable does not look again for a minute. A devnet BPO +// transition took 75s that way. +func TestForkRefreshPollsAfterABoundary(t *testing.T) { + now := time.Now() + genesis := uint64(now.Add(-384 * time.Second).Unix()) + + s := boundaryService(t, genesis, "1") + + last, ok := s.lastForkBoundary(now) + if !ok { + t.Fatal("the boundary that just passed was not reported") + } + if now.Sub(last) > 5*time.Second { + t.Fatalf("last boundary = %v, want ~now", last) + } + + if delay := s.nextForkRefreshDelay(); delay != forkSettlePoll { + t.Errorf("delay just after a boundary = %v, want the settle poll %v", delay, forkSettlePoll) + } +} + +// Outside the settle window the refresh must go back to waiting for the next +// boundary rather than polling every second forever. +func TestForkRefreshStopsPollingAfterSettleWindow(t *testing.T) { + now := time.Now() + genesis := uint64(now.Add(-(384 + 200) * time.Second).Unix()) + + s := boundaryService(t, genesis, "1") + + if delay := s.nextForkRefreshDelay(); delay == forkSettlePoll { + t.Error("still polling long after the boundary settled") + } +} diff --git a/bootnode/service.go b/bootnode/service.go index 6a24111..98e0ce3 100644 --- a/bootnode/service.go +++ b/bootnode/service.go @@ -599,18 +599,34 @@ func (s *Service) maintenanceLoop() { } } -// forkRefreshLead re-publishes just before a boundary so the record is already -// correct when peers act on the transition. +// forkRefreshLead re-publishes just after a boundary, once the new fork is what +// the clock reports. const forkRefreshLead = 500 * time.Millisecond // maxForkRefreshDelay caps the wait so a schedule that yields no future boundary // still reaches refreshForkENR at the old cadence. const maxForkRefreshDelay = time.Minute -// nextForkRefreshDelay returns how long until the next scheduled fork boundary. +// forkSettleWindow is how long after a boundary the refresh keeps polling. +// +// A boundary that has just passed is skipped by nextForkBoundary, so arming for +// the following one leaves a full maxForkRefreshDelay hole: a fire landing on the +// boundary before the digest is computable finds no change and does not look +// again for a minute. A devnet BPO transition took 75s that way. Polling until +// the change appears bounds the lag to forkSettlePoll instead. +const forkSettleWindow = 90 * time.Second + +// forkSettlePoll is the retry interval inside forkSettleWindow. +const forkSettlePoll = time.Second + +// nextForkRefreshDelay returns how long to wait before the next refresh attempt. func (s *Service) nextForkRefreshDelay() time.Duration { now := time.Now() + if last, ok := s.lastForkBoundary(now); ok && now.Sub(last) < forkSettleWindow { + return forkSettlePoll + } + next, ok := s.nextForkBoundary(now) if !ok { return maxForkRefreshDelay @@ -649,7 +665,7 @@ func (s *Service) nextForkBoundary(now time.Time) (time.Time, bool) { var next time.Time found := false - consider := func(t time.Time) { + s.eachForkBoundary(func(t time.Time) { if !t.After(now) { return } @@ -657,8 +673,32 @@ func (s *Service) nextForkBoundary(now time.Time) (time.Time, bool) { next = t found = true } - } + }) + + return next, found +} + +// lastForkBoundary returns the most recent CL or EL fork activation at or before +// now, so a refresh can keep polling until the transition is observable. +func (s *Service) lastForkBoundary(now time.Time) (time.Time, bool) { + var last time.Time + found := false + s.eachForkBoundary(func(t time.Time) { + if t.After(now) { + return + } + if !found || t.After(last) { + last = t + found = true + } + }) + + return last, found +} + +// eachForkBoundary calls fn with every scheduled fork activation time. +func (s *Service) eachForkBoundary(consider func(time.Time)) { if cfg := s.config.CLConfig; cfg != nil { genesis := cfg.GetGenesisTime() slotsPerEpoch := cfg.GetSlotsPerEpoch() @@ -686,8 +726,6 @@ func (s *Service) nextForkBoundary(now time.Time) (time.Time, bool) { } } } - - return next, found } // localIDs returns the node IDs of every discovery identity, so discovery can From 65f3f344d05d42298112c4d421f86623c813b194 Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Wed, 29 Jul 2026 19:25:57 -0500 Subject: [PATCH 32/49] test(bootnode): cover serve-all pooling one node ID into both layers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The composite-key change had no end-to-end coverage: no real peer advertises eth and eth2 together, since EL and CL run as separate identities with separate keys, so no devnet topology produces one. Serve-all is the reachable path — it pools every discovered peer into every enabled table, so under it an ordinary CL-only node occupies both layers and every peer hit the collision. Verified against the old key: only one row persists and the reload fails. --- bootnode/duallayer_persist_test.go | 162 +++++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 bootnode/duallayer_persist_test.go diff --git a/bootnode/duallayer_persist_test.go b/bootnode/duallayer_persist_test.go new file mode 100644 index 0000000..3519528 --- /dev/null +++ b/bootnode/duallayer_persist_test.go @@ -0,0 +1,162 @@ +package bootnode + +import ( + "context" + "net" + "path/filepath" + "testing" + "time" + + "github.com/ethpandaops/bootnodoor/bootnode/clconfig" + "github.com/ethpandaops/bootnodoor/bootnode/elconfig" + "github.com/ethpandaops/bootnodoor/db" + v5node "github.com/ethpandaops/bootnodoor/discv5/node" + "github.com/ethpandaops/bootnodoor/enr" + "github.com/ethpandaops/bootnodoor/nodes" +) + +// serveAllServiceAt builds a file-backed service with classification disabled, +// which is what puts one node ID into both tables. +// +// No real peer advertises eth and eth2 together — EL and CL run as separate +// identities with separate keys — so serve-all, not a dual-stack client, is how +// the same node ID reaches both layers in practice. +func serveAllServiceAt(t *testing.T, file string) (*Service, *db.Database, context.CancelFunc) { + t.Helper() + + logger := quietLogger() + database := db.NewDatabase(&db.SqliteDatabaseConfig{File: file, MaxOpenConns: 5, MaxIdleConns: 2}, logger) + if err := database.Init(); err != nil { + t.Fatalf("db init: %v", err) + } + if err := database.ApplyEmbeddedDbSchema(-2); err != nil { + t.Fatalf("db schema: %v", err) + } + + cfg := &Config{ + Logger: logger, + Database: database, + ELConfig: &elconfig.ChainConfig{}, + ELGenesisHash: [32]byte{1, 2, 3}, + ELGenesisTime: 1000, + CLConfig: &clconfig.Config{}, + ServeAll: true, + } + + key := mustKey(t) + ln, err := createLocalNode(cfg, key, net.ParseIP("1.2.3.4"), nil, 9000, nil) + if err != nil { + t.Fatalf("createLocalNode: %v", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + + s := &Service{config: cfg, ctx: ctx, enrManager: NewENRManager(cfg, key, ln, true, true)} + s.elNodeDB = nodes.NewNodeDB(ctx, database, db.LayerEL, logger) + s.clNodeDB = nodes.NewNodeDB(ctx, database, db.LayerCL, logger) + if s.elTable, err = s.createTable(ln.ID(), s.elNodeDB, "EL"); err != nil { + t.Fatalf("createTable EL: %v", err) + } + if s.clTable, err = s.createTable(ln.ID(), s.clNodeDB, "CL"); err != nil { + t.Fatalf("createTable CL: %v", err) + } + return s, database, cancel +} + +// plainCLNode is an ordinary single-layer peer: eth2 only, as a real beacon node +// advertises. +func plainCLNode(t *testing.T, ip net.IP) *v5node.Node { + t.Helper() + + key := mustKey(t) + rec := enr.New() + if err := rec.Set("ip", ip); err != nil { + t.Fatalf("set ip: %v", err) + } + if err := rec.Set("udp", uint16(9000)); err != nil { + t.Fatalf("set udp: %v", err) + } + if err := rec.Set("eth2", clconfig.EncodeETH2Field(clconfig.ForkDigest{1, 2, 3, 4}, [4]byte{}, ^uint64(0))); err != nil { + t.Fatalf("set eth2: %v", err) + } + rec.SetSeq(1) + if err := rec.Sign(key); err != nil { + t.Fatalf("sign: %v", err) + } + + n, err := v5node.New(rec) + if err != nil { + t.Fatalf("v5node.New: %v", err) + } + return n +} + +// Under serve-all every discovered peer is pooled into every enabled table, so an +// ordinary CL-only node occupies both layers. With nodeid as the sole primary key +// the second write replaced the first, so one layer was lost on every restart — +// for every peer, not just an exotic one. +func TestServeAllPeerPersistsToBothLayers(t *testing.T) { + file := filepath.Join(t.TempDir(), "serveall.db") + + s, database, cancel := serveAllServiceAt(t, file) + n := plainCLNode(t, net.IPv4(9, 9, 9, 9)) + + if !s.checkAndAddNode(n) { + t.Fatal("node was not admitted under serve-all") + } + if s.elTable.Get(n.ID()) == nil { + t.Fatal("serve-all did not pool the node into the EL table") + } + if s.clTable.Get(n.ID()) == nil { + t.Fatal("serve-all did not pool the node into the CL table") + } + + waitForRows(t, database, 2) + + id := n.ID() + if _, err := database.GetNode(db.LayerEL, id[:]); err != nil { + t.Errorf("EL row missing: %v", err) + } + if _, err := database.GetNode(db.LayerCL, id[:]); err != nil { + t.Errorf("CL row missing: %v", err) + } + + // Cancel before Close: the queue processor exits on the context, and Close + // waits for it. + cancel() + s.elNodeDB.Close() + s.clNodeDB.Close() + database.Close() + + _, reopened, cancel2 := serveAllServiceAt(t, file) + defer cancel2() + defer reopened.Close() + + elBack, err := reopened.GetNode(db.LayerEL, id[:]) + if err != nil { + t.Fatalf("EL row did not survive the restart: %v", err) + } + clBack, err := reopened.GetNode(db.LayerCL, id[:]) + if err != nil { + t.Fatalf("CL row did not survive the restart: %v", err) + } + if elBack.Layer != string(db.LayerEL) || clBack.Layer != string(db.LayerCL) { + t.Errorf("layer tags wrong after reload: el=%q cl=%q", elBack.Layer, clBack.Layer) + } +} + +func waitForRows(t *testing.T, database *db.Database, want int) { + t.Helper() + + deadline := time.Now().Add(10 * time.Second) + for { + got, err := database.CountAllNodes() + if err == nil && got >= want { + return + } + if time.Now().After(deadline) { + t.Fatalf("only %d of %d rows persisted", got, want) + } + time.Sleep(20 * time.Millisecond) + } +} From cade30c4fcedfccf193137277dabf24214614f15 Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Wed, 29 Jul 2026 20:12:52 -0500 Subject: [PATCH 33/49] fix: address review findings on lookup blocking, capability loss and metrics - probeV5Support ran inline in the lookup admission path and each probe waits a request timeout, so a 16-node NEIGHBORS response stalled the maintenance loop for over a minute. It now runs after admission, bounded to 8 in flight, and resolves the table entry so the result lands on the object the table kept. - FlatTable.Add refreshed only a newer ENR for a known node, so a peer admitted as v5-only by a completing handshake lost its v4 pointer when the v4 admission followed. Protocols are now merged into the existing entry. - Two discv5 identities share a socket and each rejects the other's packets. That is identity demultiplexing, but it was counted and displayed as other-protocol traffic. Handlers now carry a protocol label and only a real protocol change counts. - The aggregate ping RTT averaged the per-identity averages; it is now weighted by each identity's pong count. - Reset left the two dispatch counters untouched. --- bootnode/service.go | 57 +++++++++++++++++++++++++----- bootnode/stats.go | 9 +++-- discv4/service.go | 3 +- discv5/service.go | 3 +- nodes/admission_persist_test.go | 46 ++++++++++++++++++++++++ nodes/flattable.go | 20 ++++++++++- transport/dispatch_metrics_test.go | 51 ++++++++++++++++++++++++++ transport/metrics.go | 2 ++ transport/udp.go | 36 +++++++++++++++++-- 9 files changed, 210 insertions(+), 17 deletions(-) diff --git a/bootnode/service.go b/bootnode/service.go index 98e0ce3..fdb8c70 100644 --- a/bootnode/service.go +++ b/bootnode/service.go @@ -68,6 +68,9 @@ type Service struct { // ENR request tracking (prevents duplicate requests) pendingENRRequestsV4 sync.Map // map[node.ID]time.Time + // v5ProbeSem bounds concurrent v5 capability probes + v5ProbeSem chan struct{} + // Lifecycle ctx context.Context cancel context.CancelFunc @@ -106,9 +109,10 @@ func New(cfg *Config) (*Service, error) { ctx, cancel := context.WithCancel(context.Background()) s := &Service{ - config: cfg, - ctx: ctx, - cancel: cancel, + config: cfg, + ctx: ctx, + cancel: cancel, + v5ProbeSem: make(chan struct{}, maxConcurrentV5Probes), } // Resolve discovery identities (one shared, or separate EL/CL keys). @@ -1383,11 +1387,17 @@ func (s *Service) admitELLookupNode(n *nodes.Node) services.AdmissionResult { } } - if n.HasV4() && !n.HasV5() { - s.probeV5Support(n) + result := s.admitToTable(n, s.elTable, db.LayerEL) + + // After admission, and off this goroutine: each probe waits a request timeout, + // so probing a 16-node NEIGHBORS response inline stalled the lookup and the + // whole maintenance loop for over a minute. Resolving the table entry rather + // than reusing n also means the result lands on the object the table kept. + if result == services.AdmissionAccepted && n.HasV4() && !n.HasV5() { + s.scheduleV5Probe(n.ID()) } - return s.admitToTable(n, s.elTable, db.LayerEL) + return result } // admitCLLookupNode decides admission of a lookup-discovered node to the CL @@ -1429,6 +1439,32 @@ func (s *Service) admitToTable(n *nodes.Node, table *nodes.FlatTable, layer db.N // probeV5Support pings a v4-discovered node over discv5 and, on success, // attaches v5 support so lookups prefer the richer protocol. +// maxConcurrentV5Probes bounds the probes in flight so a large NEIGHBORS response +// cannot spawn one goroutine per node. +const maxConcurrentV5Probes = 8 + +// scheduleV5Probe runs a v5 probe for an admitted node without blocking the +// caller. Dropping the probe when saturated is fine: the node stays v4-only and +// the next lookup that rediscovers it tries again. +func (s *Service) scheduleV5Probe(id [32]byte) { + select { + case s.v5ProbeSem <- struct{}{}: + default: + return + } + + go func() { + defer func() { <-s.v5ProbeSem }() + + if s.elTable == nil { + return + } + if n := s.elTable.Get(id); n != nil { + s.probeV5Support(n) + } + }() +} + func (s *Service) probeV5Support(n *nodes.Node) { handler := s.getV5Handler() if handler == nil { @@ -1448,8 +1484,13 @@ func (s *Service) probeV5Support(n *nodes.Node) { if err != nil { return } - resp := <-respChan - if resp.Error != nil { + var resp *v5protocol.Response + select { + case resp = <-respChan: + case <-s.ctx.Done(): + return + } + if resp == nil || resp.Error != nil { return } diff --git a/bootnode/stats.go b/bootnode/stats.go index 5b4d46d..32368ef 100644 --- a/bootnode/stats.go +++ b/bootnode/stats.go @@ -61,9 +61,12 @@ func (s *Service) GetStats() Stats { out.Ping.PingTimeouts += p.PingTimeouts out.Ping.PingsV5 += p.PingsV5 out.Ping.PingsV4 += p.PingsV4 - if p.AverageRTT > 0 { - totalRTT += p.AverageRTT - rttSamples++ + // Weight by the sample count behind each average: EL and CL rarely + // answer the same number of pings, and averaging the averages would + // let the quieter identity move the aggregate as much as the busier one. + if p.AverageRTT > 0 && p.PongsReceived > 0 { + totalRTT += p.AverageRTT * time.Duration(p.PongsReceived) + rttSamples += p.PongsReceived } } diff --git a/discv4/service.go b/discv4/service.go index 567e376..7912b59 100644 --- a/discv4/service.go +++ b/discv4/service.go @@ -32,6 +32,7 @@ type Transport interface { protocol.Transport LocalAddr() *net.UDPAddr AddHandler(handler func(data []byte, from *net.UDPAddr, localAddr *net.UDPAddr) bool) + AddHandlerFor(protocol string, handler func(data []byte, from *net.UDPAddr, localAddr *net.UDPAddr) bool) } // Service represents a discv4 service instance. @@ -125,7 +126,7 @@ func New(config *Config, transport Transport) (*Service, error) { } // Register packet handler with transport - transport.AddHandler(s.packetHandler) + transport.AddHandlerFor("discv4", s.packetHandler) return s, nil } diff --git a/discv5/service.go b/discv5/service.go index 9e4ec15..750fd87 100644 --- a/discv5/service.go +++ b/discv5/service.go @@ -54,6 +54,7 @@ type Transport interface { protocol.Transport LocalAddr() *net.UDPAddr AddHandler(handler func(data []byte, from *net.UDPAddr, localAddr *net.UDPAddr) bool) + AddHandlerFor(protocol string, handler func(data []byte, from *net.UDPAddr, localAddr *net.UDPAddr) bool) } // New creates a new discv5 service. @@ -169,7 +170,7 @@ func New(cfg *Config, transport Transport) (*Service, error) { protocolHandler.SetTransport(transport) // Register packet handler with transport - transport.AddHandler(s.packetHandler) + transport.AddHandlerFor("discv5", s.packetHandler) return s, nil } diff --git a/nodes/admission_persist_test.go b/nodes/admission_persist_test.go index f4148c6..b9ebb41 100644 --- a/nodes/admission_persist_test.go +++ b/nodes/admission_persist_test.go @@ -8,6 +8,8 @@ import ( "time" "github.com/ethpandaops/bootnodoor/db" + v4node "github.com/ethpandaops/bootnodoor/discv4/node" + discv5node "github.com/ethpandaops/bootnodoor/discv5/node" "github.com/sirupsen/logrus" ) @@ -195,3 +197,47 @@ func TestClearDirtySnapshotKeepsUnobservedFlags(t *testing.T) { t.Error("same-bit re-mark was cleared unwritten") } } + +// A peer found over discv4 can be admitted as v5-only first, if its handshake +// completes before the v4 admission lands. Add previously refreshed only a newer +// ENR, so the v4 pointer was dropped and the peer persisted as v5-only. +func TestAddMergesProtocolCapabilities(t *testing.T) { + database := persistTestDB(t, filepath.Join(t.TempDir(), "merge.db")) + defer database.Close() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + logger := quietTableLogger() + ndb := NewNodeDB(ctx, database, db.LayerEL, logger) + table := newPersistTable(t, ndb, logger) + + v5 := makeV5At(t, net.IPv4(10, 9, 0, 1)) + first := NewFromV5(v5, ndb) + if !table.Add(first) { + t.Fatal("v5-only node was not admitted") + } + if table.Get(first.ID()).HasV4() { + t.Fatal("precondition: entry should start v5-only") + } + + // The same peer arriving over discv4, as the probe path produces it. + second := NewFromV5(v5, ndb) + second.SetV4(makeV4For(t, v5)) + if !table.Add(second) { + t.Fatal("second admission was rejected") + } + + entry := table.Get(first.ID()) + if !entry.HasV4() { + t.Error("v4 capability was lost: the table entry is still v5-only") + } + if !entry.HasV5() { + t.Error("v5 capability was dropped by the merge") + } +} + +func makeV4For(t *testing.T, v5 *discv5node.Node) *v4node.Node { + t.Helper() + return v4node.New(v5.PublicKey(), v5.Addr()) +} diff --git a/nodes/flattable.go b/nodes/flattable.go index 18d7dbb..cc743eb 100644 --- a/nodes/flattable.go +++ b/nodes/flattable.go @@ -260,6 +260,20 @@ func (t *FlatTable) Add(n *Node) bool { if existing, exists := t.activeNodes[nodeID]; exists { t.mu.Unlock() + // Adopt protocols the entry lacks. A peer found over discv4 can be admitted + // as v5-only first if its handshake completes before the v4 admission lands; + // keeping only the newer ENR would drop the v4 pointer and persist the peer + // as v5-only. + changed := false + if v4 := n.V4(); v4 != nil && !existing.HasV4() { + existing.SetV4(v4) + changed = true + } + if v5 := n.V5(); v5 != nil && !existing.HasV5() { + existing.SetV5(v5) + changed = true + } + // Update ENR if newer newSeq := n.Record().Seq() if newSeq > existing.Record().Seq() { @@ -267,8 +281,12 @@ func (t *FlatTable) Add(n *Node) bool { // Queue ENR update (preserves stats) existing.MarkDirty(DirtyENR) + changed = true + } + + if changed { if err := t.db.QueueUpdate(existing); err != nil { - t.logger.WithError(err).WithField("peerID", existing.PeerID()).Debug("failed to queue ENR update") + t.logger.WithError(err).WithField("peerID", existing.PeerID()).Debug("failed to queue node update") } if t.nodeChangedCallback != nil { diff --git a/transport/dispatch_metrics_test.go b/transport/dispatch_metrics_test.go index fa3bc9f..ed6d30b 100644 --- a/transport/dispatch_metrics_test.go +++ b/transport/dispatch_metrics_test.go @@ -66,3 +66,54 @@ func TestDispatchDistinguishesFallthroughFromUnhandled(t *testing.T) { } }) } + +// Two discv5 identities share a socket, and the first rejects packets addressed +// to the second. That is identity demultiplexing, not other-protocol traffic, so +// it must not inflate the fallthrough counter. +func TestDispatchDoesNotCountSameProtocolDemux(t *testing.T) { + accept := func(_ []byte, _ *net.UDPAddr, _ *net.UDPAddr) bool { return true } + reject := func(_ []byte, _ *net.UDPAddr, _ *net.UDPAddr) bool { return false } + + from := &net.UDPAddr{IP: net.IPv4(10, 0, 0, 1), Port: 30303} + local := &net.UDPAddr{IP: net.IPv4(10, 0, 0, 2), Port: 9000} + + logger := logrus.New() + logger.SetLevel(logrus.ErrorLevel) + + t.Run("second discv5 identity accepts", func(t *testing.T) { + tr := &UDPTransport{logger: logger, metrics: NewMetrics()} + tr.AddHandlerFor("discv5", reject) + tr.AddHandlerFor("discv5", accept) + tr.AddHandlerFor("discv4", reject) + tr.dispatchPacket([]byte("packet"), from, local) + + if got := tr.Metrics().Snapshot().PacketsFellThrough; got != 0 { + t.Errorf("PacketsFellThrough = %d, want 0 for same-protocol demux", got) + } + }) + + t.Run("discv4 accepts after discv5 identities", func(t *testing.T) { + tr := &UDPTransport{logger: logger, metrics: NewMetrics()} + tr.AddHandlerFor("discv5", reject) + tr.AddHandlerFor("discv5", reject) + tr.AddHandlerFor("discv4", accept) + tr.dispatchPacket([]byte("packet"), from, local) + + if got := tr.Metrics().Snapshot().PacketsFellThrough; got != 1 { + t.Errorf("PacketsFellThrough = %d, want 1 when the protocol changed", got) + } + }) +} + +// Reset must clear the dispatch counters too, or a caller sees stale values. +func TestResetClearsDispatchCounters(t *testing.T) { + m := NewMetrics() + m.RecordFellThrough() + m.RecordUnhandled() + m.Reset() + + got := m.Snapshot() + if got.PacketsFellThrough != 0 || got.PacketsUnhandled != 0 { + t.Errorf("after Reset: fellThrough=%d unhandled=%d, want 0/0", got.PacketsFellThrough, got.PacketsUnhandled) + } +} diff --git a/transport/metrics.go b/transport/metrics.go index 0bfa6fe..b2aeea4 100644 --- a/transport/metrics.go +++ b/transport/metrics.go @@ -122,6 +122,8 @@ func (m *Metrics) Reset() { m.sendErrors.Store(0) m.receiveErrors.Store(0) m.rateLimited.Store(0) + m.packetsFellThrough.Store(0) + m.packetsUnhandled.Store(0) } // PacketsSent returns the number of packets sent. diff --git a/transport/udp.go b/transport/udp.go index 464d207..ec46240 100644 --- a/transport/udp.go +++ b/transport/udp.go @@ -58,8 +58,9 @@ type UDPTransport struct { ipv6Conn *ipv6.PacketConn // handlers is a list of packet handlers (tried in order) - handlers []PacketHandler - handlersMu sync.RWMutex + handlers []PacketHandler + handlerProtocols []string + handlersMu sync.RWMutex // logger for debug and error messages logger logrus.FieldLogger @@ -265,9 +266,20 @@ func (t *UDPTransport) Conn() *net.UDPConn { // return err == nil // }) func (t *UDPTransport) AddHandler(handler func(data []byte, from *net.UDPAddr, localAddr *net.UDPAddr) bool) { + t.AddHandlerFor("", handler) +} + +// AddHandlerFor registers a packet handler under a protocol label. +// +// The label only affects accounting: two discv5 identities share a socket and +// each rejects the other's packets, which is identity demultiplexing rather than +// a protocol mismatch. Without the label that rejection would be reported as +// other-protocol traffic. +func (t *UDPTransport) AddHandlerFor(protocol string, handler func(data []byte, from *net.UDPAddr, localAddr *net.UDPAddr) bool) { t.handlersMu.Lock() defer t.handlersMu.Unlock() t.handlers = append(t.handlers, PacketHandler(handler)) + t.handlerProtocols = append(t.handlerProtocols, protocol) } // SendTo sends a packet to the specified address. @@ -382,18 +394,36 @@ func (t *UDPTransport) sendWithSource(data []byte, to *net.UDPAddr, from *net.UD } } +// crossedProtocol reports whether any handler before accepted was registered +// under a different protocol label. +func crossedProtocol(protocols []string, accepted int) bool { + if accepted >= len(protocols) { + return accepted > 0 + } + for i := 0; i < accepted && i < len(protocols); i++ { + if protocols[i] != protocols[accepted] { + return true + } + } + return false +} + // dispatchPacket routes a packet to the registered handlers. // // Handlers are tried in order until one returns true. func (t *UDPTransport) dispatchPacket(data []byte, from *net.UDPAddr, localAddr *net.UDPAddr) { t.handlersMu.RLock() handlers := t.handlers + protocols := t.handlerProtocols t.handlersMu.RUnlock() // Try each handler in order for i, handler := range handlers { if handler(data, from, localAddr) { - if i > 0 && t.metrics != nil { + // Only a handler for a different protocol counts as fallthrough. An + // earlier handler of the same protocol rejecting the packet is one + // identity declining another's traffic on a shared socket. + if t.metrics != nil && crossedProtocol(protocols, i) { t.metrics.RecordFellThrough() } return From 16802ba5cd5230037fb43e81d8a250578225fa00 Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Wed, 29 Jul 2026 20:20:43 -0500 Subject: [PATCH 34/49] fix: close the residuals on the probe and capability merge - The probe decision read the incoming v4-only wrapper, so a node the table already knew to be v5 was re-probed on every rediscovery and could occupy all the slots that genuinely v4-only nodes needed. The table entry decides now, and one probe per node is in flight at a time. - Resolving the entry before the ping did not close the wrong-object window: the entry can be swept or replaced during the round trip. The result is applied to whichever wrapper the table holds when the answer arrives. - The merge adopted a protocol pointer without checking record freshness. Senders use that pointer's own address, so an older record would aim the protocol at an endpoint the peer has left; adoption now requires a record at least as new. --- bootnode/service.go | 46 +++++++++++++++++++----- nodes/admission_persist_test.go | 62 +++++++++++++++++++++++++++++++++ nodes/flattable.go | 22 +++++++----- 3 files changed, 113 insertions(+), 17 deletions(-) diff --git a/bootnode/service.go b/bootnode/service.go index fdb8c70..34d2184 100644 --- a/bootnode/service.go +++ b/bootnode/service.go @@ -71,6 +71,9 @@ type Service struct { // v5ProbeSem bounds concurrent v5 capability probes v5ProbeSem chan struct{} + // v5ProbesInFlight keeps one probe per node in flight + v5ProbesInFlight sync.Map // map[[32]byte]struct{} + // Lifecycle ctx context.Context cancel context.CancelFunc @@ -1446,26 +1449,43 @@ const maxConcurrentV5Probes = 8 // scheduleV5Probe runs a v5 probe for an admitted node without blocking the // caller. Dropping the probe when saturated is fine: the node stays v4-only and // the next lookup that rediscovers it tries again. +// +// The table entry decides, not the wrapper the caller happened to hold: after a +// merge the entry may already know v5, and re-probing it on every rediscovery +// would occupy the slots that genuinely v4-only nodes need. func (s *Service) scheduleV5Probe(id [32]byte) { + if s.elTable == nil { + return + } + entry := s.elTable.Get(id) + if entry == nil || entry.HasV5() { + return + } + if _, inFlight := s.v5ProbesInFlight.LoadOrStore(id, struct{}{}); inFlight { + return + } + select { case s.v5ProbeSem <- struct{}{}: default: + s.v5ProbesInFlight.Delete(id) return } go func() { - defer func() { <-s.v5ProbeSem }() - - if s.elTable == nil { - return - } - if n := s.elTable.Get(id); n != nil { - s.probeV5Support(n) - } + defer func() { + <-s.v5ProbeSem + s.v5ProbesInFlight.Delete(id) + }() + s.probeV5Support(id, entry) }() } -func (s *Service) probeV5Support(n *nodes.Node) { +// probeV5Support pings a v4-discovered node over discv5 and records the result on +// whichever wrapper the table holds when the answer arrives — the entry can be +// swept or replaced during the round trip, and writing to a detached object would +// silently lose the capability. +func (s *Service) probeV5Support(id [32]byte, n *nodes.Node) { handler := s.getV5Handler() if handler == nil { return @@ -1494,6 +1514,14 @@ func (s *Service) probeV5Support(n *nodes.Node) { return } + // Re-resolve: this is the first moment the result can be applied, and the + // entry may have been swept or replaced while the ping was outstanding. + target := s.elTable.Get(id) + if target == nil { + return + } + n = target + n.SetV5(v5Node) s.config.Logger.WithFields(logrus.Fields{ "peerID": n.PeerID(), diff --git a/nodes/admission_persist_test.go b/nodes/admission_persist_test.go index b9ebb41..8175003 100644 --- a/nodes/admission_persist_test.go +++ b/nodes/admission_persist_test.go @@ -2,14 +2,17 @@ package nodes import ( "context" + "crypto/ecdsa" "net" "path/filepath" "testing" "time" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethpandaops/bootnodoor/db" v4node "github.com/ethpandaops/bootnodoor/discv4/node" discv5node "github.com/ethpandaops/bootnodoor/discv5/node" + "github.com/ethpandaops/bootnodoor/enr" "github.com/sirupsen/logrus" ) @@ -241,3 +244,62 @@ func makeV4For(t *testing.T, v5 *discv5node.Node) *v4node.Node { t.Helper() return v4node.New(v5.PublicKey(), v5.Addr()) } + +// Senders use the adopted protocol node's own address, so a protocol pointer from +// an older record would aim that protocol at an endpoint the peer has left. +func TestAddDoesNotAdoptProtocolFromStaleRecord(t *testing.T) { + database := persistTestDB(t, filepath.Join(t.TempDir(), "stale.db")) + defer database.Close() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + logger := quietTableLogger() + ndb := NewNodeDB(ctx, database, db.LayerEL, logger) + table := newPersistTable(t, ndb, logger) + + key, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("generate key: %v", err) + } + + newer := signedRecordAt(t, key, 5, net.IPv4(10, 8, 0, 1)) + older := signedRecordAt(t, key, 2, net.IPv4(10, 8, 0, 2)) + + newerV5, err := discv5node.New(newer) + if err != nil { + t.Fatalf("v5 from newer: %v", err) + } + existing := NewFromV5(newerV5, ndb) + if !table.Add(existing) { + t.Fatal("first admission rejected") + } + + olderV5, err := discv5node.New(older) + if err != nil { + t.Fatalf("v5 from older: %v", err) + } + stale := NewFromV5(olderV5, ndb) + stale.SetV4(v4node.New(olderV5.PublicKey(), olderV5.Addr())) + table.Add(stale) + + if table.Get(existing.ID()).HasV4() { + t.Error("adopted a v4 pointer from a record older than the entry's") + } +} + +func signedRecordAt(t *testing.T, key *ecdsa.PrivateKey, seq uint64, ip net.IP) *enr.Record { + t.Helper() + rec := enr.New() + if err := rec.Set("ip", ip); err != nil { + t.Fatalf("set ip: %v", err) + } + if err := rec.Set("udp", uint16(9000)); err != nil { + t.Fatalf("set udp: %v", err) + } + rec.SetSeq(seq) + if err := rec.Sign(key); err != nil { + t.Fatalf("sign: %v", err) + } + return rec +} diff --git a/nodes/flattable.go b/nodes/flattable.go index cc743eb..6dc9cd5 100644 --- a/nodes/flattable.go +++ b/nodes/flattable.go @@ -264,18 +264,24 @@ func (t *FlatTable) Add(n *Node) bool { // as v5-only first if its handshake completes before the v4 admission lands; // keeping only the newer ENR would drop the v4 pointer and persist the peer // as v5-only. + // + // Only from a record at least as new as the entry's: senders use the adopted + // protocol node's own address, so taking one from an older record would point + // that protocol at an endpoint the peer has already moved off. changed := false - if v4 := n.V4(); v4 != nil && !existing.HasV4() { - existing.SetV4(v4) - changed = true - } - if v5 := n.V5(); v5 != nil && !existing.HasV5() { - existing.SetV5(v5) - changed = true + newSeq := n.Record().Seq() + if newSeq >= existing.Record().Seq() { + if v4 := n.V4(); v4 != nil && !existing.HasV4() { + existing.SetV4(v4) + changed = true + } + if v5 := n.V5(); v5 != nil && !existing.HasV5() { + existing.SetV5(v5) + changed = true + } } // Update ENR if newer - newSeq := n.Record().Seq() if newSeq > existing.Record().Seq() { existing.UpdateENR(n.Record()) From e82d88b2222555b6c1204005e2d56037eb51bb6f Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Wed, 29 Jul 2026 20:27:56 -0500 Subject: [PATCH 35/49] fix(nodes): make protocol adoption and probe application atomic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adoption only ever fills an empty slot, so a pointer installed from a stale record is permanent. The freshness check and the install ran as two separate operations, letting a concurrent admission advance the entry in between and the older pointer land anyway. AdoptProtocolsFrom now does both under one hold of the node's lock. The v5 probe had the same shape: the probed node was built before the round trip and applied after it, so an ENR that advanced while the ping was outstanding left v5 traffic aimed at the address that happened to answer — and HasV5 then suppressed any corrective probe. SetV5AtSeq discards a result whose record is no longer current. --- bootnode/service.go | 7 ++- nodes/admission_persist_test.go | 38 ++++++++++++++++ nodes/flattable.go | 12 +---- nodes/node.go | 80 +++++++++++++++++++++++++++++++++ 4 files changed, 125 insertions(+), 12 deletions(-) diff --git a/bootnode/service.go b/bootnode/service.go index 34d2184..5f4a855 100644 --- a/bootnode/service.go +++ b/bootnode/service.go @@ -1494,6 +1494,7 @@ func (s *Service) probeV5Support(id [32]byte, n *nodes.Node) { if record == nil { return } + probedSeq := record.Seq() v5Node, err := nodes.NewV5NodeFromRecord(record) if err != nil { return @@ -1516,13 +1517,17 @@ func (s *Service) probeV5Support(id [32]byte, n *nodes.Node) { // Re-resolve: this is the first moment the result can be applied, and the // entry may have been swept or replaced while the ping was outstanding. + // SetV5AtSeq then discards the result if the peer moved on from the record we + // probed, rather than pinning v5 traffic to the address we happened to test. target := s.elTable.Get(id) if target == nil { return } n = target - n.SetV5(v5Node) + if !n.SetV5AtSeq(v5Node, probedSeq) { + return + } s.config.Logger.WithFields(logrus.Fields{ "peerID": n.PeerID(), "addr": n.Addr(), diff --git a/nodes/admission_persist_test.go b/nodes/admission_persist_test.go index 8175003..b0170d2 100644 --- a/nodes/admission_persist_test.go +++ b/nodes/admission_persist_test.go @@ -5,6 +5,7 @@ import ( "crypto/ecdsa" "net" "path/filepath" + "sync" "testing" "time" @@ -303,3 +304,40 @@ func signedRecordAt(t *testing.T, key *ecdsa.PrivateKey, seq uint64, ip net.IP) } return rec } + +// Adoption only ever fills an empty slot, so a pointer installed from a stale +// record is permanent. The freshness check and the install must therefore happen +// under one lock hold, or a concurrent advance can slip between them. +func TestAdoptProtocolsFromIsAtomicUnderRace(t *testing.T) { + database := persistTestDB(t, filepath.Join(t.TempDir(), "race.db")) + defer database.Close() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + ndb := NewNodeDB(ctx, database, db.LayerEL, quietTableLogger()) + + key, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("generate key: %v", err) + } + + for round := 0; round < 200; round++ { + base, _ := discv5node.New(signedRecordAt(t, key, 3, net.IPv4(10, 7, 0, 1))) + entry := NewFromV5(base, ndb) + + staleV5, _ := discv5node.New(signedRecordAt(t, key, 1, net.IPv4(10, 7, 0, 2))) + stale := NewFromV5(staleV5, ndb) + stale.SetV4(v4node.New(staleV5.PublicKey(), staleV5.Addr())) + + var wg sync.WaitGroup + wg.Add(2) + go func() { defer wg.Done(); entry.AdoptProtocolsFrom(stale) }() + go func() { defer wg.Done(); entry.UpdateENR(signedRecordAt(t, key, 9, net.IPv4(10, 7, 0, 3))) }() + wg.Wait() + + if v4 := entry.V4(); v4 != nil && v4.Addr().IP.Equal(net.IPv4(10, 7, 0, 2)) { + t.Fatalf("round %d: installed a v4 pointer from the stale record", round) + } + } +} diff --git a/nodes/flattable.go b/nodes/flattable.go index 6dc9cd5..2ab3f0e 100644 --- a/nodes/flattable.go +++ b/nodes/flattable.go @@ -268,18 +268,8 @@ func (t *FlatTable) Add(n *Node) bool { // Only from a record at least as new as the entry's: senders use the adopted // protocol node's own address, so taking one from an older record would point // that protocol at an endpoint the peer has already moved off. - changed := false + changed := existing.AdoptProtocolsFrom(n) newSeq := n.Record().Seq() - if newSeq >= existing.Record().Seq() { - if v4 := n.V4(); v4 != nil && !existing.HasV4() { - existing.SetV4(v4) - changed = true - } - if v5 := n.V5(); v5 != nil && !existing.HasV5() { - existing.SetV5(v5) - changed = true - } - } // Update ENR if newer if newSeq > existing.Record().Seq() { diff --git a/nodes/node.go b/nodes/node.go index 22df325..876973b 100644 --- a/nodes/node.go +++ b/nodes/node.go @@ -207,6 +207,86 @@ func (n *Node) SetV4(v4 *node.Node) { n.MarkDirty(DirtyProtocol) } +// AdoptProtocolsFrom installs protocol pointers this node lacks, taken from a +// wrapper for the same peer, and reports whether anything changed. +// +// The freshness check and the install happen under one lock hold. Doing them +// separately let a concurrent admission advance the record in between, so a +// pointer built from an older record could still be installed — and because +// adoption only ever fills an empty slot, that stale endpoint would then be +// permanent. +func (n *Node) AdoptProtocolsFrom(other *Node) bool { + if other == nil { + return false + } + + otherRecord := other.Record() + otherV4, otherV5 := other.V4(), other.V5() + if otherRecord == nil || (otherV4 == nil && otherV5 == nil) { + return false + } + + n.mu.Lock() + if n.enr != nil && otherRecord.Seq() < n.enr.Seq() { + n.mu.Unlock() + return false + } + + changed := false + if otherV4 != nil && n.v4Node == nil { + n.v4Node = otherV4 + changed = true + } + if otherV5 != nil && n.v5Node == nil { + n.v5Node = otherV5 + changed = true + } + stats := n.nodeStats + v4, v5 := n.v4Node, n.v5Node + n.mu.Unlock() + + if !changed { + return false + } + + if stats != nil { + n.setupSharedStatsCallback() + if otherV4 != nil && v4 != nil { + v4.SetStats(stats) + } + if otherV5 != nil && v5 != nil { + v5.SetStats(stats) + } + } + n.MarkDirty(DirtyProtocol) + return true +} + +// SetV5AtSeq installs a discv5 node only while the record it was probed from is +// still current, so a result that arrived after the peer moved is discarded +// rather than pinning traffic to the old endpoint. +func (n *Node) SetV5AtSeq(v5 *discv5node.Node, seq uint64) bool { + if v5 == nil { + return false + } + + n.mu.Lock() + if n.enr == nil || n.enr.Seq() != seq { + n.mu.Unlock() + return false + } + n.v5Node = v5 + stats := n.nodeStats + n.mu.Unlock() + + if stats != nil { + n.setupSharedStatsCallback() + v5.SetStats(stats) + } + n.MarkDirty(DirtyProtocol) + return true +} + // SetV5 sets the discv5 node and marks protocol support dirty. func (n *Node) SetV5(v5 *discv5node.Node) { n.mu.Lock() From 42367b2f853ab49b57a779a9d5902343103a1990 Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Wed, 29 Jul 2026 20:34:30 -0500 Subject: [PATCH 36/49] fix(nodes): merge protocols and record in one operation Splitting adoption from ENR advancement left them interleavable however each was guarded individually: a newer admission could adopt, an older one install its pointer against the still-old record, and the newer one then advance the record, stranding an endpoint UpdateENR does not refresh for v4. AdoptProtocolsFrom now does both under one hold and reports each outcome. Self-merge is also a no-op now. Re-admitting a table entry passes it to itself, and snapshotting its own pointers before taking the lock could reinstall a protocol a concurrent clear had just removed. --- nodes/admission_persist_test.go | 24 ++++++++++++++++ nodes/flattable.go | 14 ++------- nodes/node.go | 51 ++++++++++++++++++++++----------- 3 files changed, 60 insertions(+), 29 deletions(-) diff --git a/nodes/admission_persist_test.go b/nodes/admission_persist_test.go index b0170d2..508bf51 100644 --- a/nodes/admission_persist_test.go +++ b/nodes/admission_persist_test.go @@ -341,3 +341,27 @@ func TestAdoptProtocolsFromIsAtomicUnderRace(t *testing.T) { } } } + +// Re-admitting a table entry passes it to itself. Snapshotting its own pointers +// and reinstalling them could resurrect a protocol a concurrent clear removed. +func TestAdoptProtocolsFromSelfIsNoOp(t *testing.T) { + database := persistTestDB(t, filepath.Join(t.TempDir(), "self.db")) + defer database.Close() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + ndb := NewNodeDB(ctx, database, db.LayerEL, quietTableLogger()) + v5 := makeV5At(t, net.IPv4(10, 6, 0, 1)) + n := NewFromV5(v5, ndb) + + adopted, advanced := n.AdoptProtocolsFrom(n) + if adopted || advanced { + t.Errorf("self-merge reported changes: adopted=%v advanced=%v", adopted, advanced) + } + + n.SetV5(nil) + if _, _ = n.AdoptProtocolsFrom(n); n.HasV5() { + t.Error("self-merge resurrected a cleared protocol") + } +} diff --git a/nodes/flattable.go b/nodes/flattable.go index 2ab3f0e..08b50bf 100644 --- a/nodes/flattable.go +++ b/nodes/flattable.go @@ -268,19 +268,9 @@ func (t *FlatTable) Add(n *Node) bool { // Only from a record at least as new as the entry's: senders use the adopted // protocol node's own address, so taking one from an older record would point // that protocol at an endpoint the peer has already moved off. - changed := existing.AdoptProtocolsFrom(n) - newSeq := n.Record().Seq() + adopted, advanced := existing.AdoptProtocolsFrom(n) - // Update ENR if newer - if newSeq > existing.Record().Seq() { - existing.UpdateENR(n.Record()) - - // Queue ENR update (preserves stats) - existing.MarkDirty(DirtyENR) - changed = true - } - - if changed { + if adopted || advanced { if err := t.db.QueueUpdate(existing); err != nil { t.logger.WithError(err).WithField("peerID", existing.PeerID()).Debug("failed to queue node update") } diff --git a/nodes/node.go b/nodes/node.go index 876973b..dc36cf8 100644 --- a/nodes/node.go +++ b/nodes/node.go @@ -215,41 +215,49 @@ func (n *Node) SetV4(v4 *node.Node) { // pointer built from an older record could still be installed — and because // adoption only ever fills an empty slot, that stale endpoint would then be // permanent. -func (n *Node) AdoptProtocolsFrom(other *Node) bool { - if other == nil { - return false +func (n *Node) AdoptProtocolsFrom(other *Node) (adopted, advanced bool) { + // Self-merge is a no-op, not a re-install: a caller re-admitting a table entry + // would otherwise snapshot its own pointer, race a concurrent clear, and + // resurrect a protocol the node no longer supports. + if other == nil || other == n { + return false, false } otherRecord := other.Record() - otherV4, otherV5 := other.V4(), other.V5() - if otherRecord == nil || (otherV4 == nil && otherV5 == nil) { - return false + if otherRecord == nil { + return false, false } + otherV4, otherV5 := other.V4(), other.V5() n.mu.Lock() if n.enr != nil && otherRecord.Seq() < n.enr.Seq() { n.mu.Unlock() - return false + return false, false } - changed := false if otherV4 != nil && n.v4Node == nil { n.v4Node = otherV4 - changed = true + adopted = true } if otherV5 != nil && n.v5Node == nil { n.v5Node = otherV5 - changed = true + adopted = true } + + // Advance the record in the same hold. Leaving it to a separate UpdateENR let + // a newer admission complete its adoption, an older one install its pointer, + // and the newer one then advance the record — stranding the older endpoint, + // which UpdateENR does not refresh for v4. + if n.enr == nil || otherRecord.Seq() > n.enr.Seq() { + n.enr = otherRecord + advanced = true + } + stats := n.nodeStats v4, v5 := n.v4Node, n.v5Node n.mu.Unlock() - if !changed { - return false - } - - if stats != nil { + if adopted && stats != nil { n.setupSharedStatsCallback() if otherV4 != nil && v4 != nil { v4.SetStats(stats) @@ -258,8 +266,17 @@ func (n *Node) AdoptProtocolsFrom(other *Node) bool { v5.SetStats(stats) } } - n.MarkDirty(DirtyProtocol) - return true + if advanced && v5 != nil { + v5.UpdateENR(otherRecord) + } + + if adopted { + n.MarkDirty(DirtyProtocol) + } + if advanced { + n.MarkDirty(DirtyENR) + } + return adopted, advanced } // SetV5AtSeq installs a discv5 node only while the record it was probed from is From b2bf6fa7cde6afde778e8fa5f41fa150ab4704d9 Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Wed, 29 Jul 2026 20:39:55 -0500 Subject: [PATCH 37/49] fix(nodes): let a newer record replace a protocol pointer Adoption only filled empty slots, so a pointer taken from an early record was permanent: a later record advanced the ENR but could not replace it, and the table went on serving that endpoint. UpdateENR refreshes the v5 node, but nothing refreshes a v4 node's address, so the stale one persisted. Each pointer now records the sequence it came from, and adoption replaces one sourced from an older record. Writing the record's claimed address into the v4 node instead would undo the endpoint-proof work: the table shares that object with the discv4 handler, where only a matched PONG may move an address. --- nodes/admission_persist_test.go | 41 +++++++++++++++++++++++++++++++++ nodes/node.go | 27 ++++++++++++++++++++-- 2 files changed, 66 insertions(+), 2 deletions(-) diff --git a/nodes/admission_persist_test.go b/nodes/admission_persist_test.go index 508bf51..a21583b 100644 --- a/nodes/admission_persist_test.go +++ b/nodes/admission_persist_test.go @@ -365,3 +365,44 @@ func TestAdoptProtocolsFromSelfIsNoOp(t *testing.T) { t.Error("self-merge resurrected a cleared protocol") } } + +// A pointer installed from an early record must be replaceable, or the table +// serves that endpoint long after the peer's record has advanced past it — +// UpdateENR refreshes the v5 node but nothing refreshes v4's address. +func TestAdoptReplacesPointerFromOlderRecord(t *testing.T) { + database := persistTestDB(t, filepath.Join(t.TempDir(), "replace.db")) + defer database.Close() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + ndb := NewNodeDB(ctx, database, db.LayerEL, quietTableLogger()) + + key, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("generate key: %v", err) + } + + base, _ := discv5node.New(signedRecordAt(t, key, 1, net.IPv4(10, 5, 0, 1))) + entry := NewFromV5(base, ndb) + + oldIP, newIP := net.IPv4(10, 5, 0, 2), net.IPv4(10, 5, 0, 3) + + earlyV5, _ := discv5node.New(signedRecordAt(t, key, 5, oldIP)) + early := NewFromV5(earlyV5, ndb) + early.SetV4(v4node.New(earlyV5.PublicKey(), earlyV5.Addr())) + entry.AdoptProtocolsFrom(early) + + if got := entry.V4().Addr().IP; !got.Equal(oldIP) { + t.Fatalf("precondition: v4 addr = %v, want %v", got, oldIP) + } + + laterV5, _ := discv5node.New(signedRecordAt(t, key, 9, newIP)) + later := NewFromV5(laterV5, ndb) + later.SetV4(v4node.New(laterV5.PublicKey(), laterV5.Addr())) + entry.AdoptProtocolsFrom(later) + + if got := entry.V4().Addr().IP; !got.Equal(newIP) { + t.Errorf("v4 addr = %v, want %v: a pointer from an older record was never replaced", got, newIP) + } +} diff --git a/nodes/node.go b/nodes/node.go index dc36cf8..5e84468 100644 --- a/nodes/node.go +++ b/nodes/node.go @@ -50,6 +50,13 @@ type Node struct { v4Node *node.Node v5Node *discv5node.Node + // Record sequence each protocol pointer was taken from. Adoption fills an + // empty slot or replaces one sourced from an older record; without this a + // pointer installed early is never replaced, and its endpoint is served long + // after the peer's record has moved on. + v4Seq uint64 + v5Seq uint64 + // Network info mu sync.RWMutex addr *net.UDPAddr @@ -82,6 +89,9 @@ func NewFromV4(v4 *node.Node, nodedb *NodeDB) *Node { addr: v4.Addr(), nodeStats: nodeStats, } + if n.enr != nil { + n.v4Seq = n.enr.Seq() + } // Set up callback on shared stats to trigger DB updates n.setupSharedStatsCallback() @@ -115,6 +125,9 @@ func NewFromV5(v5 *discv5node.Node, nodedb *NodeDB) *Node { addr: v5.Addr(), nodeStats: nodeStats, } + if n.enr != nil { + n.v5Seq = n.enr.Seq() + } // Set up callback on shared stats to trigger DB updates n.setupSharedStatsCallback() @@ -196,6 +209,9 @@ func (n *Node) HasV5() bool { func (n *Node) SetV4(v4 *node.Node) { n.mu.Lock() n.v4Node = v4 + if n.enr != nil { + n.v4Seq = n.enr.Seq() + } n.mu.Unlock() if v4 != nil && n.nodeStats != nil { @@ -235,12 +251,15 @@ func (n *Node) AdoptProtocolsFrom(other *Node) (adopted, advanced bool) { return false, false } - if otherV4 != nil && n.v4Node == nil { + otherSeq := otherRecord.Seq() + if otherV4 != nil && (n.v4Node == nil || otherSeq > n.v4Seq) { n.v4Node = otherV4 + n.v4Seq = otherSeq adopted = true } - if otherV5 != nil && n.v5Node == nil { + if otherV5 != nil && (n.v5Node == nil || otherSeq > n.v5Seq) { n.v5Node = otherV5 + n.v5Seq = otherSeq adopted = true } @@ -293,6 +312,7 @@ func (n *Node) SetV5AtSeq(v5 *discv5node.Node, seq uint64) bool { return false } n.v5Node = v5 + n.v5Seq = seq stats := n.nodeStats n.mu.Unlock() @@ -308,6 +328,9 @@ func (n *Node) SetV5AtSeq(v5 *discv5node.Node, seq uint64) bool { func (n *Node) SetV5(v5 *discv5node.Node) { n.mu.Lock() n.v5Node = v5 + if n.enr != nil { + n.v5Seq = n.enr.Seq() + } n.mu.Unlock() if v5 != nil && n.nodeStats != nil { From 8cc3cc80a1bd03446701b1cde4eb5233f7e4a32a Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Wed, 29 Jul 2026 20:46:23 -0500 Subject: [PATCH 38/49] fix(nodes): label a protocol pointer with its own record sequence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pointer was stamped with its carrier's sequence, so a pointer built from an older snapshot was marked current and no later admission could replace it. The label now comes from the record the pointer itself carries. A discv4 node created from an enode has no record, so it falls back to the carrier's sequence — without that every such pointer would be labelled 0 and become unreplaceable, which a regression test caught. --- nodes/node.go | 69 +++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 51 insertions(+), 18 deletions(-) diff --git a/nodes/node.go b/nodes/node.go index 5e84468..e41a03a 100644 --- a/nodes/node.go +++ b/nodes/node.go @@ -89,9 +89,7 @@ func NewFromV4(v4 *node.Node, nodedb *NodeDB) *Node { addr: v4.Addr(), nodeStats: nodeStats, } - if n.enr != nil { - n.v4Seq = n.enr.Seq() - } + n.v4Seq = v4RecordSeq(v4) // Set up callback on shared stats to trigger DB updates n.setupSharedStatsCallback() @@ -125,9 +123,7 @@ func NewFromV5(v5 *discv5node.Node, nodedb *NodeDB) *Node { addr: v5.Addr(), nodeStats: nodeStats, } - if n.enr != nil { - n.v5Seq = n.enr.Seq() - } + n.v5Seq = v5RecordSeq(v5) // Set up callback on shared stats to trigger DB updates n.setupSharedStatsCallback() @@ -209,9 +205,7 @@ func (n *Node) HasV5() bool { func (n *Node) SetV4(v4 *node.Node) { n.mu.Lock() n.v4Node = v4 - if n.enr != nil { - n.v4Seq = n.enr.Seq() - } + n.v4Seq = protocolSeq(v4RecordSeq(v4), n.recordSeqLocked()) n.mu.Unlock() if v4 != nil && n.nodeStats != nil { @@ -251,15 +245,14 @@ func (n *Node) AdoptProtocolsFrom(other *Node) (adopted, advanced bool) { return false, false } - otherSeq := otherRecord.Seq() - if otherV4 != nil && (n.v4Node == nil || otherSeq > n.v4Seq) { + if otherV4Seq := protocolSeq(v4RecordSeq(otherV4), otherRecord.Seq()); otherV4 != nil && (n.v4Node == nil || otherV4Seq > n.v4Seq) { n.v4Node = otherV4 - n.v4Seq = otherSeq + n.v4Seq = otherV4Seq adopted = true } - if otherV5 != nil && (n.v5Node == nil || otherSeq > n.v5Seq) { + if otherV5Seq := protocolSeq(v5RecordSeq(otherV5), otherRecord.Seq()); otherV5 != nil && (n.v5Node == nil || otherV5Seq > n.v5Seq) { n.v5Node = otherV5 - n.v5Seq = otherSeq + n.v5Seq = otherV5Seq adopted = true } @@ -312,7 +305,7 @@ func (n *Node) SetV5AtSeq(v5 *discv5node.Node, seq uint64) bool { return false } n.v5Node = v5 - n.v5Seq = seq + n.v5Seq = v5RecordSeq(v5) stats := n.nodeStats n.mu.Unlock() @@ -328,9 +321,7 @@ func (n *Node) SetV5AtSeq(v5 *discv5node.Node, seq uint64) bool { func (n *Node) SetV5(v5 *discv5node.Node) { n.mu.Lock() n.v5Node = v5 - if n.enr != nil { - n.v5Seq = n.enr.Seq() - } + n.v5Seq = v5RecordSeq(v5) n.mu.Unlock() if v5 != nil && n.nodeStats != nil { @@ -684,3 +675,45 @@ func NewV5NodeFromRecord(record *enr.Record) (*discv5node.Node, error) { func NewV4NodeFromRecord(record *enr.Record, addr *net.UDPAddr) (*node.Node, error) { return node.FromENR(record, addr) } + +// v4RecordSeq and v5RecordSeq read the sequence of the record a protocol pointer +// was built from. The label has to follow the pointer, not the wrapper that +// carried it: stamping a pointer with its carrier's sequence marks an old +// endpoint as current and blocks the admission that would replace it. +func v4RecordSeq(v4 *node.Node) uint64 { + if v4 == nil { + return 0 + } + if rec := v4.ENR(); rec != nil { + return rec.Seq() + } + return 0 +} + +func v5RecordSeq(v5 *discv5node.Node) uint64 { + if v5 == nil { + return 0 + } + if rec := v5.Record(); rec != nil { + return rec.Seq() + } + return 0 +} + +// protocolSeq prefers the sequence of the record a pointer was built from, and +// falls back to its carrier's. A discv4 node created from an enode has no record +// of its own, so without the fallback every such pointer would be labelled 0 and +// no later admission could ever replace it. +func protocolSeq(pointerSeq, carrierSeq uint64) uint64 { + if pointerSeq != 0 { + return pointerSeq + } + return carrierSeq +} + +func (n *Node) recordSeqLocked() uint64 { + if n.enr == nil { + return 0 + } + return n.enr.Seq() +} From 7ca11af13d071bc27bfab0fa45d22f5980889285 Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Thu, 30 Jul 2026 09:22:04 -0500 Subject: [PATCH 39/49] fix(nodes): keep protocol adoption to filling empty slots MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An ENR sequence orders signed records. It does not order endpoint proofs, and the per-pointer sequence labels added here treated them as one thing: a discv4 pointer's ENR can advance while its proven address deliberately stays put, so labelling it by record left a proven-old address marked current and unreplaceable. Each attempt to patch that produced another provenance edge case. The table does not arbitrate endpoint freshness. checkAndAddNodeV4 hands the handler's own node to the table, promoteAddr moves that object's address on a matched PONG, and onFindNodeV4 serves the pointer — so the served address already follows proof without adoption doing anything. Adoption fills an empty slot, which is all the original v4-capability loss ever needed. Also here: ApplyProbeResult, so an asynchronous probe applies both protocol outcomes under one lock hold and only while the record it measured is still current; nil-safety in UpdateENR; SetV5AtSeq validating the pointer's record as well as the carrier's; the sequence reads hoisted out of n.mu so it never nests with a protocol node's mutex; and the removal of SetAddr, which had no callers. Part of this reached the branch as an unreviewed tool-generated commit. That has been folded in here, reviewed, and mostly reverted. --- nodes/admission_persist_test.go | 84 ----------------- nodes/node.go | 156 ++++++++++++++++--------------- nodes/node_test.go | 157 ++++++++++++++++++++++++++++++++ 3 files changed, 239 insertions(+), 158 deletions(-) diff --git a/nodes/admission_persist_test.go b/nodes/admission_persist_test.go index a21583b..88d6fc6 100644 --- a/nodes/admission_persist_test.go +++ b/nodes/admission_persist_test.go @@ -246,49 +246,6 @@ func makeV4For(t *testing.T, v5 *discv5node.Node) *v4node.Node { return v4node.New(v5.PublicKey(), v5.Addr()) } -// Senders use the adopted protocol node's own address, so a protocol pointer from -// an older record would aim that protocol at an endpoint the peer has left. -func TestAddDoesNotAdoptProtocolFromStaleRecord(t *testing.T) { - database := persistTestDB(t, filepath.Join(t.TempDir(), "stale.db")) - defer database.Close() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - logger := quietTableLogger() - ndb := NewNodeDB(ctx, database, db.LayerEL, logger) - table := newPersistTable(t, ndb, logger) - - key, err := crypto.GenerateKey() - if err != nil { - t.Fatalf("generate key: %v", err) - } - - newer := signedRecordAt(t, key, 5, net.IPv4(10, 8, 0, 1)) - older := signedRecordAt(t, key, 2, net.IPv4(10, 8, 0, 2)) - - newerV5, err := discv5node.New(newer) - if err != nil { - t.Fatalf("v5 from newer: %v", err) - } - existing := NewFromV5(newerV5, ndb) - if !table.Add(existing) { - t.Fatal("first admission rejected") - } - - olderV5, err := discv5node.New(older) - if err != nil { - t.Fatalf("v5 from older: %v", err) - } - stale := NewFromV5(olderV5, ndb) - stale.SetV4(v4node.New(olderV5.PublicKey(), olderV5.Addr())) - table.Add(stale) - - if table.Get(existing.ID()).HasV4() { - t.Error("adopted a v4 pointer from a record older than the entry's") - } -} - func signedRecordAt(t *testing.T, key *ecdsa.PrivateKey, seq uint64, ip net.IP) *enr.Record { t.Helper() rec := enr.New() @@ -365,44 +322,3 @@ func TestAdoptProtocolsFromSelfIsNoOp(t *testing.T) { t.Error("self-merge resurrected a cleared protocol") } } - -// A pointer installed from an early record must be replaceable, or the table -// serves that endpoint long after the peer's record has advanced past it — -// UpdateENR refreshes the v5 node but nothing refreshes v4's address. -func TestAdoptReplacesPointerFromOlderRecord(t *testing.T) { - database := persistTestDB(t, filepath.Join(t.TempDir(), "replace.db")) - defer database.Close() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - ndb := NewNodeDB(ctx, database, db.LayerEL, quietTableLogger()) - - key, err := crypto.GenerateKey() - if err != nil { - t.Fatalf("generate key: %v", err) - } - - base, _ := discv5node.New(signedRecordAt(t, key, 1, net.IPv4(10, 5, 0, 1))) - entry := NewFromV5(base, ndb) - - oldIP, newIP := net.IPv4(10, 5, 0, 2), net.IPv4(10, 5, 0, 3) - - earlyV5, _ := discv5node.New(signedRecordAt(t, key, 5, oldIP)) - early := NewFromV5(earlyV5, ndb) - early.SetV4(v4node.New(earlyV5.PublicKey(), earlyV5.Addr())) - entry.AdoptProtocolsFrom(early) - - if got := entry.V4().Addr().IP; !got.Equal(oldIP) { - t.Fatalf("precondition: v4 addr = %v, want %v", got, oldIP) - } - - laterV5, _ := discv5node.New(signedRecordAt(t, key, 9, newIP)) - later := NewFromV5(laterV5, ndb) - later.SetV4(v4node.New(laterV5.PublicKey(), laterV5.Addr())) - entry.AdoptProtocolsFrom(later) - - if got := entry.V4().Addr().IP; !got.Equal(newIP) { - t.Errorf("v4 addr = %v, want %v: a pointer from an older record was never replaced", got, newIP) - } -} diff --git a/nodes/node.go b/nodes/node.go index e41a03a..ee3a51e 100644 --- a/nodes/node.go +++ b/nodes/node.go @@ -50,13 +50,6 @@ type Node struct { v4Node *node.Node v5Node *discv5node.Node - // Record sequence each protocol pointer was taken from. Adoption fills an - // empty slot or replaces one sourced from an older record; without this a - // pointer installed early is never replaced, and its endpoint is served long - // after the peer's record has moved on. - v4Seq uint64 - v5Seq uint64 - // Network info mu sync.RWMutex addr *net.UDPAddr @@ -89,7 +82,6 @@ func NewFromV4(v4 *node.Node, nodedb *NodeDB) *Node { addr: v4.Addr(), nodeStats: nodeStats, } - n.v4Seq = v4RecordSeq(v4) // Set up callback on shared stats to trigger DB updates n.setupSharedStatsCallback() @@ -123,7 +115,6 @@ func NewFromV5(v5 *discv5node.Node, nodedb *NodeDB) *Node { addr: v5.Addr(), nodeStats: nodeStats, } - n.v5Seq = v5RecordSeq(v5) // Set up callback on shared stats to trigger DB updates n.setupSharedStatsCallback() @@ -161,18 +152,6 @@ func (n *Node) Addr() *net.UDPAddr { return n.addr } -// SetAddr updates the node's address. -func (n *Node) SetAddr(addr *net.UDPAddr) { - n.mu.Lock() - defer n.mu.Unlock() - n.addr = addr - - // Update protocol-specific nodes - if n.v4Node != nil { - n.v4Node.SetAddr(addr) - } -} - // V4 returns the discv4 node if available. func (n *Node) V4() *node.Node { n.mu.RLock() @@ -205,7 +184,6 @@ func (n *Node) HasV5() bool { func (n *Node) SetV4(v4 *node.Node) { n.mu.Lock() n.v4Node = v4 - n.v4Seq = protocolSeq(v4RecordSeq(v4), n.recordSeqLocked()) n.mu.Unlock() if v4 != nil && n.nodeStats != nil { @@ -217,14 +195,13 @@ func (n *Node) SetV4(v4 *node.Node) { n.MarkDirty(DirtyProtocol) } -// AdoptProtocolsFrom installs protocol pointers this node lacks, taken from a -// wrapper for the same peer, and reports whether anything changed. +// AdoptProtocolsFrom fills protocol slots this node has empty from a wrapper for +// the same peer, advances the record if the other's is newer, and reports each. // -// The freshness check and the install happen under one lock hold. Doing them -// separately let a concurrent admission advance the record in between, so a -// pointer built from an older record could still be installed — and because -// adoption only ever fills an empty slot, that stale endpoint would then be -// permanent. +// It fills only: which endpoint a protocol should use is the discovery layer's +// call, not the table's. discv4 moves an address solely on a matched PONG +// (promoteAddr), and discv5 moves its own when its record advances. A table that +// also ranked pointers would be arbitrating endpoints on weaker evidence. func (n *Node) AdoptProtocolsFrom(other *Node) (adopted, advanced bool) { // Self-merge is a no-op, not a re-install: a caller re-admitting a table entry // would otherwise snapshot its own pointer, race a concurrent clear, and @@ -238,34 +215,30 @@ func (n *Node) AdoptProtocolsFrom(other *Node) (adopted, advanced bool) { return false, false } otherV4, otherV5 := other.V4(), other.V5() + carrierSeq := otherRecord.Seq() n.mu.Lock() - if n.enr != nil && otherRecord.Seq() < n.enr.Seq() { + if n.enr != nil && carrierSeq < n.recordSeqLocked() { n.mu.Unlock() return false, false } - if otherV4Seq := protocolSeq(v4RecordSeq(otherV4), otherRecord.Seq()); otherV4 != nil && (n.v4Node == nil || otherV4Seq > n.v4Seq) { + if otherV4 != nil && n.v4Node == nil { n.v4Node = otherV4 - n.v4Seq = otherV4Seq adopted = true } - if otherV5Seq := protocolSeq(v5RecordSeq(otherV5), otherRecord.Seq()); otherV5 != nil && (n.v5Node == nil || otherV5Seq > n.v5Seq) { + if otherV5 != nil && n.v5Node == nil { n.v5Node = otherV5 - n.v5Seq = otherV5Seq adopted = true } - // Advance the record in the same hold. Leaving it to a separate UpdateENR let - // a newer admission complete its adoption, an older one install its pointer, - // and the newer one then advance the record — stranding the older endpoint, - // which UpdateENR does not refresh for v4. - if n.enr == nil || otherRecord.Seq() > n.enr.Seq() { + if n.enr == nil || carrierSeq > n.recordSeqLocked() { n.enr = otherRecord advanced = true } stats := n.nodeStats + current := n.enr v4, v5 := n.v4Node, n.v5Node n.mu.Unlock() @@ -278,8 +251,12 @@ func (n *Node) AdoptProtocolsFrom(other *Node) (adopted, advanced bool) { v5.SetStats(stats) } } - if advanced && v5 != nil { - v5.UpdateENR(otherRecord) + // Bring the v5 pointer up to the record the wrapper now holds, whether that is + // because the record advanced or because an older pointer just filled an empty + // slot. UpdateENR ignores anything not newer, and this stays outside n.mu so + // the two mutexes never nest. + if (adopted || advanced) && v5 != nil { + v5.UpdateENR(current) } if adopted { @@ -291,6 +268,60 @@ func (n *Node) AdoptProtocolsFrom(other *Node) (adopted, advanced bool) { return adopted, advanced } +// ApplyProbeResult installs or clears both protocol pointers from a completed +// probe, and reports whether anything changed. +// +// Gated on the record still being the one the probe measured: a result that +// arrived after the peer published a new record describes endpoints it may have +// left, so it must neither install nor clear. Both protocols are decided under one +// lock hold so they see the same record. +func (n *Node) ApplyProbeResult(probedSeq uint64, v4 *node.Node, v4OK bool, v5 *discv5node.Node, v5OK bool) bool { + n.mu.Lock() + if n.enr == nil || n.recordSeqLocked() != probedSeq { + n.mu.Unlock() + return false + } + + changed := false + switch { + case v4OK && v4 != nil && n.v4Node == nil: + n.v4Node = v4 + changed = true + case !v4OK && n.v4Node != nil: + n.v4Node = nil + changed = true + } + + switch { + case v5OK && v5 != nil && n.v5Node == nil: + n.v5Node = v5 + changed = true + case !v5OK && n.v5Node != nil: + n.v5Node = nil + changed = true + } + + stats := n.nodeStats + installedV4, installedV5 := n.v4Node, n.v5Node + n.mu.Unlock() + + if !changed { + return false + } + + if stats != nil { + n.setupSharedStatsCallback() + if installedV4 != nil { + installedV4.SetStats(stats) + } + if installedV5 != nil { + installedV5.SetStats(stats) + } + } + n.MarkDirty(DirtyProtocol) + return true +} + // SetV5AtSeq installs a discv5 node only while the record it was probed from is // still current, so a result that arrived after the peer moved is discarded // rather than pinning traffic to the old endpoint. @@ -298,14 +329,14 @@ func (n *Node) SetV5AtSeq(v5 *discv5node.Node, seq uint64) bool { if v5 == nil { return false } + v5Seq := v5RecordSeq(v5) n.mu.Lock() - if n.enr == nil || n.enr.Seq() != seq { + if n.enr == nil || n.enr.Seq() != seq || v5Seq != seq { n.mu.Unlock() return false } n.v5Node = v5 - n.v5Seq = v5RecordSeq(v5) stats := n.nodeStats n.mu.Unlock() @@ -321,7 +352,6 @@ func (n *Node) SetV5AtSeq(v5 *discv5node.Node, seq uint64) bool { func (n *Node) SetV5(v5 *discv5node.Node) { n.mu.Lock() n.v5Node = v5 - n.v5Seq = v5RecordSeq(v5) n.mu.Unlock() if v5 != nil && n.nodeStats != nil { @@ -543,7 +573,7 @@ func (n *Node) UpdateENR(newRecord *enr.Record) bool { // Update our ENR n.mu.Lock() - if newRecord.Seq() <= n.enr.Seq() { + if n.enr != nil && newRecord.Seq() <= n.recordSeqLocked() { n.mu.Unlock() return false } @@ -676,44 +706,22 @@ func NewV4NodeFromRecord(record *enr.Record, addr *net.UDPAddr) (*node.Node, err return node.FromENR(record, addr) } -// v4RecordSeq and v5RecordSeq read the sequence of the record a protocol pointer -// was built from. The label has to follow the pointer, not the wrapper that -// carried it: stamping a pointer with its carrier's sequence marks an old -// endpoint as current and blocks the admission that would replace it. -func v4RecordSeq(v4 *node.Node) uint64 { - if v4 == nil { +// recordSeq reads a record's sequence, treating a missing record as sequence 0. +func recordSeq(rec *enr.Record) uint64 { + if rec == nil { return 0 } - if rec := v4.ENR(); rec != nil { - return rec.Seq() - } - return 0 + return rec.Seq() } +// v5RecordSeq reads the sequence of the record a discv5 pointer was built from. func v5RecordSeq(v5 *discv5node.Node) uint64 { if v5 == nil { return 0 } - if rec := v5.Record(); rec != nil { - return rec.Seq() - } - return 0 -} - -// protocolSeq prefers the sequence of the record a pointer was built from, and -// falls back to its carrier's. A discv4 node created from an enode has no record -// of its own, so without the fallback every such pointer would be labelled 0 and -// no later admission could ever replace it. -func protocolSeq(pointerSeq, carrierSeq uint64) uint64 { - if pointerSeq != 0 { - return pointerSeq - } - return carrierSeq + return recordSeq(v5.Record()) } func (n *Node) recordSeqLocked() uint64 { - if n.enr == nil { - return 0 - } - return n.enr.Seq() + return recordSeq(n.enr) } diff --git a/nodes/node_test.go b/nodes/node_test.go index 1b4acf5..86ed953 100644 --- a/nodes/node_test.go +++ b/nodes/node_test.go @@ -1,6 +1,7 @@ package nodes import ( + "crypto/ecdsa" "net" "sync" "testing" @@ -174,3 +175,159 @@ func TestNodeNoNilDerefDuringProtocolSwap(t *testing.T) { close(stop) wg.Wait() } + +func TestUpdateENRInstallsSequenceZeroOnRecordlessNode(t *testing.T) { + key, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("generate key: %v", err) + } + addr := &net.UDPAddr{IP: net.IPv4(10, 20, 0, 1), Port: 9000} + n := NewFromV4(discv4node.New(&key.PublicKey, addr), nil) + record := signedRecordAt(t, key, 0, addr.IP) + + if !n.UpdateENR(record) { + t.Fatal("sequence-zero ENR was rejected for a node with no record") + } + if got := n.Record(); got != record { + t.Fatal("sequence-zero ENR was not installed") + } +} + +func TestSetV5AtSeqRejectsRecordlessSequenceZeroTarget(t *testing.T) { + key, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("generate key: %v", err) + } + addr := &net.UDPAddr{IP: net.IPv4(10, 20, 0, 2), Port: 9000} + n := NewFromV4(discv4node.New(&key.PublicKey, addr), nil) + probed, err := discv5node.New(signedRecordAt(t, key, 0, addr.IP)) + if err != nil { + t.Fatalf("new v5 node: %v", err) + } + + if n.SetV5AtSeq(probed, 0) { + t.Fatal("installed a probe result on a target with no current ENR") + } +} + +func TestAdoptInstallsSequenceZeroCarrierOnRecordlessNode(t *testing.T) { + key, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("generate key: %v", err) + } + addr := &net.UDPAddr{IP: net.IPv4(10, 20, 0, 13), Port: 9000} + entry := NewFromV4(discv4node.New(&key.PublicKey, addr), nil) + record := signedRecordAt(t, key, 0, addr.IP) + v5, err := discv5node.New(record) + if err != nil { + t.Fatalf("new v5 node: %v", err) + } + + adopted, advanced := entry.AdoptProtocolsFrom(NewFromV5(v5, nil)) + if !adopted || !advanced { + t.Fatalf("sequence-zero adoption = (%v, %v), want (true, true)", adopted, advanced) + } + if got := entry.Record(); got != record { + t.Fatal("sequence-zero carrier ENR was not installed") + } +} + +func TestSetV5AtSeqRejectsProbeNodeThatAdvanced(t *testing.T) { + key, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("generate key: %v", err) + } + record1 := signedRecordAt(t, key, 1, net.IPv4(10, 20, 0, 3)) + v4, err := discv4node.FromENR(record1, &net.UDPAddr{IP: record1.IP(), Port: int(record1.UDP())}) + if err != nil { + t.Fatalf("new v4 node: %v", err) + } + n := NewFromV4(v4, nil) + + probed, err := discv5node.New(record1) + if err != nil { + t.Fatalf("new v5 node: %v", err) + } + if !probed.UpdateENR(signedRecordAt(t, key, 2, net.IPv4(10, 20, 0, 4))) { + t.Fatal("advance probe node: update rejected") + } + + if n.SetV5AtSeq(probed, 1) { + t.Fatal("installed a probe node that advanced beyond the probed record") + } +} + +func signedRecordSeq(t *testing.T, key *ecdsa.PrivateKey, seq uint64, ip net.IP) *enr.Record { + t.Helper() + rec := enr.New() + if err := rec.Set("ip", ip); err != nil { + t.Fatalf("set ip: %v", err) + } + if err := rec.Set("udp", uint16(30303)); err != nil { + t.Fatalf("set udp: %v", err) + } + rec.SetSeq(seq) + if err := rec.Sign(key); err != nil { + t.Fatalf("sign: %v", err) + } + return rec +} + +// A probe describes the endpoints of the record it ran against. Applying it after +// a newer record arrived would install an endpoint the peer has already left, or +// clear a pointer that newer record brought in. +func TestApplyProbeResultRejectsStaleRecord(t *testing.T) { + key, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("generate key: %v", err) + } + + base, err := discv5node.New(signedRecordSeq(t, key, 1, net.IPv4(10, 30, 0, 1))) + if err != nil { + t.Fatalf("base v5: %v", err) + } + n := NewFromV5(base, nil) + + // The node moves on while the probe is outstanding. + if !n.UpdateENR(signedRecordSeq(t, key, 7, net.IPv4(10, 30, 0, 2))) { + t.Fatal("record did not advance") + } + + v4 := discv4node.New(base.PublicKey(), base.Addr()) + if n.ApplyProbeResult(1, v4, true, nil, false) { + t.Error("applied a probe result against a record the node had already left") + } + if n.HasV4() { + t.Error("installed a v4 pointer from a superseded probe") + } + if !n.HasV5() { + t.Error("cleared the v5 pointer from a superseded probe") + } +} + +// A probe that fails against the current record must be able to clear, and both +// protocol decisions have to land together rather than one at a time. +func TestApplyProbeResultAppliesBothDecisionsAtCurrentSeq(t *testing.T) { + key, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("generate key: %v", err) + } + + rec := signedRecordSeq(t, key, 4, net.IPv4(10, 31, 0, 1)) + base, err := discv5node.New(rec) + if err != nil { + t.Fatalf("base v5: %v", err) + } + n := NewFromV5(base, nil) + + v4 := discv4node.New(base.PublicKey(), base.Addr()) + if !n.ApplyProbeResult(4, v4, true, nil, false) { + t.Fatal("probe at the current sequence was rejected") + } + if !n.HasV4() { + t.Error("v4 confirmed by the probe was not installed") + } + if n.HasV5() { + t.Error("v5 unconfirmed by the probe was not cleared") + } +} From e0346b174f950effc3ee4dbdd863799f8170faf0 Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Thu, 30 Jul 2026 09:22:40 -0500 Subject: [PATCH 40/49] fix(services): gate protocol-support installs on record freshness CheckProtocolSupport snapshotted the record before probing and then installed pointers rebuilt from that snapshot. It needed no concurrency to go wrong: on a successful v4 ping it fetched a fresh ENR and advanced the node's record itself, then discarded the refreshed probe object and rebuilt from the record it had just superseded. The two clear paths were unguarded as well, so a probe that started before a newer admission could remove the pointer that admission installed. The probe objects themselves are now installed, and the fetched ENR is applied only after ApplyProbeResult so the sequence the install is gated on cannot move underneath it. It is attached to the object that was probed rather than to whatever V4() returns by then, which a concurrent admission may have replaced. --- services/ping.go | 73 +++++++++++++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 29 deletions(-) diff --git a/services/ping.go b/services/ping.go index 8098c68..39e3d8c 100644 --- a/services/ping.go +++ b/services/ping.go @@ -6,7 +6,10 @@ import ( "time" "github.com/ethpandaops/bootnodoor/discv4" + discv4node "github.com/ethpandaops/bootnodoor/discv4/node" + discv5node "github.com/ethpandaops/bootnodoor/discv5/node" "github.com/ethpandaops/bootnodoor/discv5/protocol" + "github.com/ethpandaops/bootnodoor/enr" nodedb "github.com/ethpandaops/bootnodoor/nodes" "github.com/sirupsen/logrus" ) @@ -243,6 +246,9 @@ func (ps *PingService) CheckProtocolSupport(n *nodedb.Node) (bool, bool, error) if record == nil { return false, false, fmt.Errorf("node has no ENR") } + // Left alone until after the install, so the sequence the install is gated on + // cannot move underneath it. + probedSeq := record.Seq() ps.logger.WithFields(logrus.Fields{ "peerID": n.PeerID(), @@ -252,6 +258,13 @@ func (ps *PingService) CheckProtocolSupport(n *nodedb.Node) (bool, bool, error) var v4Supported, v5Supported bool var v4RTT, v5RTT time.Duration + // Probe objects are kept for the install: the v4 one is refreshed in place by + // the ENR fetch below, and rebuilding from the pre-probe snapshot would discard + // exactly that refresh. + var probedV4 *discv4node.Node + var probedV5 *discv5node.Node + var refreshedRecord *enr.Record + // Test discv5 support if ps.v5Handler != nil { // Create or get v5 node @@ -265,6 +278,8 @@ func (ps *PingService) CheckProtocolSupport(n *nodedb.Node) (bool, bool, error) } } + probedV5 = v5Node + if v5Node != nil { start := time.Now() respChan, err := ps.v5Handler.SendPing(v5Node) @@ -296,6 +311,8 @@ func (ps *PingService) CheckProtocolSupport(n *nodedb.Node) (bool, bool, error) } } + probedV4 = v4Node + if v4Node != nil { start := time.Now() _, err := ps.v4Service.Ping(v4Node) @@ -308,44 +325,42 @@ func (ps *PingService) CheckProtocolSupport(n *nodedb.Node) (bool, bool, error) "rtt": v4RTT, }).Debug("v4 support confirmed") - // If v4 ping succeeded, request ENR to ensure we have latest record + // Fetched here but applied after the install: advancing the record + // mid-probe would move the very sequence the install is gated on. if enrRecord, err := ps.v4Service.RequestENR(v4Node); err == nil { - v4Node.SetENR(enrRecord) - n.UpdateENR(enrRecord) + refreshedRecord = enrRecord } } } } - // Update node with discovered protocol support - // Add v5 support if confirmed and not present - if v5Supported && n.V5() == nil { - // Create and set v5 node - if v5Node, err := nodedb.NewV5NodeFromRecord(record); err == nil { - n.SetV5(v5Node) - ps.logger.WithField("peerID", n.PeerID()).Info("added v5 support to node") - } - } - - // Remove v5 support if not confirmed but present - if !v5Supported && n.V5() != nil { - n.SetV5(nil) - ps.logger.WithField("peerID", n.PeerID()).Warn("removed v5 support from node (no longer responding)") - } - - // Add v4 support if confirmed and not present - if v4Supported && n.V4() == nil { - // Create and set v4 node - if v4Node, err := nodedb.NewV4NodeFromRecord(record, addr); err == nil { - n.SetV4(v4Node) - ps.logger.WithField("peerID", n.PeerID()).Info("added v4 support to node") + // Apply both outcomes together, and only while the record they describe is + // still the node's current one. A probe that started before a newer record + // arrived knows nothing about it, so it must neither install a superseded + // endpoint nor clear a pointer that newer record brought in. + applied := n.ApplyProbeResult(probedSeq, probedV4, v4Supported, probedV5, v5Supported) + + // Only now advance the record. Attaching it to the v4 node deliberately leaves + // that node's proven address alone, so its label keeps describing the endpoint + // the ping actually reached — which is what lets a correctly addressed pointer + // for this same record replace it later. + // Attach to the object that was probed, not to whatever V4() returns now: a + // concurrent admission may have replaced the pointer, and overwriting its ENR + // with this older response would leave that pointer's record disagreeing with + // both the node's record and its label. + if refreshedRecord != nil { + if probedV4 != nil { + probedV4.SetENR(refreshedRecord) } + n.UpdateENR(refreshedRecord) } - // Remove v4 support if not confirmed but present - if !v4Supported && n.V4() != nil { - n.SetV4(nil) - ps.logger.WithField("peerID", n.PeerID()).Warn("removed v4 support from node (no longer responding)") + if applied { + ps.logger.WithFields(logrus.Fields{ + "peerID": n.PeerID(), + "v4": v4Supported, + "v5": v5Supported, + }).Info("updated protocol support from probe") } // Update RTT with best available From 8dafd1eb88bb238d6dc327a0ad7d49d8aa9c52c4 Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Thu, 30 Jul 2026 09:56:28 -0500 Subject: [PATCH 41/49] docs: add the Kurtosis testing guide for bootnodoor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Covers what only a devnet reaches — a fork activating under load, real peers, wire evidence, and restart against a populated database — since every serious defect found in this component came from one of those rather than the unit suite. Includes the discv4 packet-size table and the shared-netns tcpdump recipe for telling who originated traffic, the --serve-all standalone setup that is the only way to exercise the composite key against real peers, and the harness traps: WAL sidecar files, Kurtosis reassigning host ports on restart, hyphenated fork names breaking naive scraping, and no extra-args hook for the packaged service. --- docs/testing-with-kurtosis.md | 388 ++++++++++++++++++++++++++++++++++ 1 file changed, 388 insertions(+) create mode 100644 docs/testing-with-kurtosis.md diff --git a/docs/testing-with-kurtosis.md b/docs/testing-with-kurtosis.md new file mode 100644 index 0000000..961674d --- /dev/null +++ b/docs/testing-with-kurtosis.md @@ -0,0 +1,388 @@ +# Testing bootnodoor against a Kurtosis devnet + +This is the end-to-end test for the bootnode itself: does it discover real clients, +classify them into the right layer tables, keep its advertised fork fields correct +across transitions, persist what it learns, and generate no traffic it should not. + +The procedure below is deliberately independent of client versions and open branches. +Put short-lived image pins and known interop failures in +[Current validation notes](#current-validation-notes), not in the procedure. + +## What only a devnet can tell you + +Unit tests cover the logic. A devnet is the only place that produces: + +- a **fork actually activating** while the daemon runs, with real clients reacting to it; +- **real peers on the other end** of discv4/discv5, including clients that are slow, + wrong, or aggressive; +- **wire evidence** — what bootnodoor actually sends, which is the only way to catch a + self-inflicted traffic loop; +- **restart behaviour** against a database with real contents. + +Every serious defect found in this component to date came from one of those four, not +from the unit suite. + +## Scratch layout + +Keep devnet state outside the repository: + +```text +/path/to/bootnodoor-devnet/ +├── network_params.yaml # steady state, all forks at genesis +├── network_params.forks.yaml # scheduled transitions +├── network_params.devnet7.yaml # pinned public-devnet images +├── sample.sh # 30s counter + ENR sampler +├── burst.sh # 5s high-resolution sampler +└── config/ # optional: ENRScout bundle +``` + +## 1. Define the client matrix + +Seven EL/CL pairs give full client coverage: + +```yaml +participants: + - { el_type: geth, cl_type: lighthouse, count: 1 } + - { el_type: nethermind, cl_type: teku, count: 1 } + - { el_type: reth, cl_type: prysm, count: 1 } + - { el_type: erigon, cl_type: caplin, count: 1 } + - { el_type: besu, cl_type: nimbus, count: 1 } + - { el_type: nimbus, cl_type: grandine, count: 1 } + - { el_type: ethrex, cl_type: lodestar, count: 1 } + +network_params: + network: kurtosis + network_id: "3151908" + seconds_per_slot: 12 + deneb_fork_epoch: 0 + electra_fork_epoch: 0 + +bootnodoor_params: + image: ethpandaops/bootnodoor:your-build + +additional_services: + - bootnodoor +``` + +Build the image under test locally and reference it by tag; the package uses it as-is if +it exists in the local Docker daemon. + +```bash +docker build -t ethpandaops/bootnodoor:my-test . +``` + +## 2. Launch + +```bash +cd /path/to/bootnodoor-devnet +kurtosis run --enclave bootnodoor-devnet \ + github.com/ethpandaops/ethereum-package --args-file network_params.yaml +``` + +Then resolve the ports and genesis, which everything else keys off: + +```bash +BN=$(kurtosis port print bootnodoor-devnet bootnodoor http | grep -oE '[0-9]+$') +CL=$(kurtosis port print bootnodoor-devnet cl-1-lighthouse-geth http | grep -oE '[0-9]+$') +GEN=$(curl -s "http://127.0.0.1:$CL/eth/v1/beacon/genesis" \ + | grep -oE '"genesis_time":"[0-9]+"' | grep -oE '[0-9]+') +``` + +## 3. Sample continuously, not at the end + +bootnodoor's interesting behaviour is transient. A counter read after the fact tells you +almost nothing; a timestamped series tells you when something started and what it +correlated with. Two samplers, both scraping the web UI: + +- **30s sampler** — fork name, digest, ENR hashes for `/enr`, `/el-enr`, `/cl-enr`, plus + node/session/packet counters. Enough to see a run's shape. +- **5s sampler** — the same counters around a transition, where a 30s gap can hide the + entire event. + +Scrape the fields by label from `/`, and note the parsing gotchas in +[Harness gotchas](#harness-gotchas). + +## 4. What to check, and what "good" looks like + +### Fork discipline + +The core invariant. Per scheduled fork: + +- exactly **one** `fork transition: re-published ENR fork fields` log line; +- the ENR sequence steps **exactly once** — flat between transitions; +- all three of `/enr`, `/el-enr`, `/cl-enr` change together; +- the fork name and `Current Digest` on the UI both advance. + +A step per refresh tick means change detection is broken. No step means the refresh is +not firing. Both are why `UpdateENR` is a no-op when nothing changed. + +A BPO changes the digest without changing the CL fork _name_ — `Fulu` stays while the +digest moves. That is correct, not a missed transition. + +### Refresh lag + +Time from the epoch boundary to the re-publish log line. This should be within about one +slot. If it varies wildly between transitions on the same run, the refresh is being +carried by a periodic backstop rather than by the boundary, which is a defect even when +the average looks acceptable — see the 2026-07-29 notes. + +### Self-inflicted traffic + +**Watch `Packets Sent`, not just `Packets Received`.** A bootnode answering queries is +normal; a bootnode _originating_ thousands of packets is not. Across a transition, the +received rate should stay at its baseline. A spike of several hundred per second means +something is looping. + +Aggregate counters cannot tell you who is talking or which direction. Capture the wire: + +```bash +C=$(docker ps --format '{{.ID}}\t{{.Names}}' | grep 'bootnodoor--' | cut -f1) +docker run --rm --net=container:$C nicolaka/netshoot \ + tcpdump -n -q -c 20000 'udp port 9000' > capture.txt +``` + +`--net=container:` shares the target's network namespace, which is what makes another +container's traffic visible at all. Then count discv4 packet types by size and direction: + +| size | type | +| ------- | ----------------- | +| 138 | PING | +| 154 | PONG | +| 104 | ENRREQUEST | +| 292–298 | ENRRESPONSE | +| 436–800 | NEIGHBORS / NODES | + +```bash +grep -oE '172\.16\.0\.11\.9000 > 172\.16\.0\.[0-9]+\.[0-9]+: UDP, length (138|104)' capture.txt \ + | awk '{split($3,a,"."); print a[4], $NF}' | sort | uniq -c | sort -rn +``` + +Healthy is single digits per peer per minute. Thousands means a retry loop. + +**Byte ratio is not packet ratio.** Outbound is dominated by NEIGHBORS/NODES replies that +are far larger than the queries provoking them, so sent/received bytes sits around 3:1 in +normal operation. That is inherent to serving discovery. The protection against +reflection is the bond/session gate on FINDNODE, not the ratio — do not read a ratio +above 1 as an amplification bug. + +### Counter sanity + +- **`Invalid Packets` should be near-zero and flat.** discv5 is registered first and + rejects anything it cannot decode, so every ordinary discv4 packet on the shared socket + falls through to discv4. Those land in `Other Protocol`. If `Invalid Packets` tracks + your traffic volume, the dispatcher accounting has regressed. +- **`Inactive Nodes` must never be negative**, and `Active` must never exceed `Total`. + Those come from different populations (memory vs database) and a negative value means + the set arithmetic broke. + +### Persistence + +Restart the container mid-run and confirm the tables reload: + +```bash +docker restart "$C" +docker logs "$C" 2>&1 | grep "loaded random nodes into active pool" +``` + +Both layers should report counts. Zero means organic discoveries were never written. + +### `--serve-all` + +This is the highest-value single test and it needs its own node, because the package +exposes no way to pass extra arguments to the packaged bootnodoor. Run a second instance +joined to the enclave network: + +```bash +GH=$(curl -s -X POST -H 'content-type: application/json' \ + --data '{"jsonrpc":"2.0","method":"eth_getBlockByNumber","params":["0x0",false],"id":1}' \ + "http://127.0.0.1:$RPC" | grep -oE '"hash":"0x[0-9a-f]+"' | grep -oE '0x[0-9a-f]+') + +docker run -d --name sa --network kt-bootnodoor-devnet \ + -v /tmp/gendata:/network-configs:ro -p 38080:8080 \ + ethpandaops/bootnodoor:my-test \ + --cl-config /network-configs/config.yaml \ + --genesis-validators-root "$(cat /tmp/gendata/genesis_validators_root.txt)" \ + --el-config /network-configs/genesis.json --el-genesis-hash "$GH" \ + --private-key "$(openssl rand -hex 32)" \ + --bind-addr 0.0.0.0 --web-ui --nodedb /nodes.db --serve-all \ + --el-bootnodes "$(curl -s http://127.0.0.1:$BN/el-enr)" \ + --cl-bootnodes "$(curl -s http://127.0.0.1:$BN/cl-enr)" +``` + +`--serve-all` pools every discovered peer into **every** enabled table, so one node ID +occupies both layers. That makes it the only configuration that exercises the +`(nodeid, layer)` composite key against real peers — a normal run cannot, because no real +client advertises both `eth` and `eth2`. EL and CL are separate identities with separate +keys; not even a unified binary publishes one record for both. + +Expect most peers in both layers, each row carrying its own layer's fork digest with the +other empty, and every `lookup complete` showing `rejected_fork=0 rejected_layer=0`. + +## 5. Fork-transition testing + +Copy the whole steady-state file and replace only `network_params`. Kurtosis takes one +`--args-file` and does not merge, so the fork file must still carry `participants` and +`additional_services: [bootnodoor]`. + +```yaml +network_params: + network: kurtosis + network_id: "3151908" + seconds_per_slot: 12 + deneb_fork_epoch: 0 + electra_fork_epoch: 1 + fulu_fork_epoch: 3 + bpo_1_epoch: 5 + bpo_1_max_blobs: 12 +``` + +Mainnet preset, 32 slots per epoch, 12s slots — one epoch is 384s, so that schedule +transitions at roughly T+6m, T+19m and T+32m and wants ~50 minutes. Include a BPO: it +moves the digest through the blob schedule rather than a fork version, reaching code an +ordinary fork does not. + +Compute the wall-clocks before you start so log lines can be correlated: + +```bash +for e in 1 3 5; do + t=$((GEN + e*32*12)) + echo "epoch $e -> $(date -r "$t" '+%H:%M:%S' 2>/dev/null || date -d "@$t" '+%H:%M:%S')" +done +``` + +### Testing against a public devnet's images + +Pin every client to the target devnet's tags, taken from that devnet's ansible inventory +(`images.yaml`). Pull them first — this is network-bound and can be done while other work +proceeds. + +Public devnets schedule their real fork far out (gloas at epoch 38 is ~4h at 12s slots). +Pull it forward to something like epoch 4 so the transition lands inside the run, and say +so in the file — the deviation matters when reading results. + +## Harness gotchas + +Every one of these cost real time. + +**SQLite is in WAL mode.** `docker cp` of `nodes.db` alone shows _no tables_ — it looks +like total data loss. Copy `nodes.db`, `nodes.db-wal` and `nodes.db-shm` together. + +**Kurtosis reassigns host ports on `docker restart`.** After restarting the bootnodoor +container, re-resolve with `kurtosis port print`; samplers pointed at the old port +silently write empty rows. + +**Kurtosis refuses to schedule when the host is busy.** "requires 100 millicores but we +will only have 0 available" means something else is consuming the box — a concurrent hive +run will do it. Never run fork-timing tests on a loaded host: missed slots look exactly +like bootnodoor bugs. + +**Scrape hyphenated fork names.** `BPO-1` contains a hyphen; a `[A-Za-z0-9]+` character +class silently skips it and matches the _next_ field, which is the digest. Symptom: the +fork column fills with hex. + +**`bootnodoor_params` has no extra-args hook.** Non-default flags such as `--serve-all` +need a standalone container on the enclave network. + +**Caplin has no separate image pin.** It ships inside erigon, so a devnet inventory has no +caplin entry and the package falls back to its default tag. Leave `cl_image` unset for the +erigon pair and note the gap. + +**Arm packet captures with care.** A shell loop building filenames from positional +parameters inside `nohup bash -c` is easy to get wrong; all three of my captures ended up +sharing one filename and firing early. Verify each armed job's command line before +walking away. + +**ENRScout needs `GENESIS_TIME`.** Kurtosis emits only `MIN_GENESIS_TIME`; append the +derived key if you run the crawler alongside. See the [ENRScout guide][enrscout-doc]. + +## Interpreting results + +- **A transition is not a fork name change.** A BPO moves the digest while the name holds. + Compare digests, not labels. +- **Rejection counters are not fork health.** On a dual-layer network most records an EL + lookup sees are consensus records; that is `rejected_layer`, not `rejected_fork`. +- **A quiet counter can still be wrong.** `Invalid Packets` sitting at 84% of received + traffic looked like an attack signal for an entire run before it turned out to be + ordinary discv4 load being miscounted. Check what a counter _means_ before treating its + value as evidence. +- **Correlate direction before assigning blame.** A traffic spike involving one peer says + nothing about which side started it. The packet sizes above resolve it in seconds. + +## Cleanup + +```bash +docker rm -f sa +cd /path/to/bootnodoor-devnet && docker compose down -v +kurtosis enclave rm -f bootnodoor-devnet +``` + +## Current validation notes + +Dated and disposable. Update as fixes land; the procedure above should stay valid. + +### Runs on 2026-07-29 / 07-30 + +Baselines from a full validation sweep — steady state, the Electra/Fulu/BPO1 schedule, and +glamsterdam devnet-7 images (13 of 14 pinned; caplin fell back to its default tag). + +Organic coverage was 5/7 EL (besu, erigon, ethrex, geth, reth) and 7/7 CL. nimbus-eth1 is +the standing EL gap. Nethermind joins but is identified only outbound. + +**A packet storm was found and fixed.** A PONG advertising a newer ENR sequence spawned an +unguarded ENR refresh; each refresh PINGs and sleeps before its ENRREQUEST, so the PONG it +provoked re-entered the same trigger with the cached sequence still stale. Peak was +~980 packets/s, with 2575 PINGs and 2574 ENRREQUESTs sent to a single geth peer in 51s. +Post-fix that peer sees a maximum of 14 and 7 over 60s. + +Two things about that defect are worth remembering as method: + +- It looked fork-correlated because forks bump many peers' sequences at once. **The actual + trigger is any ENR sequence bump**, which is a far broader exposure than the fork window + it appeared in. +- The direction was initially read backwards from aggregate counters alone. Only the + packet capture showed bootnodoor was the originator. + +**A regression was caught by the devnet after code review missed it.** A first fix armed +the fork refresh at the boundary but skipped a boundary that had just passed, falling back +to a 60s backstop; lag came out 75s / 16s / 0s — the 75s _worse_ than the ticker it +replaced. The fix polls after a boundary; re-run gave 0s / 0s / 4s. Reviewing the code +found none of this. + +Also fixed in the same sweep: organic nodes were never written to the database (`Add` +marked them dirty but nothing enqueued them); the `nodes` and `bad_nodes` tables were keyed +on `nodeid` alone despite being per-layer, so under `--serve-all` 10 of 11 peers lost a +layer on every restart; and `Invalid Packets` conflated other-protocol traffic with +malformed packets. + +### Confirmation run on 2026-07-30 + +devnet-7 images plus a standalone `--serve-all` node, both on the same build. + +Three transitions, `seq` 3→4→5, one log line each, zero errors or warnings. Six distinct +`(fork, digest)` states across the series. Storm-free throughout: ~11–13 packets/s across +every boundary, and a peak of 9 PINGs / 5 ENRREQUESTs to any single peer in the 60s around +gloas. `Invalid Packets` sat at 6 for the entire run while `Other Protocol` absorbed 8,800. +`Inactive Nodes` was 0 at every sample. Under `--serve-all`, 12 of 14 peers occupied both +layers with no errors. + +**Refresh lags were 2s, 8s and 25s.** The third exceeds one slot and is worth knowing how +to read. The UI's `Current Fork` is computed live from the wall clock, while +`Current Digest` returns a value cached until `clFilter.Update()` runs inside the refresh +(`bootnode/clconfig/filter.go` — compare `GetCurrentFork` with `GetCurrentDigest`). So a +window where the name has advanced but the digest has not *is* the refresh lag, displayed +in two fields rather than a second defect. Here the boundary timer fired on time and the +digest was simply not derivable for ~25s. Candidate follow-up, not a blocker: it is well +inside the backstop and far better than the 8/20/22s and 2/22/51s measured before the +boundary-armed refresh landed. + +### Not yet covered + +- **BPO blob semantics.** BPO-1/BPO-2 change `MAX_BLOBS_PER_BLOCK`, but both runs carried + no blob transactions, so only the digest change was exercised. Adding `spamoor` with blob + load would close this. +- **Dead nodes are still served.** `PerformSweep` demotes only when the table is over + capacity, and FINDNODE responses are not filtered on liveness. +- **DB-restored v4 pointers are detached.** `buildNodeFromDB` reconstructs its own discv4 + node rather than the handler's, so proven-address promotion never reaches it. + +[enrscout-doc]: https://github.com/mysticryuujin/enrscout/blob/main/docs/testing-with-kurtosis.md From b19a416c15fd6521caffbbcb07f4693e68669335 Mon Sep 17 00:00:00 2001 From: pk910 Date: Thu, 30 Jul 2026 15:39:03 +0000 Subject: [PATCH 42/49] discv4: always send NEIGHBORS and advertise the real TCP port An empty routing table previously produced a silent (zero-packet) response, so go-ethereum's querier waited out its full request timeout instead of returning early on the first reply. Always send >=1 NEIGHBORS packet. Also advertise the node's real TCP port from its ENR instead of copying the UDP port. --- discv4/protocol/handler.go | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/discv4/protocol/handler.go b/discv4/protocol/handler.go index 21283b1..c4e9ad9 100644 --- a/discv4/protocol/handler.go +++ b/discv4/protocol/handler.go @@ -1058,8 +1058,11 @@ func (h *Handler) sendPong(to *node.Node, addr *net.UDPAddr, localAddr *net.UDPA // sendNeighbors sends NEIGHBORS response(s). func (h *Handler) sendNeighbors(to *node.Node, addr *net.UDPAddr, localAddr *net.UDPAddr, nodes []*node.Node) error { - // Split nodes into packets of MaxNeighbors - for i := 0; i < len(nodes); i += MaxNeighbors { + // Split nodes into packets of MaxNeighbors. Always send at least one packet, + // even when we have no nodes to offer: go-ethereum's querier waits for a + // NEIGHBORS reply and only stops early once at least one arrives, so a silent + // (zero-packet) response makes it wait out the full request timeout. + for i := 0; i == 0 || i < len(nodes); i += MaxNeighbors { end := i + MaxNeighbors if end > len(nodes) { end = len(nodes) @@ -1069,10 +1072,18 @@ func (h *Handler) sendNeighbors(to *node.Node, addr *net.UDPAddr, localAddr *net nodeRecords := make([]NodeRecord, len(batch)) for j, n := range batch { + // Advertise the node's real TCP port from its ENR when known; + // only fall back to the UDP port if no ENR tcp entry is available. + tcpPort := uint16(n.Addr().Port) + if rec := n.ENR(); rec != nil { + if t := rec.TCP(); t != 0 { + tcpPort = t + } + } nodeRecords[j] = NodeRecord{ IP: n.Addr().IP, UDP: uint16(n.Addr().Port), - TCP: uint16(n.Addr().Port), + TCP: tcpPort, ID: EncodePubkey(n.PublicKey()), } } From b0240b478c369db8c1b8e805ac66b4f4522553fb Mon Sep 17 00:00:00 2001 From: pk910 Date: Thu, 30 Jul 2026 15:41:01 +0000 Subject: [PATCH 43/49] discv5: cap NODES response to <=15 nodes / <=5 packets go-ethereum honours only the first NODES packet's total and reads at most 5 packets; sigp/discv5 caps at 16 nodes. Cap the served set at 15 nodes / <=5 packets so no served node is silently dropped by a requester. --- discv5/protocol/handler.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/discv5/protocol/handler.go b/discv5/protocol/handler.go index 0c1311f..29d7569 100644 --- a/discv5/protocol/handler.go +++ b/discv5/protocol/handler.go @@ -1065,10 +1065,19 @@ func (h *Handler) handleFindNode(msg *FindNode, remoteID node.ID, from *net.UDPA }).Debug("handler: FINDNODE lookup completed via callback") } - // Split nodes into multiple packets if needed to stay under max packet size - // Each ENR is typically 200-400 bytes, so we limit to 3 nodes per packet to be safe + // Split nodes into multiple packets if needed to stay under max packet size. + // Each ENR is typically 200-400 bytes, so we limit to 3 nodes per packet to be safe. const maxNodesPerPacket = 3 + // Cap the total response so it never exceeds what real clients consume. go-ethereum + // honours only the first packet's `total` and reads at most 5 NODES packets + // (totalNodesResponseLimit); anything beyond that is dropped as unsolicited. sigp/discv5 + // caps at 16 nodes. Keep to <=5 packets / <=15 nodes so no served node is silently lost. + const maxNodesPerResponse = 15 + if len(nodes) > maxNodesPerResponse { + nodes = nodes[:maxNodesPerResponse] + } + // Calculate total number of packets needed totalPackets := (len(nodes) + maxNodesPerPacket - 1) / maxNodesPerPacket if totalPackets == 0 { From 8bcac67ad74a553d7b7fc53066bf2e4c59382176 Mon Sep 17 00:00:00 2001 From: pk910 Date: Thu, 30 Jul 2026 15:42:27 +0000 Subject: [PATCH 44/49] discv4: evict a stale node instead of dropping new peers when the map is full lookupOrCreateNode previously returned a non-retained node once the map was full, so under a flood of distinct signed IDs a genuinely new peer's inbound PING marked bond state on a discarded object and could never bond (memory-growth DoS turned into a bonding-lockout DoS). When full, evict one unbonded entry to admit the new node; bonded, endpoint-proven peers are never evicted this way. Adds regression tests for this and the two preceding NEIGHBORS fixes. --- discv4/protocol/handler.go | 24 ++++- discv4/protocol/remainder_fixes_test.go | 137 ++++++++++++++++++++++++ 2 files changed, 156 insertions(+), 5 deletions(-) create mode 100644 discv4/protocol/remainder_fixes_test.go diff --git a/discv4/protocol/handler.go b/discv4/protocol/handler.go index c4e9ad9..b3667bd 100644 --- a/discv4/protocol/handler.go +++ b/discv4/protocol/handler.go @@ -1168,12 +1168,26 @@ func (h *Handler) lookupOrCreateNode(id node.ID, pubkey *ecdsa.PublicKey, addr * n = node.New(pubkey, addr) n.UpdateLastSeen() - // Bound the map so an unauthenticated flood of distinct node IDs (for - // example fabricated NEIGHBORS records) cannot grow it without limit. Stale - // unbonded entries are reclaimed by cleanup; until a slot frees up we still - // return the node so the packet is handled, but we do not retain it. + // Bound the map so an unauthenticated flood of distinct node IDs (for example + // fabricated NEIGHBORS records, or signed PINGs from generated keys) cannot grow + // it without limit. When full, evict one unbonded entry to make room rather than + // dropping the new node: otherwise a flood that pins the map at MaxNodes would + // lock out genuine new peers (their node is never retained, so their inbound PING + // can never lead to a bond). Bonded entries are real, endpoint-proven peers and + // are never evicted here; if every entry is bonded (genuine load, not a flood) we + // leave the map as-is and return the node without retaining it. if len(h.nodes) >= h.config.MaxNodes { - return n + evicted := false + for eid, en := range h.nodes { + if !en.IsBonded() { + delete(h.nodes, eid) + evicted = true + break + } + } + if !evicted { + return n + } } h.nodes[id] = n diff --git a/discv4/protocol/remainder_fixes_test.go b/discv4/protocol/remainder_fixes_test.go new file mode 100644 index 0000000..eb4314c --- /dev/null +++ b/discv4/protocol/remainder_fixes_test.go @@ -0,0 +1,137 @@ +package protocol + +import ( + "context" + "net" + "testing" + "time" + + ethcrypto "github.com/ethereum/go-ethereum/crypto" + "github.com/ethpandaops/bootnodoor/discv4/node" + "github.com/ethpandaops/bootnodoor/enr" +) + +// captureTransport records every packet sent so a test can decode it. +type captureTransport struct{ sent [][]byte } + +func (c *captureTransport) SendTo(b []byte, _ *net.UDPAddr) error { + c.sent = append(c.sent, b) + return nil +} +func (c *captureTransport) Send(b []byte, _ *net.UDPAddr, _ *net.UDPAddr) error { + c.sent = append(c.sent, b) + return nil +} + +// TestSendNeighborsAlwaysSendsAtLeastOnePacket verifies BUG3: an empty result +// still produces exactly one (empty) NEIGHBORS packet, so a go-ethereum querier +// returns immediately instead of waiting out its request timeout on silence. +func TestSendNeighborsAlwaysSendsAtLeastOnePacket(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + key, _ := ethcrypto.GenerateKey() + ct := &captureTransport{} + h := NewHandler(ctx, HandlerConfig{PrivateKey: key}, ct) + + pub, id := makeNodeID(t) + to := h.lookupOrCreateNode(id, pub, testAddr()) + + if err := h.sendNeighbors(to, testAddr(), nil, nil); err != nil { + t.Fatalf("sendNeighbors(empty): %v", err) + } + if len(ct.sent) != 1 { + t.Fatalf("empty result sent %d packets, want exactly 1", len(ct.sent)) + } + pkt, err := DecodePacket(ct.sent[0]) + if err != nil { + t.Fatalf("decode NEIGHBORS: %v", err) + } + nb, ok := pkt.(*Neighbors) + if !ok { + t.Fatalf("wrong packet type %T", pkt) + } + if len(nb.Nodes) != 0 { + t.Fatalf("empty NEIGHBORS carried %d nodes", len(nb.Nodes)) + } +} + +// TestSendNeighborsAdvertisesEnrTCPPort verifies BUG4: the NEIGHBORS record +// advertises the node's real TCP port from its ENR, not the UDP port. +func TestSendNeighborsAdvertisesEnrTCPPort(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + key, _ := ethcrypto.GenerateKey() + ct := &captureTransport{} + h := NewHandler(ctx, HandlerConfig{PrivateKey: key}, ct) + + // Build a peer whose ENR advertises tcp=40404 at udp=30303. + peerKey, _ := ethcrypto.GenerateKey() + rec := enr.New() + _ = rec.Set("id", "v4") + _ = rec.Set("ip", net.IPv4(203, 0, 113, 7).To4()) + _ = rec.Set("udp", uint16(30303)) + _ = rec.Set("tcp", uint16(40404)) + if err := rec.Sign(peerKey); err != nil { + t.Fatalf("sign: %v", err) + } + peer := node.New(&peerKey.PublicKey, &net.UDPAddr{IP: net.IPv4(203, 0, 113, 7), Port: 30303}) + peer.SetENR(rec) + + pub, id := makeNodeID(t) + to := h.lookupOrCreateNode(id, pub, testAddr()) + + if err := h.sendNeighbors(to, testAddr(), nil, []*node.Node{peer}); err != nil { + t.Fatalf("sendNeighbors: %v", err) + } + if len(ct.sent) != 1 { + t.Fatalf("sent %d packets, want 1", len(ct.sent)) + } + pkt, _ := DecodePacket(ct.sent[0]) + nb := pkt.(*Neighbors) + if len(nb.Nodes) != 1 { + t.Fatalf("NEIGHBORS carried %d nodes, want 1", len(nb.Nodes)) + } + if nb.Nodes[0].TCP != 40404 { + t.Fatalf("advertised TCP=%d, want 40404 (the ENR tcp port, not the UDP port)", nb.Nodes[0].TCP) + } + if nb.Nodes[0].UDP != 30303 { + t.Fatalf("advertised UDP=%d, want 30303", nb.Nodes[0].UDP) + } +} + +// TestFloodDoesNotEvictBondedPeers verifies the #34 follow-up: when the node map +// is full, inserts evict a stale unbonded entry (so genuine new peers are never +// locked out) while bonded, endpoint-proven peers are retained. +func TestFloodDoesNotEvictBondedPeers(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + const maxNodes = 10 + h := NewHandler(ctx, HandlerConfig{MaxNodes: maxNodes, NodeTTL: time.Hour}, nil) + + // A genuine, bonded peer. + pub, bondedID := makeNodeID(t) + bonded := h.lookupOrCreateNode(bondedID, pub, testAddr()) + bonded.MarkPongReceived(time.Hour, testAddr()) + + // Fill the rest with unbonded nodes, then flood well past the cap. + for i := 0; i < maxNodes*20; i++ { + p, id := makeNodeID(t) + h.lookupOrCreateNode(id, p, testAddr()) + } + + if got := len(h.AllNodes()); got != maxNodes { + t.Fatalf("map not bounded under flood: got %d want %d", got, maxNodes) + } + if h.GetNode(bondedID) == nil { + t.Fatal("bonded peer was evicted by an unbonded-ID flood") + } + // A brand-new node still gets retained (evicting an unbonded entry). + p, freshID := makeNodeID(t) + h.lookupOrCreateNode(freshID, p, testAddr()) + if h.GetNode(freshID) == nil { + t.Fatal("new peer not retained when map full") + } +} From 48385e9a5cd60349126f20b708d040652c271175 Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Thu, 30 Jul 2026 11:23:01 -0500 Subject: [PATCH 45/49] fix(webui): gate pprof handler on the --pprof flag --- webui/webui.go | 9 +++++++-- webui/webui_test.go | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 webui/webui_test.go diff --git a/webui/webui.go b/webui/webui.go index 9ba75f3..c35d9e6 100644 --- a/webui/webui.go +++ b/webui/webui.go @@ -48,8 +48,7 @@ func StartHttpServer(config *types.FrontendConfig, logger logrus.FieldLogger, bo // metrics endpoint router.Handle("/metrics", promhttp.Handler()).Methods("GET") - // add pprof handler - router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux) + registerPprofHandler(router, config.Pprof) router.PathPrefix("/").Handler(frontend) @@ -79,3 +78,9 @@ func StartHttpServer(config *types.FrontendConfig, logger logrus.FieldLogger, bo } }() } + +func registerPprofHandler(router *mux.Router, enabled bool) { + if enabled { + router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux) + } +} diff --git a/webui/webui_test.go b/webui/webui_test.go new file mode 100644 index 0000000..c7b9bdd --- /dev/null +++ b/webui/webui_test.go @@ -0,0 +1,43 @@ +package webui + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/gorilla/mux" +) + +func TestRegisterPprofHandler(t *testing.T) { + tests := []struct { + name string + enabled bool + statusCode int + }{ + { + name: "disabled", + enabled: false, + statusCode: http.StatusNotFound, + }, + { + name: "enabled", + enabled: true, + statusCode: http.StatusOK, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + router := mux.NewRouter() + registerPprofHandler(router, test.enabled) + + request := httptest.NewRequest(http.MethodGet, "/debug/pprof/", nil) + response := httptest.NewRecorder() + router.ServeHTTP(response, request) + + if response.Code != test.statusCode { + t.Fatalf("expected status code %d, got %d", test.statusCode, response.Code) + } + }) + } +} From 754d7910060280599b0a288ce6f66d24754fa8d9 Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Thu, 30 Jul 2026 11:23:01 -0500 Subject: [PATCH 46/49] docs: record the 2026-07-30 kurtosis sweep and BPO blob coverage --- docs/testing-with-kurtosis.md | 121 ++++++++++++++++++++++++++++++++-- 1 file changed, 114 insertions(+), 7 deletions(-) diff --git a/docs/testing-with-kurtosis.md b/docs/testing-with-kurtosis.md index 961674d..a36924a 100644 --- a/docs/testing-with-kurtosis.md +++ b/docs/testing-with-kurtosis.md @@ -116,8 +116,9 @@ The core invariant. Per scheduled fork: A step per refresh tick means change detection is broken. No step means the refresh is not firing. Both are why `UpdateENR` is a no-op when nothing changed. -A BPO changes the digest without changing the CL fork _name_ — `Fulu` stays while the -digest moves. That is correct, not a missed transition. +A BPO changes the digest without changing the underlying CL fork version. bootnodoor +labels these blob-schedule variants as pseudo-forks (`BPO-1`, `BPO-2`, ...), so the UI +name and digest both move even though the inherited Fulu fork version stays the same. ### Refresh lag @@ -234,6 +235,23 @@ network_params: fulu_fork_epoch: 3 bpo_1_epoch: 5 bpo_1_max_blobs: 12 + bpo_1_target_blobs: 8 + +additional_services: + - bootnodoor + - spamoor + +spamoor_params: + spammers: + - name: BPO blob validation + scenario: blob-combined + config: + throughput: 30 + sidecars: 3 + max_pending: 60 + max_wallets: 40 + base_fee: 100 + blob_fee: 100 ``` Mainnet preset, 32 slots per epoch, 12s slots — one epoch is 384s, so that schedule @@ -241,6 +259,13 @@ transitions at roughly T+6m, T+19m and T+32m and wants ~50 minutes. Include a BP moves the digest through the blob schedule rather than a fork version, reaching code an ordinary fork does not. +Do not call the BPO covered merely because the digest changed. Sample the beacon block's +`body.blob_kzg_commitments` and the matching execution block's type-3 transactions and +`blobGasUsed`. With three sidecars per transaction, a pre-BPO block should reach the +Fulu limit of 9 blobs; after the example BPO it should exceed 9 and ideally reach 12 +(`blobGasUsed == 0x180000`). A fresh spammer account avoids nonce contamination when +changing sidecar count during a run. + Compute the wall-clocks before you start so log lines can be correlated: ```bash @@ -260,6 +285,28 @@ Public devnets schedule their real fork far out (gloas at epoch 38 is ~4h at 12s Pull it forward to something like epoch 4 so the transition lands inside the run, and say so in the file — the deviation matters when reading results. +## Profiling a live run + +bootnodoor supports Go pprof on the web UI listener. Start it with both `--web-ui` and +`--pprof`; without `--pprof`, `/debug/pprof/` must return 404. The package has no +extra-args hook, so profiling a Kurtosis run currently requires the standalone-container +pattern used for `--serve-all`. + +```bash +curl -fsS "http://127.0.0.1:38080/debug/pprof/profile?seconds=30" \ + -o bootnodoor-cpu.pprof +curl -fsS "http://127.0.0.1:38080/debug/pprof/heap" \ + -o bootnodoor-heap.pprof +curl -fsS "http://127.0.0.1:38080/debug/pprof/goroutine" \ + -o bootnodoor-goroutine.pprof +go install github.com/google/pprof@latest +"$(go env GOPATH)/bin/pprof" -top bootnodoor-cpu.pprof +``` + +Capture across a transition when investigating a fork-correlated load increase. pprof is +an administrative endpoint and shares `--web-host`; bind it to localhost or protect the +listener when profiling outside an isolated enclave. + ## Harness gotchas Every one of these cost real time. @@ -276,6 +323,11 @@ will only have 0 available" means something else is consuming the box — a conc run will do it. Never run fork-timing tests on a loaded host: missed slots look exactly like bootnodoor bugs. +**Schedule every fork explicitly.** Current ethereum-package defaults can put Fulu and +BPO-1 at genesis even when a file specifies only `electra_fork_epoch: 0`. Set every fork +needed by the test, including BPO target/max values, and verify the generated consensus +config before interpreting a run. + **Scrape hyphenated fork names.** `BPO-1` contains a hyphen; a `[A-Za-z0-9]+` character class silently skips it and matches the _next_ field, which is the digest. Symptom: the fork column fills with hex. @@ -297,8 +349,9 @@ derived key if you run the crawler alongside. See the [ENRScout guide][enrscout- ## Interpreting results -- **A transition is not a fork name change.** A BPO moves the digest while the name holds. - Compare digests, not labels. +- **A transition is not only a fork-version change.** A BPO inherits the active Fulu + version but changes the digest through its blob parameters; bootnodoor gives it a + `BPO-n` pseudo-fork label. Compare digests, not labels alone. - **Rejection counters are not fork health.** On a dual-layer network most records an EL lookup sees are consensus records; that is `rejected_layer`, not `rejected_fork`. - **A quiet counter can still be wrong.** `Invalid Packets` sitting at 84% of received @@ -320,6 +373,62 @@ kurtosis enclave rm -f bootnodoor-devnet Dated and disposable. Update as fixes land; the procedure above should stay valid. +### Full local validation on 2026-07-30 + +The current sweep used bootnodoor `develop` at `8dafd1e`, ENRScout `main` at +`0568233`, and a local ethereum-package integration based on the open +[bootnodoor integration PR][ethereum-package-pr] plus layer-specific `/el-enr` wiring. +The bootnodoor and both ENRScout images were built from those exact trees. Besu and Reth +used current `main` images and Nethermind used current `master`; the remaining clients +used the package's curated images. + +In the steady-state seven-pair matrix, all seven EL clients entered bootnodoor before +ENRScout started, and ENRScout identified all seven organically. The CL layer was fully +discovered and verified; six implementation names were fingerprinted, with Caplin still +the naming gap. ENRScout was seeded only with bootnodoor's two identities. This supersedes +the old 5/7 organic EL baseline: Nethermind `master` explicitly logged +`Discv5 bootnodes accepted: 1`. Nimbus EL still logged `Skipping discovery bootstrap, no +bootnodes provided` with both its old and proposed flag forms, but joined indirectly +through the live mesh. + +The released-image comparison remained useful: it reached 5/7 EL organically, then +direct outbound probes identified Nethermind and Nimbus as well. A separate `--serve-all` +instance held 15 EL and 13 CL rows before and after restart; the SQLite database retained +both layer rows under the `(nodeid, layer)` key, with zero failed or inactive rows. + +The scheduled run was Deneb at genesis, Electra at epoch 1, Fulu at epoch 3, and BPO-1 +at epoch 5. Sequence moved 2→3→4→5 exactly once per boundary. Observed refresh lag was +effectively 0s for Electra, 2s for Fulu, and at most 6s at BPO-1. Tables stayed at 8/8 +active EL and 7/7 active CL. Electra and Fulu packet captures were flat at roughly +4.1k–4.3k packets/minute across each boundary, with zero capture drops. Elevated traffic +from Nethermind `master` was client-originated discovery traffic and bootnodoor replies, +not the earlier bootnodoor-originated refresh storm. `Invalid Packets` rose slowly before +Fulu but was flat at 216 across BPO-1; there was no boundary-correlated jump. + +**BPO blob semantics are now covered.** Sustained spamoor load produced three type-3 +transactions, 9 commitments, and `blobGasUsed=0x120000` before BPO-1. The first sampled +post-BPO block (execution block 118, beacon slot 160) contained four type-3 transactions, +12 matching commitments, and `blobGasUsed=0x180000`. Its execution hash matched the +beacon payload hash. This proves the chain actually moved from the Fulu capacity of 9 to +the configured BPO-1 capacity of 12; it was not only a digest transition. + +ENRScout's audit invariant held: `fork=current` plus `fork=stale` was always 14, and all +14 nodes stayed verified. The headline view dipped from 14/0 to 0/14 current/stale at +each boundary, then recovered. BPO-1 recovered to 11/3 in 37s, 13/1 in 97s, and 14/0 in +188s. The longer tail exposed an ENRScout precedence defect: a later stale signed-ENR +observation can overwrite authenticated RLPx Status fork evidence in +`internal/nodeset/nodeset.go`, despite the nearby no-downgrade comment. Besu, Ethrex, and +Reth continued advertising older signed ENRs while Status was current, causing rows to +oscillate between `fork_source: status` and `fork_source: enr`. The companion ENRScout +guide's “no ENRScout code defect” conclusion and blanket ~60s recovery expectation are +therefore superseded. + +The same live run was profiled for 30s. CPU samples represented about 0.6% of one core, +with the largest application cost in SQLite transaction commit/fsync; sampled live heap +was about 1 MiB, and 25 goroutines showed normal UDP, database, HTTP, and maintenance +loops with no leak signature. Profiling also found that pprof had been registered even +when `--pprof` was false; the handler is now gated by the flag with a regression test. + ### Runs on 2026-07-29 / 07-30 Baselines from a full validation sweep — steady state, the Electra/Fulu/BPO1 schedule, and @@ -377,12 +486,10 @@ boundary-armed refresh landed. ### Not yet covered -- **BPO blob semantics.** BPO-1/BPO-2 change `MAX_BLOBS_PER_BLOCK`, but both runs carried - no blob transactions, so only the digest change was exercised. Adding `spamoor` with blob - load would close this. - **Dead nodes are still served.** `PerformSweep` demotes only when the table is over capacity, and FINDNODE responses are not filtered on liveness. - **DB-restored v4 pointers are detached.** `buildNodeFromDB` reconstructs its own discv4 node rather than the handler's, so proven-address promotion never reaches it. [enrscout-doc]: https://github.com/mysticryuujin/enrscout/blob/main/docs/testing-with-kurtosis.md +[ethereum-package-pr]: https://github.com/ethpandaops/ethereum-package/pull/1461 From 9c2e56ce865e806c739bc8ef1104314d79bf7199 Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Thu, 30 Jul 2026 12:40:16 -0500 Subject: [PATCH 47/49] log: demote per-peer discovery events to debug At mainnet scale these scale with client population rather than with time, and the unexpected-WHOAREYOU line is remotely triggerable. info now carries startup, fork transitions and periodic aggregates only. --- bootnode/service.go | 2 +- discv5/protocol/handler.go | 4 ++-- nodes/flattable.go | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bootnode/service.go b/bootnode/service.go index 5f4a855..42da26e 100644 --- a/bootnode/service.go +++ b/bootnode/service.go @@ -1585,7 +1585,7 @@ func (s *Service) checkAndAddNodeV4(n *v4node.Node) bool { s.config.Logger.WithFields(logrus.Fields{ "nodeID": fmt.Sprintf("%x", n.IDBytes()[:8]), "addr": n.Addr().String(), - }).Info("Added discv4 node to EL table") + }).Debug("Added discv4 node to EL table") } return true } diff --git a/discv5/protocol/handler.go b/discv5/protocol/handler.go index 29d7569..d715e7e 100644 --- a/discv5/protocol/handler.go +++ b/discv5/protocol/handler.go @@ -580,7 +580,7 @@ func (h *Handler) handleWHOAREYOUPacket(packet *Packet, from *net.UDPAddr, local "nodeID": sess.RemoteID.String()[:16], "addr": from, "age": sess.Age(), - }).Info("handler: received unexpected WHOAREYOU with no pending request") + }).Debug("handler: received unexpected WHOAREYOU with no pending request") return fmt.Errorf("no pending handshake or request for %s", from) } @@ -886,7 +886,7 @@ func (h *Handler) handleHandshakePacket(packet *Packet, from *net.UDPAddr, local h.config.Logger.WithFields(logrus.Fields{ "sourceNodeID": sourceNodeID.String()[:16], "from": from, - }).Info("handler: session established successfully") + }).Debug("handler: session established successfully") // Store node in session and call OnHandshakeComplete callback if remoteNodeFromENR != nil { diff --git a/nodes/flattable.go b/nodes/flattable.go index 08b50bf..a6333a5 100644 --- a/nodes/flattable.go +++ b/nodes/flattable.go @@ -328,12 +328,12 @@ func (t *FlatTable) Add(n *Node) bool { "addr": n.Addr(), "currentSize": currentSize + 1, "maxActive": t.maxActiveNodes, - }).Infof("added alive node to active pool (over capacity)") + }).Debugf("added alive node to active pool (over capacity)") } else { t.logger.WithFields(logrus.Fields{ "peerID": n.PeerID(), "addr": n.Addr(), - }).Info("added node to active pool") + }).Debug("added node to active pool") } // Queue ENR update to DB and mark as active From eafc97931ae39b481f5bc3be0fdd0de21661c3dc Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Thu, 30 Jul 2026 12:40:31 -0500 Subject: [PATCH 48/49] docs: untrack the kurtosis testing guide Kept as local notes; not repo content. --- .gitignore | 3 + docs/testing-with-kurtosis.md | 495 ---------------------------------- 2 files changed, 3 insertions(+), 495 deletions(-) delete mode 100644 docs/testing-with-kurtosis.md diff --git a/.gitignore b/.gitignore index 38dc976..9dcae72 100644 --- a/.gitignore +++ b/.gitignore @@ -46,3 +46,6 @@ tmp-* # Devnet files .hack/devnet/custom-* .hack/devnet/generated-* + +# Local devnet testing notes +docs/testing-with-kurtosis.md diff --git a/docs/testing-with-kurtosis.md b/docs/testing-with-kurtosis.md deleted file mode 100644 index a36924a..0000000 --- a/docs/testing-with-kurtosis.md +++ /dev/null @@ -1,495 +0,0 @@ -# Testing bootnodoor against a Kurtosis devnet - -This is the end-to-end test for the bootnode itself: does it discover real clients, -classify them into the right layer tables, keep its advertised fork fields correct -across transitions, persist what it learns, and generate no traffic it should not. - -The procedure below is deliberately independent of client versions and open branches. -Put short-lived image pins and known interop failures in -[Current validation notes](#current-validation-notes), not in the procedure. - -## What only a devnet can tell you - -Unit tests cover the logic. A devnet is the only place that produces: - -- a **fork actually activating** while the daemon runs, with real clients reacting to it; -- **real peers on the other end** of discv4/discv5, including clients that are slow, - wrong, or aggressive; -- **wire evidence** — what bootnodoor actually sends, which is the only way to catch a - self-inflicted traffic loop; -- **restart behaviour** against a database with real contents. - -Every serious defect found in this component to date came from one of those four, not -from the unit suite. - -## Scratch layout - -Keep devnet state outside the repository: - -```text -/path/to/bootnodoor-devnet/ -├── network_params.yaml # steady state, all forks at genesis -├── network_params.forks.yaml # scheduled transitions -├── network_params.devnet7.yaml # pinned public-devnet images -├── sample.sh # 30s counter + ENR sampler -├── burst.sh # 5s high-resolution sampler -└── config/ # optional: ENRScout bundle -``` - -## 1. Define the client matrix - -Seven EL/CL pairs give full client coverage: - -```yaml -participants: - - { el_type: geth, cl_type: lighthouse, count: 1 } - - { el_type: nethermind, cl_type: teku, count: 1 } - - { el_type: reth, cl_type: prysm, count: 1 } - - { el_type: erigon, cl_type: caplin, count: 1 } - - { el_type: besu, cl_type: nimbus, count: 1 } - - { el_type: nimbus, cl_type: grandine, count: 1 } - - { el_type: ethrex, cl_type: lodestar, count: 1 } - -network_params: - network: kurtosis - network_id: "3151908" - seconds_per_slot: 12 - deneb_fork_epoch: 0 - electra_fork_epoch: 0 - -bootnodoor_params: - image: ethpandaops/bootnodoor:your-build - -additional_services: - - bootnodoor -``` - -Build the image under test locally and reference it by tag; the package uses it as-is if -it exists in the local Docker daemon. - -```bash -docker build -t ethpandaops/bootnodoor:my-test . -``` - -## 2. Launch - -```bash -cd /path/to/bootnodoor-devnet -kurtosis run --enclave bootnodoor-devnet \ - github.com/ethpandaops/ethereum-package --args-file network_params.yaml -``` - -Then resolve the ports and genesis, which everything else keys off: - -```bash -BN=$(kurtosis port print bootnodoor-devnet bootnodoor http | grep -oE '[0-9]+$') -CL=$(kurtosis port print bootnodoor-devnet cl-1-lighthouse-geth http | grep -oE '[0-9]+$') -GEN=$(curl -s "http://127.0.0.1:$CL/eth/v1/beacon/genesis" \ - | grep -oE '"genesis_time":"[0-9]+"' | grep -oE '[0-9]+') -``` - -## 3. Sample continuously, not at the end - -bootnodoor's interesting behaviour is transient. A counter read after the fact tells you -almost nothing; a timestamped series tells you when something started and what it -correlated with. Two samplers, both scraping the web UI: - -- **30s sampler** — fork name, digest, ENR hashes for `/enr`, `/el-enr`, `/cl-enr`, plus - node/session/packet counters. Enough to see a run's shape. -- **5s sampler** — the same counters around a transition, where a 30s gap can hide the - entire event. - -Scrape the fields by label from `/`, and note the parsing gotchas in -[Harness gotchas](#harness-gotchas). - -## 4. What to check, and what "good" looks like - -### Fork discipline - -The core invariant. Per scheduled fork: - -- exactly **one** `fork transition: re-published ENR fork fields` log line; -- the ENR sequence steps **exactly once** — flat between transitions; -- all three of `/enr`, `/el-enr`, `/cl-enr` change together; -- the fork name and `Current Digest` on the UI both advance. - -A step per refresh tick means change detection is broken. No step means the refresh is -not firing. Both are why `UpdateENR` is a no-op when nothing changed. - -A BPO changes the digest without changing the underlying CL fork version. bootnodoor -labels these blob-schedule variants as pseudo-forks (`BPO-1`, `BPO-2`, ...), so the UI -name and digest both move even though the inherited Fulu fork version stays the same. - -### Refresh lag - -Time from the epoch boundary to the re-publish log line. This should be within about one -slot. If it varies wildly between transitions on the same run, the refresh is being -carried by a periodic backstop rather than by the boundary, which is a defect even when -the average looks acceptable — see the 2026-07-29 notes. - -### Self-inflicted traffic - -**Watch `Packets Sent`, not just `Packets Received`.** A bootnode answering queries is -normal; a bootnode _originating_ thousands of packets is not. Across a transition, the -received rate should stay at its baseline. A spike of several hundred per second means -something is looping. - -Aggregate counters cannot tell you who is talking or which direction. Capture the wire: - -```bash -C=$(docker ps --format '{{.ID}}\t{{.Names}}' | grep 'bootnodoor--' | cut -f1) -docker run --rm --net=container:$C nicolaka/netshoot \ - tcpdump -n -q -c 20000 'udp port 9000' > capture.txt -``` - -`--net=container:` shares the target's network namespace, which is what makes another -container's traffic visible at all. Then count discv4 packet types by size and direction: - -| size | type | -| ------- | ----------------- | -| 138 | PING | -| 154 | PONG | -| 104 | ENRREQUEST | -| 292–298 | ENRRESPONSE | -| 436–800 | NEIGHBORS / NODES | - -```bash -grep -oE '172\.16\.0\.11\.9000 > 172\.16\.0\.[0-9]+\.[0-9]+: UDP, length (138|104)' capture.txt \ - | awk '{split($3,a,"."); print a[4], $NF}' | sort | uniq -c | sort -rn -``` - -Healthy is single digits per peer per minute. Thousands means a retry loop. - -**Byte ratio is not packet ratio.** Outbound is dominated by NEIGHBORS/NODES replies that -are far larger than the queries provoking them, so sent/received bytes sits around 3:1 in -normal operation. That is inherent to serving discovery. The protection against -reflection is the bond/session gate on FINDNODE, not the ratio — do not read a ratio -above 1 as an amplification bug. - -### Counter sanity - -- **`Invalid Packets` should be near-zero and flat.** discv5 is registered first and - rejects anything it cannot decode, so every ordinary discv4 packet on the shared socket - falls through to discv4. Those land in `Other Protocol`. If `Invalid Packets` tracks - your traffic volume, the dispatcher accounting has regressed. -- **`Inactive Nodes` must never be negative**, and `Active` must never exceed `Total`. - Those come from different populations (memory vs database) and a negative value means - the set arithmetic broke. - -### Persistence - -Restart the container mid-run and confirm the tables reload: - -```bash -docker restart "$C" -docker logs "$C" 2>&1 | grep "loaded random nodes into active pool" -``` - -Both layers should report counts. Zero means organic discoveries were never written. - -### `--serve-all` - -This is the highest-value single test and it needs its own node, because the package -exposes no way to pass extra arguments to the packaged bootnodoor. Run a second instance -joined to the enclave network: - -```bash -GH=$(curl -s -X POST -H 'content-type: application/json' \ - --data '{"jsonrpc":"2.0","method":"eth_getBlockByNumber","params":["0x0",false],"id":1}' \ - "http://127.0.0.1:$RPC" | grep -oE '"hash":"0x[0-9a-f]+"' | grep -oE '0x[0-9a-f]+') - -docker run -d --name sa --network kt-bootnodoor-devnet \ - -v /tmp/gendata:/network-configs:ro -p 38080:8080 \ - ethpandaops/bootnodoor:my-test \ - --cl-config /network-configs/config.yaml \ - --genesis-validators-root "$(cat /tmp/gendata/genesis_validators_root.txt)" \ - --el-config /network-configs/genesis.json --el-genesis-hash "$GH" \ - --private-key "$(openssl rand -hex 32)" \ - --bind-addr 0.0.0.0 --web-ui --nodedb /nodes.db --serve-all \ - --el-bootnodes "$(curl -s http://127.0.0.1:$BN/el-enr)" \ - --cl-bootnodes "$(curl -s http://127.0.0.1:$BN/cl-enr)" -``` - -`--serve-all` pools every discovered peer into **every** enabled table, so one node ID -occupies both layers. That makes it the only configuration that exercises the -`(nodeid, layer)` composite key against real peers — a normal run cannot, because no real -client advertises both `eth` and `eth2`. EL and CL are separate identities with separate -keys; not even a unified binary publishes one record for both. - -Expect most peers in both layers, each row carrying its own layer's fork digest with the -other empty, and every `lookup complete` showing `rejected_fork=0 rejected_layer=0`. - -## 5. Fork-transition testing - -Copy the whole steady-state file and replace only `network_params`. Kurtosis takes one -`--args-file` and does not merge, so the fork file must still carry `participants` and -`additional_services: [bootnodoor]`. - -```yaml -network_params: - network: kurtosis - network_id: "3151908" - seconds_per_slot: 12 - deneb_fork_epoch: 0 - electra_fork_epoch: 1 - fulu_fork_epoch: 3 - bpo_1_epoch: 5 - bpo_1_max_blobs: 12 - bpo_1_target_blobs: 8 - -additional_services: - - bootnodoor - - spamoor - -spamoor_params: - spammers: - - name: BPO blob validation - scenario: blob-combined - config: - throughput: 30 - sidecars: 3 - max_pending: 60 - max_wallets: 40 - base_fee: 100 - blob_fee: 100 -``` - -Mainnet preset, 32 slots per epoch, 12s slots — one epoch is 384s, so that schedule -transitions at roughly T+6m, T+19m and T+32m and wants ~50 minutes. Include a BPO: it -moves the digest through the blob schedule rather than a fork version, reaching code an -ordinary fork does not. - -Do not call the BPO covered merely because the digest changed. Sample the beacon block's -`body.blob_kzg_commitments` and the matching execution block's type-3 transactions and -`blobGasUsed`. With three sidecars per transaction, a pre-BPO block should reach the -Fulu limit of 9 blobs; after the example BPO it should exceed 9 and ideally reach 12 -(`blobGasUsed == 0x180000`). A fresh spammer account avoids nonce contamination when -changing sidecar count during a run. - -Compute the wall-clocks before you start so log lines can be correlated: - -```bash -for e in 1 3 5; do - t=$((GEN + e*32*12)) - echo "epoch $e -> $(date -r "$t" '+%H:%M:%S' 2>/dev/null || date -d "@$t" '+%H:%M:%S')" -done -``` - -### Testing against a public devnet's images - -Pin every client to the target devnet's tags, taken from that devnet's ansible inventory -(`images.yaml`). Pull them first — this is network-bound and can be done while other work -proceeds. - -Public devnets schedule their real fork far out (gloas at epoch 38 is ~4h at 12s slots). -Pull it forward to something like epoch 4 so the transition lands inside the run, and say -so in the file — the deviation matters when reading results. - -## Profiling a live run - -bootnodoor supports Go pprof on the web UI listener. Start it with both `--web-ui` and -`--pprof`; without `--pprof`, `/debug/pprof/` must return 404. The package has no -extra-args hook, so profiling a Kurtosis run currently requires the standalone-container -pattern used for `--serve-all`. - -```bash -curl -fsS "http://127.0.0.1:38080/debug/pprof/profile?seconds=30" \ - -o bootnodoor-cpu.pprof -curl -fsS "http://127.0.0.1:38080/debug/pprof/heap" \ - -o bootnodoor-heap.pprof -curl -fsS "http://127.0.0.1:38080/debug/pprof/goroutine" \ - -o bootnodoor-goroutine.pprof -go install github.com/google/pprof@latest -"$(go env GOPATH)/bin/pprof" -top bootnodoor-cpu.pprof -``` - -Capture across a transition when investigating a fork-correlated load increase. pprof is -an administrative endpoint and shares `--web-host`; bind it to localhost or protect the -listener when profiling outside an isolated enclave. - -## Harness gotchas - -Every one of these cost real time. - -**SQLite is in WAL mode.** `docker cp` of `nodes.db` alone shows _no tables_ — it looks -like total data loss. Copy `nodes.db`, `nodes.db-wal` and `nodes.db-shm` together. - -**Kurtosis reassigns host ports on `docker restart`.** After restarting the bootnodoor -container, re-resolve with `kurtosis port print`; samplers pointed at the old port -silently write empty rows. - -**Kurtosis refuses to schedule when the host is busy.** "requires 100 millicores but we -will only have 0 available" means something else is consuming the box — a concurrent hive -run will do it. Never run fork-timing tests on a loaded host: missed slots look exactly -like bootnodoor bugs. - -**Schedule every fork explicitly.** Current ethereum-package defaults can put Fulu and -BPO-1 at genesis even when a file specifies only `electra_fork_epoch: 0`. Set every fork -needed by the test, including BPO target/max values, and verify the generated consensus -config before interpreting a run. - -**Scrape hyphenated fork names.** `BPO-1` contains a hyphen; a `[A-Za-z0-9]+` character -class silently skips it and matches the _next_ field, which is the digest. Symptom: the -fork column fills with hex. - -**`bootnodoor_params` has no extra-args hook.** Non-default flags such as `--serve-all` -need a standalone container on the enclave network. - -**Caplin has no separate image pin.** It ships inside erigon, so a devnet inventory has no -caplin entry and the package falls back to its default tag. Leave `cl_image` unset for the -erigon pair and note the gap. - -**Arm packet captures with care.** A shell loop building filenames from positional -parameters inside `nohup bash -c` is easy to get wrong; all three of my captures ended up -sharing one filename and firing early. Verify each armed job's command line before -walking away. - -**ENRScout needs `GENESIS_TIME`.** Kurtosis emits only `MIN_GENESIS_TIME`; append the -derived key if you run the crawler alongside. See the [ENRScout guide][enrscout-doc]. - -## Interpreting results - -- **A transition is not only a fork-version change.** A BPO inherits the active Fulu - version but changes the digest through its blob parameters; bootnodoor gives it a - `BPO-n` pseudo-fork label. Compare digests, not labels alone. -- **Rejection counters are not fork health.** On a dual-layer network most records an EL - lookup sees are consensus records; that is `rejected_layer`, not `rejected_fork`. -- **A quiet counter can still be wrong.** `Invalid Packets` sitting at 84% of received - traffic looked like an attack signal for an entire run before it turned out to be - ordinary discv4 load being miscounted. Check what a counter _means_ before treating its - value as evidence. -- **Correlate direction before assigning blame.** A traffic spike involving one peer says - nothing about which side started it. The packet sizes above resolve it in seconds. - -## Cleanup - -```bash -docker rm -f sa -cd /path/to/bootnodoor-devnet && docker compose down -v -kurtosis enclave rm -f bootnodoor-devnet -``` - -## Current validation notes - -Dated and disposable. Update as fixes land; the procedure above should stay valid. - -### Full local validation on 2026-07-30 - -The current sweep used bootnodoor `develop` at `8dafd1e`, ENRScout `main` at -`0568233`, and a local ethereum-package integration based on the open -[bootnodoor integration PR][ethereum-package-pr] plus layer-specific `/el-enr` wiring. -The bootnodoor and both ENRScout images were built from those exact trees. Besu and Reth -used current `main` images and Nethermind used current `master`; the remaining clients -used the package's curated images. - -In the steady-state seven-pair matrix, all seven EL clients entered bootnodoor before -ENRScout started, and ENRScout identified all seven organically. The CL layer was fully -discovered and verified; six implementation names were fingerprinted, with Caplin still -the naming gap. ENRScout was seeded only with bootnodoor's two identities. This supersedes -the old 5/7 organic EL baseline: Nethermind `master` explicitly logged -`Discv5 bootnodes accepted: 1`. Nimbus EL still logged `Skipping discovery bootstrap, no -bootnodes provided` with both its old and proposed flag forms, but joined indirectly -through the live mesh. - -The released-image comparison remained useful: it reached 5/7 EL organically, then -direct outbound probes identified Nethermind and Nimbus as well. A separate `--serve-all` -instance held 15 EL and 13 CL rows before and after restart; the SQLite database retained -both layer rows under the `(nodeid, layer)` key, with zero failed or inactive rows. - -The scheduled run was Deneb at genesis, Electra at epoch 1, Fulu at epoch 3, and BPO-1 -at epoch 5. Sequence moved 2→3→4→5 exactly once per boundary. Observed refresh lag was -effectively 0s for Electra, 2s for Fulu, and at most 6s at BPO-1. Tables stayed at 8/8 -active EL and 7/7 active CL. Electra and Fulu packet captures were flat at roughly -4.1k–4.3k packets/minute across each boundary, with zero capture drops. Elevated traffic -from Nethermind `master` was client-originated discovery traffic and bootnodoor replies, -not the earlier bootnodoor-originated refresh storm. `Invalid Packets` rose slowly before -Fulu but was flat at 216 across BPO-1; there was no boundary-correlated jump. - -**BPO blob semantics are now covered.** Sustained spamoor load produced three type-3 -transactions, 9 commitments, and `blobGasUsed=0x120000` before BPO-1. The first sampled -post-BPO block (execution block 118, beacon slot 160) contained four type-3 transactions, -12 matching commitments, and `blobGasUsed=0x180000`. Its execution hash matched the -beacon payload hash. This proves the chain actually moved from the Fulu capacity of 9 to -the configured BPO-1 capacity of 12; it was not only a digest transition. - -ENRScout's audit invariant held: `fork=current` plus `fork=stale` was always 14, and all -14 nodes stayed verified. The headline view dipped from 14/0 to 0/14 current/stale at -each boundary, then recovered. BPO-1 recovered to 11/3 in 37s, 13/1 in 97s, and 14/0 in -188s. The longer tail exposed an ENRScout precedence defect: a later stale signed-ENR -observation can overwrite authenticated RLPx Status fork evidence in -`internal/nodeset/nodeset.go`, despite the nearby no-downgrade comment. Besu, Ethrex, and -Reth continued advertising older signed ENRs while Status was current, causing rows to -oscillate between `fork_source: status` and `fork_source: enr`. The companion ENRScout -guide's “no ENRScout code defect” conclusion and blanket ~60s recovery expectation are -therefore superseded. - -The same live run was profiled for 30s. CPU samples represented about 0.6% of one core, -with the largest application cost in SQLite transaction commit/fsync; sampled live heap -was about 1 MiB, and 25 goroutines showed normal UDP, database, HTTP, and maintenance -loops with no leak signature. Profiling also found that pprof had been registered even -when `--pprof` was false; the handler is now gated by the flag with a regression test. - -### Runs on 2026-07-29 / 07-30 - -Baselines from a full validation sweep — steady state, the Electra/Fulu/BPO1 schedule, and -glamsterdam devnet-7 images (13 of 14 pinned; caplin fell back to its default tag). - -Organic coverage was 5/7 EL (besu, erigon, ethrex, geth, reth) and 7/7 CL. nimbus-eth1 is -the standing EL gap. Nethermind joins but is identified only outbound. - -**A packet storm was found and fixed.** A PONG advertising a newer ENR sequence spawned an -unguarded ENR refresh; each refresh PINGs and sleeps before its ENRREQUEST, so the PONG it -provoked re-entered the same trigger with the cached sequence still stale. Peak was -~980 packets/s, with 2575 PINGs and 2574 ENRREQUESTs sent to a single geth peer in 51s. -Post-fix that peer sees a maximum of 14 and 7 over 60s. - -Two things about that defect are worth remembering as method: - -- It looked fork-correlated because forks bump many peers' sequences at once. **The actual - trigger is any ENR sequence bump**, which is a far broader exposure than the fork window - it appeared in. -- The direction was initially read backwards from aggregate counters alone. Only the - packet capture showed bootnodoor was the originator. - -**A regression was caught by the devnet after code review missed it.** A first fix armed -the fork refresh at the boundary but skipped a boundary that had just passed, falling back -to a 60s backstop; lag came out 75s / 16s / 0s — the 75s _worse_ than the ticker it -replaced. The fix polls after a boundary; re-run gave 0s / 0s / 4s. Reviewing the code -found none of this. - -Also fixed in the same sweep: organic nodes were never written to the database (`Add` -marked them dirty but nothing enqueued them); the `nodes` and `bad_nodes` tables were keyed -on `nodeid` alone despite being per-layer, so under `--serve-all` 10 of 11 peers lost a -layer on every restart; and `Invalid Packets` conflated other-protocol traffic with -malformed packets. - -### Confirmation run on 2026-07-30 - -devnet-7 images plus a standalone `--serve-all` node, both on the same build. - -Three transitions, `seq` 3→4→5, one log line each, zero errors or warnings. Six distinct -`(fork, digest)` states across the series. Storm-free throughout: ~11–13 packets/s across -every boundary, and a peak of 9 PINGs / 5 ENRREQUESTs to any single peer in the 60s around -gloas. `Invalid Packets` sat at 6 for the entire run while `Other Protocol` absorbed 8,800. -`Inactive Nodes` was 0 at every sample. Under `--serve-all`, 12 of 14 peers occupied both -layers with no errors. - -**Refresh lags were 2s, 8s and 25s.** The third exceeds one slot and is worth knowing how -to read. The UI's `Current Fork` is computed live from the wall clock, while -`Current Digest` returns a value cached until `clFilter.Update()` runs inside the refresh -(`bootnode/clconfig/filter.go` — compare `GetCurrentFork` with `GetCurrentDigest`). So a -window where the name has advanced but the digest has not *is* the refresh lag, displayed -in two fields rather than a second defect. Here the boundary timer fired on time and the -digest was simply not derivable for ~25s. Candidate follow-up, not a blocker: it is well -inside the backstop and far better than the 8/20/22s and 2/22/51s measured before the -boundary-armed refresh landed. - -### Not yet covered - -- **Dead nodes are still served.** `PerformSweep` demotes only when the table is over - capacity, and FINDNODE responses are not filtered on liveness. -- **DB-restored v4 pointers are detached.** `buildNodeFromDB` reconstructs its own discv4 - node rather than the handler's, so proven-address promotion never reaches it. - -[enrscout-doc]: https://github.com/mysticryuujin/enrscout/blob/main/docs/testing-with-kurtosis.md -[ethereum-package-pr]: https://github.com/ethpandaops/ethereum-package/pull/1461 From d08b6c70ee699611b813a4f9f1407cbac4a6464d Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Thu, 30 Jul 2026 12:47:53 -0500 Subject: [PATCH 49/49] chore: drop the ignore rule for the local testing notes --- .gitignore | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitignore b/.gitignore index 9dcae72..38dc976 100644 --- a/.gitignore +++ b/.gitignore @@ -46,6 +46,3 @@ tmp-* # Devnet files .hack/devnet/custom-* .hack/devnet/generated-* - -# Local devnet testing notes -docs/testing-with-kurtosis.md