diff --git a/bootnode/clconfig/admit_test.go b/bootnode/clconfig/admit_test.go new file mode 100644 index 0000000..96b22a3 --- /dev/null +++ b/bootnode/clconfig/admit_test.go @@ -0,0 +1,187 @@ +package clconfig + +import ( + "net" + "sync" + "testing" + "time" + + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethpandaops/bootnodoor/enr" +) + +func admitTestFilter(t *testing.T) *ForkDigestFilter { + t.Helper() + + cfg := &Config{ + SecondsPerSlot: 12, + customSlotsPerEpoch: 32, + genesisForkVersion: [4]byte{0x00, 0x00, 0x00, 0x01}, + forks: []forkDefinition{ + {name: "Altair", epoch: 0, parsedVersion: [4]byte{0x01, 0x00, 0x00, 0x00}}, + }, + } + cfg.SetGenesisTime(uint64(time.Now().Unix()) - 60) + return NewForkDigestFilter(cfg, time.Hour) +} + +func recordWithEth2(t *testing.T, eth2 []byte) *enr.Record { + t.Helper() + + key, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("generate key: %v", err) + } + rec := enr.New() + if err := rec.Set("ip", net.IPv4(1, 2, 3, 4)); err != nil { + t.Fatalf("set ip: %v", err) + } + if eth2 != nil { + if err := rec.Set("eth2", eth2); err != nil { + t.Fatalf("set eth2: %v", err) + } + } + rec.SetSeq(1) + if err := rec.Sign(key); err != nil { + t.Fatalf("sign: %v", err) + } + return rec +} + +// Matches is the per-packet classification entry point, so it must decide +// without moving any counter — those numbers report admissions, and packet +// traffic dwarfs admissions by orders of magnitude. +func TestMatchesRecordsNoStats(t *testing.T) { + filter := admitTestFilter(t) + current := filter.GetCurrentForkDigest() + + cases := []struct { + name string + eth2 []byte + want bool + }{ + {"no eth2", nil, false}, + {"malformed", []byte{0x01, 0x02}, false}, + {"current digest", EncodeETH2Field(current, [4]byte{0x01, 0x00, 0x00, 0x00}, ^uint64(0)), true}, + {"unknown digest", EncodeETH2Field(ForkDigest{0xde, 0xad, 0xbe, 0xef}, [4]byte{}, 0), false}, + } + + for _, tc := range cases { + rec := recordWithEth2(t, tc.eth2) + if got := filter.Matches(rec); got != tc.want { + t.Errorf("Matches(%s) = %v, want %v", tc.name, got, tc.want) + } + } + + stats := filter.GetStats() + if stats.TotalChecks != 0 || stats.AcceptedCurrent != 0 || stats.AcceptedOld != 0 || + stats.AcceptedHistorical != 0 || stats.RejectedInvalid != 0 { + t.Fatalf("Matches moved counters: %+v", stats) + } +} + +// Admit is the only counting entry point, and TotalChecks must always equal the +// sum of the buckets — the web UI renders them in one table, so a reader must +// never see rows that do not add up. +func TestAdmitRecordsOneBucketPerCall(t *testing.T) { + filter := admitTestFilter(t) + current := filter.GetCurrentForkDigest() + + cases := []struct { + name string + eth2 []byte + wantAccept bool + wantChecks int + wantCurrent int + wantInvalid int + }{ + {"no eth2 is uncounted", nil, false, 0, 0, 0}, + {"current digest", EncodeETH2Field(current, [4]byte{0x01, 0x00, 0x00, 0x00}, ^uint64(0)), true, 1, 1, 0}, + {"unknown digest", EncodeETH2Field(ForkDigest{0xde, 0xad, 0xbe, 0xef}, [4]byte{}, 0), false, 2, 1, 1}, + {"malformed", []byte{0x01, 0x02}, false, 3, 1, 2}, + } + + for _, tc := range cases { + rec := recordWithEth2(t, tc.eth2) + if got := filter.Admit(rec); got != tc.wantAccept { + t.Errorf("Admit(%s) = %v, want %v", tc.name, got, tc.wantAccept) + } + + stats := filter.GetStats() + if stats.TotalChecks != tc.wantChecks { + t.Errorf("after %s: TotalChecks = %d, want %d", tc.name, stats.TotalChecks, tc.wantChecks) + } + if stats.AcceptedCurrent != tc.wantCurrent { + t.Errorf("after %s: AcceptedCurrent = %d, want %d", tc.name, stats.AcceptedCurrent, tc.wantCurrent) + } + if stats.RejectedInvalid != tc.wantInvalid { + t.Errorf("after %s: RejectedInvalid = %d, want %d", tc.name, stats.RejectedInvalid, tc.wantInvalid) + } + + sum := stats.AcceptedCurrent + stats.AcceptedOld + stats.AcceptedHistorical + stats.RejectedInvalid + if stats.TotalChecks != sum { + t.Errorf("after %s: TotalChecks = %d but buckets sum to %d", tc.name, stats.TotalChecks, sum) + } + } +} + +// Update mutates oldForkDigests while packets are being filtered, so the digest +// lookups must happen under the lock. Publishing the map reference and indexing +// it afterwards is a concurrent map read/write, which aborts the process. +func TestAdmitConcurrentWithUpdate(t *testing.T) { + filter := admitTestFilter(t) + + rec := recordWithEth2(t, EncodeETH2Field(ForkDigest{0x11, 0x22, 0x33, 0x44}, [4]byte{}, 0)) + + var wg sync.WaitGroup + stop := make(chan struct{}) + + // Update only writes oldForkDigests on a fork activation or a grace expiry, + // so seed an already-expired entry each round to make its cleanup loop + // delete — the same map write, just at test frequency. + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case <-stop: + return + default: + filter.mu.Lock() + filter.oldForkDigests[ForkDigest{0x11, 0x22, 0x33, 0x44}] = time.Now().Add(-2 * time.Hour) + filter.mu.Unlock() + filter.Update() + } + } + }() + + for i := 0; i < 8; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + for j := 0; j < 200; j++ { + if i%2 == 0 { + filter.Admit(rec) + } else { + filter.Matches(rec) + } + } + }(i) + } + + // The Update loop runs until the readers finish, then is joined separately. + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + time.Sleep(200 * time.Millisecond) + close(stop) + <-done + + stats := filter.GetStats() + sum := stats.AcceptedCurrent + stats.AcceptedOld + stats.AcceptedHistorical + stats.RejectedInvalid + if stats.TotalChecks != sum { + t.Fatalf("TotalChecks = %d but buckets sum to %d under concurrency", stats.TotalChecks, sum) + } +} diff --git a/bootnode/clconfig/config.go b/bootnode/clconfig/config.go index 53ba155..691b1dd 100644 --- a/bootnode/clconfig/config.go +++ b/bootnode/clconfig/config.go @@ -13,6 +13,7 @@ import ( "fmt" "math" "os" + "slices" "sort" "strings" "time" @@ -76,6 +77,29 @@ func (c *Config) getForks() []forkDefinition { return c.forks } +// ForkEpochs returns every scheduled fork epoch in ascending order, including +// BPO entries and excluding far-future placeholders. +// +// Distinct from GetAllForkDigestInfos, which deduplicates by digest: the eth2 +// next-fork tuple changes at a boundary even when the current digest does not, +// so a caller scheduling work per boundary needs the raw epochs. +func (c *Config) ForkEpochs() []uint64 { + forks := c.getForks() + + epochs := make([]uint64, 0, len(forks)) + for i := range forks { + epoch := forks[i].epoch + if epoch == math.MaxUint64 { + continue + } + if len(epochs) > 0 && epochs[len(epochs)-1] == epoch { + continue + } + epochs = append(epochs, epoch) + } + return epochs +} + // GetForkEpoch returns the epoch for a given fork name. // Returns nil if the fork is not defined. func (c *Config) GetForkEpoch(forkName string) *uint64 { @@ -549,22 +573,19 @@ func (c *Config) currentEpochNow() (uint64, bool) { // genesis, each registered fork (including BPO pseudo-forks), and each blob // schedule boundary. Sorted ascending, deduplicated. func (c *Config) forkBoundaryEpochs() []uint64 { - seen := map[uint64]bool{0: true} epochs := []uint64{0} - add := func(epoch uint64) { - if epoch != math.MaxUint64 && !seen[epoch] { - seen[epoch] = true - epochs = append(epochs, epoch) - } - } for _, fork := range c.getForks() { - add(fork.epoch) + if fork.epoch != math.MaxUint64 { + epochs = append(epochs, fork.epoch) + } } for _, entry := range c.BlobSchedule { - add(entry.Epoch) + if entry.Epoch != math.MaxUint64 { + epochs = append(epochs, entry.Epoch) + } } - sort.Slice(epochs, func(i, j int) bool { return epochs[i] < epochs[j] }) - return epochs + slices.Sort(epochs) + return slices.Compact(epochs) } // GetCurrentForkDigest returns the fork digest for the current epoch. @@ -659,14 +680,13 @@ type ForkDigestInfo struct { // for same-epoch intermediate forks (never current on the wire) are // intentionally not included. func (c *Config) GetAllForkDigests() []ForkDigest { - var digests []ForkDigest - seen := make(map[ForkDigest]bool) - for _, epoch := range c.forkBoundaryEpochs() { - digest := c.GetForkDigestForEpoch(epoch) - if !seen[digest] { - seen[digest] = true - digests = append(digests, digest) - } + infos := c.GetAllForkDigestInfos() + if len(infos) == 0 { + return nil + } + digests := make([]ForkDigest, 0, len(infos)) + for _, info := range infos { + digests = append(digests, info.Digest) } return digests } diff --git a/bootnode/clconfig/filter.go b/bootnode/clconfig/filter.go index fb90ac2..0e2ae69 100644 --- a/bootnode/clconfig/filter.go +++ b/bootnode/clconfig/filter.go @@ -65,15 +65,8 @@ type Logger interface { // - config: CL configuration for computing fork digests // - gracePeriod: How long to accept old fork digests (0 = default 60 minutes) // -// Example: -// -// config, _ := LoadConfig("config.yaml") -// filter := NewForkDigestFilter(config, 60*time.Minute) -// -// // Use as admission filter -// service, _ := discv5.New(&discv5.Config{ -// AdmissionFilter: filter.Filter(), -// }) +// Call Admit from admission paths and Matches from per-packet classification; +// only Admit moves the stats counters. func NewForkDigestFilter(config *Config, gracePeriod time.Duration) *ForkDigestFilter { if gracePeriod <= 0 { gracePeriod = DefaultGracePeriod @@ -105,104 +98,134 @@ func (f *ForkDigestFilter) SetLogger(logger Logger) { f.logger = logger } -// Filter returns an ENR admission filter function. -// -// This filter accepts ALL historically valid fork digests: -// - Current fork digest -// - Old fork digests (within grace period) -// - Any historically valid fork digest from the network -// -// Nodes with old digests are accepted into the routing table and will be -// pinged, which triggers ENR updates. Use ResponseFilter() to exclude them -// from FINDNODE responses. -// -// Example: +// clOutcome is the result of evaluating one record's fork digest. Each value +// maps to exactly one stats bucket, except outcomeNotCL which is deliberately +// uncounted. +type clOutcome int + +const ( + // outcomeNotCL is a record with no eth2 entry: an execution node, not a + // broken consensus node, so it must not move the CL counters. + outcomeNotCL clOutcome = iota + outcomeUndecodable + outcomeUnparsable + outcomeCurrent + outcomeOldInGrace + outcomeHistorical + outcomeUnknownDigest +) + +func (o clOutcome) accepted() bool { + return o == outcomeCurrent || o == outcomeOldInGrace || o == outcomeHistorical +} + +// classify evaluates a record's fork digest, touching neither stats nor logs. // -// filter := NewForkDigestFilter(config, 60*time.Minute) -// service, _ := discv5.New(&discv5.Config{ -// AdmissionFilter: filter.Filter(), -// ResponseFilter: filter.ResponseFilter(), -// }) -func (f *ForkDigestFilter) Filter(record *enr.Record) bool { - f.mu.Lock() - f.totalChecks++ - f.mu.Unlock() +// The digest lookups run inside one read-lock: Update mutates oldForkDigests, so +// publishing that map outside the lock and indexing it later is a concurrent +// map read/write, which aborts the process rather than returning an error. +func (f *ForkDigestFilter) classify(record *enr.Record) (clOutcome, ForkDigest, error) { + if record == nil { + return outcomeNotCL, ForkDigest{}, nil + } - // Get eth2 field from ENR var eth2Data []byte if err := record.Get("eth2", ð2Data); err != nil { - // No eth2 field, reject - f.mu.Lock() - f.rejectedInvalid++ - if f.logger != nil { - f.logger.Debugf("Rejected node: no eth2 field in ENR") + if !record.Has("eth2") { + return outcomeNotCL, ForkDigest{}, nil } - f.mu.Unlock() - return false + return outcomeUndecodable, ForkDigest{}, err } - // Parse fork digest (first 4 bytes only) forkDigest, err := ParseETH2Field(eth2Data) if err != nil { - // Invalid eth2 field, reject - f.mu.Lock() - f.rejectedInvalid++ - if f.logger != nil { - f.logger.Debugf("Rejected node: invalid eth2 field - %v", err) - } - f.mu.Unlock() - return false + return outcomeUnparsable, ForkDigest{}, err } f.mu.RLock() - currentDigest := f.currentForkDigest - oldDigests := f.oldForkDigests - gracePeriod := f.gracePeriod - historicalDigests := f.historicalDigests - f.mu.RUnlock() + defer f.mu.RUnlock() - // Check if matches current fork digest - if forkDigest == currentDigest { - f.mu.Lock() - f.acceptedCurrent++ - f.mu.Unlock() - return true + if forkDigest == f.currentForkDigest { + return outcomeCurrent, forkDigest, nil } - // Check if matches old fork digest within grace period - if activationTime, exists := oldDigests[forkDigest]; exists { - age := time.Since(activationTime) - if age <= gracePeriod { - f.mu.Lock() - f.acceptedOld++ - f.mu.Unlock() - return true + if activationTime, exists := f.oldForkDigests[forkDigest]; exists { + if time.Since(activationTime) <= f.gracePeriod { + return outcomeOldInGrace, forkDigest, nil } - // Grace period expired but still historically valid - fall through + // Grace period expired but the digest may still be historically valid. } - // Check if it's any historically valid fork digest - // These nodes will be added to the table and pinged (triggering ENR updates) - // but may be excluded from FINDNODE responses via ResponseFilter - if historicalDigests[forkDigest] { - f.mu.Lock() - f.acceptedHistorical++ - if f.logger != nil { - f.logger.Debugf("Accepted node with historical fork digest: %s (current: %s)", forkDigest.String(), currentDigest.String()) - } - f.mu.Unlock() - return true + if f.historicalDigests[forkDigest] { + return outcomeHistorical, forkDigest, nil + } + + return outcomeUnknownDigest, forkDigest, nil +} + +// Matches reports whether a record's fork digest is acceptable. +// +// It is pure: no counter moves and nothing is logged. Use it for per-packet +// layer classification, which happens far too often to be a stats signal. +func (f *ForkDigestFilter) Matches(record *enr.Record) bool { + outcome, _, _ := f.classify(record) + return outcome.accepted() +} + +// Admit is Matches plus stats: it is the only entry point that moves the +// counters. Call it from admission paths only, never from layer classification +// (the same contract as elconfig.ForkFilter.RecordAdmission). +// +// This filter accepts ALL historically valid fork digests: the current digest, +// old digests within the grace period, and any digest from network history. +// Nodes with old digests are accepted into the routing table and pinged, which +// triggers ENR updates. +func (f *ForkDigestFilter) Admit(record *enr.Record) bool { + outcome, forkDigest, err := f.classify(record) + f.recordOutcome(outcome, forkDigest, err) + return outcome.accepted() +} + +// recordOutcome folds one admission outcome into the stats and emits the +// matching debug line, under a single lock so TotalChecks and the buckets always +// agree for any GetStats observer. +func (f *ForkDigestFilter) recordOutcome(outcome clOutcome, forkDigest ForkDigest, err error) { + if outcome == outcomeNotCL { + return } - // Unknown fork digest, reject f.mu.Lock() - f.rejectedInvalid++ - if f.logger != nil { - f.logger.Debugf("Rejected node: unknown fork digest %s (current: %s, %d historical digests known)", - forkDigest.String(), currentDigest.String(), len(historicalDigests)) + defer f.mu.Unlock() + + f.totalChecks++ + + switch outcome { + case outcomeUndecodable: + f.rejectedInvalid++ + if f.logger != nil { + f.logger.Debugf("Rejected node: undecodable eth2 field - %v", err) + } + case outcomeUnparsable: + f.rejectedInvalid++ + if f.logger != nil { + f.logger.Debugf("Rejected node: invalid eth2 field - %v", err) + } + case outcomeCurrent: + f.acceptedCurrent++ + case outcomeOldInGrace: + f.acceptedOld++ + case outcomeHistorical: + f.acceptedHistorical++ + if f.logger != nil { + f.logger.Debugf("Accepted node with historical fork digest: %s (current: %s)", forkDigest.String(), f.currentForkDigest.String()) + } + case outcomeUnknownDigest: + f.rejectedInvalid++ + if f.logger != nil { + f.logger.Debugf("Rejected node: unknown fork digest %s (current: %s, %d historical digests known)", + forkDigest.String(), f.currentForkDigest.String(), len(f.historicalDigests)) + } } - f.mu.Unlock() - return false } // Update updates the fork digest based on the current epoch. @@ -330,18 +353,7 @@ func (f *ForkDigestFilter) ComputeEth2Field() []byte { func (f *ForkDigestFilter) nextForkInfo() ([4]byte, uint64) { const farFutureEpoch = ^uint64(0) - genesisTime := f.config.GetGenesisTime() - secondsPerSlot := f.config.SecondsPerSlot - if secondsPerSlot == 0 { - secondsPerSlot = 12 - } - - currentEpoch := uint64(0) - if genesisTime > 0 { - slotsPerEpoch := f.config.GetSlotsPerEpoch() - currentEpoch = uint64(GetCurrentEpoch(genesisTime, uint64(time.Now().Unix()), secondsPerSlot, slotsPerEpoch)) - } - + currentEpoch, _ := f.config.currentEpochNow() currentForkVersion := f.config.GetForkVersionAtEpoch(currentEpoch) for _, fork := range f.config.getForks() { @@ -364,23 +376,10 @@ func (f *ForkDigestFilter) nextForkInfo() ([4]byte, uint64) { // GetCurrentFork returns the name of the current fork. func (f *ForkDigestFilter) GetCurrentFork() string { - // Get genesis time - genesisTime := f.config.GetGenesisTime() - if genesisTime == 0 { - // No genesis time, fallback to "Unknown" + currentEpoch, ok := f.config.currentEpochNow() + if !ok { return "Unknown" } - - // Calculate current epoch - currentTime := uint64(time.Now().Unix()) - slotsPerEpoch := f.config.GetSlotsPerEpoch() - secondsPerSlot := f.config.SecondsPerSlot - if secondsPerSlot == 0 { - secondsPerSlot = 12 - } - currentEpoch := uint64(GetCurrentEpoch(genesisTime, currentTime, secondsPerSlot, slotsPerEpoch)) - - // Get fork name for current epoch return f.config.GetForkNameAtEpoch(currentEpoch) } @@ -452,47 +451,6 @@ func (f *ForkDigestFilter) GetOldDigests() map[string]time.Duration { return result } -// GetAcceptedCurrent returns the count of nodes accepted with current fork digest. -func (f *ForkDigestFilter) GetAcceptedCurrent() int { - f.mu.RLock() - defer f.mu.RUnlock() - - return f.acceptedCurrent -} - -// GetAcceptedOld returns the count of nodes accepted with old fork digests. -func (f *ForkDigestFilter) GetAcceptedOld() int { - f.mu.RLock() - defer f.mu.RUnlock() - - return f.acceptedOld -} - -// GetRejectedInvalid returns the count of nodes rejected due to invalid fork digest. -func (f *ForkDigestFilter) GetRejectedInvalid() int { - f.mu.RLock() - defer f.mu.RUnlock() - - return f.rejectedInvalid -} - -// GetAcceptedHistorical returns the count of nodes accepted on a historical -// fork digest (valid chain, not the current or grace-period fork). -func (f *ForkDigestFilter) GetAcceptedHistorical() int { - f.mu.RLock() - defer f.mu.RUnlock() - - return f.acceptedHistorical -} - -// GetTotalChecks returns the total number of filter checks performed. -func (f *ForkDigestFilter) GetTotalChecks() int { - f.mu.RLock() - defer f.mu.RUnlock() - - return f.totalChecks -} - // GetPreviousForkDigest returns the previous fork digest as a hex string. func (f *ForkDigestFilter) GetPreviousForkDigest() string { return f.config.GetPreviousForkDigest().String() diff --git a/bootnode/clconfig/filter_test.go b/bootnode/clconfig/filter_test.go index 1c5eb0e..4ad1a13 100644 --- a/bootnode/clconfig/filter_test.go +++ b/bootnode/clconfig/filter_test.go @@ -2,8 +2,12 @@ package clconfig import ( "math" + "net" "testing" "time" + + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethpandaops/bootnodoor/enr" ) func TestNextForkInfoReturnsUpcomingFork(t *testing.T) { @@ -71,6 +75,81 @@ func TestNextForkInfoFallsBackToFarFuture(t *testing.T) { } } +// TestAdmitSkipsRecordsWithoutEth2: a record with no eth2 entry is an +// execution node, not an invalid consensus node, so it must not move any +// counter (mirrors the EL side's AdmitELNode gate). A malformed eth2 +// entry still counts as invalid. +func TestAdmitSkipsRecordsWithoutEth2(t *testing.T) { + cfg := &Config{ + SecondsPerSlot: 12, + customSlotsPerEpoch: 32, + genesisForkVersion: [4]byte{0x00, 0x00, 0x00, 0x01}, + forks: []forkDefinition{ + {name: "Altair", epoch: 0, parsedVersion: [4]byte{0x01, 0x00, 0x00, 0x00}}, + }, + } + cfg.SetGenesisTime(uint64(time.Now().Unix()) - 60) + filter := NewForkDigestFilter(cfg, time.Hour) + + key, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("generate key: %v", err) + } + noEth2 := enr.New() + if err := noEth2.Set("ip", net.IPv4(1, 2, 3, 4)); err != nil { + t.Fatalf("set ip: %v", err) + } + noEth2.SetSeq(1) + if err := noEth2.Sign(key); err != nil { + t.Fatalf("sign: %v", err) + } + + if filter.Admit(noEth2) { + t.Fatal("record without eth2 passed the CL filter") + } + stats := filter.GetStats() + if stats.TotalChecks != 0 || stats.RejectedInvalid != 0 { + t.Fatalf("no-eth2 record moved counters: checks=%d rejectedInvalid=%d, want 0/0", + stats.TotalChecks, stats.RejectedInvalid) + } + + malformed := enr.New() + if err := malformed.Set("eth2", []byte{0x01, 0x02}); err != nil { + t.Fatalf("set eth2: %v", err) + } + malformed.SetSeq(1) + if err := malformed.Sign(key); err != nil { + t.Fatalf("sign: %v", err) + } + + if filter.Admit(malformed) { + t.Fatal("malformed eth2 passed the CL filter") + } + stats = filter.GetStats() + if stats.TotalChecks != 1 || stats.RejectedInvalid != 1 { + t.Fatalf("malformed eth2 counters: checks=%d rejectedInvalid=%d, want 1/1", + stats.TotalChecks, stats.RejectedInvalid) + } + + undecodable := enr.New() + if err := undecodable.Set("eth2", []uint64{1, 2}); err != nil { + t.Fatalf("set eth2: %v", err) + } + undecodable.SetSeq(1) + if err := undecodable.Sign(key); err != nil { + t.Fatalf("sign: %v", err) + } + + if filter.Admit(undecodable) { + t.Fatal("undecodable eth2 passed the CL filter") + } + stats = filter.GetStats() + if stats.TotalChecks != 2 || stats.RejectedInvalid != 2 { + t.Fatalf("undecodable eth2 counters: checks=%d rejectedInvalid=%d, want 2/2", + stats.TotalChecks, stats.RejectedInvalid) + } +} + // TestForkDigestFilterUpdateTransition verifies Update() tracks an epoch // crossing: the current digest swaps, the old digest lands in the grace map // and stays accepted, and the recomputed eth2 field reflects the new digest. diff --git a/bootnode/clfilter_test.go b/bootnode/clfilter_test.go new file mode 100644 index 0000000..2c5b9a3 --- /dev/null +++ b/bootnode/clfilter_test.go @@ -0,0 +1,117 @@ +package bootnode + +import ( + "context" + "net" + "testing" + "time" + + "github.com/ethpandaops/bootnodoor/bootnode/clconfig" + "github.com/ethpandaops/bootnodoor/db" + v5node "github.com/ethpandaops/bootnodoor/discv5/node" + "github.com/ethpandaops/bootnodoor/enr" + "github.com/ethpandaops/bootnodoor/nodes" +) + +// newCLTestService mirrors newServeAllTestService but wires the CL layer, so the +// CL fork-digest filter and its counters are reachable. +func newCLTestService(t *testing.T) *Service { + t.Helper() + + logger := quietLogger() + database := db.NewDatabase(&db.SqliteDatabaseConfig{File: ":memory:", MaxOpenConns: 5, MaxIdleConns: 2}, logger) + if err := database.Init(); err != nil { + t.Fatalf("db init: %v", err) + } + t.Cleanup(func() { database.Close() }) + if err := database.ApplyEmbeddedDbSchema(-2); err != nil { + t.Fatalf("db schema: %v", err) + } + + cfg := &Config{ + Logger: logger, + Database: database, + CLConfig: &clconfig.Config{}, + } + + key := mustKey(t) + ln, err := createLocalNode(cfg, key, net.ParseIP("1.2.3.4"), nil, 9000, nil) + if err != nil { + t.Fatalf("createLocalNode: %v", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + s := &Service{config: cfg, ctx: ctx, enrManager: NewENRManager(cfg, key, ln, false, true)} + s.clNodeDB = nodes.NewNodeDB(ctx, database, db.LayerCL, logger) + s.clTable, err = s.createTable(ln.ID(), s.clNodeDB, "CL") + if err != nil { + t.Fatalf("createTable: %v", err) + } + + return s +} + +// clNodeOnCurrentDigest builds a v5 node whose eth2 entry carries the digest the +// filter currently accepts, so it exercises the accepted path. +func clNodeOnCurrentDigest(t *testing.T, s *Service) *v5node.Node { + t.Helper() + + digest := s.enrManager.GetCLFilter().GetCurrentForkDigest() + eth2 := clconfig.EncodeETH2Field(digest, [4]byte{}, ^uint64(0)) + + key := mustKey(t) + rec := enr.New() + if err := rec.Set("ip", net.IPv4(9, 9, 9, 9)); err != nil { + t.Fatalf("set ip: %v", err) + } + if err := rec.Set("udp", uint16(9000)); err != nil { + t.Fatalf("set udp: %v", err) + } + if err := rec.Set("eth2", eth2); err != nil { + t.Fatalf("set eth2: %v", err) + } + rec.SetSeq(1) + if err := rec.Sign(key); err != nil { + t.Fatalf("sign: %v", err) + } + + n, err := v5node.New(rec) + if err != nil { + t.Fatalf("v5node.New: %v", err) + } + return n +} + +// onNodeSeen runs once per decoded discv5 message, so it must classify without +// counting: otherwise the Fork Filter card reports packet traffic rather than +// admission decisions. +func TestOnNodeSeen_LeavesCLFilterCountersUntouched(t *testing.T) { + s := newCLTestService(t) + n := clNodeOnCurrentDigest(t, s) + + for i := 0; i < 3; i++ { + s.onNodeSeen(n, time.Now()) + } + + if got := s.enrManager.GetCLFilter().GetStats().TotalChecks; got != 0 { + t.Fatalf("TotalChecks = %d after 3 onNodeSeen calls, want 0", got) + } +} + +// The counterpart guard: admission must still count, or the fix would just zero +// the UI permanently while the classification tests passed. +func TestCheckAndAddNode_RecordsCLAdmissionOnce(t *testing.T) { + s := newCLTestService(t) + n := clNodeOnCurrentDigest(t, s) + + if !s.checkAndAddNode(n) { + t.Fatal("current-digest node was not admitted to the CL table") + } + + stats := s.enrManager.GetCLFilter().GetStats() + if stats.TotalChecks != 1 || stats.AcceptedCurrent != 1 { + t.Fatalf("stats = %+v after one admission, want 1 check / 1 accepted-current", stats) + } +} diff --git a/bootnode/config.go b/bootnode/config.go index f507abb..ef947a8 100644 --- a/bootnode/config.go +++ b/bootnode/config.go @@ -112,6 +112,12 @@ type Config struct { // EnableDiscv5 enables Discovery v5 protocol (default: true) EnableDiscv5 bool + // ServeAll disables EL/CL classification and fork-ID/digest filtering. Every + // discovered node is pooled (into every enabled table) and served to every + // requester, turning the bootnode into a plain discv5 rendezvous that relays + // all peers regardless of eth/eth2 fields. Default: false (classify + filter). + ServeAll bool + // SessionLifetime is the discv5 session lifetime (default: 12 hours) SessionLifetime time.Duration @@ -120,7 +126,9 @@ type Config struct { // Discovery configuration - // EnableIPDiscovery enables automatic IP discovery from PONG responses (default: false) + // EnableIPDiscovery enables automatic IP discovery from PONG responses + // (default: true). An explicitly configured ENRIP/ENRIP6 is never overridden + // by discovery regardless of this setting. EnableIPDiscovery bool // GracePeriod is the grace period for accepting old fork digests (default: 60 minutes) @@ -151,7 +159,7 @@ func DefaultConfig() *Config { EnableDiscv5: true, SessionLifetime: 12 * time.Hour, MaxSessions: 1024, - EnableIPDiscovery: false, + EnableIPDiscovery: true, GracePeriod: 60 * time.Minute, } } diff --git a/bootnode/duallayer_persist_test.go b/bootnode/duallayer_persist_test.go new file mode 100644 index 0000000..3519528 --- /dev/null +++ b/bootnode/duallayer_persist_test.go @@ -0,0 +1,162 @@ +package bootnode + +import ( + "context" + "net" + "path/filepath" + "testing" + "time" + + "github.com/ethpandaops/bootnodoor/bootnode/clconfig" + "github.com/ethpandaops/bootnodoor/bootnode/elconfig" + "github.com/ethpandaops/bootnodoor/db" + v5node "github.com/ethpandaops/bootnodoor/discv5/node" + "github.com/ethpandaops/bootnodoor/enr" + "github.com/ethpandaops/bootnodoor/nodes" +) + +// serveAllServiceAt builds a file-backed service with classification disabled, +// which is what puts one node ID into both tables. +// +// No real peer advertises eth and eth2 together — EL and CL run as separate +// identities with separate keys — so serve-all, not a dual-stack client, is how +// the same node ID reaches both layers in practice. +func serveAllServiceAt(t *testing.T, file string) (*Service, *db.Database, context.CancelFunc) { + t.Helper() + + logger := quietLogger() + database := db.NewDatabase(&db.SqliteDatabaseConfig{File: file, MaxOpenConns: 5, MaxIdleConns: 2}, logger) + if err := database.Init(); err != nil { + t.Fatalf("db init: %v", err) + } + if err := database.ApplyEmbeddedDbSchema(-2); err != nil { + t.Fatalf("db schema: %v", err) + } + + cfg := &Config{ + Logger: logger, + Database: database, + ELConfig: &elconfig.ChainConfig{}, + ELGenesisHash: [32]byte{1, 2, 3}, + ELGenesisTime: 1000, + CLConfig: &clconfig.Config{}, + ServeAll: true, + } + + key := mustKey(t) + ln, err := createLocalNode(cfg, key, net.ParseIP("1.2.3.4"), nil, 9000, nil) + if err != nil { + t.Fatalf("createLocalNode: %v", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + + s := &Service{config: cfg, ctx: ctx, enrManager: NewENRManager(cfg, key, ln, true, true)} + s.elNodeDB = nodes.NewNodeDB(ctx, database, db.LayerEL, logger) + s.clNodeDB = nodes.NewNodeDB(ctx, database, db.LayerCL, logger) + if s.elTable, err = s.createTable(ln.ID(), s.elNodeDB, "EL"); err != nil { + t.Fatalf("createTable EL: %v", err) + } + if s.clTable, err = s.createTable(ln.ID(), s.clNodeDB, "CL"); err != nil { + t.Fatalf("createTable CL: %v", err) + } + return s, database, cancel +} + +// plainCLNode is an ordinary single-layer peer: eth2 only, as a real beacon node +// advertises. +func plainCLNode(t *testing.T, ip net.IP) *v5node.Node { + t.Helper() + + key := mustKey(t) + rec := enr.New() + if err := rec.Set("ip", ip); err != nil { + t.Fatalf("set ip: %v", err) + } + if err := rec.Set("udp", uint16(9000)); err != nil { + t.Fatalf("set udp: %v", err) + } + if err := rec.Set("eth2", clconfig.EncodeETH2Field(clconfig.ForkDigest{1, 2, 3, 4}, [4]byte{}, ^uint64(0))); err != nil { + t.Fatalf("set eth2: %v", err) + } + rec.SetSeq(1) + if err := rec.Sign(key); err != nil { + t.Fatalf("sign: %v", err) + } + + n, err := v5node.New(rec) + if err != nil { + t.Fatalf("v5node.New: %v", err) + } + return n +} + +// Under serve-all every discovered peer is pooled into every enabled table, so an +// ordinary CL-only node occupies both layers. With nodeid as the sole primary key +// the second write replaced the first, so one layer was lost on every restart — +// for every peer, not just an exotic one. +func TestServeAllPeerPersistsToBothLayers(t *testing.T) { + file := filepath.Join(t.TempDir(), "serveall.db") + + s, database, cancel := serveAllServiceAt(t, file) + n := plainCLNode(t, net.IPv4(9, 9, 9, 9)) + + if !s.checkAndAddNode(n) { + t.Fatal("node was not admitted under serve-all") + } + if s.elTable.Get(n.ID()) == nil { + t.Fatal("serve-all did not pool the node into the EL table") + } + if s.clTable.Get(n.ID()) == nil { + t.Fatal("serve-all did not pool the node into the CL table") + } + + waitForRows(t, database, 2) + + id := n.ID() + if _, err := database.GetNode(db.LayerEL, id[:]); err != nil { + t.Errorf("EL row missing: %v", err) + } + if _, err := database.GetNode(db.LayerCL, id[:]); err != nil { + t.Errorf("CL row missing: %v", err) + } + + // Cancel before Close: the queue processor exits on the context, and Close + // waits for it. + cancel() + s.elNodeDB.Close() + s.clNodeDB.Close() + database.Close() + + _, reopened, cancel2 := serveAllServiceAt(t, file) + defer cancel2() + defer reopened.Close() + + elBack, err := reopened.GetNode(db.LayerEL, id[:]) + if err != nil { + t.Fatalf("EL row did not survive the restart: %v", err) + } + clBack, err := reopened.GetNode(db.LayerCL, id[:]) + if err != nil { + t.Fatalf("CL row did not survive the restart: %v", err) + } + if elBack.Layer != string(db.LayerEL) || clBack.Layer != string(db.LayerCL) { + t.Errorf("layer tags wrong after reload: el=%q cl=%q", elBack.Layer, clBack.Layer) + } +} + +func waitForRows(t *testing.T, database *db.Database, want int) { + t.Helper() + + deadline := time.Now().Add(10 * time.Second) + for { + got, err := database.CountAllNodes() + if err == nil && got >= want { + return + } + if time.Now().After(deadline) { + t.Fatalf("only %d of %d rows persisted", got, want) + } + time.Sleep(20 * time.Millisecond) + } +} diff --git a/bootnode/duallayer_test.go b/bootnode/duallayer_test.go new file mode 100644 index 0000000..b94a306 --- /dev/null +++ b/bootnode/duallayer_test.go @@ -0,0 +1,138 @@ +package bootnode + +import ( + "context" + "net" + "testing" + "time" + + "github.com/ethpandaops/bootnodoor/bootnode/clconfig" + "github.com/ethpandaops/bootnodoor/bootnode/elconfig" + "github.com/ethpandaops/bootnodoor/db" + v5node "github.com/ethpandaops/bootnodoor/discv5/node" + "github.com/ethpandaops/bootnodoor/enr" + "github.com/ethpandaops/bootnodoor/nodes" +) + +// newDualLayerService wires both tables so a record carrying eth and eth2 is +// admitted to each. +func newDualLayerService(t *testing.T) *Service { + t.Helper() + + logger := quietLogger() + database := db.NewDatabase(&db.SqliteDatabaseConfig{File: ":memory:", MaxOpenConns: 5, MaxIdleConns: 2}, logger) + if err := database.Init(); err != nil { + t.Fatalf("db init: %v", err) + } + t.Cleanup(func() { database.Close() }) + if err := database.ApplyEmbeddedDbSchema(-2); err != nil { + t.Fatalf("db schema: %v", err) + } + + cfg := &Config{ + Logger: logger, + Database: database, + ELConfig: &elconfig.ChainConfig{}, + ELGenesisHash: [32]byte{1, 2, 3}, + ELGenesisTime: 1000, + CLConfig: &clconfig.Config{}, + } + + key := mustKey(t) + ln, err := createLocalNode(cfg, key, net.ParseIP("1.2.3.4"), nil, 9000, nil) + if err != nil { + t.Fatalf("createLocalNode: %v", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + s := &Service{config: cfg, ctx: ctx, enrManager: NewENRManager(cfg, key, ln, true, true)} + s.elNodeDB = nodes.NewNodeDB(ctx, database, db.LayerEL, logger) + s.clNodeDB = nodes.NewNodeDB(ctx, database, db.LayerCL, logger) + if s.elTable, err = s.createTable(ln.ID(), s.elNodeDB, "EL"); err != nil { + t.Fatalf("createTable EL: %v", err) + } + if s.clTable, err = s.createTable(ln.ID(), s.clNodeDB, "CL"); err != nil { + t.Fatalf("createTable CL: %v", err) + } + return s +} + +// dualLayerNode builds a v5 node advertising the fork id and fork digest this +// service currently accepts, so it is admitted to both tables. +func dualLayerNode(t *testing.T, s *Service) *v5node.Node { + t.Helper() + + forkID := s.enrManager.GetELFilter().GetCurrentForkID(StaticHead()) + digest := s.enrManager.GetCLFilter().GetCurrentForkDigest() + + key := mustKey(t) + rec := enr.New() + if err := rec.Set("ip", net.IPv4(9, 9, 9, 9)); err != nil { + t.Fatalf("set ip: %v", err) + } + if err := rec.Set("udp", uint16(9000)); err != nil { + t.Fatalf("set udp: %v", err) + } + if err := rec.Set("eth", []struct { + Hash []byte + Next uint64 + }{{Hash: forkID.Hash[:], Next: forkID.Next}}); err != nil { + t.Fatalf("set eth: %v", err) + } + if err := rec.Set("eth2", clconfig.EncodeETH2Field(digest, [4]byte{}, ^uint64(0))); err != nil { + t.Fatalf("set eth2: %v", err) + } + rec.SetSeq(1) + if err := rec.Sign(key); err != nil { + t.Fatalf("sign: %v", err) + } + + n, err := v5node.New(rec) + if err != nil { + t.Fatalf("v5node.New: %v", err) + } + return n +} + +// A dual-layer peer occupies both tables as two node objects with their own +// last-seen. The handler's SetLastSeen reaches only whichever one currently +// shares stats with the v5 node, and re-admission repoints that at a wrapper the +// tables discarded — so onNodeSeen has to refresh both itself. +func TestOnNodeSeenRefreshesBothLayers(t *testing.T) { + s := newDualLayerService(t) + n := dualLayerNode(t, s) + + // Admit with a known last-seen, so a later failure shows a stale timestamp + // rather than a zero one and cannot be mistaken for "never populated". + admitted := time.Now() + n.SetLastSeen(admitted) + + if !s.checkAndAddNode(n) { + t.Fatal("dual-layer node was not admitted") + } + el, cl := s.elTable.Get(n.ID()), s.clTable.Get(n.ID()) + if el == nil || cl == nil { + t.Fatalf("node not in both tables: el=%v cl=%v", el != nil, cl != nil) + } + if el == cl { + t.Skip("tables share one node object; this test only means something when they differ") + } + + // Re-admission, as an ENR refresh would do: repoints the v5 node's stats at a + // wrapper the tables do not hold. + s.checkAndAddNode(n) + + // One inbound packet, in the order the handler produces it. + refreshed := time.Now().Add(time.Hour) + n.SetLastSeen(refreshed) + s.onNodeSeen(n, refreshed) + + if got := s.elTable.Get(n.ID()).LastSeen(); !got.Equal(refreshed) { + t.Errorf("EL last-seen = %v, want the refreshed %v", got, refreshed) + } + if got := s.clTable.Get(n.ID()).LastSeen(); !got.Equal(refreshed) { + t.Errorf("CL last-seen = %v, want the refreshed %v", got, refreshed) + } +} diff --git a/bootnode/elconfig/filter.go b/bootnode/elconfig/filter.go index de69f2c..b4424a9 100644 --- a/bootnode/elconfig/filter.go +++ b/bootnode/elconfig/filter.go @@ -2,8 +2,9 @@ package elconfig import ( "fmt" - "hash/crc32" "math" + "slices" + "strings" "sync" "time" ) @@ -31,19 +32,11 @@ type ForkFilter struct { // special casing. forks []uint64 - // numBlockForks is the boundary index separating block forks from time - // forks in forks, including go-ethereum's rule that the sentry counts as - // a block fork when the chain has no time forks at all. + // numBlockForks indexes the first time fork in forks. Every block fork is + // passed under the static head stance, so validation starts scanning here. numBlockForks int - // blockHead is the static block head: at or past every canonical block - // fork, before the sentry. - blockHead uint64 - - // sums[i] is the checksum after passing the first i fork boundaries. - sums [][4]byte - - // allForkIDs contains the complete canonical list for display + // allForkIDs[i].Hash is the checksum after passing the first i boundaries. allForkIDs []ForkID // Admission outcomes, recorded by the admission call sites only (the @@ -79,38 +72,15 @@ func NewForkFilter(genesisHash [32]byte, config *ChainConfig, genesisTime uint64 forksByBlock, forksByTime := GatherForks(config, genesisTime) forks := append(append([]uint64{}, forksByBlock...), forksByTime...) - sums := make([][4]byte, len(forks)+1) - hash := crc32.ChecksumIEEE(genesisHash[:]) - sums[0] = checksumToBytes(hash) - for i, fork := range forks { - hash = checksumUpdate(hash, fork) - sums[i+1] = checksumToBytes(hash) - } - - blockHead := uint64(0) - if len(forksByBlock) > 0 { - blockHead = forksByBlock[len(forksByBlock)-1] - } - - numBlockForks := len(forksByBlock) forks = append(forks, math.MaxUint64) - if len(forksByTime) == 0 { - // In purely block based forks, keep the sentry out of timestamp - // territory (go-ethereum's rule). - numBlockForks++ - } - - allForkIDs := ComputeAllForkIDs(genesisHash, forksByBlock, forksByTime) return &ForkFilter{ genesisHash: genesisHash, chainConfig: config, genesisTime: genesisTime, forks: forks, - numBlockForks: numBlockForks, - blockHead: blockHead, - sums: sums, - allForkIDs: allForkIDs, + numBlockForks: len(forksByBlock), + allForkIDs: ComputeAllForkIDs(genesisHash, forksByBlock, forksByTime), } } @@ -137,16 +107,12 @@ func (f *ForkFilter) Filter(id ForkID) bool { // be completed with locally known future forks: accept (we are syncing). // 4. Reject in all other cases. func (f *ForkFilter) validate(id ForkID, now uint64) error { - for i, fork := range f.forks { - head := f.blockHead - if i >= f.numBlockForks { - head = now - } - if head >= fork { + for i := f.numBlockForks; i < len(f.forks); i++ { + if now >= f.forks[i] { continue } // Found the first unpassed fork, check the remote against it (rule #1). - if f.sums[i] == id.Hash { + if f.allForkIDs[i].Hash == id.Hash { // A remote-announced fork we have already passed means the remote // is stale (rule #1a). Every unpassed fork here is time-scheduled // (block forks are all passed under the static stance), so the @@ -158,7 +124,7 @@ func (f *ForkFilter) validate(id ForkID, now uint64) error { } // Different fork state: subset means the remote is syncing (rule #2). for j := 0; j < i; j++ { - if f.sums[j] == id.Hash { + if f.allForkIDs[j].Hash == id.Hash { if f.forks[j] != id.Next { return fmt.Errorf("remote is stale: subset checksum with next %d, want %d", id.Next, f.forks[j]) } @@ -166,8 +132,8 @@ func (f *ForkFilter) validate(id ForkID, now uint64) error { } } // Superset means we would be the one syncing (rule #3). - for j := i + 1; j < len(f.sums); j++ { - if f.sums[j] == id.Hash { + for j := i + 1; j < len(f.allForkIDs); j++ { + if f.allForkIDs[j].Hash == id.Hash { return nil } } @@ -177,8 +143,9 @@ func (f *ForkFilter) validate(id ForkID, now uint64) error { return nil } -// RecordAdmission records an admission decision for the stats surface. Call -// this from admission paths only, never from layer classification. +// RecordAdmission records an admission decision for the stats surface. Its only +// caller is ENRManager.AdmitELNode, which owns the eth-entry gate; the pure +// predicate path (ClassifyELNode) must never reach here. func (f *ForkFilter) RecordAdmission(acceptedNode bool, id ForkID) { f.statsMu.Lock() defer f.statsMu.Unlock() @@ -203,9 +170,11 @@ func (f *ForkFilter) GetStats() FilterStats { } } -// GetAllForkIDs returns all valid fork IDs for debugging. +// GetAllForkIDs returns all valid fork IDs for debugging. Copied because +// validate reads allForkIDs to decide admission; mutating it would corrupt +// peer filtering. func (f *ForkFilter) GetAllForkIDs() []ForkID { - return f.allForkIDs + return slices.Clone(f.allForkIDs) } // GetCurrentForkID calculates the current fork ID based on chain state. @@ -273,19 +242,16 @@ func (f *ForkFilter) GetAllForkIDsWithNames() []ForkIDWithName { if i+1 >= len(f.allForkIDs) { break } - name := "" - for j, n := range b.names { + names := make([]string, 0, len(b.names)) + for _, n := range b.names { if len(n) > 0 { - n = string(n[0]-32) + n[1:] - } - if j > 0 { - name += "/" + n = strings.ToUpper(n[:1]) + n[1:] } - name += n + names = append(names, n) } result = append(result, ForkIDWithName{ ForkID: f.allForkIDs[i+1], - Name: name, + Name: strings.Join(names, "/"), Activation: b.value, IsTime: b.isTime, }) diff --git a/bootnode/enr.go b/bootnode/enr.go index 50066ed..ebd33d1 100644 --- a/bootnode/enr.go +++ b/bootnode/enr.go @@ -74,7 +74,8 @@ func StaticHead() (block, timestamp uint64) { return math.MaxUint64 - 1, uint64(time.Now().Unix()) } -// UpdateENR updates the local ENR with current eth and eth2 fields. +// UpdateENR updates the local ENR with current eth and eth2 fields, reporting +// whether the record actually changed. // // It is a no-op when the computed fields already match the published record, // so periodic callers do not churn the sequence number (peers re-fetch a @@ -84,98 +85,103 @@ func StaticHead() (block, timestamp uint64) { // - On startup // - After fork transitions // - When head changes significantly (for EL fork ID Next field) -func (m *ENRManager) UpdateENR(currentBlock, currentTime uint64) error { +func (m *ENRManager) UpdateENR(currentBlock, currentTime uint64) (bool, error) { record := m.localNode.Record() - // Clone the current ENR to preserve all fields - newRecord, err := record.Clone() - if err != nil { - return fmt.Errorf("failed to clone ENR: %w", err) - } - - changed := false - // A bootnode serves no TCP, so never advertise tcp/tcp6 — including any // inherited from an ENR persisted by an older, TCP-advertising version. - newRecord.Delete("tcp") - newRecord.Delete("tcp6") + changed := record.Has("tcp") || record.Has("tcp6") - if m.servesEL && m.config.HasEL() { - forkID := m.elFilter.GetCurrentForkID(currentBlock, currentTime) - // Set eth field as a list of fork IDs - ENR.Set() will handle RLP encoding - // The eth field format is [[Hash, Next]] - a list containing fork IDs - ethField := []struct { - Hash []byte - Next uint64 - }{ - { - Hash: forkID.Hash[:], - Next: forkID.Next, - }, - } - newRecord.Set("eth", ethField) + serveEL := m.servesEL && m.config.HasEL() + serveCL := m.servesCL && m.config.HasCL() + var forkID elconfig.ForkID + switch { + case serveEL: + forkID = m.elFilter.GetCurrentForkID(currentBlock, currentTime) if current, ok := record.Eth(); !ok || len(current) == 0 || current[0].ForkID != forkID.Hash || current[0].NextForkEpoch != forkID.Next { changed = true m.config.Logger.WithField("forkID", forkID.String()).Debug("updated ENR with eth field") } - } else if record.Has("eth") { + case record.Has("eth"): // Drop any stale eth field (e.g. inherited from a reused shared ENR). - newRecord.Delete("eth") changed = true } - if m.servesCL && m.config.HasCL() { - eth2Field := m.clFilter.ComputeEth2Field() - newRecord.Set("eth2", eth2Field) + var eth2Field []byte + switch { + case serveCL: + eth2Field = m.clFilter.ComputeEth2Field() var currentEth2 []byte if err := record.Get("eth2", ¤tEth2); err != nil || !bytes.Equal(currentEth2, eth2Field) { changed = true - // eth2Field is []byte, extract first 4 bytes as fork digest for logging var forkDigest [4]byte if len(eth2Field) >= 4 { copy(forkDigest[:], eth2Field[0:4]) } m.config.Logger.WithField("forkDigest", fmt.Sprintf("%#x", forkDigest)).Debug("updated ENR with eth2 field") } - } else if record.Has("eth2") { - newRecord.Delete("eth2") + case record.Has("eth2"): changed = true } - if record.Has("tcp") || record.Has("tcp6") { - changed = true + if !changed { + return false, nil } - if !changed { - return nil + newRecord, err := record.Clone() + if err != nil { + return false, fmt.Errorf("failed to clone ENR: %w", err) + } + + newRecord.Delete("tcp") + newRecord.Delete("tcp6") + + if serveEL { + // The eth field format is [[Hash, Next]] - a list containing fork IDs. + newRecord.Set("eth", []struct { + Hash []byte + Next uint64 + }{ + { + Hash: forkID.Hash[:], + Next: forkID.Next, + }, + }) + } else { + newRecord.Delete("eth") + } + + if serveCL { + newRecord.Set("eth2", eth2Field) + } else { + newRecord.Delete("eth2") } - // Increment sequence number newRecord.SetSeq(record.Seq() + 1) - // Re-sign the record if err := newRecord.Sign(m.key); err != nil { - return fmt.Errorf("failed to sign ENR: %w", err) + return false, fmt.Errorf("failed to sign ENR: %w", err) } - // Update local node's ENR if !m.localNode.UpdateENR(newRecord) { - return fmt.Errorf("failed to update local node ENR (sequence number may be stale)") + return false, fmt.Errorf("failed to update local node ENR (sequence number may be stale)") } m.config.Logger.WithField("seq", newRecord.Seq()).Info("updated local ENR with eth/eth2 fields") - return nil + return true, nil } -// FilterELNode checks if an EL node's fork ID is valid. +// ClassifyELNode reports whether a record is an execution node on a compatible +// fork, along with the fork ID it advertised. // -// Returns true if the node should be accepted, false otherwise. -func (m *ENRManager) FilterELNode(record *enr.Record) (bool, elconfig.ForkID) { - if !m.config.HasEL() { +// It is pure: no counter moves. Use it for per-packet layer classification, and +// AdmitELNode when the result decides admission. +func (m *ENRManager) ClassifyELNode(record *enr.Record) (bool, elconfig.ForkID) { + if !m.config.HasEL() || record == nil { return false, elconfig.ForkID{} } @@ -211,16 +217,40 @@ func (m *ENRManager) FilterELNode(record *enr.Record) (bool, elconfig.ForkID) { return m.elFilter.Filter(forkID), forkID } -// FilterCLNode checks if a CL node's fork digest is valid. +// AdmitELNode is ClassifyELNode plus stats. Call it from admission paths only. // -// Returns true if the node should be accepted, false otherwise. -func (m *ENRManager) FilterCLNode(record *enr.Record) bool { +// Records with no eth entry are consensus nodes, not wrong-fork execution nodes, +// and are not counted (see services.AdmissionRejectedLayer). +func (m *ENRManager) AdmitELNode(record *enr.Record) (bool, elconfig.ForkID) { + accepted, forkID := m.ClassifyELNode(record) + + if m.elFilter != nil && record != nil && record.Has("eth") { + m.elFilter.RecordAdmission(accepted, forkID) + } + + return accepted, forkID +} + +// ClassifyCLNode reports whether a record is a consensus node on an accepted +// fork digest. +// +// It is pure: no counter moves. Use it for per-packet layer classification, and +// AdmitCLNode when the result decides admission. +func (m *ENRManager) ClassifyCLNode(record *enr.Record) bool { + if !m.config.HasCL() { + return false + } + + return m.clFilter.Matches(record) +} + +// AdmitCLNode is ClassifyCLNode plus stats. Call it from admission paths only. +func (m *ENRManager) AdmitCLNode(record *enr.Record) bool { if !m.config.HasCL() { return false } - // Use existing fork digest filter - return m.clFilter.Filter(record) + return m.clFilter.Admit(record) } // GetELFilter returns the EL fork filter (may be nil). diff --git a/bootnode/fork_boundary_test.go b/bootnode/fork_boundary_test.go new file mode 100644 index 0000000..91754d1 --- /dev/null +++ b/bootnode/fork_boundary_test.go @@ -0,0 +1,127 @@ +package bootnode + +import ( + "os" + "path/filepath" + "strconv" + "testing" + "time" + + "github.com/ethpandaops/bootnodoor/bootnode/clconfig" +) + +func boundaryService(t *testing.T, genesis uint64, electraEpoch string) *Service { + t.Helper() + + yaml := "PRESET_BASE: mainnet\n" + + "MIN_GENESIS_TIME: " + strconv.FormatUint(genesis, 10) + "\n" + + "GENESIS_DELAY: 0\n" + + "SECONDS_PER_SLOT: 12\n" + + "SLOTS_PER_EPOCH: 32\n" + + "GENESIS_FORK_VERSION: 0x10000000\n" + + "ALTAIR_FORK_VERSION: 0x20000000\nALTAIR_FORK_EPOCH: 0\n" + + "BELLATRIX_FORK_VERSION: 0x30000000\nBELLATRIX_FORK_EPOCH: 0\n" + + "CAPELLA_FORK_VERSION: 0x40000000\nCAPELLA_FORK_EPOCH: 0\n" + + "DENEB_FORK_VERSION: 0x50000000\nDENEB_FORK_EPOCH: 0\n" + + "ELECTRA_FORK_VERSION: 0x60000000\nELECTRA_FORK_EPOCH: " + electraEpoch + "\n" + + path := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(path, []byte(yaml), 0o600); err != nil { + t.Fatalf("write config: %v", err) + } + cl, err := clconfig.LoadConfig(path) + if err != nil { + t.Fatalf("load config: %v", err) + } + return &Service{config: &Config{CLConfig: cl}} +} + +// A free-running ticker left the record advertising the previous fork for up to +// its full period, so the wait has to track the next scheduled boundary. +func TestNextForkBoundaryTracksSchedule(t *testing.T) { + now := time.Now() + s := boundaryService(t, uint64(now.Unix()), "1") + + next, ok := s.nextForkBoundary(now) + if !ok { + t.Fatal("no boundary found for an epoch-1 fork") + } + wantAt := now.Add(384 * time.Second) + if diff := next.Sub(wantAt); diff > 2*time.Second || diff < -2*time.Second { + t.Errorf("boundary = %v, want ~%v", next, wantAt) + } +} + +// Boundaries already passed must not yield a negative or immediate timer. +func TestNextForkBoundaryAllPassed(t *testing.T) { + now := time.Now() + s := boundaryService(t, uint64(now.Add(-10*time.Hour).Unix()), "1") + + if _, ok := s.nextForkBoundary(now); ok { + t.Error("a past boundary was reported as upcoming") + } + if delay := s.nextForkRefreshDelay(); delay != maxForkRefreshDelay { + t.Errorf("delay = %v, want the backstop %v", delay, maxForkRefreshDelay) + } +} + +// With no genesis data no epoch has a wall clock, so the backstop must carry the +// refresh instead of the timer firing continuously. +func TestNextForkBoundaryWithoutGenesis(t *testing.T) { + s := &Service{config: &Config{CLConfig: &clconfig.Config{SecondsPerSlot: 12}}} + + if _, ok := s.nextForkBoundary(time.Now()); ok { + t.Error("a boundary was reported with no genesis time") + } + if delay := s.nextForkRefreshDelay(); delay != maxForkRefreshDelay { + t.Errorf("delay = %v, want the backstop %v", delay, maxForkRefreshDelay) + } +} + +// A distant boundary is capped so the reconciliation backstop still runs. +// Genesis is placed beyond the settle window, since genesis is itself a boundary +// and would otherwise put this inside the post-boundary polling period. +func TestNextForkRefreshDelayCapped(t *testing.T) { + now := time.Now() + s := boundaryService(t, uint64(now.Add(-2*forkSettleWindow).Unix()), "100") + + if delay := s.nextForkRefreshDelay(); delay != maxForkRefreshDelay { + t.Errorf("delay = %v, want the cap %v", delay, maxForkRefreshDelay) + } +} + +// A boundary that has just passed is skipped by nextForkBoundary, so arming for +// the following one leaves a full backstop hole: a fire landing on the boundary +// before the digest is computable does not look again for a minute. A devnet BPO +// transition took 75s that way. +func TestForkRefreshPollsAfterABoundary(t *testing.T) { + now := time.Now() + genesis := uint64(now.Add(-384 * time.Second).Unix()) + + s := boundaryService(t, genesis, "1") + + last, ok := s.lastForkBoundary(now) + if !ok { + t.Fatal("the boundary that just passed was not reported") + } + if now.Sub(last) > 5*time.Second { + t.Fatalf("last boundary = %v, want ~now", last) + } + + if delay := s.nextForkRefreshDelay(); delay != forkSettlePoll { + t.Errorf("delay just after a boundary = %v, want the settle poll %v", delay, forkSettlePoll) + } +} + +// Outside the settle window the refresh must go back to waiting for the next +// boundary rather than polling every second forever. +func TestForkRefreshStopsPollingAfterSettleWindow(t *testing.T) { + now := time.Now() + genesis := uint64(now.Add(-(384 + 200) * time.Second).Unix()) + + s := boundaryService(t, genesis, "1") + + if delay := s.nextForkRefreshDelay(); delay == forkSettlePoll { + t.Error("still polling long after the boundary settled") + } +} diff --git a/bootnode/ipdiscovery_gate_test.go b/bootnode/ipdiscovery_gate_test.go new file mode 100644 index 0000000..5c2efd4 --- /dev/null +++ b/bootnode/ipdiscovery_gate_test.go @@ -0,0 +1,48 @@ +package bootnode + +import ( + "net" + "testing" +) + +// An explicitly configured ENR address is authoritative, so peer reports must not +// move it while running. reconcileStoredENR honours this at startup; without the +// same check at runtime a configured address is overwritten and only restored on +// the next restart. +func TestUpdateENRWithDiscoveredIP_KeepsExplicitAddress(t *testing.T) { + el := &identity{key: mustKey(t), servesEL: true, bindPort: 9000, enrPort: 9000, storeKey: "local_enr"} + s := newTestService(t, []*identity{el}) + s.config.ENRIPProvided = true + + before := el.localNode.Record().IP() + if before == nil { + t.Fatal("test identity has no ENR IP to protect") + } + + s.updateENRWithDiscoveredIP(net.ParseIP("9.9.9.9"), 31000, false) + + if got := el.localNode.Record().IP(); !got.Equal(before) { + t.Fatalf("configured ENR IP was overwritten by discovery: %v -> %v", before, got) + } +} + +// The same path must still self-correct when the address was auto-detected, or +// the guard above would disable IP discovery entirely. +func TestUpdateENRWithDiscoveredIP_UpdatesAutoDetectedAddress(t *testing.T) { + el := &identity{key: mustKey(t), servesEL: true, bindPort: 9000, enrPort: 9000, storeKey: "local_enr"} + s := newTestService(t, []*identity{el}) + + s.updateENRWithDiscoveredIP(net.ParseIP("9.9.9.9"), 31000, false) + + if got := el.localNode.Record().IP(); !got.Equal(net.ParseIP("9.9.9.9")) { + t.Fatalf("auto-detected ENR IP = %v, want the discovered 9.9.9.9", got) + } +} + +// The default was declared false while nothing in the runtime path read it, so +// discovery ran unconditionally. Pin the default that the runtime gate now honours. +func TestDefaultConfigEnablesIPDiscovery(t *testing.T) { + if !DefaultConfig().EnableIPDiscovery { + t.Fatal("EnableIPDiscovery default is false; the runtime gate would disable discovery for everyone") + } +} diff --git a/bootnode/service.go b/bootnode/service.go index 9029e43..42da26e 100644 --- a/bootnode/service.go +++ b/bootnode/service.go @@ -3,7 +3,9 @@ package bootnode import ( "context" "fmt" + "math" "net" + "slices" "sync" "time" @@ -66,6 +68,12 @@ type Service struct { // ENR request tracking (prevents duplicate requests) pendingENRRequestsV4 sync.Map // map[node.ID]time.Time + // v5ProbeSem bounds concurrent v5 capability probes + v5ProbeSem chan struct{} + + // v5ProbesInFlight keeps one probe per node in flight + v5ProbesInFlight sync.Map // map[[32]byte]struct{} + // Lifecycle ctx context.Context cancel context.CancelFunc @@ -104,9 +112,10 @@ func New(cfg *Config) (*Service, error) { ctx, cancel := context.WithCancel(context.Background()) s := &Service{ - config: cfg, - ctx: ctx, - cancel: cancel, + config: cfg, + ctx: ctx, + cancel: cancel, + v5ProbeSem: make(chan struct{}, maxConcurrentV5Probes), } // Resolve discovery identities (one shared, or separate EL/CL keys). @@ -120,13 +129,23 @@ func New(cfg *Config) (*Service, error) { t.Close() } } + + // Everything created below hangs off s.ctx (NodeDB queue processors, + // protocol-handler cleanup goroutines), so cancelling tears it all down. + ok := false + defer func() { + if !ok { + cancel() + closeTransports() + } + }() + for _, id := range s.identities { if transports[id.bindPort] == nil { // JoinHostPort so an IPv6 bind addr becomes [::]:port, not :::port. listenAddr := net.JoinHostPort(cfg.BindIP.String(), fmt.Sprintf("%d", id.bindPort)) t, terr := transport.NewUDPTransport(&transport.Config{ListenAddr: listenAddr, Logger: cfg.Logger}) if terr != nil { - closeTransports() return nil, fmt.Errorf("failed to create UDP transport on port %d: %w", id.bindPort, terr) } cfg.Logger.WithField("address", listenAddr).Info("listening for discovery") @@ -143,7 +162,6 @@ func New(cfg *Config) (*Service, error) { localNode, nerr := createLocalNode(cfg, id.key, id.enrIP, id.enrIP6, id.enrPort, storedENR) if nerr != nil { - closeTransports() return nil, fmt.Errorf("failed to create local node: %w", nerr) } id.localNode = localNode @@ -154,10 +172,14 @@ func New(cfg *Config) (*Service, error) { id.enrManager = NewENRManager(cfg, id.key, localNode, id.servesEL, id.servesCL) headBlock, headTime := StaticHead() - if uerr := id.enrManager.UpdateENR(headBlock, headTime); uerr != nil { + changed, uerr := id.enrManager.UpdateENR(headBlock, headTime) + switch { + case uerr != nil: cfg.Logger.WithError(uerr).Warn("failed to update ENR with eth/eth2 fields") - } else if serr := s.storeENR(id.storeKey, localNode.Record()); serr != nil { - cfg.Logger.WithError(serr).Warn("failed to store updated ENR") + case changed: + if serr := s.storeENR(id.storeKey, localNode.Record()); serr != nil { + cfg.Logger.WithError(serr).Warn("failed to store updated ENR") + } } } @@ -166,16 +188,19 @@ func New(cfg *Config) (*Service, error) { s.localNode = primary.localNode s.enrManager = primary.enrManager - // Create IP discovery service - ipDiscoveryCfg := services.IPDiscoveryConfig{ - MinReports: 5, // Require 5 reports - MinDistinctIPs: 3, // From at least 3 distinct IPs - Logger: cfg.Logger, - OnConsensusReached: func(ip net.IP, port uint16, isIPv6 bool) { - s.updateENRWithDiscoveredIP(ip, port, isIPv6) - }, + // Create IP discovery service. Leaving it nil when disabled is the whole + // enforcement: onPongReceived already returns early on a nil service, so no + // peer report can reach consensus and rewrite the ENR. + if cfg.EnableIPDiscovery { + s.ipDiscovery = services.NewIPDiscovery(services.IPDiscoveryConfig{ + MinReports: 5, // Require 5 reports + MinDistinctIPs: 3, // From at least 3 distinct IPs + Logger: cfg.Logger, + OnConsensusReached: func(ip net.IP, port uint16, isIPv6 bool) { + s.updateENRWithDiscoveredIP(ip, port, isIPv6) + }, + }) } - s.ipDiscovery = services.NewIPDiscovery(ipDiscoveryCfg) // Create node databases for enabled layers var err error @@ -191,14 +216,12 @@ func New(cfg *Config) (*Service, error) { if cfg.HasEL() { s.elTable, err = s.createTable(s.elIdentity().localNode.ID(), s.elNodeDB, "EL") if err != nil { - closeTransports() return nil, fmt.Errorf("failed to create EL table: %w", err) } } if cfg.HasCL() { s.clTable, err = s.createTable(s.clIdentity().localNode.ID(), s.clNodeDB, "CL") if err != nil { - closeTransports() return nil, fmt.Errorf("failed to create CL table: %w", err) } } @@ -207,7 +230,6 @@ func New(cfg *Config) (*Service, error) { if cfg.EnableDiscv5 { for _, id := range s.identities { if ierr := s.initDiscv5(id); ierr != nil { - closeTransports() return nil, fmt.Errorf("failed to initialize discv5: %w", ierr) } } @@ -217,12 +239,6 @@ func New(cfg *Config) (*Service, error) { // Create the discv4 service (EL-only) on the EL identity. if cfg.EnableDiscv4 { if ierr := s.initDiscv4(s.elIdentity()); ierr != nil { - for _, id := range s.identities { - if id.discv5Service != nil { - id.discv5Service.Stop() - } - } - closeTransports() return nil, fmt.Errorf("failed to initialize discv4: %w", ierr) } } @@ -253,84 +269,8 @@ func New(cfg *Config) (*Service, error) { Layer: db.LayerEL, Alpha: 3, LookupTimeout: 30 * time.Second, - OnNodeFound: func(n *nodes.Node) services.AdmissionResult { - // Filter by fork ID before adding to table - if n.Record() != nil && s.enrManager != nil { - isEL, forkID := s.enrManager.FilterELNode(n.Record()) - if !isEL { - // A record with no eth entry is a consensus node, not an - // execution node on the wrong fork. Counting those as - // fork rejections would make a healthy dual-layer network - // look like a fork-compatibility failure. - if !n.Record().Has("eth") { - if err := cfg.Database.StoreBadNode(n.IDBytes(), db.LayerEL, "not_el"); err != nil { - cfg.Logger.WithError(err).Debug("failed to store bad node") - } - return services.AdmissionRejectedLayer - } - if elFilter := s.enrManager.GetELFilter(); elFilter != nil { - elFilter.RecordAdmission(false, forkID) - } - cfg.Logger.WithFields(logrus.Fields{ - "peerID": n.PeerID(), - "eth": forkID.String(), - }).Debug("EL lookup admission rejected: incompatible fork id") - // Mark as bad node - if err := cfg.Database.StoreBadNode(n.IDBytes(), db.LayerEL, "invalid_fork_id"); err != nil { - cfg.Logger.WithError(err).Debug("failed to store bad node") - } - return services.AdmissionRejectedFilter - } - if elFilter := s.enrManager.GetELFilter(); elFilter != nil { - elFilter.RecordAdmission(true, forkID) - } - } - - // If node was discovered via v4 (only has v4 support), immediately test for v5 support - if n.HasV4() && !n.HasV5() && s.getV5Handler() != nil { - record := n.Record() - if record != nil { - // Try to create v5 node from ENR - v5Node, err := nodes.NewV5NodeFromRecord(record) - if err == nil && s.getV5Handler() != nil { - // Ping on v5 to test support - start := time.Now() - respChan, err := s.getV5Handler().SendPing(v5Node) - if err == nil { - resp := <-respChan - rtt := time.Since(start) - if resp.Error == nil { - // v5 ping succeeded - add v5 support - n.SetV5(v5Node) - cfg.Logger.WithFields(logrus.Fields{ - "peerID": n.PeerID(), - "addr": n.Addr(), - "rtt": rtt, - }).Debug("discovered v5 support on v4-discovered node") - - // Queue protocol support update (SetV5 already marked it dirty) - if s.elNodeDB != nil { - if err := s.elNodeDB.QueueUpdate(n); err != nil { - cfg.Logger.WithError(err).Debug("failed to queue node for protocol support update") - } - } - } - } - } - } - } - - // Attempt to add to EL table - if !s.elTable.Add(n) { - return services.AdmissionRejectedPool - } - // Remove from bad nodes list if it was previously bad - if err := cfg.Database.RemoveBadNode(n.IDBytes(), db.LayerEL); err != nil { - cfg.Logger.WithError(err).Debug("failed to remove from bad nodes") - } - return services.AdmissionAccepted - }, - Logger: cfg.Logger.WithField("service", "el-lookup"), + OnNodeFound: s.admitELLookupNode, + Logger: cfg.Logger.WithField("service", "el-lookup"), }) } @@ -356,39 +296,13 @@ func New(cfg *Config) (*Service, error) { Layer: db.LayerCL, Alpha: 3, LookupTimeout: 30 * time.Second, - OnNodeFound: func(n *nodes.Node) services.AdmissionResult { - // Filter by fork digest before adding to table - if n.Record() != nil && s.enrManager != nil { - if !s.enrManager.FilterCLNode(n.Record()) { - // No eth2 entry means an execution node, not a consensus - // node on the wrong digest; keep the two distinguishable. - if !n.Record().Has("eth2") { - if err := cfg.Database.StoreBadNode(n.IDBytes(), db.LayerCL, "not_cl"); err != nil { - cfg.Logger.WithError(err).Debug("failed to store bad node") - } - return services.AdmissionRejectedLayer - } - // Mark as bad node - if err := cfg.Database.StoreBadNode(n.IDBytes(), db.LayerCL, "invalid_fork_digest"); err != nil { - cfg.Logger.WithError(err).Debug("failed to store bad node") - } - return services.AdmissionRejectedFilter - } - } - // Attempt to add to CL table - if !s.clTable.Add(n) { - return services.AdmissionRejectedPool - } - // Remove from bad nodes list if it was previously bad - if err := cfg.Database.RemoveBadNode(n.IDBytes(), db.LayerCL); err != nil { - cfg.Logger.WithError(err).Debug("failed to remove from bad nodes") - } - return services.AdmissionAccepted - }, - Logger: cfg.Logger.WithField("service", "cl-lookup"), + OnNodeFound: s.admitCLLookupNode, + Logger: cfg.Logger.WithField("service", "cl-lookup"), }) } + ok = true + return s, nil } @@ -437,9 +351,8 @@ func (s *Service) initDiscv4(id *identity) error { discv4Config.OnNodeSeen = func(n *v4node.Node, timestamp time.Time) { s.onNodeSeenV4(n, timestamp) } - discv4Config.OnPongReceived = func(from *v4node.Node, ip net.IP, port uint16) { - sourceIP := from.Addr().IP - s.onPongReceived(from.IDBytes(), sourceIP, ip, port) + discv4Config.OnPongReceived = func(from *v4node.Node, provenAddr *net.UDPAddr, ip net.IP, port uint16) { + s.onPongReceived(from.IDBytes(), provenAddr.IP, ip, port) } // OnENRRequest: discv4 service handles this internally using LocalENR from config // No callback needed - it will automatically respond with the ENR @@ -633,7 +546,14 @@ func (s *Service) maintenanceLoop() { supportCheck := time.NewTicker(30 * time.Minute) // Check protocol support every 30 minutes badNodesCleanup := time.NewTicker(24 * time.Hour) // Cleanup bad nodes once per day enrRequestCleanup := time.NewTicker(1 * time.Minute) // Cleanup stale ENR requests every minute - forkRefresh := time.NewTicker(1 * time.Minute) // Re-publish eth/eth2 when a fork activates + + // Fork fields are re-published on a timer armed for the next boundary rather + // than polled: peers see the transition immediately and re-request our record, + // so a free-running tick left us advertising the previous fork for up to its + // full period. forkReconcile is the backstop for a missing schedule, a clock + // jump, or a boundary that passed while we were starting. + forkRefresh := time.NewTimer(s.nextForkRefreshDelay()) + forkReconcile := time.NewTicker(1 * time.Minute) defer tableMaintenance.Stop() defer alivenessCheck.Stop() @@ -642,6 +562,7 @@ func (s *Service) maintenanceLoop() { defer badNodesCleanup.Stop() defer enrRequestCleanup.Stop() defer forkRefresh.Stop() + defer forkReconcile.Stop() for { select { @@ -668,6 +589,148 @@ func (s *Service) maintenanceLoop() { case <-forkRefresh.C: s.refreshForkENR() + forkRefresh.Reset(s.nextForkRefreshDelay()) + + case <-forkReconcile.C: + // Refresh before recomputing: if a boundary was missed, this is what + // corrects the record, and the delay must be measured from now. + s.refreshForkENR() + if !forkRefresh.Stop() { + select { + case <-forkRefresh.C: + default: + } + } + forkRefresh.Reset(s.nextForkRefreshDelay()) + } + } +} + +// forkRefreshLead re-publishes just after a boundary, once the new fork is what +// the clock reports. +const forkRefreshLead = 500 * time.Millisecond + +// maxForkRefreshDelay caps the wait so a schedule that yields no future boundary +// still reaches refreshForkENR at the old cadence. +const maxForkRefreshDelay = time.Minute + +// forkSettleWindow is how long after a boundary the refresh keeps polling. +// +// A boundary that has just passed is skipped by nextForkBoundary, so arming for +// the following one leaves a full maxForkRefreshDelay hole: a fire landing on the +// boundary before the digest is computable finds no change and does not look +// again for a minute. A devnet BPO transition took 75s that way. Polling until +// the change appears bounds the lag to forkSettlePoll instead. +const forkSettleWindow = 90 * time.Second + +// forkSettlePoll is the retry interval inside forkSettleWindow. +const forkSettlePoll = time.Second + +// nextForkRefreshDelay returns how long to wait before the next refresh attempt. +func (s *Service) nextForkRefreshDelay() time.Duration { + now := time.Now() + + if last, ok := s.lastForkBoundary(now); ok && now.Sub(last) < forkSettleWindow { + return forkSettlePoll + } + + next, ok := s.nextForkBoundary(now) + if !ok { + return maxForkRefreshDelay + } + + delay := next.Sub(now) + forkRefreshLead + if delay < time.Millisecond { + delay = time.Millisecond + } + if delay > maxForkRefreshDelay { + delay = maxForkRefreshDelay + } + return delay +} + +// forkOffsetSeconds returns seconds from genesis to an epoch, reporting false if +// any step would overflow. Each multiplication is checked before it happens: a +// placeholder epoch or an absurd slot length would otherwise wrap, and dividing +// by an already-wrapped product panics. +func forkOffsetSeconds(epoch, slotsPerEpoch, secondsPerSlot uint64) (uint64, bool) { + if slotsPerEpoch == 0 || secondsPerSlot == 0 { + return 0, false + } + if slotsPerEpoch > math.MaxUint64/secondsPerSlot { + return 0, false + } + epochSeconds := slotsPerEpoch * secondsPerSlot + if epoch > math.MaxUint64/epochSeconds { + return 0, false + } + return epoch * epochSeconds, true +} + +// nextForkBoundary returns the earliest CL or EL fork activation after now. +func (s *Service) nextForkBoundary(now time.Time) (time.Time, bool) { + var next time.Time + found := false + + s.eachForkBoundary(func(t time.Time) { + if !t.After(now) { + return + } + if !found || t.Before(next) { + next = t + found = true + } + }) + + return next, found +} + +// lastForkBoundary returns the most recent CL or EL fork activation at or before +// now, so a refresh can keep polling until the transition is observable. +func (s *Service) lastForkBoundary(now time.Time) (time.Time, bool) { + var last time.Time + found := false + + s.eachForkBoundary(func(t time.Time) { + if t.After(now) { + return + } + if !found || t.After(last) { + last = t + found = true + } + }) + + return last, found +} + +// eachForkBoundary calls fn with every scheduled fork activation time. +func (s *Service) eachForkBoundary(consider func(time.Time)) { + if cfg := s.config.CLConfig; cfg != nil { + genesis := cfg.GetGenesisTime() + slotsPerEpoch := cfg.GetSlotsPerEpoch() + secondsPerSlot := cfg.SecondsPerSlot + if genesis > 0 && slotsPerEpoch > 0 && secondsPerSlot > 0 { + for _, epoch := range cfg.ForkEpochs() { + offset, ok := forkOffsetSeconds(epoch, slotsPerEpoch, secondsPerSlot) + // Bound offset first: MaxInt64-offset is unsigned arithmetic and + // would wrap for an offset past MaxInt64, letting the guard pass. + if !ok || offset > math.MaxInt64 || genesis > math.MaxInt64-offset { + continue + } + consider(time.Unix(int64(genesis+offset), 0)) + } + } + } + + if s.enrManager != nil { + if filter := s.enrManager.GetELFilter(); filter != nil { + for _, fork := range filter.GetAllForkIDsWithNames() { + // Block-numbered forks have no wall clock; only timestamps do. + if fork.IsTime && fork.Activation <= math.MaxInt64 { + consider(time.Unix(int64(fork.Activation), 0)) + } + } } } } @@ -705,25 +768,35 @@ func (s *Service) refreshForkENR() { clFilter.Update() } - beforeSeq := id.localNode.Record().Seq() - if err := id.enrManager.UpdateENR(headBlock, headTime); err != nil { + changed, err := id.enrManager.UpdateENR(headBlock, headTime) + if err != nil { s.config.Logger.WithError(err).Error("failed to refresh fork fields in ENR") continue } - if id.localNode.Record().Seq() == beforeSeq { + if !changed { continue } - if err := s.storeENR(id.storeKey, id.localNode.Record()); err != nil { - s.config.Logger.WithError(err).Warn("failed to store refreshed ENR") - } - if id.servesEL && s.discv4Service != nil { - s.discv4Service.SetLocalENR(id.localNode.Record()) - } + s.publishENR(id) s.config.Logger.WithField("seq", id.localNode.Record().Seq()).Info("fork transition: re-published ENR fork fields") } } +// publishENR persists an identity's current record and pushes it to every +// service that caches a copy. The discv4 handler answers ENRRESPONSE from its +// own copy, so skipping it would keep serving a stale record. Callers must +// hold s.mu. +func (s *Service) publishENR(id *identity) { + record := id.localNode.Record() + + if err := s.storeENR(id.storeKey, record); err != nil { + s.config.Logger.WithError(err).Warn("failed to store updated ENR") + } + if id.servesEL && s.discv4Service != nil { + s.discv4Service.SetLocalENR(record) + } +} + // performTableMaintenance performs routing table maintenance. func (s *Service) performTableMaintenance() { if s.elTable != nil { @@ -886,8 +959,9 @@ func (s *Service) cleanupStaleENRRequests() { s.pendingENRRequestsV4.Range(func(key, value interface{}) bool { if timestamp, ok := value.(time.Time); ok { - if now.Sub(timestamp) > staleThreshold { - s.pendingENRRequestsV4.Delete(key) + // Delete only the entry we just judged stale: a fresh claim may have + // replaced it between the Range read and here. + if now.Sub(timestamp) > staleThreshold && s.pendingENRRequestsV4.CompareAndDelete(key, value) { cleanedCount++ } } @@ -935,29 +1009,17 @@ func (s *Service) connectELBootnodes() { // connectELBootnodeENR connects to an EL bootnode via ENR. func (s *Service) connectELBootnodeENR(record *enr.Record) { - // Convert to v5 node + // Convert to v5 node; this also rejects records missing an IP or UDP port. v5, err := v5node.New(record) if err != nil { s.config.Logger.WithError(err).Warn("failed to create v5 node from ENR") return } - // Verify ENR has required fields (IP and port) - if record.IP() == nil && record.IP6() == nil { - s.config.Logger.Warn("bootnode ENR missing IP address, skipping") - return - } - if record.UDP() == 0 { - s.config.Logger.Warn("bootnode ENR missing UDP port, skipping") - return - } - - // Filter by fork ID before adding - if s.enrManager != nil { - isEL, forkID := s.enrManager.FilterELNode(record) - if elFilter := s.enrManager.GetELFilter(); elFilter != nil { - elFilter.RecordAdmission(isEL, forkID) - } + // Filter by fork ID before adding. Serve-all must not drop a configured seed: + // rejecting the only seed leaves the table empty, so discovery never starts. + if !s.config.ServeAll && s.enrManager != nil { + isEL, forkID := s.enrManager.AdmitELNode(record) if !isEL { s.config.Logger.WithFields(logrus.Fields{ "nodeID": fmt.Sprintf("%x", v5.ID().Bytes()[:8]), @@ -967,23 +1029,8 @@ func (s *Service) connectELBootnodeENR(record *enr.Record) { } } - // Create generic node and add to table genericNode := nodes.NewFromV5(v5, s.elNodeDB) - if s.elTable != nil { - if !s.elTable.Add(genericNode) { - s.config.Logger.Debug("ENR bootnode not admitted to table, not persisting") - return - } - s.config.Logger.Info("added ENR bootnode to table") - - // Persist to database - if s.elNodeDB != nil { - genericNode.MarkDirty(nodes.DirtyFull) - if err := s.elNodeDB.QueueUpdate(genericNode); err != nil { - s.config.Logger.WithError(err).Debug("failed to queue bootnode for database update") - } - } - } + s.addBootnodeToTable(s.elTable, s.elNodeDB, genericNode, s.config.Logger.WithField("layer", "EL")) } // connectELBootnodeEnode connects to an EL bootnode via enode URL. @@ -1006,11 +1053,9 @@ func (s *Service) connectELBootnodeEnode(enodeURL *enode.Enode) { // Never dial ourselves: our own enode in the bootnode list would otherwise // race our handshake challenges against our own identity. - for _, local := range s.localIDs() { - if local == nodeID { - s.config.Logger.WithField("enode", enodeURL).Debug("skipping bootnode: it is our own identity") - return - } + if slices.Contains(s.localIDs(), [32]byte(nodeID)) { + s.config.Logger.WithField("enode", enodeURL).Debug("skipping bootnode: it is our own identity") + return } // Request ENR from the node @@ -1025,11 +1070,8 @@ func (s *Service) connectELBootnodeEnode(enodeURL *enode.Enode) { v4Node.SetENR(enrRecord) // Filter by fork ID before adding - if s.enrManager != nil { - isEL, forkID := s.enrManager.FilterELNode(enrRecord) - if elFilter := s.enrManager.GetELFilter(); elFilter != nil { - elFilter.RecordAdmission(isEL, forkID) - } + if !s.config.ServeAll && s.enrManager != nil { + isEL, forkID := s.enrManager.AdmitELNode(enrRecord) if !isEL { s.config.Logger.WithFields(logrus.Fields{ "nodeID": fmt.Sprintf("%x", nodeID[:8]), @@ -1045,21 +1087,8 @@ func (s *Service) connectELBootnodeEnode(enodeURL *enode.Enode) { // Track successful ENR exchange genericNode.IncrementSuccess() - if s.elTable != nil { - if !s.elTable.Add(genericNode) { - s.config.Logger.WithField("nodeID", fmt.Sprintf("%x", nodeID[:8])).Debug("enode bootnode not admitted to table, not persisting") - return - } - s.config.Logger.WithField("nodeID", fmt.Sprintf("%x", nodeID[:8])).Info("added enode bootnode to table") - - // Persist to database - if s.elNodeDB != nil { - genericNode.MarkDirty(nodes.DirtyFull) - if err := s.elNodeDB.QueueUpdate(genericNode); err != nil { - s.config.Logger.WithError(err).Debug("failed to queue bootnode for database update") - } - } - } + s.addBootnodeToTable(s.elTable, s.elNodeDB, genericNode, + s.config.Logger.WithField("layer", "EL").WithField("nodeID", fmt.Sprintf("%x", nodeID[:8]))) } // connectCLBootnodes connects to CL bootnodes (ENR only). @@ -1073,7 +1102,7 @@ func (s *Service) connectCLBootnodes() { continue } - // Convert to v5 node to get node ID + // Convert to v5 node; this also rejects records missing an IP or UDP port. v5, err := v5node.New(record) if err != nil { s.config.Logger.WithError(err).Warn("failed to create v5 node from ENR") @@ -1082,38 +1111,34 @@ func (s *Service) connectCLBootnodes() { nodeID := v5.ID() - // Verify ENR has required fields (IP and port) - if record.IP() == nil && record.IP6() == nil { - s.config.Logger.WithField("nodeID", fmt.Sprintf("%x", nodeID[:8])).Warn("CL bootnode ENR missing IP address, skipping") - continue - } - if record.UDP() == 0 { - s.config.Logger.WithField("nodeID", fmt.Sprintf("%x", nodeID[:8])).Warn("CL bootnode ENR missing UDP port, skipping") - continue - } - // Filter by fork digest before adding - if s.enrManager != nil && !s.enrManager.FilterCLNode(record) { + if !s.config.ServeAll && s.enrManager != nil && !s.enrManager.AdmitCLNode(record) { s.config.Logger.WithField("nodeID", fmt.Sprintf("%x", nodeID[:8])).Warn("CL bootnode ENR has invalid fork digest, skipping") continue } - // Create generic node and add to table genericNode := nodes.NewFromV5(v5, s.clNodeDB) - if s.clTable != nil { - if !s.clTable.Add(genericNode) { - s.config.Logger.WithField("nodeID", fmt.Sprintf("%x", nodeID[:8])).Debug("CL bootnode not admitted to table, not persisting") - continue - } - s.config.Logger.WithField("nodeID", fmt.Sprintf("%x", nodeID[:8])).Info("added CL ENR bootnode to table") + s.addBootnodeToTable(s.clTable, s.clNodeDB, genericNode, + s.config.Logger.WithField("layer", "CL").WithField("nodeID", fmt.Sprintf("%x", nodeID[:8]))) + } +} - // Persist to database - if s.clNodeDB != nil { - genericNode.MarkDirty(nodes.DirtyFull) - if err := s.clNodeDB.QueueUpdate(genericNode); err != nil { - s.config.Logger.WithError(err).Debug("failed to queue bootnode for database update") - } - } +// addBootnodeToTable admits a configured bootnode to a routing table and +// persists it. +func (s *Service) addBootnodeToTable(table *nodes.FlatTable, nodeDB *nodes.NodeDB, n *nodes.Node, logger logrus.FieldLogger) { + if table == nil { + return + } + if !table.Add(n) { + logger.Debug("bootnode not admitted to table, not persisting") + return + } + logger.Info("added bootnode to table") + + if nodeDB != nil { + n.MarkDirty(nodes.DirtyFull) + if err := nodeDB.QueueUpdate(n); err != nil { + logger.WithError(err).Debug("failed to queue bootnode for database update") } } } @@ -1130,7 +1155,7 @@ func (s *Service) loadStoredENR(key string) (*enr.Record, error) { // storeENR stores an identity's ENR to the database under its state key. func (s *Service) storeENR(key string, record *enr.Record) error { - data, err := record.EncodeRLP() + data, err := record.EncodeRLPBytes() if err != nil { return err } @@ -1153,16 +1178,27 @@ func (s *Service) onNodeSeen(n *v5node.Node, timestamp time.Time) { if s.enrManager != nil { nodeID := n.ID() - if isEL, _ := s.enrManager.FilterELNode(n.Record()); isEL && s.elTable != nil && s.elNodeDB != nil { - // Look up the generic node from the table + // Both layers, independently, as checkAndAddNode admits them: a dual-layer + // record becomes two node objects with their own last-seen, so refreshing + // only one lets the other age out while the peer is actively talking. + var isEL, isCL bool + if s.config.ServeAll { + isEL = s.elTable != nil + isCL = s.clTable != nil + } else { + isEL, _ = s.enrManager.ClassifyELNode(n.Record()) + isCL = s.enrManager.ClassifyCLNode(n.Record()) + } + + if isEL && s.elTable != nil && s.elNodeDB != nil { if genericNode := s.elTable.Get(nodeID); genericNode != nil { genericNode.SetLastSeen(timestamp) // This marks it dirty // Get falls back to the DB, so Add re-admits demoted nodes s.elTable.Add(genericNode) s.elNodeDB.QueueUpdate(genericNode) } - } else if s.enrManager.FilterCLNode(n.Record()) && s.clTable != nil && s.clNodeDB != nil { - // Look up the generic node from the table + } + if isCL && s.clTable != nil && s.clNodeDB != nil { if genericNode := s.clTable.Get(nodeID); genericNode != nil { genericNode.SetLastSeen(timestamp) // This marks it dirty s.clTable.Add(genericNode) @@ -1183,10 +1219,11 @@ func (s *Service) onFindNodeV5(id *identity, msg *v5protocol.FindNode, sourceNod // A shared identity serves both layers under one ID, so classify a known // requester by its ENR and serve only its layer(s); an unclassifiable known // peer gets nothing, an unknown one (no ENR yet) falls through to both. - if id.servesEL && id.servesCL && sourceNode != nil && s.enrManager != nil { + // Serve-all skips this: every requester gets nodes from every served layer. + if !s.config.ServeAll && id.servesEL && id.servesCL && sourceNode != nil && s.enrManager != nil { sourceRecord := sourceNode.Record() - serveEL, _ = s.enrManager.FilterELNode(sourceRecord) - serveCL = s.enrManager.FilterCLNode(sourceRecord) + serveEL, _ = s.enrManager.ClassifyELNode(sourceRecord) + serveCL = s.enrManager.ClassifyCLNode(sourceRecord) } if serveEL && s.elTable != nil { @@ -1282,24 +1319,28 @@ func (s *Service) requestENRV4(n *v4node.Node) { nodeID := n.ID() now := time.Now() - // Check if we already have a recent pending ENR request for this node - if val, exists := s.pendingENRRequestsV4.Load(nodeID); exists { - if timestamp, ok := val.(time.Time); ok { - // If request is less than 30 seconds old, skip (still pending) - if time.Since(timestamp) < 30*time.Second { - return - } - // Request is stale (>30s), replace it + // Claim the slot atomically: a Load followed by a Store lets two callers both + // through, and takeover of an entry older than 30s has to stay possible, so a + // bare LoadOrStore is not enough either. + for { + val, loaded := s.pendingENRRequestsV4.LoadOrStore(nodeID, now) + if !loaded { + break + } + timestamp, ok := val.(time.Time) + if ok && time.Since(timestamp) < 30*time.Second { + return + } + if s.pendingENRRequestsV4.CompareAndSwap(nodeID, val, now) { + break } } - // Mark as pending with current timestamp - s.pendingENRRequestsV4.Store(nodeID, now) - // Run in goroutine to avoid blocking packet handling go func() { - // Remove from pending when done - defer s.pendingENRRequestsV4.Delete(nodeID) + // Release only our own claim: an unconditional delete would drop the entry + // of whoever took over after our 30s window expired. + defer s.pendingENRRequestsV4.CompareAndDelete(nodeID, now) // IMPORTANT: Some clients (like reth) require bidirectional bonding before responding to ENRRequest. // Bidirectional bonding means: @@ -1328,6 +1369,179 @@ func (s *Service) requestENRV4(n *v4node.Node) { }() } +// admitELLookupNode decides admission of a lookup-discovered node to the EL +// table. +func (s *Service) admitELLookupNode(n *nodes.Node) services.AdmissionResult { + if !s.config.ServeAll && n.Record() != nil && s.enrManager != nil { + isEL, forkID := s.enrManager.AdmitELNode(n.Record()) + if !isEL { + // A record with no eth entry is a consensus node, not an + // execution node on the wrong fork. + if !n.Record().Has("eth") { + s.markBadNode(n, db.LayerEL, "not_el") + return services.AdmissionRejectedLayer + } + s.config.Logger.WithFields(logrus.Fields{ + "peerID": n.PeerID(), + "eth": forkID.String(), + }).Debug("EL lookup admission rejected: incompatible fork id") + s.markBadNode(n, db.LayerEL, "invalid_fork_id") + return services.AdmissionRejectedFilter + } + } + + result := s.admitToTable(n, s.elTable, db.LayerEL) + + // After admission, and off this goroutine: each probe waits a request timeout, + // so probing a 16-node NEIGHBORS response inline stalled the lookup and the + // whole maintenance loop for over a minute. Resolving the table entry rather + // than reusing n also means the result lands on the object the table kept. + if result == services.AdmissionAccepted && n.HasV4() && !n.HasV5() { + s.scheduleV5Probe(n.ID()) + } + + return result +} + +// admitCLLookupNode decides admission of a lookup-discovered node to the CL +// table. +func (s *Service) admitCLLookupNode(n *nodes.Node) services.AdmissionResult { + if !s.config.ServeAll && n.Record() != nil && s.enrManager != nil { + if !s.enrManager.AdmitCLNode(n.Record()) { + // No eth2 entry means an execution node, not a consensus + // node on the wrong digest; keep the two distinguishable. + if !n.Record().Has("eth2") { + s.markBadNode(n, db.LayerCL, "not_cl") + return services.AdmissionRejectedLayer + } + s.markBadNode(n, db.LayerCL, "invalid_fork_digest") + return services.AdmissionRejectedFilter + } + } + + return s.admitToTable(n, s.clTable, db.LayerCL) +} + +// markBadNode records a rejected node so it is not retried on restart. +func (s *Service) markBadNode(n *nodes.Node, layer db.NodeLayer, reason string) { + if err := s.config.Database.StoreBadNode(n.IDBytes(), layer, reason); err != nil { + s.config.Logger.WithError(err).Debug("failed to store bad node") + } +} + +// admitToTable pools an accepted node and clears any prior bad-node record. +func (s *Service) admitToTable(n *nodes.Node, table *nodes.FlatTable, layer db.NodeLayer) services.AdmissionResult { + if !table.Add(n) { + return services.AdmissionRejectedPool + } + if err := s.config.Database.RemoveBadNode(n.IDBytes(), layer); err != nil { + s.config.Logger.WithError(err).Debug("failed to remove from bad nodes") + } + return services.AdmissionAccepted +} + +// probeV5Support pings a v4-discovered node over discv5 and, on success, +// attaches v5 support so lookups prefer the richer protocol. +// maxConcurrentV5Probes bounds the probes in flight so a large NEIGHBORS response +// cannot spawn one goroutine per node. +const maxConcurrentV5Probes = 8 + +// scheduleV5Probe runs a v5 probe for an admitted node without blocking the +// caller. Dropping the probe when saturated is fine: the node stays v4-only and +// the next lookup that rediscovers it tries again. +// +// The table entry decides, not the wrapper the caller happened to hold: after a +// merge the entry may already know v5, and re-probing it on every rediscovery +// would occupy the slots that genuinely v4-only nodes need. +func (s *Service) scheduleV5Probe(id [32]byte) { + if s.elTable == nil { + return + } + entry := s.elTable.Get(id) + if entry == nil || entry.HasV5() { + return + } + if _, inFlight := s.v5ProbesInFlight.LoadOrStore(id, struct{}{}); inFlight { + return + } + + select { + case s.v5ProbeSem <- struct{}{}: + default: + s.v5ProbesInFlight.Delete(id) + return + } + + go func() { + defer func() { + <-s.v5ProbeSem + s.v5ProbesInFlight.Delete(id) + }() + s.probeV5Support(id, entry) + }() +} + +// probeV5Support pings a v4-discovered node over discv5 and records the result on +// whichever wrapper the table holds when the answer arrives — the entry can be +// swept or replaced during the round trip, and writing to a detached object would +// silently lose the capability. +func (s *Service) probeV5Support(id [32]byte, n *nodes.Node) { + handler := s.getV5Handler() + if handler == nil { + return + } + record := n.Record() + if record == nil { + return + } + probedSeq := record.Seq() + v5Node, err := nodes.NewV5NodeFromRecord(record) + if err != nil { + return + } + + start := time.Now() + respChan, err := handler.SendPing(v5Node) + if err != nil { + return + } + var resp *v5protocol.Response + select { + case resp = <-respChan: + case <-s.ctx.Done(): + return + } + if resp == nil || resp.Error != nil { + return + } + + // Re-resolve: this is the first moment the result can be applied, and the + // entry may have been swept or replaced while the ping was outstanding. + // SetV5AtSeq then discards the result if the peer moved on from the record we + // probed, rather than pinning v5 traffic to the address we happened to test. + target := s.elTable.Get(id) + if target == nil { + return + } + n = target + + if !n.SetV5AtSeq(v5Node, probedSeq) { + return + } + s.config.Logger.WithFields(logrus.Fields{ + "peerID": n.PeerID(), + "addr": n.Addr(), + "rtt": time.Since(start), + }).Debug("discovered v5 support on v4-discovered node") + + // Queue protocol support update (SetV5 already marked it dirty) + if s.elNodeDB != nil { + if err := s.elNodeDB.QueueUpdate(n); err != nil { + s.config.Logger.WithError(err).Debug("failed to queue node for protocol support update") + } + } +} + // checkAndAddNodeV4 adds a discv4 node to the EL table after filtering. func (s *Service) checkAndAddNodeV4(n *v4node.Node) bool { // Ensure we have an ENR for filtering @@ -1350,13 +1564,8 @@ func (s *Service) checkAndAddNodeV4(n *v4node.Node) bool { alreadyKnown := s.elTable.Get(n.ID()) != nil // Filter the node using ENR manager (EL-only for discv4) - if s.enrManager != nil { - filter, forkID := s.enrManager.FilterELNode(n.ENR()) - if n.ENR().Has("eth") { - if elFilter := s.enrManager.GetELFilter(); elFilter != nil { - elFilter.RecordAdmission(filter, forkID) - } - } + if !s.config.ServeAll && s.enrManager != nil { + filter, forkID := s.enrManager.AdmitELNode(n.ENR()) if !filter { s.config.Logger.WithFields(logrus.Fields{ "nodeID": fmt.Sprintf("%x", n.IDBytes()[:8]), @@ -1376,7 +1585,7 @@ func (s *Service) checkAndAddNodeV4(n *v4node.Node) bool { s.config.Logger.WithFields(logrus.Fields{ "nodeID": fmt.Sprintf("%x", n.IDBytes()[:8]), "addr": n.Addr().String(), - }).Info("Added discv4 node to EL table") + }).Debug("Added discv4 node to EL table") } return true } @@ -1390,16 +1599,15 @@ func (s *Service) checkAndAddNode(n *v5node.Node) bool { return false } - // Determine layer - isEL, elForkID := s.enrManager.FilterELNode(n.Record()) - isCL := s.enrManager.FilterCLNode(n.Record()) - // Only an execution record is an execution admission decision; counting - // consensus nodes here made the rejection counter track cross-layer - // traffic, which on a dual-layer network is most of what arrives. - if n.Record().Has("eth") { - if elFilter := s.enrManager.GetELFilter(); elFilter != nil { - elFilter.RecordAdmission(isEL, elForkID) - } + // Serve-all skips the filters rather than overriding their results: calling + // them would move admission counters for decisions never made. + var isEL, isCL bool + if s.config.ServeAll { + isEL = s.elTable != nil + isCL = s.clTable != nil + } else { + isEL, _ = s.enrManager.AdmitELNode(n.Record()) + isCL = s.enrManager.AdmitCLNode(n.Record()) } // Add to appropriate table(s) @@ -1420,7 +1628,10 @@ func (s *Service) checkAndAddNode(n *v5node.Node) bool { return added } -// filterNodesForRequester applies LAN-aware and protocol filtering. +// filterNodesForRequester applies LAN-aware and protocol filtering. It is the +// single funnel for both protocols' FINDNODE responses, so it also enforces +// that a response never repeats a node ID — a node can sit in both tables (any +// dual-stack peer, and every peer under serve-all). func (s *Service) filterNodesForRequester(nodeList []*nodes.Node, requester *net.UDPAddr, needsV5 bool) []*nodes.Node { requesterIsLAN := v5node.IsLANAddress(requester.IP) @@ -1448,6 +1659,11 @@ func (s *Service) filterNodesForRequester(nodeList []*nodes.Node, requester *net continue } + id := n.ID() + if slices.ContainsFunc(filtered, func(kept *nodes.Node) bool { return kept.ID() == id }) { + continue + } + filtered = append(filtered, n) } @@ -1616,12 +1832,24 @@ func (s *Service) onPongReceived(remoteID []byte, sourceIP net.IP, reportedIP ne port = s.primaryIdentity().enrPort } - reporterIDStr := fmt.Sprintf("%x", remoteID[:8]) - s.ipDiscovery.ReportIP(reportedIP, port, reporterIDStr, sourceIP) + // The full ID, not a prefix: this keys the distinct-reporter threshold, so a + // truncated key would let two peers count as one. + s.ipDiscovery.ReportIP(reportedIP, port, fmt.Sprintf("%x", remoteID), sourceIP) } // updateENRWithDiscoveredIP updates every identity's ENR with the discovered IP. func (s *Service) updateENRWithDiscoveredIP(ip net.IP, port uint16, isIPv6 bool) { + // An explicitly configured address is authoritative (see Config.ENRIPProvided), + // so peer reports must not move it. reconcileStoredENR already honours this at + // startup; without the same check here a configured address is overwritten + // while running and only restored on restart. + if isIPv6 && s.config.ENRIP6Provided { + return + } + if !isIPv6 && s.config.ENRIPProvided { + return + } + s.mu.Lock() defer s.mu.Unlock() @@ -1663,13 +1891,6 @@ func (s *Service) updateENRWithDiscoveredIP(ip net.IP, port uint16, isIPv6 bool) "isIPv6": isIPv6, }).Info("IP discovery: consensus reached, updated ENR") - if err := s.storeENR(id.storeKey, id.localNode.Record()); err != nil { - s.config.Logger.WithError(err).Warn("failed to store updated ENR") - } - - // Keep the discv4 service's ENR in sync (EL identity only). - if id.servesEL && s.discv4Service != nil { - s.discv4Service.SetLocalENR(id.localNode.Record()) - } + s.publishENR(id) } } diff --git a/bootnode/service_test.go b/bootnode/service_test.go index dd4d267..ead5e8d 100644 --- a/bootnode/service_test.go +++ b/bootnode/service_test.go @@ -1,6 +1,7 @@ package bootnode import ( + "context" "crypto/ecdsa" "fmt" "net" @@ -11,7 +12,9 @@ import ( "github.com/ethpandaops/bootnodoor/bootnode/clconfig" "github.com/ethpandaops/bootnodoor/bootnode/elconfig" "github.com/ethpandaops/bootnodoor/db" + v5node "github.com/ethpandaops/bootnodoor/discv5/node" "github.com/ethpandaops/bootnodoor/enr" + "github.com/ethpandaops/bootnodoor/nodes" "github.com/ethpandaops/bootnodoor/services" "github.com/sirupsen/logrus" ) @@ -51,7 +54,7 @@ func TestUpdateENR_ELOnlyDropsInheritedEth2(t *testing.T) { t.Fatalf("createLocalNode: %v", err) } - if err := NewENRManager(cfg, key, ln, true, false).UpdateENR(0, 0); err != nil { + if _, err := NewENRManager(cfg, key, ln, true, false).UpdateENR(0, 0); err != nil { t.Fatalf("UpdateENR: %v", err) } @@ -76,7 +79,7 @@ func TestUpdateENR_DropsUnservedFields(t *testing.T) { t.Fatalf("createLocalNode: %v", err) } - if err := NewENRManager(cfg, key, ln, false, false).UpdateENR(0, 0); err != nil { + if _, err := NewENRManager(cfg, key, ln, false, false).UpdateENR(0, 0); err != nil { t.Fatalf("UpdateENR: %v", err) } @@ -90,6 +93,37 @@ func TestUpdateENR_DropsUnservedFields(t *testing.T) { } } +// A record without an eth entry is a consensus node, not a wrong-fork +// execution node, so it must not move the EL admission counters. +func TestAdmitELNode_SkipsRecordsWithoutEth(t *testing.T) { + cfg := &Config{Logger: quietLogger(), ELConfig: &elconfig.ChainConfig{}, ELGenesisHash: [32]byte{1, 2, 3}, ELGenesisTime: 1000} + key := mustKey(t) + ln, err := createLocalNode(cfg, key, net.ParseIP("1.2.3.4"), nil, 9000, nil) + if err != nil { + t.Fatalf("createLocalNode: %v", err) + } + m := NewENRManager(cfg, key, ln, true, false) + + clOnly := storedENRWith(t, key, map[string][]byte{"eth2": {0xaa, 0xbb, 0xcc, 0xdd}}) + m.AdmitELNode(clOnly) + if got := m.GetELFilter().GetStats().TotalChecks; got != 0 { + t.Fatalf("TotalChecks = %d after CL-only record, want 0", got) + } + + elRec := storedENRWith(t, key, map[string][]byte{"eth": {0x01, 0x02, 0x03, 0x04}}) + m.AdmitELNode(elRec) + stats := m.GetELFilter().GetStats() + if stats.TotalChecks != 1 || stats.Rejected != 1 { + t.Fatalf("stats = %+v after eth record, want 1 check / 1 rejection", stats) + } + + // The pure path must stay silent on the same records. + m.ClassifyELNode(elRec) + if got := m.GetELFilter().GetStats().TotalChecks; got != 1 { + t.Fatalf("TotalChecks = %d after ClassifyELNode, want 1 (unchanged)", got) + } +} + // newTestService builds a minimal Service with the given identities and an // in-memory database, enough to exercise updateENRWithDiscoveredIP. func newTestService(t *testing.T, ids []*identity) *Service { @@ -550,7 +584,7 @@ func TestUpdateENR_PublishesCurrentEraForkID(t *testing.T) { mgr := NewENRManager(cfg, key, ln, true, false) headBlock, headTime := StaticHead() - if err := mgr.UpdateENR(headBlock, headTime); err != nil { + if _, err := mgr.UpdateENR(headBlock, headTime); err != nil { t.Fatalf("UpdateENR: %v", err) } @@ -584,13 +618,13 @@ func TestUpdateENR_NoSeqBumpWhenUnchanged(t *testing.T) { mgr := NewENRManager(cfg, key, ln, true, false) headBlock, headTime := StaticHead() - if err := mgr.UpdateENR(headBlock, headTime); err != nil { + if _, err := mgr.UpdateENR(headBlock, headTime); err != nil { t.Fatalf("first UpdateENR: %v", err) } seq := ln.Record().Seq() for i := 0; i < 3; i++ { - if err := mgr.UpdateENR(StaticHead()); err != nil { + if _, err := mgr.UpdateENR(StaticHead()); err != nil { t.Fatalf("repeat UpdateENR: %v", err) } } @@ -607,3 +641,124 @@ func mustChainConfig(t *testing.T, jsonCfg string) *elconfig.ChainConfig { } return cfg } + +// newServeAllTestService builds a Service with an EL table and ENR manager, +// enough to exercise checkAndAddNode admission. +func newServeAllTestService(t *testing.T, serveAll bool) *Service { + t.Helper() + + logger := quietLogger() + database := db.NewDatabase(&db.SqliteDatabaseConfig{File: ":memory:", MaxOpenConns: 5, MaxIdleConns: 2}, logger) + if err := database.Init(); err != nil { + t.Fatalf("db init: %v", err) + } + t.Cleanup(func() { database.Close() }) + if err := database.ApplyEmbeddedDbSchema(-2); err != nil { + t.Fatalf("db schema: %v", err) + } + + cfg := &Config{ + Logger: logger, + Database: database, + ELConfig: &elconfig.ChainConfig{}, + ELGenesisHash: [32]byte{1, 2, 3}, + ELGenesisTime: 1000, + ServeAll: serveAll, + } + + key := mustKey(t) + ln, err := createLocalNode(cfg, key, net.ParseIP("1.2.3.4"), nil, 9000, nil) + if err != nil { + t.Fatalf("createLocalNode: %v", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + s := &Service{config: cfg, ctx: ctx, enrManager: NewENRManager(cfg, key, ln, true, false)} + s.elNodeDB = nodes.NewNodeDB(ctx, database, db.LayerEL, logger) + s.elTable, err = s.createTable(ln.ID(), s.elNodeDB, "EL") + if err != nil { + t.Fatalf("createTable: %v", err) + } + + return s +} + +// A node advertising neither eth nor eth2 is unclassifiable: rejected in the +// default classified mode, admitted under --serve-all. +func TestServeAll_AdmitsUnclassifiableNode(t *testing.T) { + for _, tc := range []struct { + name string + serveAll bool + wantAdded bool + }{ + {name: "classified rejects", serveAll: false, wantAdded: false}, + {name: "serve-all admits", serveAll: true, wantAdded: true}, + } { + t.Run(tc.name, func(t *testing.T) { + s := newServeAllTestService(t, tc.serveAll) + + record := storedENRWith(t, mustKey(t), nil) + n, err := v5node.New(record) + if err != nil { + t.Fatalf("v5node.New: %v", err) + } + + if got := s.checkAndAddNode(n); got != tc.wantAdded { + t.Fatalf("checkAndAddNode = %v, want %v", got, tc.wantAdded) + } + }) + } +} + +// Serve-all makes no fork-based admission decision, so it must not move the EL +// admission counters. +func TestServeAll_LeavesAdmissionCountersUntouched(t *testing.T) { + s := newServeAllTestService(t, true) + + record := storedENRWith(t, mustKey(t), map[string][]byte{"eth": {0x01, 0x02, 0x03, 0x04}}) + n, err := v5node.New(record) + if err != nil { + t.Fatalf("v5node.New: %v", err) + } + + s.checkAndAddNode(n) + + if got := s.enrManager.GetELFilter().GetStats().TotalChecks; got != 0 { + t.Fatalf("TotalChecks = %d under serve-all, want 0", got) + } +} + +// A node present in both tables must be served once, not once per table. +func TestFilterNodesForRequesterDedupes(t *testing.T) { + s := newServeAllTestService(t, true) + + rec := storedENRWith(t, mustKey(t), nil) + v5, err := v5node.New(rec) + if err != nil { + t.Fatalf("v5node.New: %v", err) + } + + a := nodes.NewFromV5(v5, s.elNodeDB) + b := nodes.NewFromV5(v5, s.elNodeDB) + other := nodes.NewFromV5(mustV5Node(t), s.elNodeDB) + + requester := &net.UDPAddr{IP: net.ParseIP("8.8.8.8"), Port: 30303} + got := s.filterNodesForRequester([]*nodes.Node{a, b, other, a}, requester, true) + if len(got) != 2 { + t.Fatalf("filterNodesForRequester returned %d nodes, want 2", len(got)) + } + if got[0].ID() != a.ID() || got[1].ID() != other.ID() { + t.Errorf("filterNodesForRequester did not keep first occurrences in order") + } +} + +func mustV5Node(t *testing.T) *v5node.Node { + t.Helper() + n, err := v5node.New(storedENRWith(t, mustKey(t), nil)) + if err != nil { + t.Fatalf("v5node.New: %v", err) + } + return n +} diff --git a/bootnode/stats.go b/bootnode/stats.go index 2478a40..32368ef 100644 --- a/bootnode/stats.go +++ b/bootnode/stats.go @@ -4,6 +4,7 @@ import ( "time" v4protocol "github.com/ethpandaops/bootnodoor/discv4/protocol" + "github.com/ethpandaops/bootnodoor/discv5/session" "github.com/ethpandaops/bootnodoor/services" "github.com/ethpandaops/bootnodoor/transport" ) @@ -18,14 +19,12 @@ type Stats struct { Discv5 Discv5Stats Discv4 v4protocol.HandlerStats HasV4 bool - Sessions SessionStats + Sessions session.Stats Packets transport.MetricsSnapshot } -// Discv5Stats is the per-identity discv5 handler counters, summed. +// Discv5Stats is the deliberate subset of protocol.HandlerStats the web UI renders, summed per identity. type Discv5Stats struct { - PacketsReceived int - PacketsSent int InvalidPackets int FilteredResponses int FindNodeReceived int @@ -33,13 +32,6 @@ type Discv5Stats struct { PendingChallenges int } -// SessionStats is the discv5 session cache totals, summed per identity. -type SessionStats struct { - Total int - Active int - Expired int -} - // GetStats returns a snapshot of the service's discovery counters. func (s *Service) GetStats() Stats { var out Stats @@ -69,17 +61,18 @@ func (s *Service) GetStats() Stats { out.Ping.PingTimeouts += p.PingTimeouts out.Ping.PingsV5 += p.PingsV5 out.Ping.PingsV4 += p.PingsV4 - if p.AverageRTT > 0 { - totalRTT += p.AverageRTT - rttSamples++ + // Weight by the sample count behind each average: EL and CL rarely + // answer the same number of pings, and averaging the averages would + // let the quieter identity move the aggregate as much as the busier one. + if p.AverageRTT > 0 && p.PongsReceived > 0 { + totalRTT += p.AverageRTT * time.Duration(p.PongsReceived) + rttSamples += p.PongsReceived } } if id.discv5Service != nil { if h := id.discv5Service.Handler(); h != nil { d := h.GetStats() - out.Discv5.PacketsReceived += d.PacketsReceived - out.Discv5.PacketsSent += d.PacketsSent out.Discv5.InvalidPackets += d.InvalidPackets out.Discv5.FilteredResponses += d.FilteredResponses out.Discv5.FindNodeReceived += d.FindNodeReceived @@ -107,6 +100,8 @@ func (s *Service) GetStats() Stats { out.Packets.SendErrors += m.SendErrors out.Packets.ReceiveErrors += m.ReceiveErrors out.Packets.RateLimited += m.RateLimited + out.Packets.PacketsFellThrough += m.PacketsFellThrough + out.Packets.PacketsUnhandled += m.PacketsUnhandled } } diff --git a/cmd/bootnodoor/main.go b/cmd/bootnodoor/main.go index ac4e822..c5f7e80 100644 --- a/cmd/bootnodoor/main.go +++ b/cmd/bootnodoor/main.go @@ -59,9 +59,10 @@ var ( clEnrPort int // ENR configuration - enrIP string - enrIP6 string - enrPort int + enrIP string + enrIP6 string + enrPort int + enableIPDiscovery bool // Logging logLevel string @@ -74,6 +75,9 @@ var ( enableEL bool enableCL bool + // Rendezvous mode + serveAll bool + // WebUI flags enableWebUI bool webUIHost string @@ -136,6 +140,7 @@ func init() { rootCmd.Flags().StringVar(&enrIP, "enr-ip", "", "IPv4 address to advertise in ENR (auto-detected if not specified)") rootCmd.Flags().StringVar(&enrIP6, "enr-ip6", "", "IPv6 address to advertise in ENR (optional)") rootCmd.Flags().IntVar(&enrPort, "enr-port", 0, "UDP port to advertise in ENR (0 = use bind-port)") + rootCmd.Flags().BoolVar(&enableIPDiscovery, "enable-ip-discovery", true, "Learn the external address from peer PONG reports (never overrides --enr-ip/--enr-ip6)") // Logging rootCmd.Flags().StringVar(&logLevel, "log-level", "info", "Log level (debug, info, warn, error)") @@ -148,6 +153,9 @@ func init() { rootCmd.Flags().BoolVar(&enableEL, "enable-el", true, "Enable Execution Layer support (discv4 + discv5)") rootCmd.Flags().BoolVar(&enableCL, "enable-cl", true, "Enable Consensus Layer support (discv5)") + // Rendezvous mode + rootCmd.Flags().BoolVar(&serveAll, "serve-all", false, "Disable EL/CL classification and fork-ID filtering: pool and serve every discovered node to everyone (plain discv5 rendezvous)") + // WebUI rootCmd.Flags().BoolVar(&enableWebUI, "web-ui", false, "Enable web UI") rootCmd.Flags().StringVar(&webUIHost, "web-host", "0.0.0.0", "Web UI host") @@ -515,9 +523,11 @@ func runBootnode(cmd *cobra.Command, args []string) error { config.ENRIP6 = enrIPv6 config.ENRIPProvided = enrIP != "" config.ENRIP6Provided = enrIP6 != "" + config.EnableIPDiscovery = enableIPDiscovery config.ENRPort = enrUDPPort config.EnableDiscv4 = enableDiscv4 config.EnableDiscv5 = enableDiscv5 + config.ServeAll = serveAll config.MaxActiveNodes = maxActiveNodes config.MaxNodesPerIP = maxNodesPerIP config.Logger = logger diff --git a/db/layer_key_test.go b/db/layer_key_test.go new file mode 100644 index 0000000..de5de68 --- /dev/null +++ b/db/layer_key_test.go @@ -0,0 +1,190 @@ +package db + +import ( + "database/sql" + "testing" + "time" + + "github.com/jmoiron/sqlx" + "github.com/sirupsen/logrus" +) + +func testDB(t *testing.T) *Database { + t.Helper() + + logger := logrus.New() + logger.SetLevel(logrus.ErrorLevel) + + database := NewDatabase(&SqliteDatabaseConfig{File: ":memory:"}, logger) + if err := database.Init(); err != nil { + t.Fatalf("init: %v", err) + } + t.Cleanup(func() { database.Close() }) + if err := database.ApplyEmbeddedDbSchema(-2); err != nil { + t.Fatalf("schema: %v", err) + } + return database +} + +// A dual-layer peer occupies one row per layer. With nodeid as the sole primary +// key both upserts collide, so the second layer overwrites the first's +// layer-specific fork digest while the row keeps the first layer's tag — and +// every read filters on (nodeid, layer), so one layer silently disappears. +func TestUpsertKeepsBothLayers(t *testing.T) { + database := testDB(t) + + id := []byte("0123456789abcdef0123456789abcdef") + elDigest := []byte{0xaa, 0xaa, 0xaa, 0xaa} + clDigest := []byte{0xbb, 0xbb, 0xbb, 0xbb} + + for _, tc := range []struct { + layer NodeLayer + digest []byte + }{{LayerEL, elDigest}, {LayerCL, clDigest}} { + n := &Node{ + NodeID: id, Layer: string(tc.layer), Port: 30303, Seq: 1, + ForkDigest: tc.digest, FirstSeen: time.Now().Unix(), ENR: []byte("enr"), + } + if err := database.RunDBTransaction(func(tx *sqlx.Tx) error { + return database.UpsertNode(tx, n) + }); err != nil { + t.Fatalf("upsert %s: %v", tc.layer, err) + } + } + + el, err := database.GetNode(LayerEL, id) + if err != nil { + t.Fatalf("EL row missing after the CL upsert: %v", err) + } + cl, err := database.GetNode(LayerCL, id) + if err != nil { + t.Fatalf("CL row missing after the EL upsert: %v", err) + } + if string(el.ForkDigest) != string(elDigest) { + t.Errorf("EL fork digest = %x, want %x", el.ForkDigest, elDigest) + } + if string(cl.ForkDigest) != string(clDigest) { + t.Errorf("CL fork digest = %x, want %x", cl.ForkDigest, clDigest) + } +} + +// Same collision on the ENR-update path, which is the one admission uses. +func TestUpdateNodeENRKeepsBothLayers(t *testing.T) { + database := testDB(t) + + id := []byte("fedcba9876543210fedcba9876543210") + + for _, tc := range []struct { + layer NodeLayer + digest []byte + }{{LayerEL, []byte{1, 1, 1, 1}}, {LayerCL, []byte{2, 2, 2, 2}}} { + if err := database.RunDBTransaction(func(tx *sqlx.Tx) error { + return database.UpdateNodeENR(tx, tc.layer, id, nil, nil, 30303, 1, tc.digest, []byte("enr"), true, true) + }); err != nil { + t.Fatalf("update %s: %v", tc.layer, err) + } + } + + if _, err := database.GetNode(LayerEL, id); err != nil { + t.Errorf("EL row missing: %v", err) + } + if _, err := database.GetNode(LayerCL, id); err != nil { + t.Errorf("CL row missing: %v", err) + } + got, err := database.CountAllNodes() + if err != nil { + t.Fatalf("CountAllNodes: %v", err) + } + if got != 2 { + t.Errorf("total rows across layers = %d, want 2", got) + } +} + +// markBadNode is called per layer, and INSERT OR REPLACE keyed on nodeid alone +// drops the other layer's entry — so a peer rejected on both layers stays +// suppressed on only whichever was written last, defeating the cache that +// exists to stop repeated ENR requests. +func TestBadNodeSuppressionSurvivesBothLayers(t *testing.T) { + database := testDB(t) + + id := []byte("badbadbadbadbadbadbadbadbadbadba") + + if err := database.StoreBadNode(id, LayerEL, "invalid_fork_id"); err != nil { + t.Fatalf("store EL: %v", err) + } + if err := database.StoreBadNode(id, LayerCL, "invalid_fork_digest"); err != nil { + t.Fatalf("store CL: %v", err) + } + + elBad, _, elReason, err := database.IsBadNode(id, LayerEL, time.Hour) + if err != nil { + t.Fatalf("IsBadNode EL: %v", err) + } + clBad, _, clReason, err := database.IsBadNode(id, LayerCL, time.Hour) + if err != nil { + t.Fatalf("IsBadNode CL: %v", err) + } + + if !elBad { + t.Error("EL rejection was forgotten after the CL rejection was recorded") + } + if !clBad { + t.Error("CL rejection was forgotten after the EL rejection was recorded") + } + if elBad && elReason != "invalid_fork_id" { + t.Errorf("EL reason = %q, want invalid_fork_id", elReason) + } + if clBad && clReason != "invalid_fork_digest" { + t.Errorf("CL reason = %q, want invalid_fork_digest", clReason) + } +} + +// The full upsert supplies last_active, but its conflict clause has to assign it +// too: an already-persisted row would otherwise keep a stale or NULL timestamp +// while the DirtyFull branch cleared the flag that would have fixed it. +func TestUpsertUpdatesLastActiveOnExistingRow(t *testing.T) { + database := testDB(t) + + id := []byte("aaaabbbbccccddddeeeeffff00001111") + + if err := database.RunDBTransaction(func(tx *sqlx.Tx) error { + return database.UpdateNodeENR(tx, LayerEL, id, nil, nil, 30303, 1, []byte{1, 2, 3, 4}, []byte("enr"), true, true) + }); err != nil { + t.Fatalf("seed row: %v", err) + } + + active := time.Now().Unix() + n := &Node{ + NodeID: id, Layer: string(LayerEL), Port: 30303, Seq: 2, + ForkDigest: []byte{1, 2, 3, 4}, FirstSeen: 1000, ENR: []byte("enr2"), + LastActive: sql.NullInt64{Valid: true, Int64: active}, + } + if err := database.RunDBTransaction(func(tx *sqlx.Tx) error { + return database.UpsertNode(tx, n) + }); err != nil { + t.Fatalf("upsert: %v", err) + } + + stored, err := database.GetNode(LayerEL, id) + if err != nil { + t.Fatalf("load: %v", err) + } + if !stored.LastActive.Valid || stored.LastActive.Int64 != active { + t.Errorf("last_active = %v, want %d", stored.LastActive, active) + } + + // A caller with no timestamp must not blank the stored one. + n.LastActive = sql.NullInt64{} + if err := database.RunDBTransaction(func(tx *sqlx.Tx) error { + return database.UpsertNode(tx, n) + }); err != nil { + t.Fatalf("second upsert: %v", err) + } + stored, err = database.GetNode(LayerEL, id) + if err != nil { + t.Fatalf("reload: %v", err) + } + if !stored.LastActive.Valid || stored.LastActive.Int64 != active { + t.Errorf("last_active was blanked by an upsert without a timestamp: %v", stored.LastActive) + } +} diff --git a/db/migration_down_test.go b/db/migration_down_test.go new file mode 100644 index 0000000..907b9fc --- /dev/null +++ b/db/migration_down_test.go @@ -0,0 +1,78 @@ +package db + +import ( + "path/filepath" + "testing" + + "github.com/jmoiron/sqlx" + "github.com/pressly/goose/v3" + "github.com/sirupsen/logrus" +) + +// The down migration collapses two rows into a nodeid primary key, so it has to +// actually run and has to pick a row deterministically rather than erroring on +// the constraint. +func TestLayerKeyDownMigrationCollapsesDeterministically(t *testing.T) { + file := filepath.Join(t.TempDir(), "down.db") + + logger := logrus.New() + logger.SetLevel(logrus.ErrorLevel) + + database := NewDatabase(&SqliteDatabaseConfig{File: file}, logger) + if err := database.Init(); err != nil { + t.Fatalf("init: %v", err) + } + defer database.Close() + if err := database.ApplyEmbeddedDbSchema(-2); err != nil { + t.Fatalf("schema: %v", err) + } + + dualID := []byte("dddddddddddddddddddddddddddddddd") + clOnlyID := []byte("11111111111111111111111111111111") + + for _, tc := range []struct { + id []byte + layer NodeLayer + }{{dualID, LayerEL}, {dualID, LayerCL}, {clOnlyID, LayerCL}} { + n := &Node{ + NodeID: tc.id, Layer: string(tc.layer), Port: 30303, Seq: 1, + ForkDigest: []byte{1, 2, 3, 4}, FirstSeen: 1000, ENR: []byte("enr-" + string(tc.layer)), + } + if err := database.RunDBTransaction(func(tx *sqlx.Tx) error { + return database.UpsertNode(tx, n) + }); err != nil { + t.Fatalf("seed %s: %v", tc.layer, err) + } + } + + goose.SetLogger(&gooseLogger{logger: logger}) + goose.SetBaseFS(embedSchema) + if err := goose.SetDialect("sqlite3"); err != nil { + t.Fatalf("dialect: %v", err) + } + if err := goose.Down(database.writerDb.DB, "schema"); err != nil { + t.Fatalf("down migration failed to run: %v", err) + } + + var rows []struct { + NodeID []byte `db:"nodeid"` + Layer string `db:"layer"` + } + if err := database.ReaderDb.Select(&rows, "SELECT nodeid, layer FROM nodes ORDER BY layer"); err != nil { + t.Fatalf("select after down: %v", err) + } + if len(rows) != 2 { + t.Fatalf("rows after collapse = %d, want 2", len(rows)) + } + + byID := map[string]string{} + for _, r := range rows { + byID[string(r.NodeID)] = r.Layer + } + if got := byID[string(dualID)]; got != string(LayerEL) { + t.Errorf("dual-layer node collapsed to layer %q, want el", got) + } + if got := byID[string(clOnlyID)]; got != string(LayerCL) { + t.Errorf("cl-only node collapsed to layer %q, want cl", got) + } +} diff --git a/db/migration_layer_key_test.go b/db/migration_layer_key_test.go new file mode 100644 index 0000000..0f54c43 --- /dev/null +++ b/db/migration_layer_key_test.go @@ -0,0 +1,98 @@ +package db + +import ( + "path/filepath" + "testing" + + "github.com/jmoiron/sqlx" + "github.com/sirupsen/logrus" +) + +const schemaBeforeLayerKey = 20251106015541 + +func openAt(t *testing.T, file string, version int64) *Database { + t.Helper() + + logger := logrus.New() + logger.SetLevel(logrus.ErrorLevel) + + database := NewDatabase(&SqliteDatabaseConfig{File: file}, logger) + if err := database.Init(); err != nil { + t.Fatalf("init: %v", err) + } + if err := database.ApplyEmbeddedDbSchema(version); err != nil { + t.Fatalf("schema %d: %v", version, err) + } + return database +} + +// The up migration rebuilds both tables, so it must carry existing rows across +// rather than silently starting empty. +func TestLayerKeyMigrationPreservesExistingRows(t *testing.T) { + file := filepath.Join(t.TempDir(), "nodes.db") + + elID := []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + clID := []byte("cccccccccccccccccccccccccccccccc") + + // Seeded with raw SQL: the Go upserts now target the composite key and + // cannot write the pre-migration schema. + old := openAt(t, file, schemaBeforeLayerKey) + if err := old.RunDBTransaction(func(tx *sqlx.Tx) error { + for _, tc := range []struct { + id []byte + layer NodeLayer + }{{elID, LayerEL}, {clID, LayerCL}} { + if _, err := tx.Exec( + `INSERT INTO nodes (nodeid, layer, port, seq, fork_digest, first_seen, enr, has_v4, has_v5) + VALUES (?, ?, 30303, 7, ?, 1000, ?, 1, 1)`, + tc.id, string(tc.layer), []byte{9, 9, 9, 9}, []byte("enr-"+string(tc.layer))); err != nil { + return err + } + } + _, err := tx.Exec( + `INSERT INTO bad_nodes (nodeid, layer, rejected_at, reason) VALUES (?, ?, ?, ?)`, + elID, string(LayerEL), 1000, "invalid_fork_id") + return err + }); err != nil { + t.Fatalf("seed: %v", err) + } + old.Close() + + migrated := openAt(t, file, -2) + defer migrated.Close() + + el, err := migrated.GetNode(LayerEL, elID) + if err != nil { + t.Fatalf("EL row lost by the migration: %v", err) + } + if el.Seq != 7 || string(el.ENR) != "enr-el" { + t.Errorf("EL row mangled: seq=%d enr=%q", el.Seq, el.ENR) + } + if _, err := migrated.GetNode(LayerCL, clID); err != nil { + t.Errorf("CL row lost by the migration: %v", err) + } + if isBad, _, reason, err := migrated.IsBadNode(elID, LayerEL, 0); err != nil { + t.Errorf("IsBadNode: %v", err) + } else if !isBad || reason != "invalid_fork_id" { + t.Errorf("bad node lost by the migration: isBad=%v reason=%q", isBad, reason) + } + + // The point of the migration: both layers now coexist for one ID. + for _, layer := range []NodeLayer{LayerEL, LayerCL} { + n := &Node{ + NodeID: elID, Layer: string(layer), Port: 30303, Seq: 8, + ForkDigest: []byte{1, 2, 3, 4}, FirstSeen: 1000, ENR: []byte("enr"), + } + if err := migrated.RunDBTransaction(func(tx *sqlx.Tx) error { + return migrated.UpsertNode(tx, n) + }); err != nil { + t.Fatalf("post-migration upsert %s: %v", layer, err) + } + } + if _, err := migrated.GetNode(LayerEL, elID); err != nil { + t.Errorf("EL row missing after dual-layer upsert: %v", err) + } + if _, err := migrated.GetNode(LayerCL, elID); err != nil { + t.Errorf("CL row missing after dual-layer upsert: %v", err) + } +} diff --git a/db/nodes.go b/db/nodes.go index e15a1a5..3619215 100644 --- a/db/nodes.go +++ b/db/nodes.go @@ -107,6 +107,14 @@ func (d *Database) CountNodes(layer NodeLayer) (int, error) { return count, err } +// GetNodeIDs returns the node IDs persisted for a specific layer. +func (d *Database) GetNodeIDs(layer NodeLayer) ([][]byte, error) { + d.trackQuery() + var ids [][]byte + err := d.ReaderDb.Select(&ids, "SELECT nodeid FROM nodes WHERE layer = $1", string(layer)) + return ids, err +} + // CountAllNodes returns the total number of nodes (all layers). func (d *Database) CountAllNodes() (int, error) { d.trackQuery() @@ -134,13 +142,15 @@ func (d *Database) UpsertNode(tx *sqlx.Tx, node *Node) error { _, err := tx.Exec(` INSERT INTO nodes (nodeid, layer, ip, ipv6, port, seq, fork_digest, first_seen, last_seen, last_active, enr, has_v4, has_v5, success_count, failure_count, avg_rtt) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16) - ON CONFLICT(nodeid) DO UPDATE SET + ON CONFLICT(nodeid, layer) DO UPDATE SET ip = excluded.ip, ipv6 = excluded.ipv6, port = excluded.port, seq = excluded.seq, fork_digest = excluded.fork_digest, last_seen = excluded.last_seen, + -- COALESCE so a caller without a timestamp cannot blank a stored one. + last_active = COALESCE(excluded.last_active, nodes.last_active), enr = excluded.enr, has_v4 = excluded.has_v4, has_v5 = excluded.has_v5, @@ -159,7 +169,7 @@ func (d *Database) UpdateNodeENR(tx *sqlx.Tx, layer NodeLayer, nodeID []byte, ip _, err := tx.Exec(` INSERT INTO nodes (nodeid, layer, ip, ipv6, port, seq, fork_digest, first_seen, last_seen, last_active, enr, has_v4, has_v5, success_count, failure_count, avg_rtt) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, NULL, NULL, $9, $10, $11, 0, 0, 0) - ON CONFLICT(nodeid) DO UPDATE SET + ON CONFLICT(nodeid, layer) DO UPDATE SET ip = excluded.ip, ipv6 = excluded.ipv6, port = excluded.port, diff --git a/db/schema/20260729200000_node_layer_key.sql b/db/schema/20260729200000_node_layer_key.sql new file mode 100644 index 0000000..546f9a6 --- /dev/null +++ b/db/schema/20260729200000_node_layer_key.sql @@ -0,0 +1,117 @@ +-- +goose Up +-- +goose StatementBegin + +-- Reads filter on (nodeid, layer), so keying on nodeid alone let a dual-layer +-- peer's second write replace its first. SQLite cannot alter a primary key, +-- hence the rebuild. Rows already collapsed are carried over as-is; the lost +-- layer returns only on rediscovery. + +CREATE TABLE "nodes_new" ( + "nodeid" BLOB NOT NULL, + "layer" TEXT NOT NULL, + "ip" BLOB, + "ipv6" BLOB, + "port" INTEGER, + "seq" INTEGER, + "fork_digest" BLOB, + "first_seen" INTEGER, + "last_seen" INTEGER, + "last_active" INTEGER, + "enr" BLOB, + "has_v4" INTEGER DEFAULT 0, + "has_v5" INTEGER DEFAULT 1, + "success_count" INTEGER DEFAULT 0, + "failure_count" INTEGER DEFAULT 0, + "avg_rtt" INTEGER DEFAULT 0, + PRIMARY KEY ("nodeid", "layer") +); + +INSERT INTO "nodes_new" SELECT + nodeid, layer, ip, ipv6, port, seq, fork_digest, first_seen, last_seen, + last_active, enr, has_v4, has_v5, success_count, failure_count, avg_rtt +FROM "nodes"; + +DROP TABLE "nodes"; +ALTER TABLE "nodes_new" RENAME TO "nodes"; + +CREATE INDEX IF NOT EXISTS "idx_nodes_layer" ON "nodes" ("layer"); +CREATE INDEX IF NOT EXISTS "idx_nodes_last_active" ON "nodes" ("last_active" DESC); +CREATE INDEX IF NOT EXISTS "idx_nodes_fork_digest" ON "nodes" ("fork_digest"); +CREATE INDEX IF NOT EXISTS "idx_nodes_layer_last_active" ON "nodes" ("layer", "last_active" DESC); + +CREATE TABLE "bad_nodes_new" ( + "nodeid" BLOB NOT NULL, + "layer" TEXT NOT NULL, + "rejected_at" INTEGER NOT NULL, + "reason" TEXT, + PRIMARY KEY ("nodeid", "layer") +); + +INSERT INTO "bad_nodes_new" SELECT nodeid, layer, rejected_at, reason FROM "bad_nodes"; + +DROP TABLE "bad_nodes"; +ALTER TABLE "bad_nodes_new" RENAME TO "bad_nodes"; + +CREATE INDEX IF NOT EXISTS "idx_bad_nodes_layer" ON "bad_nodes" ("layer"); +CREATE INDEX IF NOT EXISTS "idx_bad_nodes_rejected_at" ON "bad_nodes" ("rejected_at"); + +-- +goose StatementEnd +-- +goose Down +-- +goose StatementBegin + +-- Lossy by necessity: two rows cannot both fit a nodeid primary key. Keeping +-- the EL row makes the collapse deterministic rather than insertion-ordered. + +CREATE TABLE "nodes_old" ( + "nodeid" BLOB PRIMARY KEY, + "layer" TEXT NOT NULL, + "ip" BLOB, + "ipv6" BLOB, + "port" INTEGER, + "seq" INTEGER, + "fork_digest" BLOB, + "first_seen" INTEGER, + "last_seen" INTEGER, + "last_active" INTEGER, + "enr" BLOB, + "has_v4" INTEGER DEFAULT 0, + "has_v5" INTEGER DEFAULT 1, + "success_count" INTEGER DEFAULT 0, + "failure_count" INTEGER DEFAULT 0, + "avg_rtt" INTEGER DEFAULT 0 +); + +INSERT INTO "nodes_old" SELECT + nodeid, layer, ip, ipv6, port, seq, fork_digest, first_seen, last_seen, + last_active, enr, has_v4, has_v5, success_count, failure_count, avg_rtt +FROM "nodes" +WHERE layer = 'el' + OR nodeid NOT IN (SELECT nodeid FROM "nodes" WHERE layer = 'el'); + +DROP TABLE "nodes"; +ALTER TABLE "nodes_old" RENAME TO "nodes"; + +CREATE INDEX IF NOT EXISTS "idx_nodes_layer" ON "nodes" ("layer"); +CREATE INDEX IF NOT EXISTS "idx_nodes_last_active" ON "nodes" ("last_active" DESC); +CREATE INDEX IF NOT EXISTS "idx_nodes_fork_digest" ON "nodes" ("fork_digest"); +CREATE INDEX IF NOT EXISTS "idx_nodes_layer_last_active" ON "nodes" ("layer", "last_active" DESC); + +CREATE TABLE "bad_nodes_old" ( + "nodeid" BLOB PRIMARY KEY, + "layer" TEXT NOT NULL, + "rejected_at" INTEGER NOT NULL, + "reason" TEXT +); + +INSERT INTO "bad_nodes_old" SELECT nodeid, layer, rejected_at, reason +FROM "bad_nodes" +WHERE layer = 'el' + OR nodeid NOT IN (SELECT nodeid FROM "bad_nodes" WHERE layer = 'el'); + +DROP TABLE "bad_nodes"; +ALTER TABLE "bad_nodes_old" RENAME TO "bad_nodes"; + +CREATE INDEX IF NOT EXISTS "idx_bad_nodes_layer" ON "bad_nodes" ("layer"); +CREATE INDEX IF NOT EXISTS "idx_bad_nodes_rejected_at" ON "bad_nodes" ("rejected_at"); + +-- +goose StatementEnd diff --git a/discv4/node/node.go b/discv4/node/node.go index c85fa6b..1262115 100644 --- a/discv4/node/node.go +++ b/discv4/node/node.go @@ -55,6 +55,14 @@ type Node struct { bondExpiration time.Time consecutiveTimeout uint32 // Bond-specific consecutive timeout counter + // bondedIPs maps a proven remote IP to when its bond expires. Keyed per IP + // because a bond proves reachability at one address only: addr is rewritten + // from whatever source last sent us a packet, so serving requests on the + // strength of a bond earned elsewhere lets a spoofed source reflect our + // replies at a third party. Ports are excluded so a NAT remap does not + // silently drop a peer mid-bond. + bondedIPs map[string]time.Time + // Statistics (shared with generic node wrapper) stats *stats.SharedStats @@ -192,6 +200,24 @@ func (n *Node) SetENR(record *enr.Record) { n.mu.Unlock() } +// UpdateENR installs the record only if it is newer than the current one, so +// a replayed response cannot roll the node back to an older record. +func (n *Node) UpdateENR(record *enr.Record) bool { + if record == nil { + return false + } + + n.mu.Lock() + defer n.mu.Unlock() + + if n.enr != nil && record.Seq() <= n.enr.Seq() { + return false + } + n.enr = record + + return true +} + // statsRef returns the current shared stats pointer for use outside the lock. func (n *Node) statsRef() *stats.SharedStats { n.mu.RLock() @@ -252,6 +278,21 @@ func (n *Node) IsBonded() bool { return true } +// IsBondedFrom reports whether this node proved reachability at addr's IP and +// that proof is still valid. Inbound request handlers must use this rather than +// IsBonded, so a bond earned at one address cannot serve replies to another. +func (n *Node) IsBondedFrom(addr *net.UDPAddr) bool { + if addr == nil || addr.IP == nil { + return false + } + + n.bondMu.RLock() + defer n.bondMu.RUnlock() + + expiry, ok := n.bondedIPs[addr.IP.String()] + return ok && time.Now().Before(expiry) +} + // MarkPingSent records that we sent a PING to this node. func (n *Node) MarkPingSent() { now := time.Now() @@ -287,8 +328,11 @@ func (n *Node) MarkPongSent() { // MarkPongReceived records that we received a PONG from this node. // -// This establishes or renews the bond. -func (n *Node) MarkPongReceived(bondDuration time.Duration) { +// This establishes or renews the bond. provenAddr is the address the answered +// PING was sent to, not the PONG's source: the source is attacker-chosen on a +// spoofed packet, so binding the bond to it would prove nothing. Pass nil only +// where no endpoint was proven. +func (n *Node) MarkPongReceived(bondDuration time.Duration, provenAddr *net.UDPAddr) { now := time.Now() n.bondMu.Lock() @@ -296,6 +340,17 @@ func (n *Node) MarkPongReceived(bondDuration time.Duration) { n.bondStatus = BondStatusBonded n.bondExpiration = now.Add(bondDuration) n.consecutiveTimeout = 0 + if provenAddr != nil && provenAddr.IP != nil { + if n.bondedIPs == nil { + n.bondedIPs = make(map[string]time.Time) + } + for ip, expiry := range n.bondedIPs { + if now.After(expiry) { + delete(n.bondedIPs, ip) + } + } + n.bondedIPs[provenAddr.IP.String()] = now.Add(bondDuration) + } n.bondMu.Unlock() n.statsRef().ResetFailureCount() diff --git a/discv4/protocol/endpoint_proof_test.go b/discv4/protocol/endpoint_proof_test.go new file mode 100644 index 0000000..38ff8fb --- /dev/null +++ b/discv4/protocol/endpoint_proof_test.go @@ -0,0 +1,289 @@ +package protocol + +import ( + "context" + "net" + "sync" + "testing" + "time" + + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethpandaops/bootnodoor/discv4/node" +) + +// recordingTransport captures the destinations we send to, so a test can assert +// that no reply was reflected at a spoofed address. +type recordingTransport struct { + mu sync.Mutex + sent []string +} + +func (r *recordingTransport) SendTo(_ []byte, to *net.UDPAddr) error { + r.mu.Lock() + defer r.mu.Unlock() + r.sent = append(r.sent, to.String()) + return nil +} + +func (r *recordingTransport) Send(_ []byte, to *net.UDPAddr, _ *net.UDPAddr) error { + return r.SendTo(nil, to) +} + +func (r *recordingTransport) sentTo(addr *net.UDPAddr) bool { + r.mu.Lock() + defer r.mu.Unlock() + for _, s := range r.sent { + if s == addr.String() { + return true + } + } + return false +} + +func proofHandler(t *testing.T) (*Handler, *recordingTransport, context.CancelFunc) { + t.Helper() + key, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("generate key: %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + tr := &recordingTransport{} + return NewHandler(ctx, HandlerConfig{PrivateKey: key, LocalAddr: testAddr()}, tr), tr, cancel +} + +// bondAt drives a full PING/PONG exchange so the node ends up bonded at addr, +// the way production does: register the PING we sent, then answer it. +func bondAt(t *testing.T, h *Handler, n *node.Node, addr *net.UDPAddr) { + t.Helper() + + n.SetAddr(addr) + hash := []byte("ping-hash-" + addr.String()) + if _, err := h.addPendingRequest(hash, n, PingPacket, n.Addr()); err != nil { + t.Fatalf("addPendingRequest: %v", err) + } + pong := &Pong{ReplyTok: hash, Expiration: MakeExpiration(20 * time.Second)} + if err := h.handlePong(n, addr, pong); err != nil { + t.Fatalf("handlePong: %v", err) + } + if !n.IsBondedFrom(addr) { + t.Fatalf("node not bonded at %s after a matched PONG", addr) + } +} + +// A bond proves reachability at one address only. Serving FINDNODE from any +// other source lets an attacker who bonded legitimately spoof a victim's source +// and have us reflect the much larger NEIGHBORS at that victim. +func TestFindnodeFromUnbondedAddressRejected(t *testing.T) { + h, tr, cancel := proofHandler(t) + defer cancel() + + n, _ := makeKeyedNode(t, 30303) + attacker := &net.UDPAddr{IP: net.IPv4(1, 2, 3, 4), Port: 30303} + bondAt(t, h, n, attacker) + + victim := &net.UDPAddr{IP: net.IPv4(203, 0, 113, 9), Port: 30303} + err := h.handleFindnode(n, victim, testAddr(), &Findnode{Expiration: MakeExpiration(20 * time.Second)}) + if err == nil { + t.Fatal("FINDNODE from an unbonded source address was served") + } + if tr.sentTo(victim) { + t.Fatal("reflected a reply at the spoofed victim address") + } + if h.GetStats().UnbondedFindnode == 0 { + t.Error("unbondedFindnode counter did not move") + } +} + +// The legitimate case must still work, or the gate has simply broken discovery. +func TestFindnodeFromBondedAddressServed(t *testing.T) { + h, _, cancel := proofHandler(t) + defer cancel() + + n, _ := makeKeyedNode(t, 30303) + addr := &net.UDPAddr{IP: net.IPv4(1, 2, 3, 4), Port: 30303} + bondAt(t, h, n, addr) + + if err := h.handleFindnode(n, addr, testAddr(), &Findnode{Expiration: MakeExpiration(20 * time.Second)}); err != nil { + t.Fatalf("FINDNODE from the bonded address was refused: %v", err) + } +} + +// One node ID can legitimately bond over both address families, so a per-IP bond +// must not let the second exchange invalidate the first. +func TestDualStackPeerKeepsBothBonds(t *testing.T) { + h, _, cancel := proofHandler(t) + defer cancel() + + n, _ := makeKeyedNode(t, 30303) + v4 := &net.UDPAddr{IP: net.IPv4(1, 2, 3, 4), Port: 30303} + v6 := &net.UDPAddr{IP: net.ParseIP("2001:db8::1"), Port: 30303} + + bondAt(t, h, n, v4) + bondAt(t, h, n, v6) + + if !n.IsBondedFrom(v4) { + t.Error("IPv4 bond was lost when the IPv6 bond was established") + } + if !n.IsBondedFrom(v6) { + t.Error("IPv6 bond was not established") + } +} + +// Receiving a PING proves nothing about the source: we pong whatever address the +// packet claimed, so bonding here would bond a spoofed victim. +func TestInboundPingDoesNotBond(t *testing.T) { + h, _, cancel := proofHandler(t) + defer cancel() + + n, _ := makeKeyedNode(t, 30303) + addr := &net.UDPAddr{IP: net.IPv4(5, 6, 7, 8), Port: 30303} + n.SetAddr(addr) + + ping := &Ping{Version: 4, Expiration: MakeExpiration(20 * time.Second)} + if err := h.handlePing(n, addr, testAddr(), ping, []byte("hash")); err != nil { + t.Fatalf("handlePing: %v", err) + } + + if n.IsBondedFrom(addr) { + t.Fatal("an inbound PING alone established a bond") + } + if n.LastSeen().IsZero() { + t.Error("MarkPingReceived did not take effect") + } +} + +// A PONG must not be matched by a token belonging to a different request type: +// peers know the hashes of the packets we send them. +func TestPongMatchingRejectsNonPingRequest(t *testing.T) { + h, _, cancel := proofHandler(t) + defer cancel() + + n, _ := makeKeyedNode(t, 30303) + addr := n.Addr() + + called := 0 + h.config.OnPongReceived = func(*node.Node, *net.UDPAddr, net.IP, uint16) { called++ } + + hash := []byte("enr-request-hash") + if _, err := h.addPendingRequest(hash, n, ENRRequestPacket, n.Addr()); err != nil { + t.Fatalf("addPendingRequest: %v", err) + } + + pong := &Pong{ + ReplyTok: hash, + To: NewEndpoint(&net.UDPAddr{IP: net.IPv4(9, 9, 9, 9), Port: 30303}, 0), + Expiration: MakeExpiration(20 * time.Second), + } + if err := h.handlePong(n, addr, pong); err != nil { + t.Fatalf("handlePong: %v", err) + } + + if n.IsBondedFrom(addr) { + t.Error("a PONG matching an ENRREQUEST token established a bond") + } + if called != 0 { + t.Errorf("OnPongReceived fired %d times for a non-PING match", called) + } +} + +// The match is consumed once, so a replayed PONG cannot cast repeated +// external-IP votes off a single PING. +func TestReplayedPongAppliesSideEffectsOnce(t *testing.T) { + h, _, cancel := proofHandler(t) + defer cancel() + + n, _ := makeKeyedNode(t, 30303) + addr := n.Addr() + + called := 0 + h.config.OnPongReceived = func(*node.Node, *net.UDPAddr, net.IP, uint16) { called++ } + + hash := []byte("ping-hash") + if _, err := h.addPendingRequest(hash, n, PingPacket, n.Addr()); err != nil { + t.Fatalf("addPendingRequest: %v", err) + } + + pong := &Pong{ + ReplyTok: hash, + To: NewEndpoint(&net.UDPAddr{IP: net.IPv4(9, 9, 9, 9), Port: 30303}, 0), + Expiration: MakeExpiration(20 * time.Second), + } + for i := 0; i < 3; i++ { + if err := h.handlePong(n, addr, pong); err != nil { + t.Fatalf("handlePong %d: %v", i, err) + } + } + + if called != 1 { + t.Fatalf("OnPongReceived fired %d times for a replayed PONG, want 1", called) + } +} + +// The proven address must reach the IP-discovery callback directly. Reading it +// back off the node would hand over whatever address the last inbound packet +// set, which is attacker-controlled and undoes the endpoint proof. +func TestPongCallbackReceivesProvenAddress(t *testing.T) { + h, _, cancel := proofHandler(t) + defer cancel() + + n, _ := makeKeyedNode(t, 30303) + sentTo := n.Addr() + + var got *net.UDPAddr + h.config.OnPongReceived = func(_ *node.Node, provenAddr *net.UDPAddr, _ net.IP, _ uint16) { + got = provenAddr + } + + hash := []byte("ping-hash") + if _, err := h.addPendingRequest(hash, n, PingPacket, sentTo); err != nil { + t.Fatalf("addPendingRequest: %v", err) + } + + // A concurrent packet from the same identity rewrites the node's address + // before the PONG is processed, exactly as getOrCreateNode does. + n.SetAddr(&net.UDPAddr{IP: net.IPv4(203, 0, 113, 9), Port: 30303}) + + pong := &Pong{ + ReplyTok: hash, + To: NewEndpoint(&net.UDPAddr{IP: net.IPv4(9, 9, 9, 9), Port: 30303}, 0), + Expiration: MakeExpiration(20 * time.Second), + } + if err := h.handlePong(n, sentTo, pong); err != nil { + t.Fatalf("handlePong: %v", err) + } + + if got == nil { + t.Fatal("OnPongReceived never fired for a solicited PONG") + } + if !got.IP.Equal(sentTo.IP) { + t.Fatalf("callback got %s, want the proven %s", got.IP, sentTo.IP) + } +} + +// A PONG whose source is not the address the PING went to proves only that +// somebody received that PING, which is what the spoofing attack relies on. +func TestPongFromWrongSourceRejected(t *testing.T) { + h, _, cancel := proofHandler(t) + defer cancel() + + n, _ := makeKeyedNode(t, 30303) + sentTo := n.Addr() + + hash := []byte("ping-hash") + if _, err := h.addPendingRequest(hash, n, PingPacket, n.Addr()); err != nil { + t.Fatalf("addPendingRequest: %v", err) + } + + victim := &net.UDPAddr{IP: net.IPv4(203, 0, 113, 9), Port: 30303} + pong := &Pong{ReplyTok: hash, Expiration: MakeExpiration(20 * time.Second)} + if err := h.handlePong(n, victim, pong); err != nil { + t.Fatalf("handlePong: %v", err) + } + + if n.IsBondedFrom(victim) { + t.Fatal("a PONG spoofed from a victim address bonded that address") + } + if n.IsBondedFrom(sentTo) { + t.Fatal("a PONG from the wrong source bonded the PING destination") + } +} diff --git a/discv4/protocol/enr_refresh_test.go b/discv4/protocol/enr_refresh_test.go new file mode 100644 index 0000000..5d45b53 --- /dev/null +++ b/discv4/protocol/enr_refresh_test.go @@ -0,0 +1,409 @@ +package protocol + +import ( + "context" + "crypto/ecdsa" + "net" + "sync" + "testing" + "time" + + "github.com/ethpandaops/bootnodoor/discv4/node" +) + +// scriptedPeer decodes what the handler sends and answers it the way a real peer +// would. recordingTransport cannot be used here: it discards the packet bytes, +// and a PONG-triggered refresh emits nothing unless its PINGs are answered. +type scriptedPeer struct { + t *testing.T + h *Handler + key *ecdsa.PrivateKey + + mu sync.Mutex + pings int + enrReqs int + enrSeq uint64 + stopped bool + bumpEachPong bool + pongAddr *net.UDPAddr +} + +func (p *scriptedPeer) SendTo(data []byte, to *net.UDPAddr) error { + packet, _, hash, err := Decode(data) + if err != nil { + return nil + } + + p.mu.Lock() + if p.stopped { + p.mu.Unlock() + return nil + } + switch packet.(type) { + case *Ping: + p.pings++ + // Bump on PING only, so every round's PONG advertises strictly more than + // the record the previous round installed. Bumping on the ENRREQUEST too + // would put the installed record ahead and the loop would not re-arm. + if p.bumpEachPong { + p.enrSeq++ + } + case *ENRRequest: + p.enrReqs++ + } + seq := p.enrSeq + p.mu.Unlock() + + var reply []byte + switch packet.(type) { + case *Ping: + reply, _ = EncodePacket(p.key, &Pong{ + To: Endpoint{IP: net.IPv4(127, 0, 0, 1), UDP: 30303}, + ReplyTok: hash, + Expiration: MakeExpiration(20 * time.Second), + ENRSeq: seq, + }) + case *ENRRequest: + reply, _ = EncodePacket(p.key, &ENRResponse{ + ReplyTok: hash, + Record: signedV4Record(p.t, p.key, seq), + }) + } + + if reply != nil { + go func() { + if err := p.h.HandlePacket(reply, p.pongAddr, nil); err != nil { + p.t.Logf("reply not accepted: %v", err) + } + }() + } + return nil +} + +func (p *scriptedPeer) Send(data []byte, to *net.UDPAddr, _ *net.UDPAddr) error { + return p.SendTo(data, to) +} + +func (p *scriptedPeer) counts() (int, int) { + p.mu.Lock() + defer p.mu.Unlock() + return p.pings, p.enrReqs +} + +func (p *scriptedPeer) stop() { + p.mu.Lock() + p.stopped = true + p.mu.Unlock() +} + +// A PONG advertising a sequence above the cached record starts an ENR refresh. +// That refresh PINGs, and its PONG re-enters handlePong with the cached sequence +// still stale, so an unguarded trigger spawns another refresh at RTT speed — +// thousands of PING/ENRREQUEST pairs against one peer in the devnet capture. +func TestPongDrivenENRRefreshRunsOnce(t *testing.T) { + peer := &scriptedPeer{t: t, enrSeq: 5, pongAddr: &net.UDPAddr{IP: net.IPv4(1, 2, 3, 4), Port: 30303}} + + h, cancel := newScriptedHandler(t, peer) + defer cancel() + peer.h = h + + n, key := makeKeyedNode(t, 30303) + peer.key = key + if !n.UpdateENR(signedV4Record(t, key, 1)) { + t.Fatal("seed record was not installed") + } + h.nodesMu.Lock() + h.nodes[n.ID()] = n + h.nodesMu.Unlock() + + req, err := h.addPendingRequest([]byte("seed-ping-hash"), n, PingPacket, n.Addr()) + if err != nil { + t.Fatalf("addPendingRequest: %v", err) + } + + pong := &Pong{ + To: Endpoint{IP: net.IPv4(127, 0, 0, 1), UDP: 30303}, + ReplyTok: req.RequestHash, + Expiration: MakeExpiration(20 * time.Second), + ENRSeq: 5, + } + data, err := EncodePacket(key, pong) + if err != nil { + t.Fatalf("encode pong: %v", err) + } + if err := h.HandlePacket(data, n.Addr(), nil); err != nil { + t.Fatalf("handle pong: %v", err) + } + + time.Sleep(2 * time.Second) + peer.stop() + pings, enrReqs := peer.counts() + + if enrReqs > 1 { + t.Errorf("ENRREQUESTs sent = %d, want at most 1 (refresh not coalesced)", enrReqs) + } + if pings > 2 { + t.Errorf("PINGs sent = %d, want at most 2 (refresh not coalesced)", pings) + } + t.Logf("pings=%d enrRequests=%d", pings, enrReqs) +} + +func newScriptedHandler(t *testing.T, peer *scriptedPeer) (*Handler, func()) { + t.Helper() + + ctx, cancel := context.WithCancel(context.Background()) + h := NewHandler(ctx, HandlerConfig{ + PrivateKey: mustHandlerKey(t), + LocalAddr: &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 30304}, + BondExpiration: time.Hour, + NodeTTL: time.Hour, + ExpirationWindow: 20 * time.Second, + }, peer) + return h, cancel +} + +func mustHandlerKey(t *testing.T) *ecdsa.PrivateKey { + t.Helper() + _, key := makeKeyedNode(t, 1) + return key +} + +func (p *scriptedPeer) setSeq(seq uint64) { + p.mu.Lock() + p.enrSeq = seq + p.mu.Unlock() +} + +func deliverPong(t *testing.T, h *Handler, n *node.Node, key *ecdsa.PrivateKey, token []byte, seq uint64) { + t.Helper() + + req, err := h.addPendingRequest(token, n, PingPacket, n.Addr()) + if err != nil { + t.Fatalf("addPendingRequest: %v", err) + } + data, err := EncodePacket(key, &Pong{ + To: Endpoint{IP: net.IPv4(127, 0, 0, 1), UDP: 30303}, + ReplyTok: req.RequestHash, + Expiration: MakeExpiration(20 * time.Second), + ENRSeq: seq, + }) + if err != nil { + t.Fatalf("encode pong: %v", err) + } + if err := h.HandlePacket(data, n.Addr(), nil); err != nil { + t.Fatalf("handle pong: %v", err) + } +} + +// A sequence advertised after the running attempt started is real new data, so it +// must produce exactly one more refresh — coalescing must not swallow it. +func TestENRRefreshRearmsForBumpDuringRefresh(t *testing.T) { + peer := &scriptedPeer{t: t, enrSeq: 5, pongAddr: &net.UDPAddr{IP: net.IPv4(1, 2, 3, 4), Port: 30303}} + + h, cancel := newScriptedHandler(t, peer) + defer cancel() + peer.h = h + + n, key := makeKeyedNode(t, 30303) + peer.key = key + if !n.UpdateENR(signedV4Record(t, key, 1)) { + t.Fatal("seed record was not installed") + } + h.nodesMu.Lock() + h.nodes[n.ID()] = n + h.nodesMu.Unlock() + + deliverPong(t, h, n, key, []byte("first-ping-hash"), 5) + + // Ping sleeps 500ms before the ENRREQUEST, so this lands mid-refresh. + time.Sleep(100 * time.Millisecond) + peer.setSeq(6) + deliverPong(t, h, n, key, []byte("second-ping-hash"), 6) + + time.Sleep(3 * time.Second) + peer.stop() + _, enrReqs := peer.counts() + + if enrReqs != 2 { + t.Errorf("ENRREQUESTs sent = %d, want 2 (one per observed bump)", enrReqs) + } +} + +// Eviction removes the refresh state; a refresh completing afterwards must not +// resurrect an entry, or the map grows for every peer that ever left. +func TestENRRefreshStateClearedOnEviction(t *testing.T) { + peer := &scriptedPeer{t: t, enrSeq: 5, pongAddr: &net.UDPAddr{IP: net.IPv4(1, 2, 3, 4), Port: 30303}} + + h, cancel := newScriptedHandler(t, peer) + defer cancel() + peer.h = h + + n, key := makeKeyedNode(t, 30303) + peer.key = key + if !n.UpdateENR(signedV4Record(t, key, 1)) { + t.Fatal("seed record was not installed") + } + + h.startENRRefresh(n, 5) + + h.enrRefreshMu.Lock() + delete(h.enrRefresh, n.ID()) + h.enrRefreshMu.Unlock() + + time.Sleep(2 * time.Second) + peer.stop() + + h.enrRefreshMu.Lock() + _, present := h.enrRefresh[n.ID()] + h.enrRefreshMu.Unlock() + + if present { + t.Error("refresh state was recreated after eviction") + } +} + +// Concurrent triggers must not race on the refresh map. +func TestENRRefreshConcurrentTriggers(t *testing.T) { + peer := &scriptedPeer{t: t, enrSeq: 5, pongAddr: &net.UDPAddr{IP: net.IPv4(1, 2, 3, 4), Port: 30303}} + + h, cancel := newScriptedHandler(t, peer) + defer cancel() + peer.h = h + + n, key := makeKeyedNode(t, 30303) + peer.key = key + if !n.UpdateENR(signedV4Record(t, key, 1)) { + t.Fatal("seed record was not installed") + } + + var wg sync.WaitGroup + for i := 0; i < 50; i++ { + wg.Add(1) + go func(seq uint64) { + defer wg.Done() + h.startENRRefresh(n, seq) + }(uint64(5 + i%3)) + } + wg.Wait() + + time.Sleep(1500 * time.Millisecond) + peer.stop() +} + +// A peer that advertises a higher sequence in every PONG could otherwise re-arm +// the refresh forever, holding a goroutine and generating traffic until shutdown. +func TestENRRefreshBoundedAgainstEndlessBumps(t *testing.T) { + peer := &scriptedPeer{t: t, enrSeq: 5, pongAddr: &net.UDPAddr{IP: net.IPv4(1, 2, 3, 4), Port: 30303}, bumpEachPong: true} + + h, cancel := newScriptedHandler(t, peer) + defer cancel() + peer.h = h + + n, key := makeKeyedNode(t, 30303) + peer.key = key + if !n.UpdateENR(signedV4Record(t, key, 1)) { + t.Fatal("seed record was not installed") + } + h.nodesMu.Lock() + h.nodes[n.ID()] = n + h.nodesMu.Unlock() + + deliverPong(t, h, n, key, []byte("bump-ping-hash"), 5) + + time.Sleep(4 * time.Second) + peer.stop() + _, enrReqs := peer.counts() + + // Literal, not maxENRRefreshRounds: comparing against the constant under test + // makes the assertion vacuous when the bound is raised. + if enrReqs > 4 { + t.Errorf("ENRREQUESTs sent = %d, want at most 4 rounds per claim", enrReqs) + } + + h.enrRefreshMu.Lock() + inFlight := h.enrRefresh[n.ID()].inFlight + h.enrRefreshMu.Unlock() + if inFlight { + t.Error("refresh still marked in flight after the round bound was reached") + } +} + +// The round bound is per claim, so without a cooldown a peer could open a fresh +// claim immediately and sustain the same rate one PING at a time. +func TestENRRefreshCooldownAfterRoundsExhausted(t *testing.T) { + peer := &scriptedPeer{t: t, enrSeq: 5, pongAddr: &net.UDPAddr{IP: net.IPv4(1, 2, 3, 4), Port: 30303}, bumpEachPong: true} + + h, cancel := newScriptedHandler(t, peer) + defer cancel() + peer.h = h + + n, key := makeKeyedNode(t, 30303) + peer.key = key + if !n.UpdateENR(signedV4Record(t, key, 1)) { + t.Fatal("seed record was not installed") + } + h.nodesMu.Lock() + h.nodes[n.ID()] = n + h.nodesMu.Unlock() + + deliverPong(t, h, n, key, []byte("cooldown-ping-1"), 5) + time.Sleep(3500 * time.Millisecond) + + _, afterFirst := peer.counts() + + h.enrRefreshMu.Lock() + cooling := time.Now().Before(h.enrRefresh[n.ID()].cooldownUntil) + h.enrRefreshMu.Unlock() + if !cooling { + t.Fatal("no cooldown was set after the rounds were exhausted") + } + + // A fresh trigger during the cooldown must not open another claim. + deliverPong(t, h, n, key, []byte("cooldown-ping-2"), 99) + time.Sleep(1500 * time.Millisecond) + peer.stop() + + _, afterSecond := peer.counts() + if afterSecond != afterFirst { + t.Errorf("ENRREQUESTs went %d -> %d during the cooldown, want no new claim", afterFirst, afterSecond) + } +} + +// A bump seen during a cooldown is remembered but not fetched, so the cached +// record would stay stale unless another PONG happened to arrive later. +func TestENRRefreshResumesBumpDeferredByCooldown(t *testing.T) { + peer := &scriptedPeer{t: t, enrSeq: 5, pongAddr: &net.UDPAddr{IP: net.IPv4(1, 2, 3, 4), Port: 30303}} + + h, cancel := newScriptedHandler(t, peer) + defer cancel() + peer.h = h + + n, key := makeKeyedNode(t, 30303) + peer.key = key + if !n.UpdateENR(signedV4Record(t, key, 1)) { + t.Fatal("seed record was not installed") + } + h.nodesMu.Lock() + h.nodes[n.ID()] = n + h.nodesMu.Unlock() + + h.enrRefreshMu.Lock() + h.enrRefresh[n.ID()] = &enrRefreshState{highestSeenSeq: 9, targetSeq: 1} + h.enrRefreshMu.Unlock() + + peer.setSeq(9) + h.resumeDeferredENRRefreshes() + + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + if _, enrReqs := peer.counts(); enrReqs > 0 { + break + } + time.Sleep(50 * time.Millisecond) + } + peer.stop() + + if _, enrReqs := peer.counts(); enrReqs == 0 { + t.Error("a bump deferred by cooldown was never fetched") + } +} diff --git a/discv4/protocol/handle_packet_test.go b/discv4/protocol/handle_packet_test.go new file mode 100644 index 0000000..0418c84 --- /dev/null +++ b/discv4/protocol/handle_packet_test.go @@ -0,0 +1,195 @@ +package protocol + +import ( + "crypto/ecdsa" + "net" + "testing" + "time" + + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethpandaops/bootnodoor/discv4/node" +) + +// encodeFrom builds a real signed packet from a peer, so tests can drive +// HandlePacket end to end rather than calling handlers directly. +func encodeFrom(t *testing.T, key *ecdsa.PrivateKey, msg Packet) ([]byte, []byte) { + t.Helper() + data, hash, err := Encode(key, msg) + if err != nil { + t.Fatalf("Encode: %v", err) + } + return data, hash +} + +// A peer's claimed source address must not become the node's canonical address: +// every sender reads it and sendNeighbors republishes it, so an unauthenticated +// packet could otherwise steer our traffic and poison what we tell others. +func TestHandlePacketDoesNotMoveCanonicalAddress(t *testing.T) { + h, _, cancel := proofHandler(t) + defer cancel() + + peerKey, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("generate key: %v", err) + } + knownAddr := &net.UDPAddr{IP: net.IPv4(198, 51, 100, 5), Port: 30303} + n := node.New(&peerKey.PublicKey, knownAddr) + + h.nodesMu.Lock() + h.nodes[n.ID()] = n + h.nodesMu.Unlock() + + spoofed := &net.UDPAddr{IP: net.IPv4(203, 0, 113, 9), Port: 40404} + data, _ := encodeFrom(t, peerKey, &Findnode{ + Target: EncodePubkey(&peerKey.PublicKey), + Expiration: MakeExpiration(20 * time.Second), + }) + _ = h.HandlePacket(data, spoofed, testAddr()) + + if got := n.Addr().String(); got != knownAddr.String() { + t.Fatalf("canonical address moved to %s on an unauthenticated packet, want %s", got, knownAddr) + } +} + +// An expired packet must not refresh liveness or fire OnNodeSeen. The node has to +// pre-exist, because creating one stamps LastSeen. +func TestHandlePacketExpiredTouchesNothing(t *testing.T) { + h, _, cancel := proofHandler(t) + defer cancel() + + seen := 0 + h.config.OnNodeSeen = func(*node.Node, time.Time) { seen++ } + + peerKey, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("generate key: %v", err) + } + addr := &net.UDPAddr{IP: net.IPv4(198, 51, 100, 5), Port: 30303} + n := node.New(&peerKey.PublicKey, addr) + + h.nodesMu.Lock() + h.nodes[n.ID()] = n + h.nodesMu.Unlock() + + before := n.LastSeen() + time.Sleep(5 * time.Millisecond) + + data, _ := encodeFrom(t, peerKey, &Findnode{ + Target: EncodePubkey(&peerKey.PublicKey), + Expiration: uint64(time.Now().Add(-time.Minute).Unix()), + }) + _ = h.HandlePacket(data, addr, testAddr()) + + if !n.LastSeen().Equal(before) { + t.Error("expired packet refreshed liveness") + } + if seen != 0 { + t.Errorf("expired packet fired OnNodeSeen %d times", seen) + } +} + +// An unbonded FINDNODE is refused, so it must not admit the node either — that is +// the callback which can spawn outbound traffic toward an unproven address. +func TestHandlePacketUnbondedFindnodeDoesNotAdmit(t *testing.T) { + h, _, cancel := proofHandler(t) + defer cancel() + + seen := 0 + h.config.OnNodeSeen = func(*node.Node, time.Time) { seen++ } + + peerKey, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("generate key: %v", err) + } + addr := &net.UDPAddr{IP: net.IPv4(198, 51, 100, 5), Port: 30303} + + data, _ := encodeFrom(t, peerKey, &Findnode{ + Target: EncodePubkey(&peerKey.PublicKey), + Expiration: MakeExpiration(20 * time.Second), + }) + _ = h.HandlePacket(data, addr, testAddr()) + + if seen != 0 { + t.Fatalf("unbonded FINDNODE fired OnNodeSeen %d times, want 0", seen) + } +} + +// The blackhole regression test. Removing the address rewrite means a peer that +// moves is only reachable if the reciprocal PING goes to the source we just +// ponged; otherwise it is pinged at its old address forever, never bonds, and is +// refused service permanently. +func TestHandlePacketMovedPeerRebondsAtNewAddress(t *testing.T) { + h, tr, cancel := proofHandler(t) + defer cancel() + + peerKey, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("generate key: %v", err) + } + oldAddr := &net.UDPAddr{IP: net.IPv4(198, 51, 100, 5), Port: 30303} + newAddr := &net.UDPAddr{IP: net.IPv4(198, 51, 100, 77), Port: 30303} + n := node.New(&peerKey.PublicKey, oldAddr) + + h.nodesMu.Lock() + h.nodes[n.ID()] = n + h.nodesMu.Unlock() + + // The peer pings us from its new address. + data, _ := encodeFrom(t, peerKey, &Ping{ + Version: 4, + From: NewEndpoint(newAddr, 0), + To: NewEndpoint(testAddr(), 0), + Expiration: MakeExpiration(20 * time.Second), + }) + if err := h.HandlePacket(data, newAddr, testAddr()); err != nil { + t.Fatalf("HandlePacket(ping): %v", err) + } + + // The reciprocal PING is sent from a goroutine. + deadline := time.Now().Add(2 * time.Second) + var pingHash []byte + for time.Now().Before(deadline) { + if req := h.findPendingPingTo(n.ID(), newAddr); req != nil { + pingHash = req.RequestHash + break + } + time.Sleep(5 * time.Millisecond) + } + if pingHash == nil { + t.Fatalf("no PING was sent to the peer's new address; destinations: %v", tr.sent) + } + + // Its PONG from the new address proves the endpoint. + pongData, _ := encodeFrom(t, peerKey, &Pong{ + To: NewEndpoint(testAddr(), 0), + ReplyTok: pingHash, + Expiration: MakeExpiration(20 * time.Second), + }) + if err := h.HandlePacket(pongData, newAddr, testAddr()); err != nil { + t.Fatalf("HandlePacket(pong): %v", err) + } + + if !n.IsBondedFrom(newAddr) { + t.Error("peer did not bond at its new address") + } + if got := n.Addr().String(); got != newAddr.String() { + t.Errorf("canonical address = %s after a proven PONG, want %s", got, newAddr) + } +} + +// findPendingPingTo reports the pending PING sent to addr, for tests that need the +// reply token of a PING the handler emitted itself. +func (h *Handler) findPendingPingTo(id node.ID, addr *net.UDPAddr) *PendingRequest { + h.requestsMu.RLock() + defer h.requestsMu.RUnlock() + + for _, reqs := range h.requests { + for _, req := range reqs { + if req.PacketType == PingPacket && req.ToNode != nil && req.ToNode.ID() == id && + req.DestIP != nil && req.DestIP.Equal(addr.IP) { + return req + } + } + } + return nil +} diff --git a/discv4/protocol/handler.go b/discv4/protocol/handler.go index fc1c72b..b3667bd 100644 --- a/discv4/protocol/handler.go +++ b/discv4/protocol/handler.go @@ -5,6 +5,7 @@ import ( "crypto/ecdsa" "fmt" "net" + "slices" "sync" "time" @@ -37,7 +38,12 @@ type OnNodeSeenCallback func(n *node.Node, timestamp time.Time) // OnPongReceivedCallback is called when a PONG response is received. // The ip and port parameters contain our external address as seen by the remote peer. -type OnPongReceivedCallback func(from *node.Node, ip net.IP, port uint16) +// +// provenAddr is the address the answered PING was sent to. Callers must use it +// rather than from.Addr(), which any later inbound packet rewrites, including a +// spoofed one; attributing a report to that address would undo the endpoint +// proof this callback is gated on. +type OnPongReceivedCallback func(from *node.Node, provenAddr *net.UDPAddr, ip net.IP, port uint16) // Handler handles incoming and outgoing discv4 protocol messages. // @@ -61,9 +67,17 @@ type Handler struct { nodesMu sync.RWMutex nodes map[node.ID]*node.Node - // Pending requests (hash -> PendingRequest) + // In-flight PONG-driven ENR refreshes, keyed by node ID. The refresh cannot + // update the cached sequence before its own PING is answered, so without this + // every PONG on the way re-triggers it. + enrRefreshMu sync.Mutex + enrRefresh map[node.ID]*enrRefreshState + + // Pending requests, keyed by packet hash + destination node ID: the hash + // alone aliases across peers (deterministic signatures, 1s Expiration + // granularity), and identical requests to one peer share a key's slice. requestsMu sync.RWMutex - requests map[string]*PendingRequest + requests map[string][]*PendingRequest // Pending multi-packet FINDNODE responses pendingNeighborsMu sync.RWMutex @@ -132,6 +146,11 @@ type PendingRequest struct { // ToNode is the destination node ToNode *node.Node + // DestIP is the IP the request was sent to, snapshotted at send time; see + // lookupOrCreateNode. ToNode.Addr() cannot serve here because promoteAddr can + // move it between send and response. + DestIP net.IP + // PacketType is the type of request PacketType byte @@ -223,7 +242,8 @@ func NewHandler(ctx context.Context, config HandlerConfig, transport Transport) ctx: ctx, transport: transport, nodes: make(map[node.ID]*node.Node), - requests: make(map[string]*PendingRequest), + enrRefresh: make(map[node.ID]*enrRefreshState), + requests: make(map[string][]*PendingRequest), pendingNeighbors: make(map[string]*PendingNeighborsResponse), localENR: config.LocalENR, } @@ -268,17 +288,11 @@ func (h *Handler) HandlePacket(data []byte, from *net.UDPAddr, localAddr *net.UD fromNodeID := node.PubkeyToID(pubkey) - // Get or create node - fromNode := h.getOrCreateNode(fromNodeID, pubkey, from) - - // Update last seen - fromNode.UpdateLastSeen() - fromNode.IncrementPacketsReceived() - - // Call OnNodeSeen callback - if h.config.OnNodeSeen != nil { - h.config.OnNodeSeen(fromNode, time.Now()) - } + // Look up the node without promoting this packet's source to its canonical + // address, and without touching liveness or firing OnNodeSeen: none of that is + // warranted before the handler has checked expiration and solicitation. Each + // handler states its own gate and calls noteSeen/noteProven itself. + fromNode := h.lookupOrCreateNode(fromNodeID, pubkey, from) // Dispatch by packet type switch p := packet.(type) { @@ -315,6 +329,15 @@ func (h *Handler) handlePing(fromNode *node.Node, from *net.UDPAddr, localAddr * return ErrExpired } + h.noteSeen(fromNode) + + // Admission and its outbound traffic need a proven source; an already-bonded + // peer has one. Otherwise the reciprocal PING below proves it a moment later + // and its PONG runs noteProven then. + if fromNode.IsBondedFrom(from) { + h.noteProven(fromNode) + } + // Mark ping received fromNode.MarkPingReceived() @@ -330,9 +353,11 @@ func (h *Handler) handlePing(fromNode *node.Node, from *net.UDPAddr, localAddr * return err } - // Mark node as bonded: they pinged us, we ponged them. - // This allows THEM to query US with FINDNODE immediately. - fromNode.MarkPongReceived(h.config.BondExpiration) + // Receiving a PING grants no bond: we ponged whatever address the packet + // claimed, which proves nothing if that source was spoofed. Bonding here + // would let an attacker bond a victim's address and then have us reflect + // NEIGHBORS at it. The bond is established by the reciprocal PING below, + // when its PONG comes back from the address we sent it to. // IMPORTANT: For bidirectional bonding (required by strict clients like reth for ENRRequest), // we also need to establish that WE can reach THEM, not just that they can reach us. @@ -343,11 +368,12 @@ func (h *Handler) handlePing(fromNode *node.Node, from *net.UDPAddr, localAddr * lastPingSent := fromNode.LastPingSent() timeSinceLastPing := time.Since(lastPingSent) - // Only spawn goroutine if we're actually going to ping (don't create unnecessary goroutines) if timeSinceLastPing > 100*time.Millisecond { - // Send PING back in goroutine to establish bidirectional bond + // Ping the source we just ponged, not the canonical address: a peer that + // moved is only reachable at its new address, and its PONG from there is + // what proves the new endpoint. go func() { - if _, err := h.Ping(fromNode); err != nil { + if _, err := h.pingTo(fromNode, from); err != nil { logrus.WithFields(logrus.Fields{ "node_id": fmt.Sprintf("%x", fromNode.IDBytes()[:8]), "error": err, @@ -373,32 +399,219 @@ func (h *Handler) handlePong(fromNode *node.Node, from *net.UDPAddr, pong *Pong) return ErrExpired } - // Mark pong received (establishes bond) - fromNode.MarkPongReceived(h.config.BondExpiration) + h.noteSeen(fromNode) - // Call OnPongReceived callback with the IP and port reported in the PONG - // The To field in PONG contains our address as seen by the remote peer - if h.config.OnPongReceived != nil && pong.To.IP != nil && pong.To.UDP > 0 { - h.config.OnPongReceived(fromNode, pong.To.IP, pong.To.UDP) + // Nothing below may run for a PONG we did not solicit from this address: it + // establishes a bond, casts a vote in the external-IP election that rewrites + // our published ENR, and can trigger outbound ENR traffic. + req := h.consumePendingPing(pong.ReplyTok, fromNode.ID(), from) + if req == nil { + return nil } - // Match to pending request - req := h.getPendingRequest(string(pong.ReplyTok)) - if req != nil { - h.deliverResponse(req, pong) + // Bind the bond to the address we proved, not the packet's source. + provenAddr := &net.UDPAddr{IP: req.DestIP, Port: from.Port} + h.promoteAddr(fromNode, provenAddr) + h.promoteAddr(req.ToNode, provenAddr) + fromNode.MarkPongReceived(h.config.BondExpiration, provenAddr) + h.noteProven(fromNode) + + // The To field in PONG contains our address as seen by the remote peer. + if h.config.OnPongReceived != nil && pong.To.IP != nil && pong.To.UDP > 0 { + h.config.OnPongReceived(fromNode, provenAddr, pong.To.IP, pong.To.UDP) } + h.deliverResponse(req, pong) + // Check if remote node has newer ENR if pong.ENRSeq > 0 && fromNode.ENR() != nil { if pong.ENRSeq > fromNode.ENR().Seq() { - // Request updated ENR - go h.RequestENR(fromNode) + h.startENRRefresh(fromNode, pong.ENRSeq) } } return nil } +// maxENRRefreshRetries bounds retries after a failed refresh so a peer that +// never answers cannot keep one running. +const maxENRRefreshRetries = 2 + +// maxENRRefreshRounds bounds the rounds one claim may run. Re-arming on a +// sequence observed mid-refresh is otherwise unbounded: a peer that advertises a +// higher sequence in every PONG keeps the goroutine and its PING/ENRREQUEST +// traffic alive indefinitely. Past the bound the claim ends, and a later PONG has +// to open a fresh one. +const maxENRRefreshRounds = 4 + +// enrRefreshCooldown follows a claim that exhausted its rounds. +const enrRefreshCooldown = 30 * time.Second + +// enrRefreshMinInterval separates consecutive claims for one peer. Bounding +// rounds alone is not enough: a peer can advertise one increment per claim, let +// it finish in a single round, and reopen on the next PONG, sustaining the same +// ENRREQUEST rate without ever exhausting a claim. +const enrRefreshMinInterval = 5 * time.Second + +// enrRefreshState tracks one peer's automatic ENR refresh. +type enrRefreshState struct { + inFlight bool + + // targetSeq is what the running attempt is fetching; highestSeenSeq is the + // largest advertised since. Only highestSeenSeq > targetSeq means a genuinely + // newer record appeared mid-refresh and another round is warranted. Comparing + // against the installed record instead would also retry after a failed or + // stale response, which never terminates. + targetSeq uint64 + highestSeenSeq uint64 + + retries int + rounds int + + // cooldownUntil applies after a claim exhausts its rounds. Without it the + // bound is per claim only: one inbound PING earns a reciprocal PING, whose + // higher-sequence PONG opens a fresh claim, so a peer could sustain the same + // ENRREQUEST rate with one packet per claim. + cooldownUntil time.Time +} + +// startENRRefresh claims the refresh for a peer and runs at most one at a time. +// The claim is taken here rather than inside RequestENR because a goroutine +// descheduled past the winner's release would otherwise become a new winner — +// under exactly the load this is meant to prevent. +func (h *Handler) startENRRefresh(n *node.Node, advertisedSeq uint64) { + id := n.ID() + + h.enrRefreshMu.Lock() + state := h.enrRefresh[id] + if state == nil { + state = &enrRefreshState{} + h.enrRefresh[id] = state + } + if advertisedSeq > state.highestSeenSeq { + state.highestSeenSeq = advertisedSeq + } + if state.inFlight || time.Now().Before(state.cooldownUntil) { + h.enrRefreshMu.Unlock() + return + } + state.inFlight = true + state.targetSeq = state.highestSeenSeq + state.retries = 0 + state.rounds = 0 + h.enrRefreshMu.Unlock() + + go h.runENRRefresh(n) +} + +// runENRRefresh fetches a peer's record, repeating only for a sequence observed +// after the current attempt started or a bounded number of failures. +func (h *Handler) runENRRefresh(n *node.Node) { + id := n.ID() + + for { + _, err := h.RequestENR(n) + + h.enrRefreshMu.Lock() + state := h.enrRefresh[id] + if state == nil { + h.enrRefreshMu.Unlock() + return + } + + state.rounds++ + + if state.highestSeenSeq > state.targetSeq && state.rounds < maxENRRefreshRounds { + state.targetSeq = state.highestSeenSeq + state.retries = 0 + h.enrRefreshMu.Unlock() + continue + } + + if err != nil && state.retries < maxENRRefreshRetries && state.rounds < maxENRRefreshRounds { + state.retries++ + // ExpirationWindow can be zero in a bare config; a zero backoff would + // make the retry immediate rather than delayed. + unit := h.config.ExpirationWindow + if unit <= 0 { + unit = 20 * time.Second + } + backoff := time.Duration(state.retries) * unit + h.enrRefreshMu.Unlock() + + select { + case <-time.After(backoff): + case <-h.ctx.Done(): + h.releaseENRRefresh(id) + return + } + continue + } + + cooldown := enrRefreshMinInterval + if state.rounds >= maxENRRefreshRounds { + cooldown = enrRefreshCooldown + } + state.cooldownUntil = time.Now().Add(cooldown) + state.inFlight = false + state.retries = 0 + h.enrRefreshMu.Unlock() + return + } +} + +// resumeDeferredENRRefreshes starts refreshes for peers whose newer sequence was +// observed during a cooldown. Without this the bump is remembered but never +// fetched unless another PONG happens to arrive after the cooldown expires. +func (h *Handler) resumeDeferredENRRefreshes() { + now := time.Now() + + type pending struct { + id node.ID + seq uint64 + } + var due []pending + + h.enrRefreshMu.Lock() + for id, state := range h.enrRefresh { + if state.inFlight || now.Before(state.cooldownUntil) { + continue + } + if state.highestSeenSeq > state.targetSeq { + due = append(due, pending{id, state.highestSeenSeq}) + } + } + h.enrRefreshMu.Unlock() + + if len(due) == 0 { + return + } + + h.nodesMu.RLock() + resume := make([]*node.Node, 0, len(due)) + for _, p := range due { + if n, ok := h.nodes[p.id]; ok && n.ENR() != nil && p.seq > n.ENR().Seq() { + resume = append(resume, n) + } + } + h.nodesMu.RUnlock() + + for _, n := range resume { + h.startENRRefresh(n, 0) + } +} + +// releaseENRRefresh clears the in-flight claim without recreating a state entry +// that eviction has already removed. +func (h *Handler) releaseENRRefresh(id node.ID) { + h.enrRefreshMu.Lock() + if state := h.enrRefresh[id]; state != nil { + state.inFlight = false + state.retries = 0 + } + h.enrRefreshMu.Unlock() +} + // handleFindnode processes a FINDNODE request. func (h *Handler) handleFindnode(fromNode *node.Node, from *net.UDPAddr, localAddr *net.UDPAddr, findnode *Findnode) error { logrus.WithFields(logrus.Fields{ @@ -413,14 +626,18 @@ func (h *Handler) handleFindnode(fromNode *node.Node, from *net.UDPAddr, localAd return ErrExpired } - // Check if node is bonded - if !fromNode.IsBonded() { + h.noteSeen(fromNode) + + // Bonded at this source address specifically: a bond earned elsewhere would + // let a spoofed source have us reflect NEIGHBORS at a third party. + if !fromNode.IsBondedFrom(from) { h.incrementUnbondedFindnode() logrus.WithField("node_id", fmt.Sprintf("%x", fromNode.IDBytes()[:8])). Debug("Rejected FINDNODE from unbonded node") return fmt.Errorf("node not bonded") } + h.noteProven(fromNode) h.incrementFindnodeRequestsRecv() // Call callback to get nodes @@ -448,22 +665,29 @@ func (h *Handler) handleNeighbors(fromNode *node.Node, from *net.UDPAddr, neighb return ErrExpired } - h.incrementFindnodeResponsesRecv() + h.noteSeen(fromNode) // Only accept NEIGHBORS in response to a FINDNODE we actually sent to this - // node. Dropping unsolicited NEIGHBORS prevents a peer we never queried from - // making us accumulate node records without bound. - matchedReq := h.findPendingFindnode(fromNode.ID()) + // address. Dropping unsolicited NEIGHBORS prevents a peer we never queried + // from making us accumulate node records without bound; requiring the source + // to be the address queried stops a peer answering from a spoofed one. + // NEIGHBORS carries no reply token, so the match is by node ID plus endpoint. + matchedReq := h.findPendingFindnode(fromNode.ID(), from) if matchedReq == nil { return nil } + // Counted after the gate: this reports responses to our queries, so counting + // unsolicited packets here would let any peer inflate it. + h.noteProven(fromNode) + h.incrementFindnodeResponsesRecv() + // Accumulate the response, keyed by the matched request's hash so each // FINDNODE gets exactly one entry and a fresh request never collides with // a delivered one. Room is reserved before decoding, so records past the // cap are never persisted in the global node map, even when packets are // dispatched concurrently. - key := string(matchedReq.RequestHash) + key := requestKey(matchedReq.RequestHash, matchedReq.ToNode.ID()) h.pendingNeighborsMu.Lock() pending := h.pendingNeighbors[key] @@ -497,7 +721,7 @@ func (h *Handler) handleNeighbors(fromNode *node.Node, from *net.UDPAddr, neighb } nodeID := node.PubkeyToID(pubkey) - nodes = append(nodes, h.getOrCreateNode(nodeID, pubkey, addr)) + nodes = append(nodes, h.lookupOrCreateNode(nodeID, pubkey, addr)) } h.pendingNeighborsMu.Lock() @@ -544,12 +768,14 @@ func (h *Handler) handleENRRequest(fromNode *node.Node, from *net.UDPAddr, local return ErrExpired } + h.noteSeen(fromNode) + // IMPORTANT: Check if node is bonded (bidirectional bond required) // This prevents amplification attacks and matches reth's behavior. // Only respond to ENRRequest if we've established a bidirectional bond: // - We sent them a PING - // - They sent us a PONG - if !fromNode.IsBonded() { + // - They sent us a PONG from this address + if !fromNode.IsBondedFrom(from) { logrus.WithFields(logrus.Fields{ "from": from.String(), "node_id": fmt.Sprintf("%x", fromNode.IDBytes()[:8]), @@ -557,6 +783,8 @@ func (h *Handler) handleENRRequest(fromNode *node.Node, from *net.UDPAddr, local return fmt.Errorf("node not bonded") } + h.noteProven(fromNode) + // Call callback if h.config.OnENRRequest != nil { if err := h.config.OnENRRequest(fromNode); err != nil { @@ -576,12 +804,39 @@ func (h *Handler) handleENRResponse(fromNode *node.Node, from *net.UDPAddr, resp "enr_seq": resp.Record.Seq(), }).Debug("Received ENRRESPONSE") - // Update node's ENR - fromNode.SetENR(resp.Record) + h.noteSeen(fromNode) + + // Only a response to an ENRREQUEST we actually sent to this address may touch + // any state: ENRRESPONSE carries no expiration, so an unsolicited replay could + // otherwise roll the node back to an older record. The type and destination + // must both match, or a peer could answer with the hash of some other packet + // we sent it and resolve the wrong waiter. + reqs := h.pendingRequestsFrom(resp.ReplyTok, fromNode.ID(), from, ENRRequestPacket) + if len(reqs) == 0 { + return nil + } - // Match to pending request - req := h.getPendingRequest(string(resp.ReplyTok)) - if req != nil { + // Bind the record to the sender's identity before installing it, so a + // matched response cannot attach another node's ENR to this node. + if resp.Record == nil { + return nil + } + pub := resp.Record.PublicKey() + if pub == nil || node.PubkeyToID(pub) != fromNode.ID() { + logrus.WithFields(logrus.Fields{ + "from": from.String(), + "node_id": fmt.Sprintf("%x", fromNode.IDBytes()[:8]), + }).Debug("Dropping ENRRESPONSE: record does not match sender identity") + return nil + } + + fromNode.UpdateENR(resp.Record) + + // After UpdateENR, so OnNodeSeen sees the record and admits the node instead of + // requesting an ENR it already has. + h.noteProven(fromNode) + + for _, req := range reqs { h.deliverResponse(req, resp.Record) } @@ -590,15 +845,25 @@ func (h *Handler) handleENRResponse(fromNode *node.Node, from *net.UDPAddr, resp // Sending Methods -// Ping sends a PING request to a node. +// Ping sends a PING request to a node at its canonical address. func (h *Handler) Ping(n *node.Node) (*Pong, error) { + return h.pingTo(n, n.Addr()) +} + +// pingTo sends a PING to an explicit destination. +// +// handlePing uses it to ping back the source it just ponged, rather than the +// node's canonical address. That is what lets a peer which moved re-prove its new +// endpoint: without it, a moved peer would be pinged only at its old address, never +// answer, and so never bond or be served again. +func (h *Handler) pingTo(n *node.Node, destAddr *net.UDPAddr) (*Pong, error) { // Build PING message ping := &Ping{ Version: 4, // A bootnode serves no RLPx, so it advertises tcp-port 0; the recipient's // tcp is not the sender's to set (spec: to = [ip, udp-port, 0]). From: NewEndpoint(h.config.LocalAddr, 0), - To: NewEndpoint(n.Addr(), 0), + To: NewEndpoint(destAddr, 0), Expiration: MakeExpiration(h.config.ExpirationWindow), } @@ -614,11 +879,14 @@ func (h *Handler) Ping(n *node.Node) (*Pong, error) { } // Register pending request; removal is deferred so every exit path clears it. - req := h.addPendingRequest(hash, n, PingPacket) - defer h.removePendingRequest(string(hash)) + req, err := h.addPendingRequest(hash, n, PingPacket, destAddr) + if err != nil { + return nil, err + } + defer h.removePendingRequest(req) // Send packet - if err := h.transport.SendTo(packet, n.Addr()); err != nil { + if err := h.transport.SendTo(packet, destAddr); err != nil { return nil, err } @@ -674,11 +942,15 @@ func (h *Handler) Findnode(n *node.Node, target []byte) ([]*node.Node, error) { // Register pending request. Removal is deferred so every exit path clears // it: a completed request left in the map keeps matching later NEIGHBORS // from that node and reopens collection windows until cleanup runs. - req := h.addPendingRequest(hash, n, FindnodePacket) - defer h.removePendingRequest(string(hash)) + destAddr := n.Addr() + req, err := h.addPendingRequest(hash, n, FindnodePacket, destAddr) + if err != nil { + return nil, err + } + defer h.removePendingRequest(req) // Send packet - if err := h.transport.SendTo(packet, n.Addr()); err != nil { + if err := h.transport.SendTo(packet, destAddr); err != nil { return nil, err } @@ -725,11 +997,15 @@ func (h *Handler) RequestENR(n *node.Node) (*enr.Record, error) { } // Register pending request; removal is deferred so every exit path clears it. - pendingReq := h.addPendingRequest(hash, n, ENRRequestPacket) - defer h.removePendingRequest(string(hash)) + destAddr := n.Addr() + pendingReq, err := h.addPendingRequest(hash, n, ENRRequestPacket, destAddr) + if err != nil { + return nil, err + } + defer h.removePendingRequest(pendingReq) // Send packet - if err := h.transport.SendTo(packet, n.Addr()); err != nil { + if err := h.transport.SendTo(packet, destAddr); err != nil { return nil, err } @@ -782,8 +1058,11 @@ func (h *Handler) sendPong(to *node.Node, addr *net.UDPAddr, localAddr *net.UDPA // sendNeighbors sends NEIGHBORS response(s). func (h *Handler) sendNeighbors(to *node.Node, addr *net.UDPAddr, localAddr *net.UDPAddr, nodes []*node.Node) error { - // Split nodes into packets of MaxNeighbors - for i := 0; i < len(nodes); i += MaxNeighbors { + // Split nodes into packets of MaxNeighbors. Always send at least one packet, + // even when we have no nodes to offer: go-ethereum's querier waits for a + // NEIGHBORS reply and only stops early once at least one arrives, so a silent + // (zero-packet) response makes it wait out the full request timeout. + for i := 0; i == 0 || i < len(nodes); i += MaxNeighbors { end := i + MaxNeighbors if end > len(nodes) { end = len(nodes) @@ -793,10 +1072,18 @@ func (h *Handler) sendNeighbors(to *node.Node, addr *net.UDPAddr, localAddr *net nodeRecords := make([]NodeRecord, len(batch)) for j, n := range batch { + // Advertise the node's real TCP port from its ENR when known; + // only fall back to the UDP port if no ENR tcp entry is available. + tcpPort := uint16(n.Addr().Port) + if rec := n.ENR(); rec != nil { + if t := rec.TCP(); t != 0 { + tcpPort = t + } + } nodeRecords[j] = NodeRecord{ IP: n.Addr().IP, UDP: uint16(n.Addr().Port), - TCP: uint16(n.Addr().Port), + TCP: tcpPort, ID: EncodePubkey(n.PublicKey()), } } @@ -851,17 +1138,27 @@ func (h *Handler) sendENRResponse(to *node.Node, addr *net.UDPAddr, localAddr *n // Node Management -// getOrCreateNode gets an existing node or creates a new one. -func (h *Handler) getOrCreateNode(id node.ID, pubkey *ecdsa.PublicKey, addr *net.UDPAddr) *node.Node { +// lookupOrCreateNode returns the tracked node for id, creating one at addr if the +// id is unknown. +// +// addr is used ONLY when creating: an existing node's canonical address is never +// rewritten here, because addr is either an unauthenticated packet source or an +// address a peer claimed in a NEIGHBORS record. Every sender reads that address, +// and sendNeighbors republishes it, so letting either source set it would steer +// our outbound traffic and let a peer poison what we publish about a third party. +// Only promoteAddr, on a proven endpoint, may move it. +func (h *Handler) lookupOrCreateNode(id node.ID, pubkey *ecdsa.PublicKey, addr *net.UDPAddr) *node.Node { + h.nodesMu.RLock() + n, exists := h.nodes[id] + h.nodesMu.RUnlock() + if exists { + return n + } + h.nodesMu.Lock() defer h.nodesMu.Unlock() - n, exists := h.nodes[id] - if exists { - // Update address if changed - if n.Addr().String() != addr.String() { - n.SetAddr(addr) - } + if n, exists := h.nodes[id]; exists { return n } @@ -871,18 +1168,72 @@ func (h *Handler) getOrCreateNode(id node.ID, pubkey *ecdsa.PublicKey, addr *net n = node.New(pubkey, addr) n.UpdateLastSeen() - // Bound the map so an unauthenticated flood of distinct node IDs (for - // example fabricated NEIGHBORS records) cannot grow it without limit. Stale - // unbonded entries are reclaimed by cleanup; until a slot frees up we still - // return the node so the packet is handled, but we do not retain it. + // Bound the map so an unauthenticated flood of distinct node IDs (for example + // fabricated NEIGHBORS records, or signed PINGs from generated keys) cannot grow + // it without limit. When full, evict one unbonded entry to make room rather than + // dropping the new node: otherwise a flood that pins the map at MaxNodes would + // lock out genuine new peers (their node is never retained, so their inbound PING + // can never lead to a bond). Bonded entries are real, endpoint-proven peers and + // are never evicted here; if every entry is bonded (genuine load, not a flood) we + // leave the map as-is and return the node without retaining it. if len(h.nodes) >= h.config.MaxNodes { - return n + evicted := false + for eid, en := range h.nodes { + if !en.IsBonded() { + delete(h.nodes, eid) + evicted = true + break + } + } + if !evicted { + return n + } } h.nodes[id] = n return n } +// promoteAddr installs a proven endpoint as n's canonical address. Only +// handlePong may call it, for the address a matched PING was sent to; see +// lookupOrCreateNode for why nothing else may move it. +func (h *Handler) promoteAddr(n *node.Node, proven *net.UDPAddr) { + if n == nil || proven == nil || proven.IP == nil { + return + } + if n.Addr().String() == proven.String() { + return + } + + n.SetAddr(proven) + + logrus.WithFields(logrus.Fields{ + "node_id": fmt.Sprintf("%x", n.IDBytes()[:8]), + "addr": proven.String(), + }).Debug("promoted proven endpoint to canonical address") +} + +// noteSeen refreshes identity-scoped liveness. +// +// Safe for any non-expired packet: the signature authenticates the identity, so a +// peer can only refresh its own liveness. Withholding it until the source is +// proven would evict a peer that is actively signing packets but whose bond has +// lapsed, and it would come back with no proven addresses at all. +func (h *Handler) noteSeen(n *node.Node) { + n.UpdateLastSeen() + n.IncrementPacketsReceived() +} + +// noteProven fires OnNodeSeen, which admits the node to the routing table and can +// spawn outbound PING/ENRREQUEST traffic toward it. It requires a proven or +// solicited source, so it sits behind each handler's gate while noteSeen runs +// ahead of it. +func (h *Handler) noteProven(n *node.Node) { + if h.config.OnNodeSeen != nil { + h.config.OnNodeSeen(n, time.Now()) + } +} + // GetNode returns a node by ID. func (h *Handler) GetNode(id node.ID) *node.Node { h.nodesMu.RLock() @@ -904,11 +1255,27 @@ func (h *Handler) AllNodes() []*node.Node { // Request Tracking -// addPendingRequest registers a new pending request. -func (h *Handler) addPendingRequest(hash []byte, toNode *node.Node, packetType byte) *PendingRequest { +// requestKey scopes a pending request to its destination, since the packet +// hash alone aliases across peers (see the requests field). +func requestKey(hash []byte, id node.ID) string { + return string(hash) + string(id[:]) +} + +// addPendingRequest registers a new pending request. A second FINDNODE to a +// peer with one already in flight is rejected: NEIGHBORS carries no reply +// token, so two in-flight FINDNODEs to one peer cannot be told apart. +// destAddr must be the address the caller sends the packet to, captured once, +// so the recorded and actual destinations cannot diverge. +func (h *Handler) addPendingRequest(hash []byte, toNode *node.Node, packetType byte, destAddr *net.UDPAddr) (*PendingRequest, error) { + var destIP net.IP + if destAddr != nil && destAddr.IP != nil { + destIP = append(net.IP(nil), destAddr.IP...) + } + req := &PendingRequest{ RequestHash: hash, ToNode: toNode, + DestIP: destIP, PacketType: packetType, CreatedAt: time.Now(), Timeout: time.Now().Add(h.config.RequestTimeout), @@ -916,37 +1283,126 @@ func (h *Handler) addPendingRequest(hash []byte, toNode *node.Node, packetType b } h.requestsMu.Lock() - h.requests[string(hash)] = req - h.requestsMu.Unlock() + defer h.requestsMu.Unlock() - return req + if packetType == FindnodePacket && h.pendingFindnodeLocked(toNode.ID()) != nil { + return nil, fmt.Errorf("findnode already in flight to %x", toNode.IDBytes()[:8]) + } + + key := requestKey(hash, toNode.ID()) + h.requests[key] = append(h.requests[key], req) + + return req, nil } -// getPendingRequest retrieves a pending request by hash. -func (h *Handler) getPendingRequest(hash string) *PendingRequest { +// pendingRequestsFrom returns the pending requests of the given type that match +// this reply token and were sent to this address. +func (h *Handler) pendingRequestsFrom(replyTok []byte, id node.ID, from *net.UDPAddr, packetType byte) []*PendingRequest { + if from == nil || from.IP == nil { + return nil + } + h.requestsMu.RLock() defer h.requestsMu.RUnlock() - return h.requests[hash] + + var out []*PendingRequest + for _, req := range h.requests[requestKey(replyTok, id)] { + if req.PacketType == packetType && req.DestIP != nil && req.DestIP.Equal(from.IP) { + out = append(out, req) + } + } + return out } -// findPendingFindnode returns a pending FINDNODE request awaiting a response +// consumePendingPing removes and returns the pending PING this PONG answers, or +// nil if there is none. +// +// Three properties beyond "a token matched" are required before a PONG may +// mutate state, and all three are enforced here so no caller can forget one: +// +// - from must be the IP the PING was sent to. The token alone proves only that +// somebody received that PING; an attacker who receives it at their own +// address can replay it with a victim's source and bond the victim. +// - the request must be a PING. Peers know the hashes of packets we sent them, +// so an ENRREQUEST or FINDNODE hash would otherwise match as a reply token. +// - the entry is deleted here, under the same lock, so a replayed PONG finds +// nothing and the side effects run at most once per PING. +func (h *Handler) consumePendingPing(replyTok []byte, id node.ID, from *net.UDPAddr) *PendingRequest { + if from == nil || from.IP == nil { + return nil + } + + key := requestKey(replyTok, id) + + h.requestsMu.Lock() + defer h.requestsMu.Unlock() + + reqs := h.requests[key] + for i, req := range reqs { + if req.PacketType != PingPacket || req.DestIP == nil || !req.DestIP.Equal(from.IP) { + continue + } + + reqs = slices.Delete(reqs, i, i+1) + if len(reqs) == 0 { + delete(h.requests, key) + } else { + h.requests[key] = reqs + } + return req + } + + return nil +} + +// findPendingFindnode returns the pending FINDNODE request awaiting a response // from the given node, or nil if none exists. -func (h *Handler) findPendingFindnode(id node.ID) *PendingRequest { +func (h *Handler) findPendingFindnode(id node.ID, from *net.UDPAddr) *PendingRequest { + if from == nil || from.IP == nil { + return nil + } + h.requestsMu.RLock() defer h.requestsMu.RUnlock() - for _, req := range h.requests { - if req.PacketType == FindnodePacket && req.ToNode != nil && req.ToNode.ID() == id { - return req + + req := h.pendingFindnodeLocked(id) + if req == nil || req.DestIP == nil || !req.DestIP.Equal(from.IP) { + return nil + } + return req +} + +func (h *Handler) pendingFindnodeLocked(id node.ID) *PendingRequest { + for _, reqs := range h.requests { + for _, req := range reqs { + if req.PacketType == FindnodePacket && req.ToNode != nil && req.ToNode.ID() == id { + return req + } } } return nil } -// removePendingRequest removes a pending request. -func (h *Handler) removePendingRequest(hash string) { +// removePendingRequest removes one pending request, leaving other waiters on +// the same key in place so one caller's cleanup cannot orphan another's. +func (h *Handler) removePendingRequest(req *PendingRequest) { + if req == nil || req.ToNode == nil { + return + } + key := requestKey(req.RequestHash, req.ToNode.ID()) + h.requestsMu.Lock() - delete(h.requests, hash) - h.requestsMu.Unlock() + defer h.requestsMu.Unlock() + + reqs := h.requests[key] + if i := slices.Index(reqs, req); i >= 0 { + reqs = slices.Delete(reqs, i, i+1) + } + if len(reqs) == 0 { + delete(h.requests, key) + } else { + h.requests[key] = reqs + } } // deliverResponse hands a response to a waiting request without blocking. @@ -974,6 +1430,7 @@ func (h *Handler) cleanupLoop() { select { case <-ticker.C: h.cleanup() + h.resumeDeferredENRRefreshes() case <-h.ctx.Done(): return } @@ -986,9 +1443,12 @@ func (h *Handler) cleanup() { // Clean up expired requests h.requestsMu.Lock() - for hash, req := range h.requests { - if now.After(req.Timeout) { - delete(h.requests, hash) + for key, reqs := range h.requests { + kept := slices.DeleteFunc(reqs, func(req *PendingRequest) bool { return now.After(req.Timeout) }) + if len(kept) == 0 { + delete(h.requests, key) + } else { + h.requests[key] = kept } } h.requestsMu.Unlock() @@ -1004,14 +1464,43 @@ func (h *Handler) cleanup() { // Evict stale, unbonded nodes so the map stays bounded. Bonded nodes are // kept until their bond expires, after which IsBonded reports false and they - // become eligible here. + // become eligible here. Scanning under the read lock keeps a full-map sweep + // from stalling every inbound packet in getOrCreateNode. + stale := h.staleNodes(now) + if len(stale) == 0 { + return + } + h.nodesMu.Lock() - for id, n := range h.nodes { - if !n.IsBonded() && now.Sub(n.LastSeen()) > h.config.NodeTTL { + evicted := make([]node.ID, 0, len(stale)) + for _, id := range stale { + // Re-check: a node may have been seen again since the scan. + if n, ok := h.nodes[id]; ok && !n.IsBonded() && now.Sub(n.LastSeen()) > h.config.NodeTTL { delete(h.nodes, id) + evicted = append(evicted, id) } } h.nodesMu.Unlock() + + h.enrRefreshMu.Lock() + for _, id := range evicted { + delete(h.enrRefresh, id) + } + h.enrRefreshMu.Unlock() +} + +// staleNodes returns the IDs of unbonded nodes past their TTL. +func (h *Handler) staleNodes(now time.Time) []node.ID { + h.nodesMu.RLock() + defer h.nodesMu.RUnlock() + + var stale []node.ID + for id, n := range h.nodes { + if !n.IsBonded() && now.Sub(n.LastSeen()) > h.config.NodeTTL { + stale = append(stale, id) + } + } + return stale } // Statistics @@ -1078,7 +1567,10 @@ func (h *Handler) GetStats() HandlerStats { knownNodes := len(h.nodes) h.nodesMu.RUnlock() h.requestsMu.RLock() - pendingRequests := len(h.requests) + pendingRequests := 0 + for _, reqs := range h.requests { + pendingRequests += len(reqs) + } h.requestsMu.RUnlock() h.pendingNeighborsMu.RLock() pendingNeighbors := len(h.pendingNeighbors) @@ -1101,24 +1593,6 @@ func (h *Handler) GetStats() HandlerStats { } } -// Stats returns current statistics as a map, for callers that render it -// generically. -func (h *Handler) Stats() map[string]interface{} { - s := h.GetStats() - return map[string]interface{}{ - "packets_received": s.PacketsReceived, - "packets_sent": s.PacketsSent, - "invalid_packets": s.InvalidPackets, - "expired_packets": s.ExpiredPackets, - "unbonded_findnode": s.UnbondedFindnode, - "findnode_requests_recv": s.FindnodeRequestsRecv, - "findnode_responses_recv": s.FindnodeResponsesRecv, - "known_nodes": s.KnownNodes, - "pending_requests": s.PendingRequests, - "pending_neighbors": s.PendingNeighbors, - } -} - // LocalRecord returns the ENR the handler currently advertises. func (h *Handler) LocalRecord() *enr.Record { h.localENRMu.RLock() diff --git a/discv4/protocol/handler_test.go b/discv4/protocol/handler_test.go index d21528f..bc90d8a 100644 --- a/discv4/protocol/handler_test.go +++ b/discv4/protocol/handler_test.go @@ -39,7 +39,7 @@ func TestGetOrCreateNodeRespectsMaxNodes(t *testing.T) { for i := 0; i < maxNodes*5; i++ { pub, id := makeNodeID(t) - h.getOrCreateNode(id, pub, testAddr()) + h.lookupOrCreateNode(id, pub, testAddr()) } if got := len(h.AllNodes()); got != maxNodes { @@ -56,11 +56,11 @@ func TestCleanupEvictsStaleUnbondedNodes(t *testing.T) { h := NewHandler(ctx, HandlerConfig{MaxNodes: 1000, NodeTTL: 20 * time.Millisecond}, nil) pubStale, idStale := makeNodeID(t) - h.getOrCreateNode(idStale, pubStale, testAddr()) + h.lookupOrCreateNode(idStale, pubStale, testAddr()) pubBonded, idBonded := makeNodeID(t) - bonded := h.getOrCreateNode(idBonded, pubBonded, testAddr()) - bonded.MarkPongReceived(time.Hour) // establish a live bond + bonded := h.lookupOrCreateNode(idBonded, pubBonded, testAddr()) + bonded.MarkPongReceived(time.Hour, testAddr()) // establish a live bond time.Sleep(40 * time.Millisecond) // age both past NodeTTL @@ -84,7 +84,7 @@ func TestCleanupReclaimsFloodedNodes(t *testing.T) { for i := 0; i < 500; i++ { pub, id := makeNodeID(t) - h.getOrCreateNode(id, pub, testAddr()) + h.lookupOrCreateNode(id, pub, testAddr()) } if got := len(h.AllNodes()); got != 500 { t.Fatalf("setup: expected 500 tracked nodes, got %d", got) diff --git a/discv4/protocol/pending_neighbors_test.go b/discv4/protocol/pending_neighbors_test.go index b811080..b2a556c 100644 --- a/discv4/protocol/pending_neighbors_test.go +++ b/discv4/protocol/pending_neighbors_test.go @@ -70,7 +70,9 @@ func TestNeighborsAccumulationCapped(t *testing.T) { defer cancel() from := makeDiscv4Node(t) - h.addPendingRequest([]byte("req"), from, FindnodePacket) + if _, err := h.addPendingRequest([]byte("req"), from, FindnodePacket, from.Addr()); err != nil { + t.Fatalf("addPendingRequest: %v", err) + } // Pre-build packets so no slow key generation happens between the calls and // the read (the delivery goroutine deletes the entry after the window). @@ -85,7 +87,7 @@ func TestNeighborsAccumulationCapped(t *testing.T) { } h.pendingNeighborsMu.RLock() - pending := h.pendingNeighbors["req"] + pending := h.pendingNeighbors[requestKey([]byte("req"), from.ID())] h.pendingNeighborsMu.RUnlock() if pending == nil { t.Fatal("expected a pending entry for the matched FINDNODE") @@ -102,7 +104,10 @@ func TestNeighborsDeliveredToWaiter(t *testing.T) { defer cancel() from := makeDiscv4Node(t) - req := h.addPendingRequest([]byte("req"), from, FindnodePacket) + req, err := h.addPendingRequest([]byte("req"), from, FindnodePacket, from.Addr()) + if err != nil { + t.Fatalf("addPendingRequest: %v", err) + } if err := h.handleNeighbors(from, from.Addr(), makeNeighbors(t, 5)); err != nil { t.Fatal(err) @@ -152,7 +157,9 @@ func TestNeighborsCapAppliesBeforeNodePersistence(t *testing.T) { defer cancel() from := makeDiscv4Node(t) - h.addPendingRequest([]byte("req"), from, FindnodePacket) + if _, err := h.addPendingRequest([]byte("req"), from, FindnodePacket, from.Addr()); err != nil { + t.Fatalf("addPendingRequest: %v", err) + } if err := h.handleNeighbors(from, from.Addr(), makeNeighbors(t, maxNeighborsPerResponse+20)); err != nil { t.Fatal(err) @@ -178,7 +185,7 @@ func TestFreshNodeSurvivesCleanup(t *testing.T) { t.Fatalf("generate key: %v", err) } id := node.PubkeyToID(&key.PublicKey) - h.getOrCreateNode(id, &key.PublicKey, &net.UDPAddr{IP: net.IPv4(9, 9, 9, 9), Port: 30303}) + h.lookupOrCreateNode(id, &key.PublicKey, &net.UDPAddr{IP: net.IPv4(9, 9, 9, 9), Port: 30303}) h.cleanup() @@ -206,7 +213,7 @@ func TestFindnodeRemovesCompletedRequest(t *testing.T) { h := NewHandler(ctx, HandlerConfig{PrivateKey: key}, stubTransport{}) to := makeDiscv4Node(t) - to.MarkPongReceived(time.Hour) + to.MarkPongReceived(time.Hour, to.Addr()) target := EncodePubkey(&key.PublicKey) type result struct { @@ -220,7 +227,7 @@ func TestFindnodeRemovesCompletedRequest(t *testing.T) { }() deadline := time.Now().Add(2 * time.Second) - for h.findPendingFindnode(to.ID()) == nil { + for h.findPendingFindnode(to.ID(), to.Addr()) == nil { if time.Now().After(deadline) { t.Fatal("pending FINDNODE never registered") } @@ -250,7 +257,9 @@ func TestNeighborsPersistenceCapExactUnderConcurrency(t *testing.T) { defer cancel() from := makeDiscv4Node(t) - h.addPendingRequest([]byte("req"), from, FindnodePacket) + if _, err := h.addPendingRequest([]byte("req"), from, FindnodePacket, from.Addr()); err != nil { + t.Fatalf("addPendingRequest: %v", err) + } packets := make([]*Neighbors, 6) for i := range packets { @@ -285,7 +294,10 @@ func TestNeighborsAfterDeliveryPersistNothing(t *testing.T) { defer cancel() from := makeDiscv4Node(t) - req := h.addPendingRequest([]byte("req"), from, FindnodePacket) + req, err := h.addPendingRequest([]byte("req"), from, FindnodePacket, from.Addr()) + if err != nil { + t.Fatalf("addPendingRequest: %v", err) + } if err := h.handleNeighbors(from, from.Addr(), makeNeighbors(t, 2)); err != nil { t.Fatal(err) @@ -311,7 +323,7 @@ func TestNeighborsAfterDeliveryPersistNothing(t *testing.T) { t.Fatalf("a post-delivery packet persisted %d records, want 0", after-before) } h.pendingNeighborsMu.RLock() - pending := h.pendingNeighbors["req"] + pending := h.pendingNeighbors[requestKey([]byte("req"), from.ID())] h.pendingNeighborsMu.RUnlock() if pending == nil || !pending.Closed || len(pending.Nodes) != 2 { t.Fatalf("tombstone state = %+v, want closed with the delivered 2 nodes", pending) diff --git a/discv4/protocol/pending_request_test.go b/discv4/protocol/pending_request_test.go new file mode 100644 index 0000000..bf09332 --- /dev/null +++ b/discv4/protocol/pending_request_test.go @@ -0,0 +1,261 @@ +package protocol + +import ( + "crypto/ecdsa" + "net" + "testing" + "time" + + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethpandaops/bootnodoor/discv4/node" + "github.com/ethpandaops/bootnodoor/enr" +) + +func makeKeyedNode(t *testing.T, port int) (*node.Node, *ecdsa.PrivateKey) { + t.Helper() + key, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("generate key: %v", err) + } + return node.New(&key.PublicKey, &net.UDPAddr{IP: net.IPv4(1, 2, 3, 4), Port: port}), key +} + +func signedV4Record(t *testing.T, key *ecdsa.PrivateKey, seq uint64) *enr.Record { + t.Helper() + rec := enr.New() + if err := rec.Set("ip", net.IPv4(1, 2, 3, 4)); err != nil { + t.Fatalf("set ip: %v", err) + } + if err := rec.Set("udp", uint16(30303)); err != nil { + t.Fatalf("set udp: %v", err) + } + rec.SetSeq(seq) + if err := rec.Sign(key); err != nil { + t.Fatalf("sign: %v", err) + } + return rec +} + +func expectResponse(t *testing.T, req *PendingRequest) interface{} { + t.Helper() + select { + case resp := <-req.ResponseChan: + return resp + case <-time.After(2 * time.Second): + t.Fatal("expected a delivered response") + return nil + } +} + +func expectNoResponse(t *testing.T, req *PendingRequest) { + t.Helper() + select { + case resp := <-req.ResponseChan: + t.Fatalf("unexpected response delivered: %v", resp) + default: + } +} + +// TestIdenticalRequestsToDifferentPeersDoNotAlias covers the collision that +// poisoned live routing tables: ENRREQUEST carries only a 1s-granularity +// expiration and signatures are deterministic, so same-second requests to +// different peers share a packet hash. A response from one peer must resolve +// only that peer's request and must not touch the other node's ENR. +func TestIdenticalRequestsToDifferentPeersDoNotAlias(t *testing.T) { + h, cancel := newTestHandler(t) + defer cancel() + + nodeA, _ := makeKeyedNode(t, 30301) + nodeB, keyB := makeKeyedNode(t, 30302) + hash := []byte("same-second-packet") + + reqA, err := h.addPendingRequest(hash, nodeA, ENRRequestPacket, nodeA.Addr()) + if err != nil { + t.Fatalf("addPendingRequest A: %v", err) + } + reqB, err := h.addPendingRequest(hash, nodeB, ENRRequestPacket, nodeB.Addr()) + if err != nil { + t.Fatalf("addPendingRequest B: %v", err) + } + + recB := signedV4Record(t, keyB, 7) + if err := h.handleENRResponse(nodeB, nodeB.Addr(), &ENRResponse{ReplyTok: hash, Record: recB}); err != nil { + t.Fatalf("handleENRResponse: %v", err) + } + + if got := expectResponse(t, reqB); got != recB { + t.Fatalf("request B received %v, want node B's record", got) + } + expectNoResponse(t, reqA) + if nodeA.ENR() != nil { + t.Fatal("node B's response installed a record on node A") + } + if nodeB.ENR() != recB { + t.Fatal("node B's record was not installed on node B") + } +} + +// TestSamePeerDuplicateRequestsBothComplete covers concurrent identical +// requests to one peer (the lookup fires RequestENR per neighbor before +// dedup): every waiter gets the response, and removing one request must not +// orphan the other's pending entry. +func TestSamePeerDuplicateRequestsBothComplete(t *testing.T) { + h, cancel := newTestHandler(t) + defer cancel() + + n, key := makeKeyedNode(t, 30301) + hash := []byte("same-second-packet") + + req1, err := h.addPendingRequest(hash, n, ENRRequestPacket, n.Addr()) + if err != nil { + t.Fatalf("addPendingRequest 1: %v", err) + } + req2, err := h.addPendingRequest(hash, n, ENRRequestPacket, n.Addr()) + if err != nil { + t.Fatalf("addPendingRequest 2: %v", err) + } + + h.removePendingRequest(req1) + if got := len(h.pendingRequestsFrom(hash, n.ID(), n.Addr(), ENRRequestPacket)); got != 1 { + t.Fatalf("after removing one duplicate, %d pending remain, want 1", got) + } + + rec := signedV4Record(t, key, 3) + if err := h.handleENRResponse(n, n.Addr(), &ENRResponse{ReplyTok: hash, Record: rec}); err != nil { + t.Fatalf("handleENRResponse: %v", err) + } + if got := expectResponse(t, req2); got != rec { + t.Fatalf("surviving request received %v, want the record", got) + } +} + +// TestUnsolicitedENRResponseDoesNotMutate covers replay: ENRRESPONSE has no +// expiration, so a response matching no pending request must not touch the +// node's ENR at all. +func TestUnsolicitedENRResponseDoesNotMutate(t *testing.T) { + h, cancel := newTestHandler(t) + defer cancel() + + n, key := makeKeyedNode(t, 30301) + newer := signedV4Record(t, key, 9) + n.SetENR(newer) + + older := signedV4Record(t, key, 2) + if err := h.handleENRResponse(n, n.Addr(), &ENRResponse{ReplyTok: []byte("nothing-pending"), Record: older}); err != nil { + t.Fatalf("handleENRResponse: %v", err) + } + if n.ENR() != newer { + t.Fatal("unsolicited response replaced the node's ENR") + } +} + +// TestStaleENRResponseNotInstalled covers rollback through a matched request: +// an equal-or-lower-sequence record must not replace a newer one. +func TestStaleENRResponseNotInstalled(t *testing.T) { + h, cancel := newTestHandler(t) + defer cancel() + + n, key := makeKeyedNode(t, 30301) + newer := signedV4Record(t, key, 9) + n.SetENR(newer) + + hash := []byte("pending") + req, err := h.addPendingRequest(hash, n, ENRRequestPacket, n.Addr()) + if err != nil { + t.Fatalf("addPendingRequest: %v", err) + } + + stale := signedV4Record(t, key, 9) + if err := h.handleENRResponse(n, n.Addr(), &ENRResponse{ReplyTok: hash, Record: stale}); err != nil { + t.Fatalf("handleENRResponse: %v", err) + } + if n.ENR() != newer { + t.Fatal("equal-sequence response replaced the node's ENR") + } + if got := expectResponse(t, req); got != stale { + t.Fatalf("waiter received %v, want the response record", got) + } +} + +// TestMismatchedIdentityENRResponseDropped: a matched response whose record is +// signed by a different key must neither install nor be delivered. +func TestMismatchedIdentityENRResponseDropped(t *testing.T) { + h, cancel := newTestHandler(t) + defer cancel() + + n, _ := makeKeyedNode(t, 30301) + _, otherKey := makeKeyedNode(t, 30302) + + hash := []byte("pending") + req, err := h.addPendingRequest(hash, n, ENRRequestPacket, n.Addr()) + if err != nil { + t.Fatalf("addPendingRequest: %v", err) + } + + foreign := signedV4Record(t, otherKey, 5) + if err := h.handleENRResponse(n, n.Addr(), &ENRResponse{ReplyTok: hash, Record: foreign}); err != nil { + t.Fatalf("handleENRResponse: %v", err) + } + if n.ENR() != nil { + t.Fatal("foreign record was installed") + } + expectNoResponse(t, req) +} + +// TestIdenticalFindnodeToDifferentPeersSeparateAccumulators: colliding +// FINDNODE hashes to different peers must keep separate NEIGHBORS +// accumulators and deliver each peer's response to its own request. +func TestIdenticalFindnodeToDifferentPeersSeparateAccumulators(t *testing.T) { + h, cancel := newTestHandler(t) + defer cancel() + + nodeA, _ := makeKeyedNode(t, 30301) + nodeB, _ := makeKeyedNode(t, 30302) + hash := []byte("same-target-same-second") + + reqA, err := h.addPendingRequest(hash, nodeA, FindnodePacket, nodeA.Addr()) + if err != nil { + t.Fatalf("addPendingRequest A: %v", err) + } + reqB, err := h.addPendingRequest(hash, nodeB, FindnodePacket, nodeB.Addr()) + if err != nil { + t.Fatalf("addPendingRequest B: %v", err) + } + + if err := h.handleNeighbors(nodeA, nodeA.Addr(), makeNeighbors(t, 2)); err != nil { + t.Fatalf("handleNeighbors: %v", err) + } + + nodes, ok := expectResponse(t, reqA).([]*node.Node) + if !ok || len(nodes) != 2 { + t.Fatalf("request A received %v, want 2 nodes", nodes) + } + expectNoResponse(t, reqB) + + h.pendingNeighborsMu.RLock() + _, sharedKey := h.pendingNeighbors[string(hash)] + bEntry := h.pendingNeighbors[requestKey(hash, nodeB.ID())] + h.pendingNeighborsMu.RUnlock() + if sharedKey { + t.Fatal("accumulator stored under the bare hash key") + } + if bEntry != nil { + t.Fatal("node A's NEIGHBORS created an accumulator for node B's request") + } +} + +// TestSecondFindnodeToSamePeerRejected: NEIGHBORS has no reply token, so two +// in-flight FINDNODEs to one peer cannot be told apart and the second must be +// refused. +func TestSecondFindnodeToSamePeerRejected(t *testing.T) { + h, cancel := newTestHandler(t) + defer cancel() + + n, _ := makeKeyedNode(t, 30301) + if _, err := h.addPendingRequest([]byte("hash-1"), n, FindnodePacket, n.Addr()); err != nil { + t.Fatalf("first findnode: %v", err) + } + if _, err := h.addPendingRequest([]byte("hash-2"), n, FindnodePacket, n.Addr()); err == nil { + t.Fatal("second in-flight findnode to the same peer was accepted") + } +} diff --git a/discv4/protocol/remainder_fixes_test.go b/discv4/protocol/remainder_fixes_test.go new file mode 100644 index 0000000..eb4314c --- /dev/null +++ b/discv4/protocol/remainder_fixes_test.go @@ -0,0 +1,137 @@ +package protocol + +import ( + "context" + "net" + "testing" + "time" + + ethcrypto "github.com/ethereum/go-ethereum/crypto" + "github.com/ethpandaops/bootnodoor/discv4/node" + "github.com/ethpandaops/bootnodoor/enr" +) + +// captureTransport records every packet sent so a test can decode it. +type captureTransport struct{ sent [][]byte } + +func (c *captureTransport) SendTo(b []byte, _ *net.UDPAddr) error { + c.sent = append(c.sent, b) + return nil +} +func (c *captureTransport) Send(b []byte, _ *net.UDPAddr, _ *net.UDPAddr) error { + c.sent = append(c.sent, b) + return nil +} + +// TestSendNeighborsAlwaysSendsAtLeastOnePacket verifies BUG3: an empty result +// still produces exactly one (empty) NEIGHBORS packet, so a go-ethereum querier +// returns immediately instead of waiting out its request timeout on silence. +func TestSendNeighborsAlwaysSendsAtLeastOnePacket(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + key, _ := ethcrypto.GenerateKey() + ct := &captureTransport{} + h := NewHandler(ctx, HandlerConfig{PrivateKey: key}, ct) + + pub, id := makeNodeID(t) + to := h.lookupOrCreateNode(id, pub, testAddr()) + + if err := h.sendNeighbors(to, testAddr(), nil, nil); err != nil { + t.Fatalf("sendNeighbors(empty): %v", err) + } + if len(ct.sent) != 1 { + t.Fatalf("empty result sent %d packets, want exactly 1", len(ct.sent)) + } + pkt, err := DecodePacket(ct.sent[0]) + if err != nil { + t.Fatalf("decode NEIGHBORS: %v", err) + } + nb, ok := pkt.(*Neighbors) + if !ok { + t.Fatalf("wrong packet type %T", pkt) + } + if len(nb.Nodes) != 0 { + t.Fatalf("empty NEIGHBORS carried %d nodes", len(nb.Nodes)) + } +} + +// TestSendNeighborsAdvertisesEnrTCPPort verifies BUG4: the NEIGHBORS record +// advertises the node's real TCP port from its ENR, not the UDP port. +func TestSendNeighborsAdvertisesEnrTCPPort(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + key, _ := ethcrypto.GenerateKey() + ct := &captureTransport{} + h := NewHandler(ctx, HandlerConfig{PrivateKey: key}, ct) + + // Build a peer whose ENR advertises tcp=40404 at udp=30303. + peerKey, _ := ethcrypto.GenerateKey() + rec := enr.New() + _ = rec.Set("id", "v4") + _ = rec.Set("ip", net.IPv4(203, 0, 113, 7).To4()) + _ = rec.Set("udp", uint16(30303)) + _ = rec.Set("tcp", uint16(40404)) + if err := rec.Sign(peerKey); err != nil { + t.Fatalf("sign: %v", err) + } + peer := node.New(&peerKey.PublicKey, &net.UDPAddr{IP: net.IPv4(203, 0, 113, 7), Port: 30303}) + peer.SetENR(rec) + + pub, id := makeNodeID(t) + to := h.lookupOrCreateNode(id, pub, testAddr()) + + if err := h.sendNeighbors(to, testAddr(), nil, []*node.Node{peer}); err != nil { + t.Fatalf("sendNeighbors: %v", err) + } + if len(ct.sent) != 1 { + t.Fatalf("sent %d packets, want 1", len(ct.sent)) + } + pkt, _ := DecodePacket(ct.sent[0]) + nb := pkt.(*Neighbors) + if len(nb.Nodes) != 1 { + t.Fatalf("NEIGHBORS carried %d nodes, want 1", len(nb.Nodes)) + } + if nb.Nodes[0].TCP != 40404 { + t.Fatalf("advertised TCP=%d, want 40404 (the ENR tcp port, not the UDP port)", nb.Nodes[0].TCP) + } + if nb.Nodes[0].UDP != 30303 { + t.Fatalf("advertised UDP=%d, want 30303", nb.Nodes[0].UDP) + } +} + +// TestFloodDoesNotEvictBondedPeers verifies the #34 follow-up: when the node map +// is full, inserts evict a stale unbonded entry (so genuine new peers are never +// locked out) while bonded, endpoint-proven peers are retained. +func TestFloodDoesNotEvictBondedPeers(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + const maxNodes = 10 + h := NewHandler(ctx, HandlerConfig{MaxNodes: maxNodes, NodeTTL: time.Hour}, nil) + + // A genuine, bonded peer. + pub, bondedID := makeNodeID(t) + bonded := h.lookupOrCreateNode(bondedID, pub, testAddr()) + bonded.MarkPongReceived(time.Hour, testAddr()) + + // Fill the rest with unbonded nodes, then flood well past the cap. + for i := 0; i < maxNodes*20; i++ { + p, id := makeNodeID(t) + h.lookupOrCreateNode(id, p, testAddr()) + } + + if got := len(h.AllNodes()); got != maxNodes { + t.Fatalf("map not bounded under flood: got %d want %d", got, maxNodes) + } + if h.GetNode(bondedID) == nil { + t.Fatal("bonded peer was evicted by an unbonded-ID flood") + } + // A brand-new node still gets retained (evicting an unbonded entry). + p, freshID := makeNodeID(t) + h.lookupOrCreateNode(freshID, p, testAddr()) + if h.GetNode(freshID) == nil { + t.Fatal("new peer not retained when map full") + } +} diff --git a/discv4/protocol/response_delivery_test.go b/discv4/protocol/response_delivery_test.go index c2052a5..1b1d651 100644 --- a/discv4/protocol/response_delivery_test.go +++ b/discv4/protocol/response_delivery_test.go @@ -21,7 +21,11 @@ func TestDeliverResponseNeverBlocks(t *testing.T) { h, cancel := newTestHandler(t) defer cancel() - req := h.addPendingRequest([]byte("reqhash"), nil, PingPacket) + dest := makeDiscv4Node(t) + req, err := h.addPendingRequest([]byte("reqhash"), dest, PingPacket, dest.Addr()) + if err != nil { + t.Fatalf("addPendingRequest: %v", err) + } const dups = 200 var wg sync.WaitGroup @@ -63,7 +67,11 @@ func TestDuplicateResponsesWaiterGetsOneNoLeak(t *testing.T) { defer cancel() hash := []byte("reqhash") - req := h.addPendingRequest(hash, nil, PingPacket) + to := makeDiscv4Node(t) + req, err := h.addPendingRequest(hash, to, PingPacket, to.Addr()) + if err != nil { + t.Fatalf("addPendingRequest: %v", err) + } got := make(chan interface{}, 1) var waiter sync.WaitGroup @@ -71,7 +79,7 @@ func TestDuplicateResponsesWaiterGetsOneNoLeak(t *testing.T) { go func() { defer waiter.Done() resp := <-req.ResponseChan - h.removePendingRequest(string(hash)) + h.removePendingRequest(req) got <- resp }() @@ -81,7 +89,7 @@ func TestDuplicateResponsesWaiterGetsOneNoLeak(t *testing.T) { for i := 0; i < dups; i++ { go func() { defer wg.Done() - if r := h.getPendingRequest(string(hash)); r != nil { + for _, r := range h.pendingRequestsFrom(hash, to.ID(), to.Addr(), PingPacket) { h.deliverResponse(r, "pong") } }() diff --git a/discv4/service.go b/discv4/service.go index abc2168..7912b59 100644 --- a/discv4/service.go +++ b/discv4/service.go @@ -32,6 +32,7 @@ type Transport interface { protocol.Transport LocalAddr() *net.UDPAddr AddHandler(handler func(data []byte, from *net.UDPAddr, localAddr *net.UDPAddr) bool) + AddHandlerFor(protocol string, handler func(data []byte, from *net.UDPAddr, localAddr *net.UDPAddr) bool) } // Service represents a discv4 service instance. @@ -125,7 +126,7 @@ func New(config *Config, transport Transport) (*Service, error) { } // Register packet handler with transport - transport.AddHandler(s.packetHandler) + transport.AddHandlerFor("discv4", s.packetHandler) return s, nil } @@ -382,22 +383,6 @@ func (s *Service) Handler() *protocol.Handler { return s.handler } -// Statistics - -// Stats returns service statistics. -func (s *Service) Stats() map[string]interface{} { - handler := s.Handler() - if handler == nil { - return map[string]interface{}{} - } - - stats := handler.Stats() - - // Note: Transport stats are not included since transport is managed externally - - return stats -} - // Utility Methods // ParseNodeFromEnode creates a node from an enode:// URL. diff --git a/discv5/ipdiscovery.go b/discv5/ipdiscovery.go deleted file mode 100644 index bf6bb17..0000000 --- a/discv5/ipdiscovery.go +++ /dev/null @@ -1,479 +0,0 @@ -package discv5 - -import ( - "fmt" - "net" - "sync" - "time" - - "github.com/sirupsen/logrus" -) - -// DefaultMinReports is the minimum number of PONG responses needed before considering IP valid -const DefaultMinReports = 5 - -// DefaultMajorityThreshold is the percentage threshold for IP consensus (0.0-1.0) -const DefaultMajorityThreshold = 0.75 - -// DefaultReportExpiry is how long to keep IP reports before expiring them -const DefaultReportExpiry = 30 * time.Minute - -// DefaultRecentWindow is the time window to consider reports "recent" for IP change detection -const DefaultRecentWindow = 5 * time.Minute - -// IPDiscovery tracks external IP addresses and ports reported by peers via PONG messages. -// -// It implements a consensus mechanism to detect the node's public IP address and port: -// - Collects IP:Port from PONG responses (shows our address as seen by remote peer) -// - Tracks IPv4 and IPv6 independently (separate consensus for each) -// - Requires minimum number of reports before considering an address valid -// - Requires majority threshold (e.g., 75%) for consensus -// - Expires old reports to handle IP/port changes -type IPDiscovery struct { - // mu protects the internal state - mu sync.RWMutex - - // ipv4Reports maps "IP:Port" string to report info for IPv4 - ipv4Reports map[string]*ipReport - - // ipv6Reports maps "IP:Port" string to report info for IPv6 - ipv6Reports map[string]*ipReport - - // currentConsensusIPv4 is the IPv4 address that reached consensus - currentConsensusIPv4 net.IP - - // currentConsensusIPv4Port is the IPv4 port that reached consensus - currentConsensusIPv4Port uint16 - - // currentConsensusIPv6 is the IPv6 address that reached consensus - currentConsensusIPv6 net.IP - - // currentConsensusIPv6Port is the IPv6 port that reached consensus - currentConsensusIPv6Port uint16 - - // config - minReports int // Minimum reports needed - majorityThreshold float64 // Threshold for majority (0.0-1.0) - reportExpiry time.Duration // How long to keep reports - recentWindow time.Duration // Time window for recent reports - onConsensusReached func(ip net.IP, port uint16, isIPv6 bool) // Callback when consensus is reached - logger logrus.FieldLogger - - // stats - totalReportsIPv4 int - totalReportsIPv6 int - consensusReachedIPv4 bool - consensusReachedIPv6 bool -} - -// ipReport tracks reports for a specific IP:Port combination -type ipReport struct { - ip net.IP - port uint16 - count int - firstSeen time.Time - lastSeen time.Time - reporterIDs []string // Track which peers reported this (for debugging) -} - -// IPDiscoveryConfig contains configuration for IP discovery -type IPDiscoveryConfig struct { - // MinReports is the minimum number of PONG responses needed (default: 3) - MinReports int - - // MajorityThreshold is the percentage needed for consensus (default: 0.75) - MajorityThreshold float64 - - // ReportExpiry is how long to keep reports (default: 30 minutes) - ReportExpiry time.Duration - - // RecentWindow is the time window to consider reports "recent" (default: 5 minutes) - // Used for detecting IP changes - recent reports get priority - RecentWindow time.Duration - - // OnConsensusReached is called when IP:Port consensus is reached or changes - // isIPv6 indicates whether this is an IPv6 address (true) or IPv4 (false) - OnConsensusReached func(ip net.IP, port uint16, isIPv6 bool) - - // Logger for debug messages - Logger logrus.FieldLogger -} - -// NewIPDiscovery creates a new IP discovery service. -func NewIPDiscovery(cfg IPDiscoveryConfig) *IPDiscovery { - if cfg.MinReports <= 0 { - cfg.MinReports = DefaultMinReports - } - if cfg.MajorityThreshold <= 0 || cfg.MajorityThreshold > 1.0 { - cfg.MajorityThreshold = DefaultMajorityThreshold - } - if cfg.ReportExpiry <= 0 { - cfg.ReportExpiry = DefaultReportExpiry - } - if cfg.RecentWindow <= 0 { - cfg.RecentWindow = DefaultRecentWindow - } - if cfg.Logger == nil { - cfg.Logger = logrus.New() - } - - return &IPDiscovery{ - ipv4Reports: make(map[string]*ipReport), - ipv6Reports: make(map[string]*ipReport), - minReports: cfg.MinReports, - majorityThreshold: cfg.MajorityThreshold, - reportExpiry: cfg.ReportExpiry, - recentWindow: cfg.RecentWindow, - onConsensusReached: cfg.OnConsensusReached, - logger: cfg.Logger, - } -} - -// ReportIP records an IP address and port from a PONG response. -// -// Parameters: -// - ip: The IP address as reported by the remote peer -// - port: The port as reported by the remote peer -// - reporterID: The node ID of the peer that sent the PONG (for tracking) -func (ipd *IPDiscovery) ReportIP(ip net.IP, port uint16, reporterID string) { - if ip == nil || ip.IsLoopback() || ip.IsUnspecified() { - // Ignore invalid IPs - return - } - - if port == 0 { - // Ignore invalid ports - return - } - - // Determine if IPv4 or IPv6 - isIPv6 := ip.To4() == nil - - ipd.mu.Lock() - defer ipd.mu.Unlock() - - // Clean up expired reports first - ipd.cleanupExpiredLocked() - - // Use "IP:Port" as the key - addrKey := fmt.Sprintf("%s:%d", ip.String(), port) - now := time.Now() - - // Select appropriate reports map - var reports map[string]*ipReport - var totalReports *int - if isIPv6 { - reports = ipd.ipv6Reports - totalReports = &ipd.totalReportsIPv6 - } else { - reports = ipd.ipv4Reports - totalReports = &ipd.totalReportsIPv4 - } - - // Get or create report for this IP:Port - report, exists := reports[addrKey] - if !exists { - report = &ipReport{ - ip: ip, - port: port, - firstSeen: now, - reporterIDs: make([]string, 0), - } - reports[addrKey] = report - } - - // Update report - report.count++ - report.lastSeen = now - report.reporterIDs = append(report.reporterIDs, reporterID) - *totalReports++ - - ipd.logger.WithFields(logrus.Fields{ - "addr": addrKey, - "ipv6": isIPv6, - "count": report.count, - "reporter": reporterID[:16], - "totalReports": *totalReports, - }).Debug("IP discovery: received address report") - - // Check for consensus (check both IPv4 and IPv6) - ipd.checkConsensusLocked() -} - -// checkConsensusLocked checks if an IP:Port has reached consensus for both IPv4 and IPv6. -// Must be called with lock held. -// -// This function handles both initial consensus and address changes: -// - For initial consensus: requires minimum reports and majority threshold -// - For address changes: prioritizes recent reports to detect when IP or port has changed -func (ipd *IPDiscovery) checkConsensusLocked() { - // Check IPv4 consensus - ipd.checkConsensusForFamilyLocked(false) - - // Check IPv6 consensus - ipd.checkConsensusForFamilyLocked(true) -} - -// checkConsensusForFamilyLocked checks consensus for a specific address family (IPv4 or IPv6). -// Must be called with lock held. -func (ipd *IPDiscovery) checkConsensusForFamilyLocked(isIPv6 bool) { - now := time.Now() - - // Select appropriate maps and state - var reports map[string]*ipReport - var currentConsensusIP *net.IP - var currentConsensusPort *uint16 - var consensusReached *bool - var totalReports *int - familyName := "IPv4" - - if isIPv6 { - reports = ipd.ipv6Reports - currentConsensusIP = &ipd.currentConsensusIPv6 - currentConsensusPort = &ipd.currentConsensusIPv6Port - consensusReached = &ipd.consensusReachedIPv6 - totalReports = &ipd.totalReportsIPv6 - familyName = "IPv6" - } else { - reports = ipd.ipv4Reports - currentConsensusIP = &ipd.currentConsensusIPv4 - currentConsensusPort = &ipd.currentConsensusIPv4Port - consensusReached = &ipd.consensusReachedIPv4 - totalReports = &ipd.totalReportsIPv4 - } - - // Separate recent reports from all reports - recentReports := make(map[string]int) - allReports := make(map[string]int) - - for addrKey, report := range reports { - allReports[addrKey] = report.count - - // Count reports within the recent window - if now.Sub(report.lastSeen) <= ipd.recentWindow { - recentReports[addrKey] = report.count - } - } - - // Calculate totals - totalCount := 0 - for _, count := range allReports { - totalCount += count - } - - totalRecentCount := 0 - for _, count := range recentReports { - totalRecentCount += count - } - - // Need minimum reports before considering consensus - if totalCount < ipd.minReports { - return - } - - // Current consensus address key - currentAddrKey := "" - if *currentConsensusIP != nil && *currentConsensusPort != 0 { - currentAddrKey = fmt.Sprintf("%s:%d", (*currentConsensusIP).String(), *currentConsensusPort) - } - - // If we already have consensus, check recent reports for address changes - if *consensusReached && totalRecentCount >= ipd.minReports { - // Find address with most recent reports - var maxRecentAddr string - maxRecentCount := 0 - for addrKey, count := range recentReports { - if count > maxRecentCount { - maxRecentCount = count - maxRecentAddr = addrKey - } - } - - // Check if recent reports show consensus on a DIFFERENT address - if maxRecentAddr != "" && maxRecentAddr != currentAddrKey { - recentMajority := float64(maxRecentCount) / float64(totalRecentCount) - - if recentMajority >= ipd.majorityThreshold { - // Address change detected! - newReport := reports[maxRecentAddr] - - ipd.logger.WithFields(logrus.Fields{ - "family": familyName, - "oldAddr": currentAddrKey, - "newAddr": maxRecentAddr, - "recentCount": maxRecentCount, - "recentTotal": totalRecentCount, - "recentMajority": recentMajority, - }).Warn("IP discovery: address change detected") - - // Clear old reports to prevent flip-flopping - for k := range reports { - delete(reports, k) - } - - // Re-add only the report for the new address - if newReport != nil { - reports[maxRecentAddr] = newReport - } - - *currentConsensusIP = newReport.ip - *currentConsensusPort = newReport.port - *totalReports = maxRecentCount - - // Call callback for address change - if ipd.onConsensusReached != nil { - ip := newReport.ip - port := newReport.port - go ipd.onConsensusReached(ip, port, isIPv6) - } - return - } - } - } - - // Check for initial consensus or stable consensus on all reports - var maxReport *ipReport - maxCount := 0 - for _, report := range reports { - if report.count > maxCount { - maxCount = report.count - maxReport = report - } - } - - if maxReport == nil { - return - } - - // Check if it meets majority threshold - majority := float64(maxReport.count) / float64(totalCount) - if majority >= ipd.majorityThreshold { - // Consensus reached! - addrChanged := !*consensusReached || - *currentConsensusIP == nil || - !maxReport.ip.Equal(*currentConsensusIP) || - maxReport.port != *currentConsensusPort - - if addrChanged { - ipd.logger.WithFields(logrus.Fields{ - "family": familyName, - "addr": fmt.Sprintf("%s:%d", maxReport.ip.String(), maxReport.port), - "count": maxReport.count, - "total": totalCount, - "majority": majority, - "threshold": ipd.majorityThreshold, - }).Info("IP discovery: consensus reached") - - *currentConsensusIP = maxReport.ip - *currentConsensusPort = maxReport.port - *consensusReached = true - - // Call callback if provided - if ipd.onConsensusReached != nil { - // Call in goroutine to avoid blocking - ip := maxReport.ip - port := maxReport.port - go ipd.onConsensusReached(ip, port, isIPv6) - } - } - } -} - -// cleanupExpiredLocked removes reports older than reportExpiry. -// Must be called with lock held. -func (ipd *IPDiscovery) cleanupExpiredLocked() { - now := time.Now() - - // Clean up IPv4 reports - for addrKey, report := range ipd.ipv4Reports { - if now.Sub(report.lastSeen) > ipd.reportExpiry { - delete(ipd.ipv4Reports, addrKey) - ipd.logger.WithField("addr", addrKey).Debug("IP discovery: expired old IPv4 report") - } - } - - // Clean up IPv6 reports - for addrKey, report := range ipd.ipv6Reports { - if now.Sub(report.lastSeen) > ipd.reportExpiry { - delete(ipd.ipv6Reports, addrKey) - ipd.logger.WithField("addr", addrKey).Debug("IP discovery: expired old IPv6 report") - } - } -} - -// GetConsensusIP returns the current consensus IPv4 address, or nil if no consensus. -// For IPv6, this returns nil. Use GetStats() for complete information. -func (ipd *IPDiscovery) GetConsensusIP() net.IP { - ipd.mu.RLock() - defer ipd.mu.RUnlock() - return ipd.currentConsensusIPv4 -} - -// GetStats returns statistics about IP discovery. -type IPDiscoveryStats struct { - TotalReportsIPv4 int - TotalReportsIPv6 int - UniqueIPv4Addrs int - UniqueIPv6Addrs int - ConsensusReachedIPv4 bool - ConsensusReachedIPv6 bool - ConsensusIPv4Addr string // "IP:Port" format - ConsensusIPv6Addr string // "IP:Port" format - IPv4Reports map[string]int // "IP:Port" -> count - IPv6Reports map[string]int // "IP:Port" -> count -} - -// GetStats returns current statistics. -func (ipd *IPDiscovery) GetStats() IPDiscoveryStats { - ipd.mu.RLock() - defer ipd.mu.RUnlock() - - stats := IPDiscoveryStats{ - TotalReportsIPv4: ipd.totalReportsIPv4, - TotalReportsIPv6: ipd.totalReportsIPv6, - UniqueIPv4Addrs: len(ipd.ipv4Reports), - UniqueIPv6Addrs: len(ipd.ipv6Reports), - ConsensusReachedIPv4: ipd.consensusReachedIPv4, - ConsensusReachedIPv6: ipd.consensusReachedIPv6, - IPv4Reports: make(map[string]int), - IPv6Reports: make(map[string]int), - } - - if ipd.currentConsensusIPv4 != nil && ipd.currentConsensusIPv4Port > 0 { - stats.ConsensusIPv4Addr = fmt.Sprintf("%s:%d", ipd.currentConsensusIPv4.String(), ipd.currentConsensusIPv4Port) - } - - if ipd.currentConsensusIPv6 != nil && ipd.currentConsensusIPv6Port > 0 { - stats.ConsensusIPv6Addr = fmt.Sprintf("%s:%d", ipd.currentConsensusIPv6.String(), ipd.currentConsensusIPv6Port) - } - - for addrKey, report := range ipd.ipv4Reports { - stats.IPv4Reports[addrKey] = report.count - } - - for addrKey, report := range ipd.ipv6Reports { - stats.IPv6Reports[addrKey] = report.count - } - - return stats -} - -// Reset clears all reports and resets consensus state. -// This can be used when the node's network changes. -func (ipd *IPDiscovery) Reset() { - ipd.mu.Lock() - defer ipd.mu.Unlock() - - ipd.ipv4Reports = make(map[string]*ipReport) - ipd.ipv6Reports = make(map[string]*ipReport) - ipd.currentConsensusIPv4 = nil - ipd.currentConsensusIPv4Port = 0 - ipd.currentConsensusIPv6 = nil - ipd.currentConsensusIPv6Port = 0 - ipd.consensusReachedIPv4 = false - ipd.consensusReachedIPv6 = false - ipd.totalReportsIPv4 = 0 - ipd.totalReportsIPv6 = 0 - - ipd.logger.Info("IP discovery: reset all reports") -} diff --git a/discv5/node/node.go b/discv5/node/node.go index 4eebd88..183d69f 100644 --- a/discv5/node/node.go +++ b/discv5/node/node.go @@ -107,25 +107,9 @@ func New(record *enr.Record) (*Node, error) { } id := PubkeyToID(pubKey) - // Extract IP address - ip := record.IP() - if ip == nil { - ip = record.IP6() - } - if ip == nil { - return nil, fmt.Errorf("node: ENR missing IP address") - } - - // Extract UDP port - udpPort := record.UDP() - if udpPort == 0 { - return nil, fmt.Errorf("node: ENR missing UDP port") - } - - // Create UDP address - addr := &net.UDPAddr{ - IP: ip, - Port: int(udpPort), + addr, err := udpEndpoint(record) + if err != nil { + return nil, err } // Extract optional TCP port @@ -144,6 +128,27 @@ func New(record *enr.Record) (*Node, error) { }, nil } +// udpEndpoint extracts the discovery endpoint from a record, keeping the port +// paired with its address family: ip goes with udp, ip6 with udp6 (falling +// back to udp, which dual-stack records share across both families). +func udpEndpoint(record *enr.Record) (*net.UDPAddr, error) { + ip := record.IP() + port := record.UDP() + if ip == nil { + ip = record.IP6() + if p := record.UDP6(); p != 0 { + port = p + } + } + if ip == nil { + return nil, fmt.Errorf("node: ENR missing IP address") + } + if port == 0 { + return nil, fmt.Errorf("node: ENR missing UDP port") + } + return &net.UDPAddr{IP: ip, Port: int(port)}, nil +} + // ID returns the node's unique identifier. func (n *Node) ID() ID { return n.id @@ -282,16 +287,8 @@ func (n *Node) UpdateENR(newRecord *enr.Record) bool { n.record = newRecord // Update network address if changed - ip := newRecord.IP() - if ip == nil { - ip = newRecord.IP6() - } - udpPort := newRecord.UDP() - if ip != nil && udpPort != 0 { - n.addr = &net.UDPAddr{ - IP: ip, - Port: int(udpPort), - } + if addr, err := udpEndpoint(newRecord); err == nil { + n.addr = addr } n.tcpPort = newRecord.TCP() diff --git a/discv5/node/node_test.go b/discv5/node/node_test.go index 8f9d8c1..3d4d0a3 100644 --- a/discv5/node/node_test.go +++ b/discv5/node/node_test.go @@ -31,6 +31,110 @@ func signedRecord(t *testing.T, seq uint64, port uint16) *enr.Record { return rec } +// TestNewIPv6OnlyRecord covers records carrying only ip6/udp6: the port must +// fall back to "udp6" or every IPv6-only bootnode is rejected. +func TestNewIPv6OnlyRecord(t *testing.T) { + key, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("generate key: %v", err) + } + rec := enr.New() + ip6 := net.ParseIP("2001:db8::1") + if err := rec.Set("ip6", ip6); err != nil { + t.Fatalf("set ip6: %v", err) + } + if err := rec.Set("udp6", uint16(30304)); err != nil { + t.Fatalf("set udp6: %v", err) + } + rec.SetSeq(1) + if err := rec.Sign(key); err != nil { + t.Fatalf("sign: %v", err) + } + + n, err := New(rec) + if err != nil { + t.Fatalf("New rejected IPv6-only record: %v", err) + } + if got := n.UDPPort(); got != 30304 { + t.Fatalf("UDPPort() = %d, want 30304", got) + } + if !n.IP().Equal(ip6) { + t.Fatalf("IP() = %v, want %v", n.IP(), ip6) + } +} + +// TestNewMismatchedFamilyRejected covers a record with an IPv4 address but +// only an IPv6 port: no complete endpoint exists, so pairing them would send +// packets to an unrelated port. +func TestNewMismatchedFamilyRejected(t *testing.T) { + key, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("generate key: %v", err) + } + rec := enr.New() + if err := rec.Set("ip", net.IPv4(3, 3, 3, 3)); err != nil { + t.Fatalf("set ip: %v", err) + } + if err := rec.Set("udp6", uint16(30304)); err != nil { + t.Fatalf("set udp6: %v", err) + } + rec.SetSeq(1) + if err := rec.Sign(key); err != nil { + t.Fatalf("sign: %v", err) + } + + if _, err := New(rec); err == nil { + t.Fatal("New accepted an ip record with only a udp6 port") + } +} + +// TestUpdateENRRefreshesIPv6Endpoint covers refreshing a node with a +// higher-sequence IPv6-only record: the address must follow the record. +func TestUpdateENRRefreshesIPv6Endpoint(t *testing.T) { + key, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("generate key: %v", err) + } + rec := enr.New() + if err := rec.Set("ip", net.IPv4(3, 3, 3, 3)); err != nil { + t.Fatalf("set ip: %v", err) + } + if err := rec.Set("udp", uint16(30303)); err != nil { + t.Fatalf("set udp: %v", err) + } + rec.SetSeq(1) + if err := rec.Sign(key); err != nil { + t.Fatalf("sign: %v", err) + } + n, err := New(rec) + if err != nil { + t.Fatalf("New: %v", err) + } + + ip6 := net.ParseIP("2001:db8::2") + newRec := enr.New() + if err := newRec.Set("ip6", ip6); err != nil { + t.Fatalf("set ip6: %v", err) + } + if err := newRec.Set("udp6", uint16(30305)); err != nil { + t.Fatalf("set udp6: %v", err) + } + newRec.SetSeq(2) + if err := newRec.Sign(key); err != nil { + t.Fatalf("sign: %v", err) + } + + if !n.UpdateENR(newRec) { + t.Fatal("UpdateENR rejected higher-sequence record") + } + if got := n.UDPPort(); got != 30305 { + t.Fatalf("UDPPort() = %d after update, want 30305", got) + } + if !n.IP().Equal(ip6) { + t.Fatalf("IP() = %v after update, want %v", n.IP(), ip6) + } +} + // TestNodeConcurrentENRAndStatsAccess exercises UpdateENR and SetStats against // every reader of the guarded fields. Under the race detector it fails if any // access to record, addr, tcpPort or the stats pointer is unsynchronized. diff --git a/discv5/protocol/handler.go b/discv5/protocol/handler.go index 476553c..d715e7e 100644 --- a/discv5/protocol/handler.go +++ b/discv5/protocol/handler.go @@ -431,27 +431,12 @@ func (h *Handler) handleOrdinaryPacket(packet *Packet, from *net.UDPAddr, localA var srcNodeID node.ID copy(srcNodeID[:], packet.SrcID) - // Look up session by node ID first (most efficient and handles IP changes) + // Sessions are keyed by the sender's node ID and every creation site uses the + // correct ID, so this always hits when a session exists. There is deliberately + // no address fallback: srcID is unauthenticated, so matching a session by + // source address alone let anyone who could guess a peer's IP:port reach the + // failure path below and tear that peer's session down. sess := h.config.Sessions.Get(srcNodeID) - - // If session exists, verify and update address if needed - if sess != nil { - // Check if the address has changed - if sess.RemoteAddr.String() != from.String() { - h.config.Logger.WithFields(logrus.Fields{ - "nodeID": srcNodeID.String()[:16], - "oldAddr": sess.RemoteAddr, - "newAddr": from, - }).Info("handler: node address changed, updating session") - - // Update the session's remote address - sess.UpdateAddr(from) - } - } else { - // No session by node ID, try lookup by address (slower fallback) - sess = h.config.Sessions.GetByAddr(from) - } - if sess == nil { // No session exists, send WHOAREYOU challenge h.config.Logger.WithFields(logrus.Fields{ @@ -470,22 +455,39 @@ func (h *Handler) handleOrdinaryPacket(packet *Packet, from *net.UDPAddr, localA packet.Message, ) if err != nil { - // Decryption failed - session is corrupted/expired - // Delete the session immediately to force a new handshake - h.config.Sessions.Delete(sess.RemoteID) + // Keep the session. Anyone can send an undecryptable packet naming this + // node ID, so deleting here let an attacker who knows only a peer's public + // ID destroy that peer's session at will. A peer that genuinely lost its + // keys recovers via the handshake, and Cache.Put replaces this entry by + // node ID when it lands, so deletion buys nothing. + // + // The challenge still goes to the packet source: the legitimate "restarted + // and lost my keys" case is a random packet, which by definition fails to + // decrypt, and the source is the only address such a peer is reachable at. + // Answering at sess.Addr() instead would be a reflection primitive. h.config.Logger.WithFields(logrus.Fields{ "nodeID": sess.RemoteID.String()[:16], "addr": from, "error": err, - }).Debug("handler: decryption failed, deleted session and sending WHOAREYOU") + }).Debug("handler: decryption failed, keeping session and sending WHOAREYOU") - // Extract dest node ID from packet srcID and send WHOAREYOU - if len(packet.SrcID) != 32 { - return fmt.Errorf("no source node ID in packet") - } - var destNodeID node.ID - copy(destNodeID[:], packet.SrcID) - return h.sendWHOAREYOU(from, destNodeID, packet.Header.Nonce, localAddr) + return h.sendWHOAREYOU(from, srcNodeID, packet.Header.Nonce, localAddr) + } + + // Only now is the sender proven: AES-GCM over the header authenticates + // possession of the session key, and the source address is not part of the + // AAD, so a NAT-rebound peer decrypts fine from its new address. + // Zone is part of the comparison because Cache.GetByAddr matches on the full + // address string; ignoring it here would let the two disagree for scoped IPv6 + // and strand a peer that changed interface. + if cur := sess.Addr(); cur == nil || cur.Port != from.Port || cur.Zone != from.Zone || !cur.IP.Equal(from.IP) { + h.config.Logger.WithFields(logrus.Fields{ + "nodeID": srcNodeID.String()[:16], + "oldAddr": sess.Addr(), + "newAddr": from, + }).Info("handler: node address changed, updating session") + + sess.UpdateAddr(from) } // Decode message from plaintext @@ -556,16 +558,29 @@ func (h *Handler) handleWHOAREYOUPacket(packet *Packet, from *net.UDPAddr, local return fmt.Errorf("no pending handshake for %s", from) } + // Answering this challenge derives fresh keys and replaces the session, so + // a WHOAREYOU nobody authenticated must not reach that path: otherwise + // anyone able to reach us from this address could swap a working session + // for keys the real peer never agreed to. Only a peer that actually + // received one of our packets can quote its nonce. + if !sess.SentNonce(packet.Header.Nonce) { + h.config.Logger.WithFields(logrus.Fields{ + "nodeID": sess.RemoteID.String()[:16], + "addr": from, + }).Debug("handler: ignoring WHOAREYOU referencing a nonce we never sent") + return fmt.Errorf("unsolicited WHOAREYOU from %s", from) + } + // Look up pending request for this node to get the message to replay pendingReq := h.requests.GetPendingRequestForNode(sess.RemoteID) if pendingReq == nil { - // No pending request either - just delete stale session + // Keep the session: the handshake replaces it by node ID if the peer + // really did lose its keys, so deleting here only helps an attacker. h.config.Logger.WithFields(logrus.Fields{ "nodeID": sess.RemoteID.String()[:16], "addr": from, "age": sess.Age(), - }).Info("handler: received unexpected WHOAREYOU with no pending request, deleting stale session") - h.config.Sessions.Delete(sess.RemoteID) + }).Debug("handler: received unexpected WHOAREYOU with no pending request") return fmt.Errorf("no pending handshake or request for %s", from) } @@ -587,8 +602,9 @@ func (h *Handler) handleWHOAREYOUPacket(packet *Packet, from *net.UDPAddr, local h.pendingHandshakes[pendingKey] = pending h.mu.Unlock() - // Delete the stale session - we'll create a new one during handshake - h.config.Sessions.Delete(sess.RemoteID) + // The stale session is not deleted here: sendHandshakePacket's Put replaces + // it by node ID once the new keys exist, so deleting first only widens the + // window in which the peer has no session at all. // Continue processing the WHOAREYOU with our pending handshake // After handshake completes, the message will be sent with the SAME request ID, @@ -648,7 +664,7 @@ func (h *Handler) handleWHOAREYOUPacket(packet *Packet, from *net.UDPAddr, local var enrBytes []byte localENR := h.config.LocalNode.Record() if packet.Challenge.ENRSeq == 0 || packet.Challenge.ENRSeq < localENR.Seq() { - enrBytes, err = localENR.EncodeRLP() + enrBytes, err = localENR.EncodeRLPBytes() if err != nil { h.config.Logger.WithError(err).Warn("handler: failed to encode ENR") } else { @@ -870,7 +886,7 @@ func (h *Handler) handleHandshakePacket(packet *Packet, from *net.UDPAddr, local h.config.Logger.WithFields(logrus.Fields{ "sourceNodeID": sourceNodeID.String()[:16], "from": from, - }).Info("handler: session established successfully") + }).Debug("handler: session established successfully") // Store node in session and call OnHandshakeComplete callback if remoteNodeFromENR != nil { @@ -977,12 +993,21 @@ func (h *Handler) handlePong(msg *Pong, remoteID node.ID, from *net.UDPAddr, rem "nodeID": remoteID, }).Debug("handler: received PONG") - // Match with pending request - h.requests.MatchResponse(msg.RequestID, remoteID, msg) + // An unsolicited PONG must not reach the side effects below: OnPongReceived + // casts a vote in the external-IP election that rewrites our published ENR, + // and the ENR branch triggers outbound traffic. + req := h.requests.MatchResponse(msg.RequestID, remoteID, msg) + if req == nil { + return nil + } - // Call OnPongReceived callback with the source IP and the IP/port reported in the PONG - // The IP and Port fields in PONG contain our address as seen by the remote peer - if h.config.OnPongReceived != nil && len(msg.IP) > 0 && msg.Port > 0 { + // The IP-discovery vote additionally requires the PONG to come from the address + // we pinged. A session peer can send an authenticated PONG from a forged source, + // and from.IP is what feeds the distinct-reporter threshold. Delivery above + // stays source-agnostic so NAT rebinding and mobile peers keep working; only + // this vote needs the endpoint proven. + if h.config.OnPongReceived != nil && len(msg.IP) > 0 && msg.Port > 0 && + req.DestAddr != nil && req.DestAddr.IP.Equal(from.IP) { reportedIP := net.IP(msg.IP) h.config.OnPongReceived(remoteID, from.IP, reportedIP, msg.Port) } @@ -1040,10 +1065,19 @@ func (h *Handler) handleFindNode(msg *FindNode, remoteID node.ID, from *net.UDPA }).Debug("handler: FINDNODE lookup completed via callback") } - // Split nodes into multiple packets if needed to stay under max packet size - // Each ENR is typically 200-400 bytes, so we limit to 3 nodes per packet to be safe + // Split nodes into multiple packets if needed to stay under max packet size. + // Each ENR is typically 200-400 bytes, so we limit to 3 nodes per packet to be safe. const maxNodesPerPacket = 3 + // Cap the total response so it never exceeds what real clients consume. go-ethereum + // honours only the first packet's `total` and reads at most 5 NODES packets + // (totalNodesResponseLimit); anything beyond that is dropped as unsolicited. sigp/discv5 + // caps at 16 nodes. Keep to <=5 packets / <=15 nodes so no served node is silently lost. + const maxNodesPerResponse = 15 + if len(nodes) > maxNodesPerResponse { + nodes = nodes[:maxNodesPerResponse] + } + // Calculate total number of packets needed totalPackets := (len(nodes) + maxNodesPerPacket - 1) / maxNodesPerPacket if totalPackets == 0 { @@ -1137,122 +1171,7 @@ func (h *Handler) handleTalkResp(msg *TalkResp, remoteID node.ID, from *net.UDPA // to send arbitrary messages through the protocol handler. // remoteNode is optional - if provided, it will be stored in pending handshakes for WHOAREYOU responses. func (h *Handler) SendMessage(msg Message, remoteID node.ID, to *net.UDPAddr, remoteNode *node.Node) error { - // Look up session - sess := h.config.Sessions.Get(remoteID) - - var packetBytes []byte - var err error - - if sess == nil { - // No session - send random packet to trigger WHOAREYOU from receiver - - // Store pending message for handshake completion - // Include the node object if we have it (needed for handshake) - handshakeKey := makeHandshakeKey(remoteID, to) - now := time.Now() - pending := &PendingHandshake{ - Message: msg, - ToNode: remoteNode, - ToAddr: to, - ToNodeID: remoteID, - CreatedAt: now, - LastRetry: now, - RetryCount: 0, - MaxRetries: 3, // Retry up to 3 times before giving up - } - - h.mu.Lock() - accepted := h.addPendingHandshake(handshakeKey, pending) - h.mu.Unlock() - - if !accepted { - return fmt.Errorf("pending handshake limit reached") - } - - // Log if we don't have node info for potential handshake - if remoteNode == nil { - h.config.Logger.WithField("remoteID", remoteID).Debug("handler: sending random packet without node info, may fail handshake if WHOAREYOU received") - } - - // Encode random packet (go-ethereum style) - // This will be 91 bytes: IV(16) + header(23) + authdata(32) + random(20) - packetBytes, err = EncodeRandomPacket(h.config.LocalNode.ID(), remoteID) - if err != nil { - return fmt.Errorf("failed to encode random packet: %w", err) - } - } else { - // Have session - encrypt and send normally - - // Encode message plaintext: message-type (1 byte) + RLP-encoded message - msgBytes, err := msg.Encode() - if err != nil { - return fmt.Errorf("failed to encode message: %w", err) - } - - // Build plaintext: message type + message data - plaintext := make([]byte, 1+len(msgBytes)) - plaintext[0] = msg.Type() - copy(plaintext[1:], msgBytes) - - // Generate nonce - nonce, err := crypto.GenerateRandomBytes(12) - if err != nil { - return fmt.Errorf("failed to generate nonce: %w", err) - } - - // Get local node ID - localNodeID := h.config.LocalNode.ID() - - // Authdata for ordinary message with session: srcID (32 bytes) - authdata := localNodeID[:] - - // Build unmasked header data for GCM authentication - // This returns: maskingIV, unmasked headerData (IV || header || authdata) - maskingIV, headerData, err := BuildOrdinaryHeaderData(localNodeID, nonce, authdata) - if err != nil { - return fmt.Errorf("failed to build header data: %w", err) - } - - // Encrypt message using session key - // GCM uses unmasked headerData as additional authenticated data - ciphertext, err := session.EncryptMessage(sess.EncryptionKey(), nonce, headerData, plaintext) - if err != nil { - return fmt.Errorf("failed to encrypt message: %w", err) - } - - // Now encode the full packet with the encrypted message - // This uses the same maskingIV to ensure consistency - packetBytes, err = EncodeOrdinaryPacket(localNodeID, remoteID, maskingIV, nonce, authdata, ciphertext) - if err != nil { - return fmt.Errorf("failed to encode ordinary packet: %w", err) - } - } - - // Send via UDP transport - h.mu.RLock() - transport := h.transport - h.mu.RUnlock() - - if transport == nil { - return fmt.Errorf("transport not initialized") - } - - if err := transport.SendTo(packetBytes, to); err != nil { - return fmt.Errorf("failed to send packet: %w", err) - } - - h.mu.Lock() - h.packetsSent++ - h.mu.Unlock() - - h.config.Logger.WithFields(logrus.Fields{ - "type": msg.Type(), - "to": to, - "nodeID": remoteID, - "size": len(packetBytes), - }).Trace("sent message") - - return nil + return h.SendMessageFrom(msg, remoteID, to, remoteNode, nil) } // SendMessageFrom sends a message to a remote node from a specific local address. @@ -1351,6 +1270,10 @@ func (h *Handler) SendMessageFrom(msg Message, remoteID node.ID, to *net.UDPAddr if err != nil { return fmt.Errorf("failed to encode ordinary packet: %w", err) } + + // Remembered so a WHOAREYOU quoting this nonce can be told apart from a + // forged one; answering a forged challenge would replace the session keys. + sess.RecordSentNonce(nonce) } // Send via UDP transport from the specified local address @@ -1627,11 +1550,15 @@ func (h *Handler) SendPing(n *node.Node) (<-chan *Response, error) { ENRSeq: h.config.LocalNode.Record().Seq(), } + // One read of the address for both the record and the send: a newer ENR can + // move it in between, and the endpoint proof needs them to agree. + destAddr := n.Addr() + // Register pending request (store message and node for replay if session becomes stale) - respChan := h.requests.AddRequest(requestID, n, ping) + respChan := h.requests.AddRequest(requestID, n, ping, destAddr) // Send PING (pass node object so it's available for handshake if needed) - if err := h.SendMessage(ping, n.ID(), n.Addr(), n); err != nil { + if err := h.SendMessage(ping, n.ID(), destAddr, n); err != nil { h.config.Logger.WithFields(logrus.Fields{ "to": n.Addr(), "nodeID": n.ID(), @@ -1800,11 +1727,13 @@ func (h *Handler) SendFindNode(n *node.Node, distances []uint) (<-chan *Response Distances: distances, } + destAddr := n.Addr() + // Register pending request (store message and node for replay if session becomes stale) - respChan := h.requests.AddRequest(requestID, n, findNode) + respChan := h.requests.AddRequest(requestID, n, findNode, destAddr) // Send FINDNODE (pass node object so it's available for handshake if needed) - if err := h.SendMessage(findNode, n.ID(), n.Addr(), n); err != nil { + if err := h.SendMessage(findNode, n.ID(), destAddr, n); err != nil { h.config.Logger.WithFields(logrus.Fields{ "to": n.Addr(), "nodeID": n.ID(), diff --git a/discv5/protocol/handler_test.go b/discv5/protocol/handler_test.go index c507385..111e7e3 100644 --- a/discv5/protocol/handler_test.go +++ b/discv5/protocol/handler_test.go @@ -22,7 +22,7 @@ func TestResolveHandshakeSender(t *testing.T) { t.Fatalf("create node: %v", err) } - encoded, err := record.EncodeRLP() + encoded, err := record.EncodeRLPBytes() if err != nil { t.Fatalf("encode record: %v", err) } diff --git a/discv5/protocol/messages.go b/discv5/protocol/messages.go index 9a095d2..b9f0be1 100644 --- a/discv5/protocol/messages.go +++ b/discv5/protocol/messages.go @@ -169,7 +169,7 @@ func (n *Nodes) Encode() ([]byte, error) { // Encode each ENR record and wrap in rlp.RawValue to prevent double-encoding records := make([]interface{}, len(n.Records)) for i, record := range n.Records { - encoded, err := record.EncodeRLP() + encoded, err := record.EncodeRLPBytes() if err != nil { return nil, fmt.Errorf("failed to encode ENR %d: %w", i, err) } @@ -248,7 +248,7 @@ func (r *RegTopic) Type() byte { // Encode returns the RLP encoding of the REGTOPIC message func (r *RegTopic) Encode() ([]byte, error) { - enrBytes, err := r.ENR.EncodeRLP() + enrBytes, err := r.ENR.EncodeRLPBytes() if err != nil { return nil, fmt.Errorf("failed to encode ENR: %w", err) } diff --git a/discv5/protocol/request.go b/discv5/protocol/request.go index 9bf787b..cc12224 100644 --- a/discv5/protocol/request.go +++ b/discv5/protocol/request.go @@ -1,6 +1,7 @@ package protocol import ( + "net" "sync" "time" @@ -41,6 +42,11 @@ type PendingRequest struct { // Message is the original message that was sent (for replay after re-handshake) Message Message + // DestAddr is the address this request was sent to, captured at send time. + // Node.Addr() cannot verify a response's origin because a newer ENR moves it, + // including one supplied by the peer being verified. + DestAddr *net.UDPAddr + // Timeout is when the request expires Timeout time.Time @@ -101,8 +107,12 @@ func NewRequestTracker(timeout time.Duration) *RequestTracker { // // The message and node parameters are stored for replay if the session becomes stale. // +// destAddr must be the address the caller sends this request to, captured once: +// n.Addr() is derived from the ENR and moves when a newer record arrives, so +// reading it again at send time can record an endpoint the request never went to. +// // Returns a channel that will receive the response or timeout. -func (rt *RequestTracker) AddRequest(requestID []byte, n *node.Node, msg Message) <-chan *Response { +func (rt *RequestTracker) AddRequest(requestID []byte, n *node.Node, msg Message, destAddr *net.UDPAddr) <-chan *Response { rt.mu.Lock() defer rt.mu.Unlock() @@ -113,6 +123,7 @@ func (rt *RequestTracker) AddRequest(requestID []byte, n *node.Node, msg Message NodeID: n.ID(), Node: n, Message: msg, + DestAddr: destAddr, Timeout: now.Add(rt.timeout), ResponseChan: make(chan *Response, 1), Retries: 0, @@ -129,22 +140,52 @@ func (rt *RequestTracker) AddRequest(requestID []byte, n *node.Node, msg Message return req.ResponseChan } +// respondsTo reports whether resp is the response type request expects. +// +// An unknown request type matches nothing: a new request/response pair must be +// registered here deliberately rather than defaulting to accepting any reply. +func respondsTo(request, resp Message) bool { + if request == nil || resp == nil { + return false + } + + switch request.Type() { + case PingMsg: + return resp.Type() == PongMsg + case FindNodeMsg: + return resp.Type() == NodesMsg + case TalkReqMsg: + return resp.Type() == TalkRespMsg + default: + return false + } +} + // MatchResponse matches a response to a pending request. // -// Returns true if the request was matched and notified. -func (rt *RequestTracker) MatchResponse(requestID []byte, nodeID node.ID, msg Message) bool { +// Returns the matched request, or nil if there was none. Callers that gate a side +// effect on where the request was sent need DestAddr from the result. +func (rt *RequestTracker) MatchResponse(requestID []byte, nodeID node.ID, msg Message) *PendingRequest { rt.mu.Lock() defer rt.mu.Unlock() key := string(requestID) req, exists := rt.requests[key] if !exists { - return false + return nil } // Verify node ID matches if req.NodeID != nodeID { - return false + return nil + } + + // The response must be the kind this request asked for. Request IDs are ours + // but the peer learns them, so without this a PONG can match a pending + // FINDNODE: it would both fire the PONG side effects and consume the entry + // below, silently stranding the lookup that was waiting on it. + if !respondsTo(req.Message, msg) { + return nil } // Handle multi-packet NODES responses @@ -166,7 +207,7 @@ func (rt *RequestTracker) MatchResponse(requestID []byte, nodeID node.ID, msg Me // If we haven't received all packets yet, keep waiting if req.ReceivedCount < req.ExpectedTotal { - return true + return req } // All packets received, send accumulated response @@ -189,7 +230,7 @@ func (rt *RequestTracker) MatchResponse(requestID []byte, nodeID node.ID, msg Me delete(rt.requests, key) close(req.ResponseChan) - return true + return req } // handleTimeout handles request timeout. diff --git a/discv5/protocol/request_match_test.go b/discv5/protocol/request_match_test.go new file mode 100644 index 0000000..ddcc1cd --- /dev/null +++ b/discv5/protocol/request_match_test.go @@ -0,0 +1,85 @@ +package protocol + +import ( + "net" + "testing" + "time" + + "github.com/ethpandaops/bootnodoor/discv5/node" +) + +// Request IDs are ours but the peer learns them, so a response must also be the +// kind the request asked for. Otherwise a PONG matches a pending FINDNODE: it +// fires the PONG side effects and consumes the entry, stranding the lookup. +func TestMatchResponseRejectsWrongResponseType(t *testing.T) { + rt := NewRequestTracker(time.Second) + + n, err := node.New(signedRecord(t, generateKey(t), 1, nil)) + if err != nil { + t.Fatalf("node.New: %v", err) + } + + peerAddr := &net.UDPAddr{IP: net.IPv4(203, 0, 113, 1), Port: 30303} + requestID := []byte{0x01, 0x02, 0x03, 0x04} + ch := rt.AddRequest(requestID, n, &FindNode{RequestID: requestID, Distances: []uint{1}}, peerAddr) + + pong := &Pong{RequestID: requestID, IP: []byte{9, 9, 9, 9}, Port: 30303} + if rt.MatchResponse(requestID, n.ID(), pong) != nil { + t.Fatal("a PONG matched a pending FINDNODE") + } + + select { + case resp := <-ch: + t.Fatalf("pending FINDNODE was resolved by a PONG: %+v", resp) + default: + } + + nodes := &Nodes{RequestID: requestID, Total: 1} + if rt.MatchResponse(requestID, n.ID(), nodes) == nil { + t.Fatal("the matching NODES response was rejected") + } +} + +// MatchResponse must hand back the destination so handlePong can tell a PONG that +// came from the address we pinged apart from one with a forged source. Sessions +// are keyed by node ID and accept packets from a changed address, so the source +// alone proves nothing about where the request went. +func TestMatchResponseReportsRequestDestination(t *testing.T) { + rt := NewRequestTracker(time.Second) + + n, err := node.New(signedRecord(t, generateKey(t), 1, nil)) + if err != nil { + t.Fatalf("node.New: %v", err) + } + + peerAddr := &net.UDPAddr{IP: net.IPv4(198, 51, 100, 7), Port: 30303} + requestID := []byte{0x11, 0x22} + rt.AddRequest(requestID, n, &Ping{RequestID: requestID}, peerAddr) + + req := rt.MatchResponse(requestID, n.ID(), &Pong{RequestID: requestID}) + if req == nil { + t.Fatal("a PONG did not match its pending PING") + } + if req.DestAddr == nil || !req.DestAddr.IP.Equal(peerAddr.IP) { + t.Fatalf("DestAddr = %v, want the address the PING was sent to (%v)", req.DestAddr, peerAddr.IP) + } +} + +// The PING/PONG pair must still match, or gating handlePong on this would drop +// every legitimate PONG. +func TestMatchResponseAcceptsPongForPing(t *testing.T) { + rt := NewRequestTracker(time.Second) + + n, err := node.New(signedRecord(t, generateKey(t), 1, nil)) + if err != nil { + t.Fatalf("node.New: %v", err) + } + + peerAddr := &net.UDPAddr{IP: net.IPv4(203, 0, 113, 1), Port: 30303} + requestID := []byte{0x0a, 0x0b} + rt.AddRequest(requestID, n, &Ping{RequestID: requestID}, peerAddr) + + if rt.MatchResponse(requestID, n.ID(), &Pong{RequestID: requestID}) == nil { + t.Fatal("a PONG did not match its pending PING") + } +} diff --git a/discv5/protocol/session_proof_test.go b/discv5/protocol/session_proof_test.go new file mode 100644 index 0000000..53081e1 --- /dev/null +++ b/discv5/protocol/session_proof_test.go @@ -0,0 +1,318 @@ +package protocol + +import ( + "context" + "net" + "sync" + "testing" + "time" + + "github.com/ethpandaops/bootnodoor/discv5/node" + "github.com/ethpandaops/bootnodoor/discv5/session" + "github.com/sirupsen/logrus" +) + +type sentPacket struct { + data []byte + to *net.UDPAddr +} + +// recordingTransport captures where packets went, so a test can assert a +// challenge was not reflected at the victim's address. +type recordingTransport struct { + mu sync.Mutex + sent []sentPacket +} + +func (r *recordingTransport) SendTo(data []byte, to *net.UDPAddr) error { + r.mu.Lock() + defer r.mu.Unlock() + r.sent = append(r.sent, sentPacket{data: data, to: to}) + return nil +} + +func (r *recordingTransport) Send(data []byte, to *net.UDPAddr, _ *net.UDPAddr) error { + return r.SendTo(data, to) +} + +func (r *recordingTransport) destinations() []string { + r.mu.Lock() + defer r.mu.Unlock() + out := make([]string, 0, len(r.sent)) + for _, p := range r.sent { + out = append(out, p.to.String()) + } + return out +} + +func sessionHandler(t *testing.T) (*Handler, *recordingTransport, *session.Cache, context.CancelFunc) { + t.Helper() + + logger := logrus.New() + logger.SetLevel(logrus.PanicLevel) + + localKey := generateKey(t) + localNode, err := node.New(signedRecord(t, localKey, 1, nil)) + if err != nil { + t.Fatalf("node.New: %v", err) + } + + cache := session.NewCache(16, time.Hour, logger) + ctx, cancel := context.WithCancel(context.Background()) + h := NewHandler(ctx, HandlerConfig{ + LocalNode: localNode, + Sessions: cache, + PrivateKey: localKey, + Logger: logger, + }) + + tr := &recordingTransport{} + h.SetTransport(tr) + return h, tr, cache, cancel +} + +// victimSession installs a session for a peer reachable at addr, returning the +// peer's node ID and the keys the session was built with. +func victimSession(t *testing.T, cache *session.Cache, addr *net.UDPAddr) (*node.Node, *session.Session) { + t.Helper() + + peerKey := generateKey(t) + peerNode, err := node.New(signedRecord(t, peerKey, 1, nil)) + if err != nil { + t.Fatalf("node.New: %v", err) + } + + keys := &session.SessionKeys{ + InitiatorKey: []byte("0123456789abcdef"), + RecipientKey: []byte("fedcba9876543210"), + } + sess := session.NewSession(peerNode.ID(), addr, keys, false, time.Hour) + sess.SetNode(peerNode) + cache.Put(sess) + + return peerNode, sess +} + +// garbageOrdinaryPacket builds a syntactically valid ordinary packet that names +// srcID but whose ciphertext cannot decrypt against any real session key. +func garbageOrdinaryPacket(t *testing.T, srcID, destID node.ID) []byte { + t.Helper() + + nonce := make([]byte, 12) + for i := range nonce { + nonce[i] = byte(i + 1) + } + + authdata := srcID[:] + maskingIV, _, err := BuildOrdinaryHeaderData(srcID, nonce, authdata) + if err != nil { + t.Fatalf("BuildOrdinaryHeaderData: %v", err) + } + + data, err := EncodeOrdinaryPacket(srcID, destID, maskingIV, nonce, authdata, []byte("not-a-valid-ciphertext")) + if err != nil { + t.Fatalf("EncodeOrdinaryPacket: %v", err) + } + return data +} + +// authenticPacket encrypts msg with the session key so it decrypts successfully, +// letting a test vary only the source address. +func authenticPacket(t *testing.T, sess *session.Session, srcID, destID node.ID, msg Message) []byte { + t.Helper() + + msgBytes, err := msg.Encode() + if err != nil { + t.Fatalf("msg.Encode: %v", err) + } + plaintext := make([]byte, 1+len(msgBytes)) + plaintext[0] = msg.Type() + copy(plaintext[1:], msgBytes) + + nonce := make([]byte, 12) + for i := range nonce { + nonce[i] = byte(i + 9) + } + + authdata := srcID[:] + maskingIV, headerData, err := BuildOrdinaryHeaderData(srcID, nonce, authdata) + if err != nil { + t.Fatalf("BuildOrdinaryHeaderData: %v", err) + } + + ciphertext, err := session.EncryptMessage(sess.DecryptionKey(), nonce, headerData, plaintext) + if err != nil { + t.Fatalf("EncryptMessage: %v", err) + } + + data, err := EncodeOrdinaryPacket(srcID, destID, maskingIV, nonce, authdata, ciphertext) + if err != nil { + t.Fatalf("EncodeOrdinaryPacket: %v", err) + } + return data +} + +// A session peer can send a correctly encrypted PONG from a forged source, and +// that source is what feeds the distinct-reporter threshold in IP discovery. The +// vote must require the PONG to come from the address the PING was sent to, while +// delivery stays source-agnostic so NAT rebinding still works. +func TestPongVoteRequiresRequestDestination(t *testing.T) { + for _, tc := range []struct { + name string + fromAddr *net.UDPAddr + wantVote bool + }{ + {"from the pinged address", &net.UDPAddr{IP: net.IPv4(198, 51, 100, 5), Port: 30303}, true}, + {"from a forged source", &net.UDPAddr{IP: net.IPv4(203, 0, 113, 9), Port: 40404}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + h, _, cache, cancel := sessionHandler(t) + defer cancel() + + pingedAddr := &net.UDPAddr{IP: net.IPv4(198, 51, 100, 5), Port: 30303} + peer, sess := victimSession(t, cache, pingedAddr) + + votes := 0 + h.config.OnPongReceived = func(node.ID, net.IP, net.IP, uint16) { votes++ } + + requestID := []byte{0x07, 0x08} + h.requests.AddRequest(requestID, peer, &Ping{RequestID: requestID}, pingedAddr) + + pong := &Pong{RequestID: requestID, IP: []byte{9, 9, 9, 9}, Port: 30303} + data := authenticPacket(t, sess, peer.ID(), h.config.LocalNode.ID(), pong) + + if err := h.HandleIncomingPacket(data, tc.fromAddr, pingedAddr); err != nil { + t.Fatalf("HandleIncomingPacket: %v", err) + } + + if tc.wantVote && votes != 1 { + t.Fatalf("votes = %d for a PONG from the pinged address, want 1", votes) + } + if !tc.wantVote && votes != 0 { + t.Fatalf("votes = %d for a PONG from a forged source, want 0", votes) + } + }) + } +} + +// A node ID is public information from an ENR, so an attacker who knows only that +// must not be able to move a peer's session to an address of their choosing. +func TestSpoofedOrdinaryPacketDoesNotMigrateSession(t *testing.T) { + h, _, cache, cancel := sessionHandler(t) + defer cancel() + + victimAddr := &net.UDPAddr{IP: net.IPv4(198, 51, 100, 5), Port: 30303} + victim, sess := victimSession(t, cache, victimAddr) + victimID := victim.ID() + + attackerAddr := &net.UDPAddr{IP: net.IPv4(203, 0, 113, 9), Port: 40404} + data := garbageOrdinaryPacket(t, victimID, h.config.LocalNode.ID()) + _ = h.HandleIncomingPacket(data, attackerAddr, victimAddr) + + if got := sess.Addr().String(); got != victimAddr.String() { + t.Fatalf("session migrated to %s on an unauthenticated packet, want %s", got, victimAddr) + } +} + +// Deleting on decrypt failure let anyone holding a peer's public node ID destroy +// that peer's session, repeatably. The session must survive. +func TestDecryptFailureKeepsSession(t *testing.T) { + h, tr, cache, cancel := sessionHandler(t) + defer cancel() + + victimAddr := &net.UDPAddr{IP: net.IPv4(198, 51, 100, 5), Port: 30303} + victim, _ := victimSession(t, cache, victimAddr) + victimID := victim.ID() + + attackerAddr := &net.UDPAddr{IP: net.IPv4(203, 0, 113, 9), Port: 40404} + data := garbageOrdinaryPacket(t, victimID, h.config.LocalNode.ID()) + + for i := 0; i < 5; i++ { + _ = h.HandleIncomingPacket(data, attackerAddr, victimAddr) + if cache.Get(victimID) == nil { + t.Fatalf("session destroyed by unauthenticated packet %d", i+1) + } + } + + // The challenge answers the packet source, which is the only address a peer + // that genuinely lost its keys could be reached at. + for _, dst := range tr.destinations() { + if dst == victimAddr.String() { + t.Fatal("challenge was reflected at the victim's address") + } + } +} + +// WHOAREYOU is unauthenticated and answering one replaces the session keys, so a +// challenge quoting a nonce we never sent must be ignored. +func TestSpoofedWhoareyouDoesNotReplaceSession(t *testing.T) { + h, _, cache, cancel := sessionHandler(t) + defer cancel() + + victimAddr := &net.UDPAddr{IP: net.IPv4(198, 51, 100, 5), Port: 30303} + victim, sess := victimSession(t, cache, victimAddr) + victimID := victim.ID() + + keysBefore := sess.EncryptionKey() + + nonce := make([]byte, 12) + for i := range nonce { + nonce[i] = 0xAA + } + data, _, err := EncodeWHOAREYOUPacket(h.config.LocalNode.ID(), nonce, &WHOAREYOUChallenge{ + IDNonce: make([]byte, 16), + ENRSeq: 0, + }) + if err != nil { + t.Fatalf("EncodeWHOAREYOUPacket: %v", err) + } + + _ = h.HandleIncomingPacket(data, victimAddr, victimAddr) + + after := cache.Get(victimID) + if after == nil { + t.Fatal("spoofed WHOAREYOU destroyed the session") + } + if string(after.EncryptionKey()) != string(keysBefore) { + t.Fatal("spoofed WHOAREYOU replaced the session keys") + } +} + +// With a request in flight the forged challenge reaches handshake recovery, which +// derives new keys and replaces the session by node ID. Asserting a session still +// exists is not enough here — it exists but the real peer cannot read it. +func TestSpoofedWhoareyouWithPendingRequestKeepsKeys(t *testing.T) { + h, _, cache, cancel := sessionHandler(t) + defer cancel() + + victimAddr := &net.UDPAddr{IP: net.IPv4(198, 51, 100, 5), Port: 30303} + victim, sess := victimSession(t, cache, victimAddr) + victimID := victim.ID() + + keysBefore := string(sess.EncryptionKey()) + + requestID := []byte{0x01, 0x02, 0x03, 0x04} + h.requests.AddRequest(requestID, victim, &Ping{RequestID: requestID}, victimAddr) + + nonce := make([]byte, 12) + for i := range nonce { + nonce[i] = 0xBB + } + data, _, err := EncodeWHOAREYOUPacket(h.config.LocalNode.ID(), nonce, &WHOAREYOUChallenge{ + IDNonce: make([]byte, 16), + ENRSeq: 0, + }) + if err != nil { + t.Fatalf("EncodeWHOAREYOUPacket: %v", err) + } + + _ = h.HandleIncomingPacket(data, victimAddr, victimAddr) + + after := cache.Get(victimID) + if after == nil { + t.Fatal("spoofed WHOAREYOU destroyed the session") + } + if string(after.EncryptionKey()) != keysBefore { + t.Fatal("spoofed WHOAREYOU replaced the session keys via handshake recovery") + } +} diff --git a/discv5/service.go b/discv5/service.go index 0b64d1f..750fd87 100644 --- a/discv5/service.go +++ b/discv5/service.go @@ -54,6 +54,7 @@ type Transport interface { protocol.Transport LocalAddr() *net.UDPAddr AddHandler(handler func(data []byte, from *net.UDPAddr, localAddr *net.UDPAddr) bool) + AddHandlerFor(protocol string, handler func(data []byte, from *net.UDPAddr, localAddr *net.UDPAddr) bool) } // New creates a new discv5 service. @@ -169,7 +170,7 @@ func New(cfg *Config, transport Transport) (*Service, error) { protocolHandler.SetTransport(transport) // Register packet handler with transport - transport.AddHandler(s.packetHandler) + transport.AddHandlerFor("discv5", s.packetHandler) return s, nil } @@ -367,10 +368,12 @@ func (s *Service) TalkReq(n *node.Node, protocolName string, request []byte) ([] Request: request, } + destAddr := n.Addr() + // Register pending request and send - respChan := s.handler.Requests().AddRequest(requestID, n, talkReq) + respChan := s.handler.Requests().AddRequest(requestID, n, talkReq, destAddr) - if err := s.handler.SendMessage(talkReq, n.ID(), n.Addr(), n); err != nil { + if err := s.handler.SendMessage(talkReq, n.ID(), destAddr, n); err != nil { s.handler.Requests().CancelRequest(requestID) return nil, fmt.Errorf("failed to send talkreq: %w", err) } diff --git a/discv5/session/cache.go b/discv5/session/cache.go index 3536f74..792e938 100644 --- a/discv5/session/cache.go +++ b/discv5/session/cache.go @@ -109,8 +109,11 @@ func (c *Cache) Put(session *Session) { c.mu.Lock() defer c.mu.Unlock() - // Check if we need to evict - if len(c.sessions) >= c.maxSessions { + // Replacing an existing key frees no slot, so only evict when this Put grows + // the map. Handshake recovery replaces a retained session by node ID; without + // this check it would evict an unrelated live peer and leave the map short. + _, replacing := c.sessions[session.RemoteID] + if !replacing && len(c.sessions) >= c.maxSessions { // Find and remove the least recently used session c.evictLRU() } @@ -118,7 +121,7 @@ func (c *Cache) Put(session *Session) { // Store the session c.sessions[session.RemoteID] = session - c.logger.WithField("nodeID", session.RemoteID).WithField("addr", session.RemoteAddr).WithField("lifetime", c.sessionLifetime).Trace("cached new session") + c.logger.WithField("nodeID", session.RemoteID).WithField("addr", session.Addr()).WithField("lifetime", c.sessionLifetime).Trace("cached new session") } // Delete removes a session from the cache. @@ -216,7 +219,7 @@ func (c *Cache) GetByAddr(addr *net.UDPAddr) *Session { defer c.mu.RUnlock() for _, session := range c.sessions { - if session.RemoteAddr.String() == addr.String() { + if session.Addr().String() == addr.String() { if !session.IsExpired() { session.Touch() return session diff --git a/discv5/session/cache_test.go b/discv5/session/cache_test.go new file mode 100644 index 0000000..c6854d5 --- /dev/null +++ b/discv5/session/cache_test.go @@ -0,0 +1,70 @@ +package session + +import ( + "net" + "testing" + "time" + + "github.com/ethpandaops/bootnodoor/discv5/node" + "github.com/sirupsen/logrus" +) + +func quietCache(t *testing.T, maxSessions int) *Cache { + t.Helper() + logger := logrus.New() + logger.SetLevel(logrus.PanicLevel) + return NewCache(maxSessions, time.Hour, logger) +} + +func testSession(id byte, addr *net.UDPAddr) *Session { + var nodeID node.ID + nodeID[0] = id + keys := &SessionKeys{ + InitiatorKey: []byte("0123456789abcdef"), + RecipientKey: []byte("fedcba9876543210"), + } + return NewSession(nodeID, addr, keys, false, time.Hour) +} + +// Replacing an existing session frees no slot, so it must not evict anyone. +// Handshake recovery replaces a retained session by node ID; evicting on that +// path would drop an unrelated live peer and leave the cache below capacity. +func TestPutReplacingDoesNotEvict(t *testing.T) { + cache := quietCache(t, 2) + addr := &net.UDPAddr{IP: net.IPv4(198, 51, 100, 1), Port: 30303} + + first := testSession(1, addr) + second := testSession(2, addr) + cache.Put(first) + cache.Put(second) + + if cache.Count() != 2 { + t.Fatalf("Count = %d after filling, want 2", cache.Count()) + } + + // Recovery touches the stale entry, then replaces it with fresh keys. + cache.Get(first.RemoteID) + cache.Put(testSession(1, addr)) + + if cache.Count() != 2 { + t.Errorf("Count = %d after replacing an existing session, want 2", cache.Count()) + } + if cache.Get(second.RemoteID) == nil { + t.Error("replacing one session evicted an unrelated peer") + } +} + +// Adding a genuinely new session at capacity must still evict, or the cache +// would grow without bound. +func TestPutNewAtCapacityEvicts(t *testing.T) { + cache := quietCache(t, 2) + addr := &net.UDPAddr{IP: net.IPv4(198, 51, 100, 1), Port: 30303} + + cache.Put(testSession(1, addr)) + cache.Put(testSession(2, addr)) + cache.Put(testSession(3, addr)) + + if cache.Count() > 2 { + t.Fatalf("Count = %d, want the cache bounded at 2", cache.Count()) + } +} diff --git a/discv5/session/session.go b/discv5/session/session.go index 2cd1d20..a73b93a 100644 --- a/discv5/session/session.go +++ b/discv5/session/session.go @@ -2,6 +2,7 @@ package session import ( "net" + "slices" "sync" "time" @@ -20,8 +21,10 @@ type Session struct { // RemoteID is the node ID of the remote peer RemoteID node.ID - // RemoteAddr is the network address of the remote peer - RemoteAddr *net.UDPAddr + // remoteAddr is the network address of the remote peer. Unexported because it + // is mutated by UpdateAddr under mu while packets are handled concurrently; + // read it through Addr(). + remoteAddr *net.UDPAddr // Node is the full node information (ENR, etc.) // This allows protocol operations to access node data without a separate table lookup @@ -42,6 +45,9 @@ type Session struct { // LastUsed is the last time this session was used LastUsed time.Time + // sentNonces holds the nonces of recent ordinary packets we sent, oldest first. + sentNonces []string + // mu protects mutable fields mu sync.RWMutex } @@ -69,7 +75,7 @@ func NewSession( return &Session{ RemoteID: remoteID, - RemoteAddr: remoteAddr, + remoteAddr: remoteAddr, Keys: keys, IsInitiator: isInitiator, CreatedAt: now, @@ -109,14 +115,62 @@ func (s *Session) SetNode(n *node.Node) { s.Node = n } +// maxSentNonces bounds the remembered nonces. A WHOAREYOU answers a packet we +// sent moments ago, so the window only has to cover the traffic we can send to +// one peer within a request lifetime; sized well above that, because being too +// small silently drops a legitimate peer's restart recovery until its next +// packet, while being generous costs a few hundred bytes per session. +const maxSentNonces = 64 + +// RecordSentNonce remembers the nonce of an ordinary packet we sent on this +// session, so a WHOAREYOU claiming to answer it can be verified. +func (s *Session) RecordSentNonce(nonce []byte) { + if len(nonce) == 0 { + return + } + + s.mu.Lock() + defer s.mu.Unlock() + + s.sentNonces = append(s.sentNonces, string(nonce)) + if len(s.sentNonces) > maxSentNonces { + s.sentNonces = s.sentNonces[1:] + } +} + +// SentNonce reports whether nonce belongs to a packet we sent on this session. +// +// WHOAREYOU is unauthenticated, and answering one replaces this session's keys, +// so a forged challenge must not be able to reach that path. Only a peer that +// actually received one of our packets can quote its nonce back. +func (s *Session) SentNonce(nonce []byte) bool { + if len(nonce) == 0 { + return false + } + + s.mu.RLock() + defer s.mu.RUnlock() + + return slices.Contains(s.sentNonces, string(nonce)) +} + +// Addr returns the remote address for this session. +func (s *Session) Addr() *net.UDPAddr { + s.mu.RLock() + defer s.mu.RUnlock() + + return s.remoteAddr +} + // UpdateAddr updates the remote address for this session. // -// This is called when we detect that a node has moved to a different IP address. +// Only call this once the sender is authenticated: an unauthenticated packet +// naming this node ID must not be able to steer where the session points. func (s *Session) UpdateAddr(addr *net.UDPAddr) { s.mu.Lock() defer s.mu.Unlock() - s.RemoteAddr = addr + s.remoteAddr = addr } // GetNode returns the node reference for this session. @@ -181,14 +235,13 @@ func (s *Session) TimeUntilExpiry() time.Duration { // String returns a human-readable representation of the session. func (s *Session) String() string { - s.mu.RLock() - defer s.mu.RUnlock() - role := "recipient" if s.IsInitiator { role = "initiator" } + // Age and IdleTime take the read lock themselves; holding it here as well + // would be a recursive RLock, which deadlocks if a writer queues in between. return "Session{" + "RemoteID: " + s.RemoteID.String() + ", Role: " + role + diff --git a/enr/encoding.go b/enr/encoding.go index de89888..3da1854 100644 --- a/enr/encoding.go +++ b/enr/encoding.go @@ -9,7 +9,7 @@ import ( "github.com/ethereum/go-ethereum/rlp" ) -// EncodeRLP returns the RLP encoding of the record. +// EncodeRLPBytes returns the RLP encoding of the record. // // The encoding format is: [signature, seq, k1, v1, k2, v2, ...] // where keys are sorted lexicographically. @@ -18,7 +18,17 @@ import ( // when the record is modified. // // Returns ErrRecordTooLarge if the encoded record exceeds 300 bytes. -func (r *Record) EncodeRLP() ([]byte, error) { +func (r *Record) EncodeRLPBytes() ([]byte, error) { + // A bootnode serves the same closest-node set to every requester, so the hot + // records are re-encoded concurrently; taking the write lock for a cache read + // serialises that on one mutex. + r.mu.RLock() + cached := r.raw + r.mu.RUnlock() + if len(cached) > 0 { + return cached, nil + } + r.mu.Lock() defer r.mu.Unlock() @@ -38,6 +48,20 @@ func (r *Record) EncodeRLP() ([]byte, error) { return encoded, nil } +// EncodeRLP implements rlp.Encoder. Without it a nested Record encodes as an +// empty list, since every field is unexported. Receiver must stay a pointer +// (unlike go-ethereum's value receiver): Record holds a mutex. +func (r *Record) EncodeRLP(w io.Writer) error { + encoded, err := r.EncodeRLPBytes() + if err != nil { + return err + } + + _, err = w.Write(encoded) + + return err +} + // DecodeRLPBytes decodes an RLP-encoded record from a byte slice. // // The input must be a valid RLP list containing: @@ -54,6 +78,10 @@ func (r *Record) EncodeRLP() ([]byte, error) { // // Handle error // } func (r *Record) DecodeRLPBytes(data []byte) error { + if len(data) > MaxRecordSize { + return ErrRecordTooLarge + } + r.mu.Lock() defer r.mu.Unlock() @@ -139,7 +167,7 @@ func (r *Record) DecodeRLP(s *rlp.Stream) error { // // Example output: "enr:-IS4QHCYrYZ..." func (r *Record) EncodeBase64() (string, error) { - encoded, err := r.EncodeRLP() + encoded, err := r.EncodeRLPBytes() if err != nil { return "", err } diff --git a/enr/encoding_test.go b/enr/encoding_test.go new file mode 100644 index 0000000..977dde5 --- /dev/null +++ b/enr/encoding_test.go @@ -0,0 +1,170 @@ +package enr + +import ( + "bytes" + "crypto/ecdsa" + "net" + "testing" + + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/p2p/enode" + gethenr "github.com/ethereum/go-ethereum/p2p/enr" + "github.com/ethereum/go-ethereum/rlp" +) + +// enrResponse mirrors discv4's ENRRESPONSE layout: a Record nested in a struct +// alongside other fields, which is the shape that encoded as an empty list. +type enrResponse struct { + ReplyTok []byte + Record *Record +} + +func signedRecord(t *testing.T) (*Record, *ecdsa.PrivateKey) { + t.Helper() + + privKey, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("Failed to generate key: %v", err) + } + + record := New() + record.Set("ip", net.IPv4(192, 168, 1, 1)) + record.Set("udp", uint16(9000)) + + if err := record.Sign(privKey); err != nil { + t.Fatalf("Failed to sign record: %v", err) + } + + return record, privKey +} + +// TestNestedRecordEncoding tests that a Record nested in a struct serializes as +// its own record encoding rather than an empty list. +func TestNestedRecordEncoding(t *testing.T) { + record, _ := signedRecord(t) + + want, err := record.EncodeRLPBytes() + if err != nil { + t.Fatalf("Failed to encode record: %v", err) + } + + encoded, err := rlp.EncodeToBytes(&enrResponse{ReplyTok: []byte{0xaa}, Record: record}) + if err != nil { + t.Fatalf("Failed to encode response: %v", err) + } + + if !bytes.Contains(encoded, want) { + t.Fatalf("Nested record not present in encoding: got %x, want it to contain %x", encoded, want) + } + + var decoded enrResponse + if err := rlp.DecodeBytes(encoded, &decoded); err != nil { + t.Fatalf("Failed to decode response: %v", err) + } + + if decoded.Record.UDP() != record.UDP() { + t.Errorf("UDP port mismatch: got %d, want %d", decoded.Record.UDP(), record.UDP()) + } + + if decoded.Record.Seq() != record.Seq() { + t.Errorf("Sequence mismatch: got %d, want %d", decoded.Record.Seq(), record.Seq()) + } +} + +// TestRecordSliceEncoding tests that records in a slice each carry their own +// encoding. +func TestRecordSliceEncoding(t *testing.T) { + record, _ := signedRecord(t) + + want, err := record.EncodeRLPBytes() + if err != nil { + t.Fatalf("Failed to encode record: %v", err) + } + + encoded, err := rlp.EncodeToBytes([]*Record{record}) + if err != nil { + t.Fatalf("Failed to encode slice: %v", err) + } + + if !bytes.Contains(encoded, want) { + t.Fatalf("Record not present in slice encoding: got %x, want it to contain %x", encoded, want) + } +} + +// TestNestedRecordDecodesInGoEthereum tests that a nested record survives +// go-ethereum's decoder, which rejected our ENRRESPONSE with "record contains +// less than two list elements". +func TestNestedRecordDecodesInGoEthereum(t *testing.T) { + record, privKey := signedRecord(t) + + encoded, err := rlp.EncodeToBytes(&enrResponse{ReplyTok: []byte{0xaa}, Record: record}) + if err != nil { + t.Fatalf("Failed to encode response: %v", err) + } + + var decoded struct { + ReplyTok []byte + Record gethenr.Record + } + if err := rlp.DecodeBytes(encoded, &decoded); err != nil { + t.Fatalf("go-ethereum failed to decode response: %v", err) + } + + n, err := enode.New(enode.ValidSchemes, &decoded.Record) + if err != nil { + t.Fatalf("go-ethereum failed to build node from record: %v", err) + } + + if n.ID() != enode.PubkeyToIDV4(&privKey.PublicKey) { + t.Errorf("Node ID mismatch: got %v, want %v", n.ID(), enode.PubkeyToIDV4(&privKey.PublicKey)) + } + + if n.UDP() != int(record.UDP()) { + t.Errorf("UDP port mismatch: got %d, want %d", n.UDP(), record.UDP()) + } +} + +// TestDecodeRejectsOversizedRecord tests that the 300-byte limit is enforced on +// ingest, not just on encode. +func TestDecodeRejectsOversizedRecord(t *testing.T) { + oversized, err := rlp.EncodeToBytes([]interface{}{ + make([]byte, 64), + uint64(1), + "padding", + make([]byte, MaxRecordSize), + }) + if err != nil { + t.Fatalf("Failed to build oversized payload: %v", err) + } + + if len(oversized) <= MaxRecordSize { + t.Fatalf("Payload is %d bytes, expected more than %d", len(oversized), MaxRecordSize) + } + + if err := New().DecodeRLPBytes(oversized); err != ErrRecordTooLarge { + t.Errorf("DecodeRLPBytes error = %v, want %v", err, ErrRecordTooLarge) + } + + if _, err := Load(oversized); err != ErrRecordTooLarge { + t.Errorf("Load error = %v, want %v", err, ErrRecordTooLarge) + } +} + +// TestDecodeAcceptsRecordAtSizeLimit tests that the size check does not reject +// compliant records. +func TestDecodeAcceptsRecordAtSizeLimit(t *testing.T) { + record, _ := signedRecord(t) + + encoded, err := record.EncodeRLPBytes() + if err != nil { + t.Fatalf("Failed to encode record: %v", err) + } + + if len(encoded) > MaxRecordSize { + t.Fatalf("Record is %d bytes, expected at most %d", len(encoded), MaxRecordSize) + } + + if err := New().DecodeRLPBytes(encoded); err != nil { + t.Errorf("DecodeRLPBytes error = %v, want nil", err) + } +} diff --git a/enr/record.go b/enr/record.go index 3584821..e04e230 100644 --- a/enr/record.go +++ b/enr/record.go @@ -112,7 +112,7 @@ func (r *Record) SetSeq(seq uint64) { // clone.Set("ip", newIP) func (r *Record) Clone() (*Record, error) { // Encode the current record to RLP bytes - data, err := r.EncodeRLP() + data, err := r.EncodeRLPBytes() if err != nil { return nil, fmt.Errorf("failed to encode record for cloning: %w", err) } @@ -583,7 +583,7 @@ func (r *Record) encode() ([]byte, error) { // This is useful for interoperability with go-ethereum's p2p stack. // Returns nil if the record cannot be converted (missing required fields). func (r *Record) ToEnode() *enode.Node { - encoded, err := r.EncodeRLP() + encoded, err := r.EncodeRLPBytes() if err != nil { return nil } diff --git a/enr/record_test.go b/enr/record_test.go index 541a297..fabc6a4 100644 --- a/enr/record_test.go +++ b/enr/record_test.go @@ -43,7 +43,7 @@ func TestRecordEncoding(t *testing.T) { t.Fatalf("Failed to sign record: %v", err) } - encoded, err := original.EncodeRLP() + encoded, err := original.EncodeRLPBytes() if err != nil { t.Fatalf("Failed to encode record: %v", err) } @@ -269,6 +269,6 @@ func BenchmarkRecordEncoding(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - record.EncodeRLP() + record.EncodeRLPBytes() } } diff --git a/nodes/admission_persist_test.go b/nodes/admission_persist_test.go new file mode 100644 index 0000000..88d6fc6 --- /dev/null +++ b/nodes/admission_persist_test.go @@ -0,0 +1,324 @@ +package nodes + +import ( + "context" + "crypto/ecdsa" + "net" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethpandaops/bootnodoor/db" + v4node "github.com/ethpandaops/bootnodoor/discv4/node" + discv5node "github.com/ethpandaops/bootnodoor/discv5/node" + "github.com/ethpandaops/bootnodoor/enr" + "github.com/sirupsen/logrus" +) + +func persistTestDB(t *testing.T, file string) *db.Database { + t.Helper() + + logger := logrus.New() + logger.SetLevel(logrus.ErrorLevel) + + database := db.NewDatabase(&db.SqliteDatabaseConfig{File: file}, logger) + if err := database.Init(); err != nil { + t.Fatalf("init: %v", err) + } + if err := database.ApplyEmbeddedDbSchema(-2); err != nil { + t.Fatalf("schema: %v", err) + } + return database +} + +func quietTableLogger() logrus.FieldLogger { + logger := logrus.New() + logger.SetLevel(logrus.ErrorLevel) + return logger +} + +func newPersistTable(t *testing.T, ndb *NodeDB, logger logrus.FieldLogger) *FlatTable { + t.Helper() + + table, err := NewFlatTable(FlatTableConfig{DB: ndb, MaxActiveNodes: 10, Logger: logger}) + if err != nil { + t.Fatalf("new table: %v", err) + } + return table +} + +// Admission puts a node in the active pool and marks it dirty, but nothing ever +// enqueued it, so organically discovered nodes were never written at all. +func TestAdmissionPersistsNode(t *testing.T) { + database := persistTestDB(t, filepath.Join(t.TempDir(), "admit.db")) + defer database.Close() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + logger := quietTableLogger() + ndb := NewNodeDB(ctx, database, db.LayerCL, logger) + table := newPersistTable(t, ndb, logger) + + n := NewFromV5(makeV5At(t, net.IPv4(10, 0, 0, 7)), ndb) + if !table.Add(n) { + t.Fatal("node was not admitted") + } + + deadline := time.Now().Add(5 * time.Second) + for ndb.Count() == 0 { + if time.Now().After(deadline) { + t.Fatal("admitted node was never persisted") + } + time.Sleep(20 * time.Millisecond) + } +} + +// A node admitted immediately before shutdown must not be lost: the consumer +// abandons the channel backlog on ctx cancellation, so Close has to drain it. +func TestAdmissionSurvivesImmediateClose(t *testing.T) { + file := filepath.Join(t.TempDir(), "close.db") + database := persistTestDB(t, file) + + ctx, cancel := context.WithCancel(context.Background()) + logger := quietTableLogger() + ndb := NewNodeDB(ctx, database, db.LayerCL, logger) + table := newPersistTable(t, ndb, logger) + + n := NewFromV5(makeV5At(t, net.IPv4(10, 0, 0, 9)), ndb) + if !table.Add(n) { + t.Fatal("node was not admitted") + } + + cancel() + ndb.Close() + database.Close() + + reopened := persistTestDB(t, file) + defer reopened.Close() + + count, err := reopened.CountNodes(db.LayerCL) + if err != nil { + t.Fatalf("count: %v", err) + } + if count != 1 { + t.Errorf("persisted nodes after immediate close = %d, want 1", count) + } +} + +// QueueUpdate must refuse work once Close has begun rather than accept it into a +// queue nobody will drain. +func TestQueueUpdateRejectedAfterClose(t *testing.T) { + database := persistTestDB(t, filepath.Join(t.TempDir(), "gate.db")) + defer database.Close() + + ctx, cancel := context.WithCancel(context.Background()) + logger := quietTableLogger() + ndb := NewNodeDB(ctx, database, db.LayerCL, logger) + + cancel() + ndb.Close() + + n := NewFromV5(makeV5At(t, net.IPv4(10, 0, 0, 11)), ndb) + n.MarkDirty(DirtyFull) + if err := ndb.QueueUpdate(n); err == nil { + t.Error("QueueUpdate accepted a node after Close; it will never be written") + } +} + +// The DirtyFull branch clears every remaining flag after upserting, so the +// upsert itself has to carry last_active or the DirtyLastActive set during +// admission is discarded and the row sorts as the most inactive. +func TestFullUpsertPersistsLastActive(t *testing.T) { + database := persistTestDB(t, filepath.Join(t.TempDir(), "active.db")) + defer database.Close() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + logger := quietTableLogger() + ndb := NewNodeDB(ctx, database, db.LayerCL, logger) + + n := NewFromV5(makeV5At(t, net.IPv4(10, 0, 0, 21)), ndb) + n.SetLastActive(time.Now()) + n.MarkDirty(DirtyFull) + if err := ndb.QueueUpdate(n); err != nil { + t.Fatalf("queue: %v", err) + } + + deadline := time.Now().Add(5 * time.Second) + for ndb.Count() == 0 { + if time.Now().After(deadline) { + t.Fatal("node was never persisted") + } + time.Sleep(20 * time.Millisecond) + } + + id := n.IDBytes() + stored, err := database.GetNode(db.LayerCL, id[:]) + if err != nil { + t.Fatalf("load: %v", err) + } + if !stored.LastActive.Valid || stored.LastActive.Int64 == 0 { + t.Error("last_active was written as NULL by the full upsert") + } +} + +// batchUpdate snapshots the dirty flags, writes, then clears. Clearing +// everything discarded any flag marked while the write was in flight, because +// that caller saw the node already queued and did not enqueue it again. +func TestClearDirtySnapshotKeepsUnobservedFlags(t *testing.T) { + database := persistTestDB(t, filepath.Join(t.TempDir(), "flags.db")) + defer database.Close() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + ndb := NewNodeDB(ctx, database, db.LayerCL, quietTableLogger()) + n := NewFromV5(makeV5At(t, net.IPv4(10, 4, 0, 1)), ndb) + + n.MarkDirty(DirtyENR) + observed, gen := n.DirtySnapshot() + n.MarkDirty(DirtyLastActive) + + if remaining := n.ClearDirtySnapshot(observed, gen); !remaining { + t.Error("ClearDirtySnapshot reported nothing left, so the later mark would not be requeued") + } + if got := n.GetDirtyFlags(); got&DirtyLastActive == 0 { + t.Error("a flag marked after the snapshot was cleared unwritten") + } + + // Re-marking the same bit must also survive: the generation moved, so the + // observed write cannot be assumed to cover the newer value. + observed, gen = n.DirtySnapshot() + n.MarkDirty(DirtyLastActive) + if remaining := n.ClearDirtySnapshot(observed, gen); !remaining { + t.Error("a same-bit re-mark during the write was dropped") + } + if got := n.GetDirtyFlags(); got&DirtyLastActive == 0 { + t.Error("same-bit re-mark was cleared unwritten") + } +} + +// A peer found over discv4 can be admitted as v5-only first, if its handshake +// completes before the v4 admission lands. Add previously refreshed only a newer +// ENR, so the v4 pointer was dropped and the peer persisted as v5-only. +func TestAddMergesProtocolCapabilities(t *testing.T) { + database := persistTestDB(t, filepath.Join(t.TempDir(), "merge.db")) + defer database.Close() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + logger := quietTableLogger() + ndb := NewNodeDB(ctx, database, db.LayerEL, logger) + table := newPersistTable(t, ndb, logger) + + v5 := makeV5At(t, net.IPv4(10, 9, 0, 1)) + first := NewFromV5(v5, ndb) + if !table.Add(first) { + t.Fatal("v5-only node was not admitted") + } + if table.Get(first.ID()).HasV4() { + t.Fatal("precondition: entry should start v5-only") + } + + // The same peer arriving over discv4, as the probe path produces it. + second := NewFromV5(v5, ndb) + second.SetV4(makeV4For(t, v5)) + if !table.Add(second) { + t.Fatal("second admission was rejected") + } + + entry := table.Get(first.ID()) + if !entry.HasV4() { + t.Error("v4 capability was lost: the table entry is still v5-only") + } + if !entry.HasV5() { + t.Error("v5 capability was dropped by the merge") + } +} + +func makeV4For(t *testing.T, v5 *discv5node.Node) *v4node.Node { + t.Helper() + return v4node.New(v5.PublicKey(), v5.Addr()) +} + +func signedRecordAt(t *testing.T, key *ecdsa.PrivateKey, seq uint64, ip net.IP) *enr.Record { + t.Helper() + rec := enr.New() + if err := rec.Set("ip", ip); err != nil { + t.Fatalf("set ip: %v", err) + } + if err := rec.Set("udp", uint16(9000)); err != nil { + t.Fatalf("set udp: %v", err) + } + rec.SetSeq(seq) + if err := rec.Sign(key); err != nil { + t.Fatalf("sign: %v", err) + } + return rec +} + +// Adoption only ever fills an empty slot, so a pointer installed from a stale +// record is permanent. The freshness check and the install must therefore happen +// under one lock hold, or a concurrent advance can slip between them. +func TestAdoptProtocolsFromIsAtomicUnderRace(t *testing.T) { + database := persistTestDB(t, filepath.Join(t.TempDir(), "race.db")) + defer database.Close() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + ndb := NewNodeDB(ctx, database, db.LayerEL, quietTableLogger()) + + key, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("generate key: %v", err) + } + + for round := 0; round < 200; round++ { + base, _ := discv5node.New(signedRecordAt(t, key, 3, net.IPv4(10, 7, 0, 1))) + entry := NewFromV5(base, ndb) + + staleV5, _ := discv5node.New(signedRecordAt(t, key, 1, net.IPv4(10, 7, 0, 2))) + stale := NewFromV5(staleV5, ndb) + stale.SetV4(v4node.New(staleV5.PublicKey(), staleV5.Addr())) + + var wg sync.WaitGroup + wg.Add(2) + go func() { defer wg.Done(); entry.AdoptProtocolsFrom(stale) }() + go func() { defer wg.Done(); entry.UpdateENR(signedRecordAt(t, key, 9, net.IPv4(10, 7, 0, 3))) }() + wg.Wait() + + if v4 := entry.V4(); v4 != nil && v4.Addr().IP.Equal(net.IPv4(10, 7, 0, 2)) { + t.Fatalf("round %d: installed a v4 pointer from the stale record", round) + } + } +} + +// Re-admitting a table entry passes it to itself. Snapshotting its own pointers +// and reinstalling them could resurrect a protocol a concurrent clear removed. +func TestAdoptProtocolsFromSelfIsNoOp(t *testing.T) { + database := persistTestDB(t, filepath.Join(t.TempDir(), "self.db")) + defer database.Close() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + ndb := NewNodeDB(ctx, database, db.LayerEL, quietTableLogger()) + v5 := makeV5At(t, net.IPv4(10, 6, 0, 1)) + n := NewFromV5(v5, ndb) + + adopted, advanced := n.AdoptProtocolsFrom(n) + if adopted || advanced { + t.Errorf("self-merge reported changes: adopted=%v advanced=%v", adopted, advanced) + } + + n.SetV5(nil) + if _, _ = n.AdoptProtocolsFrom(n); n.HasV5() { + t.Error("self-merge resurrected a cleared protocol") + } +} diff --git a/nodes/flattable.go b/nodes/flattable.go index caf22c7..a6333a5 100644 --- a/nodes/flattable.go +++ b/nodes/flattable.go @@ -260,13 +260,20 @@ func (t *FlatTable) Add(n *Node) bool { if existing, exists := t.activeNodes[nodeID]; exists { t.mu.Unlock() - // Update ENR if newer - newSeq := n.Record().Seq() - if newSeq > existing.Record().Seq() { - existing.UpdateENR(n.Record()) - - // Queue ENR update (preserves stats) - existing.MarkDirty(DirtyENR) + // Adopt protocols the entry lacks. A peer found over discv4 can be admitted + // as v5-only first if its handshake completes before the v4 admission lands; + // keeping only the newer ENR would drop the v4 pointer and persist the peer + // as v5-only. + // + // Only from a record at least as new as the entry's: senders use the adopted + // protocol node's own address, so taking one from an older record would point + // that protocol at an endpoint the peer has already moved off. + adopted, advanced := existing.AdoptProtocolsFrom(n) + + if adopted || advanced { + if err := t.db.QueueUpdate(existing); err != nil { + t.logger.WithError(err).WithField("peerID", existing.PeerID()).Debug("failed to queue node update") + } if t.nodeChangedCallback != nil { t.nodeChangedCallback(existing) @@ -321,17 +328,20 @@ func (t *FlatTable) Add(n *Node) bool { "addr": n.Addr(), "currentSize": currentSize + 1, "maxActive": t.maxActiveNodes, - }).Infof("added alive node to active pool (over capacity)") + }).Debugf("added alive node to active pool (over capacity)") } else { t.logger.WithFields(logrus.Fields{ "peerID": n.PeerID(), "addr": n.Addr(), - }).Info("added node to active pool") + }).Debug("added node to active pool") } // Queue ENR update to DB and mark as active n.MarkDirty(DirtyENR) n.SetLastActive(time.Now()) + if err := t.db.QueueUpdate(n); err != nil { + t.logger.WithError(err).WithField("peerID", n.PeerID()).Debug("failed to queue admitted node") + } if t.nodeChangedCallback != nil { t.nodeChangedCallback(n) @@ -727,16 +737,31 @@ func (t *FlatTable) ActiveSize() int { } // GetStats returns statistics about the table. +// +// Active comes from memory and persisted from the database, so the two are +// counted as sets rather than subtracted: an admission whose write has not +// landed yet would otherwise report more active than total. func (t *FlatTable) GetStats() TableStats { + persisted := t.db.PersistedIDs() + t.mu.RLock() defer t.mu.RUnlock() activeCount := len(t.activeNodes) - totalCount := t.db.Count() + + inactiveCount := 0 + totalCount := activeCount + for _, id := range persisted { + if _, active := t.activeNodes[id]; !active { + inactiveCount++ + totalCount++ + } + } return TableStats{ TotalNodes: totalCount, ActiveNodes: activeCount, + InactiveNodes: inactiveCount, AdmissionRejections: t.admissionRejections, IPLimitRejections: t.ipLimitRejections, DeadNodesRemoved: t.deadNodesRemoved, diff --git a/nodes/node.go b/nodes/node.go index 59a88d7..ee3a51e 100644 --- a/nodes/node.go +++ b/nodes/node.go @@ -64,6 +64,7 @@ type Node struct { // Dirty tracking for database updates dirtyMu sync.Mutex dirtyFields DirtyFlags + dirtyGen uint64 } // NewFromV4 creates a generic Node from a discv4 node. @@ -139,11 +140,9 @@ func (n *Node) PublicKey() *ecdsa.PublicKey { return n.pubKey } -// ENR returns the node's ENR record. +// ENR returns the node's ENR record. Alias for Record. func (n *Node) ENR() *enr.Record { - n.mu.RLock() - defer n.mu.RUnlock() - return n.enr + return n.Record() } // Addr returns the node's UDP address. @@ -153,18 +152,6 @@ func (n *Node) Addr() *net.UDPAddr { return n.addr } -// SetAddr updates the node's address. -func (n *Node) SetAddr(addr *net.UDPAddr) { - n.mu.Lock() - defer n.mu.Unlock() - n.addr = addr - - // Update protocol-specific nodes - if n.v4Node != nil { - n.v4Node.SetAddr(addr) - } -} - // V4 returns the discv4 node if available. func (n *Node) V4() *node.Node { n.mu.RLock() @@ -208,6 +195,159 @@ func (n *Node) SetV4(v4 *node.Node) { n.MarkDirty(DirtyProtocol) } +// AdoptProtocolsFrom fills protocol slots this node has empty from a wrapper for +// the same peer, advances the record if the other's is newer, and reports each. +// +// It fills only: which endpoint a protocol should use is the discovery layer's +// call, not the table's. discv4 moves an address solely on a matched PONG +// (promoteAddr), and discv5 moves its own when its record advances. A table that +// also ranked pointers would be arbitrating endpoints on weaker evidence. +func (n *Node) AdoptProtocolsFrom(other *Node) (adopted, advanced bool) { + // Self-merge is a no-op, not a re-install: a caller re-admitting a table entry + // would otherwise snapshot its own pointer, race a concurrent clear, and + // resurrect a protocol the node no longer supports. + if other == nil || other == n { + return false, false + } + + otherRecord := other.Record() + if otherRecord == nil { + return false, false + } + otherV4, otherV5 := other.V4(), other.V5() + carrierSeq := otherRecord.Seq() + + n.mu.Lock() + if n.enr != nil && carrierSeq < n.recordSeqLocked() { + n.mu.Unlock() + return false, false + } + + if otherV4 != nil && n.v4Node == nil { + n.v4Node = otherV4 + adopted = true + } + if otherV5 != nil && n.v5Node == nil { + n.v5Node = otherV5 + adopted = true + } + + if n.enr == nil || carrierSeq > n.recordSeqLocked() { + n.enr = otherRecord + advanced = true + } + + stats := n.nodeStats + current := n.enr + v4, v5 := n.v4Node, n.v5Node + n.mu.Unlock() + + if adopted && stats != nil { + n.setupSharedStatsCallback() + if otherV4 != nil && v4 != nil { + v4.SetStats(stats) + } + if otherV5 != nil && v5 != nil { + v5.SetStats(stats) + } + } + // Bring the v5 pointer up to the record the wrapper now holds, whether that is + // because the record advanced or because an older pointer just filled an empty + // slot. UpdateENR ignores anything not newer, and this stays outside n.mu so + // the two mutexes never nest. + if (adopted || advanced) && v5 != nil { + v5.UpdateENR(current) + } + + if adopted { + n.MarkDirty(DirtyProtocol) + } + if advanced { + n.MarkDirty(DirtyENR) + } + return adopted, advanced +} + +// ApplyProbeResult installs or clears both protocol pointers from a completed +// probe, and reports whether anything changed. +// +// Gated on the record still being the one the probe measured: a result that +// arrived after the peer published a new record describes endpoints it may have +// left, so it must neither install nor clear. Both protocols are decided under one +// lock hold so they see the same record. +func (n *Node) ApplyProbeResult(probedSeq uint64, v4 *node.Node, v4OK bool, v5 *discv5node.Node, v5OK bool) bool { + n.mu.Lock() + if n.enr == nil || n.recordSeqLocked() != probedSeq { + n.mu.Unlock() + return false + } + + changed := false + switch { + case v4OK && v4 != nil && n.v4Node == nil: + n.v4Node = v4 + changed = true + case !v4OK && n.v4Node != nil: + n.v4Node = nil + changed = true + } + + switch { + case v5OK && v5 != nil && n.v5Node == nil: + n.v5Node = v5 + changed = true + case !v5OK && n.v5Node != nil: + n.v5Node = nil + changed = true + } + + stats := n.nodeStats + installedV4, installedV5 := n.v4Node, n.v5Node + n.mu.Unlock() + + if !changed { + return false + } + + if stats != nil { + n.setupSharedStatsCallback() + if installedV4 != nil { + installedV4.SetStats(stats) + } + if installedV5 != nil { + installedV5.SetStats(stats) + } + } + n.MarkDirty(DirtyProtocol) + return true +} + +// SetV5AtSeq installs a discv5 node only while the record it was probed from is +// still current, so a result that arrived after the peer moved is discarded +// rather than pinning traffic to the old endpoint. +func (n *Node) SetV5AtSeq(v5 *discv5node.Node, seq uint64) bool { + if v5 == nil { + return false + } + v5Seq := v5RecordSeq(v5) + + n.mu.Lock() + if n.enr == nil || n.enr.Seq() != seq || v5Seq != seq { + n.mu.Unlock() + return false + } + n.v5Node = v5 + stats := n.nodeStats + n.mu.Unlock() + + if stats != nil { + n.setupSharedStatsCallback() + v5.SetStats(stats) + } + n.MarkDirty(DirtyProtocol) + return true +} + // SetV5 sets the discv5 node and marks protocol support dirty. func (n *Node) SetV5(v5 *discv5node.Node) { n.mu.Lock() @@ -433,7 +573,7 @@ func (n *Node) UpdateENR(newRecord *enr.Record) bool { // Update our ENR n.mu.Lock() - if newRecord.Seq() <= n.enr.Seq() { + if n.enr != nil && newRecord.Seq() <= n.recordSeqLocked() { n.mu.Unlock() return false } @@ -497,6 +637,7 @@ func (n *Node) CalculateScore(forkInfo *ForkScoringInfo) float64 { func (n *Node) MarkDirty(flags DirtyFlags) { n.dirtyMu.Lock() n.dirtyFields |= flags + n.dirtyGen++ n.dirtyMu.Unlock() } @@ -515,6 +656,29 @@ func (n *Node) ClearDirtyFlags() { n.dirtyMu.Unlock() } +// DirtySnapshot returns the current flags and a generation that changes on every +// subsequent MarkDirty, so a writer can tell whether anything was marked while it +// was working. +func (n *Node) DirtySnapshot() (DirtyFlags, uint64) { + n.dirtyMu.Lock() + defer n.dirtyMu.Unlock() + return n.dirtyFields, n.dirtyGen +} + +// ClearDirtySnapshot clears the snapshotted flags and reports whether the node is +// still dirty. Clearing is skipped entirely when the generation moved: the same +// bit may have been re-marked for a newer value, which is indistinguishable from +// the one just written, so the field would otherwise be dropped unwritten. +func (n *Node) ClearDirtySnapshot(flags DirtyFlags, gen uint64) bool { + n.dirtyMu.Lock() + defer n.dirtyMu.Unlock() + if n.dirtyGen != gen { + return true + } + n.dirtyFields &^= flags + return n.dirtyFields != 0 +} + // LastActive returns the last active timestamp. func (n *Node) LastActive() time.Time { n.mu.RLock() @@ -541,3 +705,23 @@ func NewV5NodeFromRecord(record *enr.Record) (*discv5node.Node, error) { func NewV4NodeFromRecord(record *enr.Record, addr *net.UDPAddr) (*node.Node, error) { return node.FromENR(record, addr) } + +// recordSeq reads a record's sequence, treating a missing record as sequence 0. +func recordSeq(rec *enr.Record) uint64 { + if rec == nil { + return 0 + } + return rec.Seq() +} + +// v5RecordSeq reads the sequence of the record a discv5 pointer was built from. +func v5RecordSeq(v5 *discv5node.Node) uint64 { + if v5 == nil { + return 0 + } + return recordSeq(v5.Record()) +} + +func (n *Node) recordSeqLocked() uint64 { + return recordSeq(n.enr) +} diff --git a/nodes/node_test.go b/nodes/node_test.go index 1b4acf5..86ed953 100644 --- a/nodes/node_test.go +++ b/nodes/node_test.go @@ -1,6 +1,7 @@ package nodes import ( + "crypto/ecdsa" "net" "sync" "testing" @@ -174,3 +175,159 @@ func TestNodeNoNilDerefDuringProtocolSwap(t *testing.T) { close(stop) wg.Wait() } + +func TestUpdateENRInstallsSequenceZeroOnRecordlessNode(t *testing.T) { + key, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("generate key: %v", err) + } + addr := &net.UDPAddr{IP: net.IPv4(10, 20, 0, 1), Port: 9000} + n := NewFromV4(discv4node.New(&key.PublicKey, addr), nil) + record := signedRecordAt(t, key, 0, addr.IP) + + if !n.UpdateENR(record) { + t.Fatal("sequence-zero ENR was rejected for a node with no record") + } + if got := n.Record(); got != record { + t.Fatal("sequence-zero ENR was not installed") + } +} + +func TestSetV5AtSeqRejectsRecordlessSequenceZeroTarget(t *testing.T) { + key, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("generate key: %v", err) + } + addr := &net.UDPAddr{IP: net.IPv4(10, 20, 0, 2), Port: 9000} + n := NewFromV4(discv4node.New(&key.PublicKey, addr), nil) + probed, err := discv5node.New(signedRecordAt(t, key, 0, addr.IP)) + if err != nil { + t.Fatalf("new v5 node: %v", err) + } + + if n.SetV5AtSeq(probed, 0) { + t.Fatal("installed a probe result on a target with no current ENR") + } +} + +func TestAdoptInstallsSequenceZeroCarrierOnRecordlessNode(t *testing.T) { + key, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("generate key: %v", err) + } + addr := &net.UDPAddr{IP: net.IPv4(10, 20, 0, 13), Port: 9000} + entry := NewFromV4(discv4node.New(&key.PublicKey, addr), nil) + record := signedRecordAt(t, key, 0, addr.IP) + v5, err := discv5node.New(record) + if err != nil { + t.Fatalf("new v5 node: %v", err) + } + + adopted, advanced := entry.AdoptProtocolsFrom(NewFromV5(v5, nil)) + if !adopted || !advanced { + t.Fatalf("sequence-zero adoption = (%v, %v), want (true, true)", adopted, advanced) + } + if got := entry.Record(); got != record { + t.Fatal("sequence-zero carrier ENR was not installed") + } +} + +func TestSetV5AtSeqRejectsProbeNodeThatAdvanced(t *testing.T) { + key, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("generate key: %v", err) + } + record1 := signedRecordAt(t, key, 1, net.IPv4(10, 20, 0, 3)) + v4, err := discv4node.FromENR(record1, &net.UDPAddr{IP: record1.IP(), Port: int(record1.UDP())}) + if err != nil { + t.Fatalf("new v4 node: %v", err) + } + n := NewFromV4(v4, nil) + + probed, err := discv5node.New(record1) + if err != nil { + t.Fatalf("new v5 node: %v", err) + } + if !probed.UpdateENR(signedRecordAt(t, key, 2, net.IPv4(10, 20, 0, 4))) { + t.Fatal("advance probe node: update rejected") + } + + if n.SetV5AtSeq(probed, 1) { + t.Fatal("installed a probe node that advanced beyond the probed record") + } +} + +func signedRecordSeq(t *testing.T, key *ecdsa.PrivateKey, seq uint64, ip net.IP) *enr.Record { + t.Helper() + rec := enr.New() + if err := rec.Set("ip", ip); err != nil { + t.Fatalf("set ip: %v", err) + } + if err := rec.Set("udp", uint16(30303)); err != nil { + t.Fatalf("set udp: %v", err) + } + rec.SetSeq(seq) + if err := rec.Sign(key); err != nil { + t.Fatalf("sign: %v", err) + } + return rec +} + +// A probe describes the endpoints of the record it ran against. Applying it after +// a newer record arrived would install an endpoint the peer has already left, or +// clear a pointer that newer record brought in. +func TestApplyProbeResultRejectsStaleRecord(t *testing.T) { + key, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("generate key: %v", err) + } + + base, err := discv5node.New(signedRecordSeq(t, key, 1, net.IPv4(10, 30, 0, 1))) + if err != nil { + t.Fatalf("base v5: %v", err) + } + n := NewFromV5(base, nil) + + // The node moves on while the probe is outstanding. + if !n.UpdateENR(signedRecordSeq(t, key, 7, net.IPv4(10, 30, 0, 2))) { + t.Fatal("record did not advance") + } + + v4 := discv4node.New(base.PublicKey(), base.Addr()) + if n.ApplyProbeResult(1, v4, true, nil, false) { + t.Error("applied a probe result against a record the node had already left") + } + if n.HasV4() { + t.Error("installed a v4 pointer from a superseded probe") + } + if !n.HasV5() { + t.Error("cleared the v5 pointer from a superseded probe") + } +} + +// A probe that fails against the current record must be able to clear, and both +// protocol decisions have to land together rather than one at a time. +func TestApplyProbeResultAppliesBothDecisionsAtCurrentSeq(t *testing.T) { + key, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("generate key: %v", err) + } + + rec := signedRecordSeq(t, key, 4, net.IPv4(10, 31, 0, 1)) + base, err := discv5node.New(rec) + if err != nil { + t.Fatalf("base v5: %v", err) + } + n := NewFromV5(base, nil) + + v4 := discv4node.New(base.PublicKey(), base.Addr()) + if !n.ApplyProbeResult(4, v4, true, nil, false) { + t.Fatal("probe at the current sequence was rejected") + } + if !n.HasV4() { + t.Error("v4 confirmed by the probe was not installed") + } + if n.HasV5() { + t.Error("v5 unconfirmed by the probe was not cleared") + } +} diff --git a/nodes/nodedb.go b/nodes/nodedb.go index 1fcad76..e4c321c 100644 --- a/nodes/nodedb.go +++ b/nodes/nodedb.go @@ -28,6 +28,7 @@ type NodeDB struct { updateQueue chan *Node updateQueueSet map[[32]byte]*Node // Tracks pending updates by nodeID updateQueueLock sync.Mutex + closing bool // Set under updateQueueLock so no write is accepted after Close starts draining // Stats tracking stats NodeDBStats @@ -78,6 +79,10 @@ func (ndb *NodeDB) QueueUpdate(n *Node) error { ndb.updateQueueLock.Lock() defer ndb.updateQueueLock.Unlock() + if ndb.closing { + return fmt.Errorf("node db is closing") + } + // Check if there's already a pending update for this node if _, ok := ndb.updateQueueSet[nodeID]; ok { // Node already queued - dirty flags will accumulate automatically @@ -125,10 +130,15 @@ func (ndb *NodeDB) processUpdateQueue() { ticker := time.NewTicker(1000 * time.Millisecond) defer ticker.Stop() + // Failures requeue their nodes so nothing is lost, which on a persistent + // database error would otherwise spin: the requeue refills the batch and the + // next pass runs immediately. Back off between consecutive failures instead. + failures := 0 + for { select { case <-ndb.ctx.Done(): - // Process remaining batch + batch = ndb.drainQueue(batch) if len(batch) > 0 { ndb.batchUpdate(batch) } @@ -139,7 +149,7 @@ func (ndb *NodeDB) processUpdateQueue() { // Process when batch reaches 50 items if len(batch) >= 50 { - ndb.batchUpdate(batch) + failures = ndb.runBatch(batch, failures) batch = batch[:0] time.Sleep(10 * time.Millisecond) // Avoid hammering DB } @@ -147,17 +157,54 @@ func (ndb *NodeDB) processUpdateQueue() { case <-ticker.C: // Process any pending items if len(batch) > 0 { - ndb.batchUpdate(batch) + failures = ndb.runBatch(batch, failures) batch = batch[:0] } } } } +// maxBatchBackoff caps the delay after repeated batch failures. +const maxBatchBackoff = 5 * time.Second + +// runBatch writes a batch and sleeps proportionally to how many consecutive +// batches have failed, returning the updated count. +func (ndb *NodeDB) runBatch(batch []*Node, failures int) int { + if ndb.batchUpdate(batch) { + return 0 + } + + failures++ + + backoff := time.Duration(failures) * 100 * time.Millisecond + if backoff > maxBatchBackoff { + backoff = maxBatchBackoff + } + + select { + case <-time.After(backoff): + case <-ndb.ctx.Done(): + } + return failures +} + +// drainQueue moves everything currently queued into batch without blocking. +func (ndb *NodeDB) drainQueue(batch []*Node) []*Node { + for { + select { + case node := <-ndb.updateQueue: + batch = append(batch, node) + default: + return batch + } + } +} + // batchUpdate performs a batch update of nodes. -func (ndb *NodeDB) batchUpdate(nodes []*Node) { +// batchUpdate writes a batch and reports whether the transaction committed. +func (ndb *NodeDB) batchUpdate(nodes []*Node) bool { if len(nodes) == 0 { - return + return true } ndb.logger.WithFields(logrus.Fields{ @@ -165,10 +212,19 @@ func (ndb *NodeDB) batchUpdate(nodes []*Node) { "layer": ndb.layer, }).Debug("processing batch update") + type written struct { + node *Node + flags DirtyFlags + gen uint64 + } + var processed []written + err := ndb.db.RunDBTransaction(func(tx *sqlx.Tx) error { + processed = processed[:0] for _, node := range nodes { - dirtyFlags := node.GetDirtyFlags() + dirtyFlags, dirtyGen := node.DirtySnapshot() nodeID := node.ID() + writeFailed := false ndb.logger.WithFields(logrus.Fields{ "nodeID": fmt.Sprintf("%x", nodeID[:8]), @@ -182,8 +238,7 @@ func (ndb *NodeDB) batchUpdate(nodes []*Node) { ndb.logger.WithError(err).WithField("nodeID", fmt.Sprintf("%x", nodeID[:8])).Error("failed to upsert node in batch") continue } - // Full upsert covers everything, clear all dirty flags - node.ClearDirtyFlags() + processed = append(processed, written{node, dirtyFlags, dirtyGen}) continue } @@ -192,6 +247,7 @@ func (ndb *NodeDB) batchUpdate(nodes []*Node) { ndb.logger.WithField("nodeID", fmt.Sprintf("%x", nodeID[:8])).Debug("updating ENR") if err := ndb.updateNodeENRTx(tx, node); err != nil { ndb.logger.WithError(err).WithField("nodeID", fmt.Sprintf("%x", nodeID[:8])).Error("failed to update ENR in batch") + writeFailed = true } } @@ -200,6 +256,7 @@ func (ndb *NodeDB) batchUpdate(nodes []*Node) { ndb.logger.WithField("nodeID", fmt.Sprintf("%x", nodeID[:8])).Debug("updating stats") if err := ndb.updateNodeStatsTx(tx, node); err != nil { ndb.logger.WithError(err).WithField("nodeID", fmt.Sprintf("%x", nodeID[:8])).Error("failed to update stats in batch") + writeFailed = true } } @@ -209,6 +266,7 @@ func (ndb *NodeDB) batchUpdate(nodes []*Node) { if !lastActive.IsZero() { if err := ndb.db.UpdateNodeLastActive(tx, ndb.layer, nodeID[:], lastActive.Unix()); err != nil { ndb.logger.WithError(err).WithField("nodeID", fmt.Sprintf("%x", nodeID[:8])).Error("failed to update last_active in batch") + writeFailed = true } } } @@ -219,6 +277,7 @@ func (ndb *NodeDB) batchUpdate(nodes []*Node) { if !lastSeen.IsZero() { if err := ndb.db.UpdateNodeLastSeen(tx, ndb.layer, nodeID[:], lastSeen.Unix()); err != nil { ndb.logger.WithError(err).WithField("nodeID", fmt.Sprintf("%x", nodeID[:8])).Error("failed to update last_seen in batch") + writeFailed = true } } } @@ -227,16 +286,21 @@ func (ndb *NodeDB) batchUpdate(nodes []*Node) { if dirtyFlags&DirtyProtocol != 0 { if err := ndb.updateNodeProtocolSupportTx(tx, nodeID, node.HasV4(), node.HasV5()); err != nil { ndb.logger.WithError(err).WithField("nodeID", fmt.Sprintf("%x", nodeID[:8])).Error("failed to update protocol support in batch") + writeFailed = true } } - // Clear dirty flags after successful update - node.ClearDirtyFlags() + if !writeFailed { + processed = append(processed, written{node, dirtyFlags, dirtyGen}) + } } return nil }) if err != nil { + // Nothing reached the database, so no snapshot may be cleared: the flags + // are all that will bring these nodes back on a later pass. + processed = processed[:0] ndb.logger.WithError(err).Error("failed to commit batch transaction") } else { ndb.logger.WithFields(logrus.Fields{ @@ -252,10 +316,43 @@ func (ndb *NodeDB) batchUpdate(nodes []*Node) { } ndb.updateQueueLock.Unlock() + // Clear after the set entry is gone, so a caller that marked the node while + // the write was running either enqueued itself or is requeued here. Clearing + // first would let its QueueUpdate be swallowed as already-queued and then have + // the entry deleted underneath it. + persisted := make(map[[32]byte]bool, len(processed)) + requeue := make([]*Node, 0, len(nodes)) + + for _, p := range processed { + persisted[p.node.ID()] = true + if p.node.ClearDirtySnapshot(p.flags, p.gen) { + requeue = append(requeue, p.node) + } + } + + // A node whose write failed keeps its flags, but it is out of the queue set + // now, so nothing would bring it back until an unrelated update marked it. + for _, node := range nodes { + if !persisted[node.ID()] { + requeue = append(requeue, node) + } + } + + for _, node := range requeue { + if err := ndb.QueueUpdate(node); err != nil { + ndb.logger.WithError(err).Debug("failed to requeue node after batch") + } + } + // Track processed updates ndb.statsLock.Lock() ndb.stats.ProcessedUpdates += int64(len(nodes)) ndb.statsLock.Unlock() + + // Row-level failures leave the callback returning nil, so a committed + // transaction is not proof every node landed. Reporting success on a partial + // batch would reset the backoff while the requeued nodes retry immediately. + return err == nil && len(processed) == len(nodes) } // updateNodeENRTx updates only ENR info within a transaction. @@ -297,7 +394,7 @@ func (ndb *NodeDB) updateNodeENRTx(tx *sqlx.Tx, n *Node) error { } } - enrBytes, err := n.ENR().EncodeRLP() + enrBytes, err := n.ENR().EncodeRLPBytes() if err != nil { return fmt.Errorf("failed to encode ENR: %w", err) } @@ -335,7 +432,7 @@ func (ndb *NodeDB) upsertNodeTx(tx *sqlx.Tx, n *Node) error { port := n.Addr().Port seq := n.ENR().Seq() - enrBytes, err := n.ENR().EncodeRLP() + enrBytes, err := n.ENR().EncodeRLPBytes() if err != nil { return fmt.Errorf("failed to encode ENR: %w", err) } @@ -349,6 +446,14 @@ func (ndb *NodeDB) upsertNodeTx(tx *sqlx.Tx, n *Node) error { lastSeen.Int64 = stats.LastSeen.Unix() } + // The DirtyFull branch clears every other flag once this upsert runs, so a + // DirtyLastActive set alongside it would otherwise be dropped. + lastActive := sql.NullInt64{} + if t := n.LastActive(); !t.IsZero() { + lastActive.Valid = true + lastActive.Int64 = t.Unix() + } + // Extract fork digest based on layer var forkDigest []byte if ndb.layer == db.LayerEL { @@ -383,7 +488,7 @@ func (ndb *NodeDB) upsertNodeTx(tx *sqlx.Tx, n *Node) error { ForkDigest: forkDigest, FirstSeen: firstSeen, LastSeen: lastSeen, - LastActive: sql.NullInt64{}, // Updated separately + LastActive: lastActive, ENR: enrBytes, HasV4: n.HasV4(), HasV5: n.HasV5(), @@ -541,9 +646,21 @@ func (ndb *NodeDB) LoadRandom(limit int) ([]*Node, error) { } // Close stops the update queue processor and waits for pending updates. +// +// The processor exits on context cancellation, and Stop cancels before calling +// here, so a producer can still enqueue after the processor is gone. Refusing +// new work first and flushing afterwards is what makes that write-or-reject +// rather than a silent drop. func (ndb *NodeDB) Close() { - // Wait for queue processor to finish + ndb.updateQueueLock.Lock() + ndb.closing = true + ndb.updateQueueLock.Unlock() + ndb.wg.Wait() + + if batch := ndb.drainQueue(nil); len(batch) > 0 { + ndb.batchUpdate(batch) + } } // GetStats returns current database statistics. @@ -572,6 +689,26 @@ func (ndb *NodeDB) List() []*Node { return nodes } +// PersistedIDs returns the node IDs persisted for this layer. +func (ndb *NodeDB) PersistedIDs() [][32]byte { + rows, err := ndb.db.GetNodeIDs(ndb.layer) + if err != nil { + ndb.logger.WithError(err).Warn("failed to list persisted node ids") + return nil + } + + ids := make([][32]byte, 0, len(rows)) + for _, raw := range rows { + if len(raw) != 32 { + continue + } + var id [32]byte + copy(id[:], raw) + ids = append(ids, id) + } + return ids +} + // Count returns the total number of nodes in the database. func (ndb *NodeDB) Count() int { count, err := ndb.db.CountNodes(ndb.layer) diff --git a/nodes/stats_population_test.go b/nodes/stats_population_test.go new file mode 100644 index 0000000..2d8606e --- /dev/null +++ b/nodes/stats_population_test.go @@ -0,0 +1,117 @@ +package nodes + +import ( + "context" + "net" + "path/filepath" + "testing" + "time" + + "github.com/ethpandaops/bootnodoor/db" +) + +// TotalNodes came from the database and ActiveNodes from memory, so consumers +// subtracting them could report more active than total and a negative inactive +// count. The three populations are asserted exactly, because the inequality +// alone is also satisfied by reporting total == active and inactive == 0. +func TestGetStatsCountsPopulationsExactly(t *testing.T) { + database := persistTestDB(t, filepath.Join(t.TempDir(), "pop.db")) + defer database.Close() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + logger := quietTableLogger() + ndb := NewNodeDB(ctx, database, db.LayerCL, logger) + table := newPersistTable(t, ndb, logger) + + // Persisted but never admitted: two rows written straight through the queue. + for i := 0; i < 2; i++ { + n := NewFromV5(makeV5At(t, net.IPv4(10, 1, 0, byte(i+1))), ndb) + n.MarkDirty(DirtyFull) + if err := ndb.QueueUpdate(n); err != nil { + t.Fatalf("queue: %v", err) + } + } + waitForPersisted(t, ndb, 2) + + // Admitted, and therefore also persisted: overlapping population. + admitted := make([]*Node, 0, 3) + for i := 0; i < 3; i++ { + n := NewFromV5(makeV5At(t, net.IPv4(10, 2, 0, byte(i+1))), ndb) + if !table.Add(n) { + t.Fatalf("node %d not admitted", i) + } + admitted = append(admitted, n) + } + waitForPersisted(t, ndb, 5) + + stats := table.GetStats() + if stats.ActiveNodes != 3 { + t.Errorf("ActiveNodes = %d, want 3", stats.ActiveNodes) + } + if stats.TotalNodes != 5 { + t.Errorf("TotalNodes = %d, want 5 (union of persisted and active)", stats.TotalNodes) + } + if stats.InactiveNodes != 2 { + t.Errorf("InactiveNodes = %d, want 2 (persisted but not active)", stats.InactiveNodes) + } + + // Demotion drops a node from the active pool while its row remains. + table.mu.Lock() + delete(table.activeNodes, admitted[0].ID()) + table.mu.Unlock() + + stats = table.GetStats() + if stats.ActiveNodes != 2 { + t.Errorf("after demotion ActiveNodes = %d, want 2", stats.ActiveNodes) + } + if stats.TotalNodes != 5 { + t.Errorf("after demotion TotalNodes = %d, want 5", stats.TotalNodes) + } + if stats.InactiveNodes != 3 { + t.Errorf("after demotion InactiveNodes = %d, want 3", stats.InactiveNodes) + } +} + +// An active node whose write has not landed yet must not make active exceed +// total, which is what produced the negative count in the devnet run. +func TestGetStatsHoldsInvariantBeforePersistence(t *testing.T) { + database := persistTestDB(t, filepath.Join(t.TempDir(), "lag.db")) + defer database.Close() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + logger := quietTableLogger() + ndb := NewNodeDB(ctx, database, db.LayerCL, logger) + table := newPersistTable(t, ndb, logger) + + for i := 0; i < 4; i++ { + n := NewFromV5(makeV5At(t, net.IPv4(10, 3, 0, byte(i+1))), ndb) + table.mu.Lock() + table.activeNodes[n.ID()] = n + table.ipLimiter.Add(n) + table.mu.Unlock() + } + + stats := table.GetStats() + if stats.ActiveNodes > stats.TotalNodes { + t.Errorf("ActiveNodes %d > TotalNodes %d", stats.ActiveNodes, stats.TotalNodes) + } + if stats.InactiveNodes < 0 { + t.Errorf("InactiveNodes = %d, want >= 0", stats.InactiveNodes) + } +} + +func waitForPersisted(t *testing.T, ndb *NodeDB, want int) { + t.Helper() + + deadline := time.Now().Add(5 * time.Second) + for ndb.Count() < want { + if time.Now().After(deadline) { + t.Fatalf("only %d of %d nodes persisted", ndb.Count(), want) + } + time.Sleep(20 * time.Millisecond) + } +} diff --git a/nodes/types.go b/nodes/types.go index dda838a..5b84bc3 100644 --- a/nodes/types.go +++ b/nodes/types.go @@ -36,6 +36,7 @@ type NodeChangedCallback func(*Node) type TableStats struct { TotalNodes int ActiveNodes int + InactiveNodes int AdmissionRejections int IPLimitRejections int DeadNodesRemoved int diff --git a/services/ipdiscovery.go b/services/ipdiscovery.go index eefc8fd..c6ab92e 100644 --- a/services/ipdiscovery.go +++ b/services/ipdiscovery.go @@ -75,12 +75,14 @@ type IPDiscovery struct { // ipReport tracks reports for a specific IP:Port combination type ipReport struct { - ip net.IP - port uint16 - count int - firstSeen time.Time - lastSeen time.Time - reporterIDs []string // Track which peers reported this (for debugging) + ip net.IP + port uint16 + count int + firstSeen time.Time + lastSeen time.Time + // Counted by identity as well as by source IP: source addresses are + // spoofable, so one node ID must not satisfy the thresholds on its own. + reporterIDs map[string]int // Track distinct reporter node IDs -> count reporterIPs map[string]int // Track distinct reporter IPs -> count } @@ -199,7 +201,7 @@ func (ipd *IPDiscovery) ReportIP(ip net.IP, port uint16, reporterID string, repo ip: ip, port: port, firstSeen: now, - reporterIDs: make([]string, 0), + reporterIDs: make(map[string]int), reporterIPs: make(map[string]int), } reports[addrKey] = report @@ -208,7 +210,7 @@ func (ipd *IPDiscovery) ReportIP(ip net.IP, port uint16, reporterID string, repo // Update report report.count++ report.lastSeen = now - report.reporterIDs = append(report.reporterIDs, reporterID) + report.reporterIDs[reporterID]++ // Track reporter IP reporterIPStr := reporterIP.String() @@ -328,9 +330,11 @@ func (ipd *IPDiscovery) checkConsensusForFamilyLocked(isIPv6 bool) { if maxRecentAddr != "" && maxRecentAddr != currentAddrKey && maxRecentReport != nil { recentMajority := float64(maxRecentCount) / float64(totalRecentCount) distinctIPCount := len(maxRecentReport.reporterIPs) + distinctIDCount := len(maxRecentReport.reporterIDs) - // Enforce distinct IP threshold for address changes too - if recentMajority >= ipd.majorityThreshold && distinctIPCount >= ipd.minDistinctIPs { + // Enforce both distinct thresholds for address changes too + if recentMajority >= ipd.majorityThreshold && + distinctIPCount >= ipd.minDistinctIPs && distinctIDCount >= ipd.minDistinctIPs { // Address change detected! ipd.logger.WithFields(logrus.Fields{ "family": familyName, @@ -381,13 +385,15 @@ func (ipd *IPDiscovery) checkConsensusForFamilyLocked(isIPv6 bool) { // Check distinct IP count distinctIPCount := len(maxReport.reporterIPs) - if distinctIPCount < ipd.minDistinctIPs { + distinctIDCount := len(maxReport.reporterIDs) + if distinctIPCount < ipd.minDistinctIPs || distinctIDCount < ipd.minDistinctIPs { ipd.logger.WithFields(logrus.Fields{ "family": familyName, "addr": fmt.Sprintf("%s:%d", maxReport.ip.String(), maxReport.port), "distinctIPs": distinctIPCount, + "distinctIDs": distinctIDCount, "minDistinct": ipd.minDistinctIPs, - }).Debug("IP discovery: insufficient distinct reporter IPs") + }).Debug("IP discovery: insufficient distinct reporters") return } diff --git a/services/ipdiscovery_test.go b/services/ipdiscovery_test.go new file mode 100644 index 0000000..cda440f --- /dev/null +++ b/services/ipdiscovery_test.go @@ -0,0 +1,70 @@ +package services + +import ( + "fmt" + "net" + "testing" + "time" + + "github.com/sirupsen/logrus" +) + +func quietIPDiscovery(t *testing.T) (*IPDiscovery, <-chan string) { + t.Helper() + + logger := logrus.New() + logger.SetLevel(logrus.PanicLevel) + + reached := make(chan string, 4) + ipd := NewIPDiscovery(IPDiscoveryConfig{ + MinReports: 5, + MinDistinctIPs: 3, + Logger: logger, + OnConsensusReached: func(ip net.IP, port uint16, isIPv6 bool) { + reached <- fmt.Sprintf("%s:%d", ip.String(), port) + }, + }) + return ipd, reached +} + +// A single peer must not reach consensus on its own, however many source +// addresses it appears to report from: UDP source IPs are spoofable, so +// distinct-IP alone is not a measure of independent opinions. +func TestConsensusRequiresDistinctReporters(t *testing.T) { + ipd, reached := quietIPDiscovery(t) + + external := net.ParseIP("203.0.113.7") + for i := 0; i < 8; i++ { + reporterIP := net.IPv4(198, 51, 100, byte(1+i%4)) + ipd.ReportIP(external, 30303, "same-reporter-node-id-0000000000", reporterIP) + } + + // The callback is dispatched with `go`, so a non-blocking receive here would + // pass even when consensus fired. + select { + case addr := <-reached: + t.Fatalf("one reporter reached consensus on %s across spoofed source IPs", addr) + case <-time.After(500 * time.Millisecond): + } +} + +// The same report volume from genuinely distinct peers must still reach +// consensus, so the new gate does not simply disable IP discovery. +func TestConsensusReachedWithDistinctReporters(t *testing.T) { + ipd, reached := quietIPDiscovery(t) + + external := net.ParseIP("203.0.113.7") + for i := 0; i < 6; i++ { + reporterIP := net.IPv4(198, 51, 100, byte(1+i)) + ipd.ReportIP(external, 30303, fmt.Sprintf("reporter-node-id-%026d", i), reporterIP) + } + + select { + case addr := <-reached: + if addr != "203.0.113.7:30303" { + t.Fatalf("consensus on %s, want 203.0.113.7:30303", addr) + } + case <-time.After(2 * time.Second): + t.Fatal("distinct reporters did not reach consensus") + } +} diff --git a/services/lookup.go b/services/lookup.go index 2f062b0..e23fd46 100644 --- a/services/lookup.go +++ b/services/lookup.go @@ -13,6 +13,7 @@ import ( "crypto/rand" "fmt" mathrand "math/rand" + "slices" "sort" "sync" "time" @@ -123,12 +124,7 @@ type Config struct { // isLocal reports whether id belongs to one of our own identities. The // parameter is the raw array so both discv4 and discv5 node IDs can be passed. func (ls *LookupService) isLocal(id [32]byte) bool { - for _, local := range ls.config.LocalIDs { - if local == id { - return true - } - } - return false + return slices.Contains(ls.config.LocalIDs, id) } // discoveries accumulates the records observed during a single lookup, keeping diff --git a/services/lookup_test.go b/services/lookup_test.go index 29164c0..00f47fb 100644 --- a/services/lookup_test.go +++ b/services/lookup_test.go @@ -288,7 +288,11 @@ func TestPingServiceStatsRace(t *testing.T) { defer writers.Done() for j := 0; j < 200; j++ { ps.countPingSent() - ps.countProtocol(j%2 == 0) + if j%2 == 0 { + ps.countV5Ping() + } else { + ps.countV4Ping() + } ps.countPong(time.Millisecond) ps.countTimeout() } diff --git a/services/ping.go b/services/ping.go index 087dbbb..39e3d8c 100644 --- a/services/ping.go +++ b/services/ping.go @@ -6,7 +6,10 @@ import ( "time" "github.com/ethpandaops/bootnodoor/discv4" + discv4node "github.com/ethpandaops/bootnodoor/discv4/node" + discv5node "github.com/ethpandaops/bootnodoor/discv5/node" "github.com/ethpandaops/bootnodoor/discv5/protocol" + "github.com/ethpandaops/bootnodoor/enr" nodedb "github.com/ethpandaops/bootnodoor/nodes" "github.com/sirupsen/logrus" ) @@ -65,7 +68,7 @@ func (ps *PingService) Ping(n *nodedb.Node) (bool, time.Duration, error) { // Try discv5 first if available if v5Node := n.V5(); v5Node != nil && ps.v5Handler != nil { - ps.countProtocol(true) + ps.countV5Ping() respChan, err := ps.v5Handler.SendPing(v5Node) if err != nil { // Failed to send ping - only increment failure if no v4 fallback available @@ -136,7 +139,7 @@ func (ps *PingService) Ping(n *nodedb.Node) (bool, time.Duration, error) { // Try discv4 fallback if available if v4Node := n.V4(); v4Node != nil && ps.v4Service != nil { - ps.countProtocol(false) + ps.countV4Ping() pong, err := ps.v4Service.Ping(v4Node) rtt := time.Since(start) @@ -243,6 +246,9 @@ func (ps *PingService) CheckProtocolSupport(n *nodedb.Node) (bool, bool, error) if record == nil { return false, false, fmt.Errorf("node has no ENR") } + // Left alone until after the install, so the sequence the install is gated on + // cannot move underneath it. + probedSeq := record.Seq() ps.logger.WithFields(logrus.Fields{ "peerID": n.PeerID(), @@ -252,6 +258,13 @@ func (ps *PingService) CheckProtocolSupport(n *nodedb.Node) (bool, bool, error) var v4Supported, v5Supported bool var v4RTT, v5RTT time.Duration + // Probe objects are kept for the install: the v4 one is refreshed in place by + // the ENR fetch below, and rebuilding from the pre-probe snapshot would discard + // exactly that refresh. + var probedV4 *discv4node.Node + var probedV5 *discv5node.Node + var refreshedRecord *enr.Record + // Test discv5 support if ps.v5Handler != nil { // Create or get v5 node @@ -265,6 +278,8 @@ func (ps *PingService) CheckProtocolSupport(n *nodedb.Node) (bool, bool, error) } } + probedV5 = v5Node + if v5Node != nil { start := time.Now() respChan, err := ps.v5Handler.SendPing(v5Node) @@ -296,6 +311,8 @@ func (ps *PingService) CheckProtocolSupport(n *nodedb.Node) (bool, bool, error) } } + probedV4 = v4Node + if v4Node != nil { start := time.Now() _, err := ps.v4Service.Ping(v4Node) @@ -308,44 +325,42 @@ func (ps *PingService) CheckProtocolSupport(n *nodedb.Node) (bool, bool, error) "rtt": v4RTT, }).Debug("v4 support confirmed") - // If v4 ping succeeded, request ENR to ensure we have latest record + // Fetched here but applied after the install: advancing the record + // mid-probe would move the very sequence the install is gated on. if enrRecord, err := ps.v4Service.RequestENR(v4Node); err == nil { - v4Node.SetENR(enrRecord) - n.UpdateENR(enrRecord) + refreshedRecord = enrRecord } } } } - // Update node with discovered protocol support - // Add v5 support if confirmed and not present - if v5Supported && n.V5() == nil { - // Create and set v5 node - if v5Node, err := nodedb.NewV5NodeFromRecord(record); err == nil { - n.SetV5(v5Node) - ps.logger.WithField("peerID", n.PeerID()).Info("added v5 support to node") - } - } - - // Remove v5 support if not confirmed but present - if !v5Supported && n.V5() != nil { - n.SetV5(nil) - ps.logger.WithField("peerID", n.PeerID()).Warn("removed v5 support from node (no longer responding)") - } - - // Add v4 support if confirmed and not present - if v4Supported && n.V4() == nil { - // Create and set v4 node - if v4Node, err := nodedb.NewV4NodeFromRecord(record, addr); err == nil { - n.SetV4(v4Node) - ps.logger.WithField("peerID", n.PeerID()).Info("added v4 support to node") + // Apply both outcomes together, and only while the record they describe is + // still the node's current one. A probe that started before a newer record + // arrived knows nothing about it, so it must neither install a superseded + // endpoint nor clear a pointer that newer record brought in. + applied := n.ApplyProbeResult(probedSeq, probedV4, v4Supported, probedV5, v5Supported) + + // Only now advance the record. Attaching it to the v4 node deliberately leaves + // that node's proven address alone, so its label keeps describing the endpoint + // the ping actually reached — which is what lets a correctly addressed pointer + // for this same record replace it later. + // Attach to the object that was probed, not to whatever V4() returns now: a + // concurrent admission may have replaced the pointer, and overwriting its ENR + // with this older response would leave that pointer's record disagreeing with + // both the node's record and its label. + if refreshedRecord != nil { + if probedV4 != nil { + probedV4.SetENR(refreshedRecord) } + n.UpdateENR(refreshedRecord) } - // Remove v4 support if not confirmed but present - if !v4Supported && n.V4() != nil { - n.SetV4(nil) - ps.logger.WithField("peerID", n.PeerID()).Warn("removed v4 support from node (no longer responding)") + if applied { + ps.logger.WithFields(logrus.Fields{ + "peerID": n.PeerID(), + "v4": v4Supported, + "v5": v5Supported, + }).Info("updated protocol support from probe") } // Update RTT with best available @@ -441,13 +456,15 @@ func (ps *PingService) countPingSent() { ps.mu.Unlock() } -func (ps *PingService) countProtocol(isV5 bool) { +func (ps *PingService) countV5Ping() { ps.mu.Lock() - if isV5 { - ps.pingsV5++ - } else { - ps.pingsV4++ - } + ps.pingsV5++ + ps.mu.Unlock() +} + +func (ps *PingService) countV4Ping() { + ps.mu.Lock() + ps.pingsV4++ ps.mu.Unlock() } diff --git a/transport/dispatch_metrics_test.go b/transport/dispatch_metrics_test.go new file mode 100644 index 0000000..ed6d30b --- /dev/null +++ b/transport/dispatch_metrics_test.go @@ -0,0 +1,119 @@ +package transport + +import ( + "net" + "testing" + + "github.com/sirupsen/logrus" +) + +func dispatchTestTransport(t *testing.T, handlers ...PacketHandler) *UDPTransport { + t.Helper() + + logger := logrus.New() + logger.SetLevel(logrus.ErrorLevel) + + return &UDPTransport{logger: logger, metrics: NewMetrics(), handlers: handlers} +} + +// discv5 registers first and rejects anything it cannot decode, so a normal +// discv4 packet is only recognised on the second attempt. Counting the first +// handler's rejection as "invalid" made 84% of ordinary traffic look invalid, so +// the distinction has to be made here, where the final outcome is known. +func TestDispatchDistinguishesFallthroughFromUnhandled(t *testing.T) { + accept := func(_ []byte, _ *net.UDPAddr, _ *net.UDPAddr) bool { return true } + reject := func(_ []byte, _ *net.UDPAddr, _ *net.UDPAddr) bool { return false } + + from := &net.UDPAddr{IP: net.IPv4(10, 0, 0, 1), Port: 30303} + local := &net.UDPAddr{IP: net.IPv4(10, 0, 0, 2), Port: 9000} + + t.Run("first handler accepts", func(t *testing.T) { + tr := dispatchTestTransport(t, accept, reject) + tr.dispatchPacket([]byte("packet"), from, local) + + got := tr.Metrics().Snapshot() + if got.PacketsFellThrough != 0 { + t.Errorf("PacketsFellThrough = %d, want 0", got.PacketsFellThrough) + } + if got.PacketsUnhandled != 0 { + t.Errorf("PacketsUnhandled = %d, want 0", got.PacketsUnhandled) + } + }) + + t.Run("second handler accepts", func(t *testing.T) { + tr := dispatchTestTransport(t, reject, accept) + tr.dispatchPacket([]byte("packet"), from, local) + + got := tr.Metrics().Snapshot() + if got.PacketsFellThrough != 1 { + t.Errorf("PacketsFellThrough = %d, want 1", got.PacketsFellThrough) + } + if got.PacketsUnhandled != 0 { + t.Errorf("PacketsUnhandled = %d, want 0", got.PacketsUnhandled) + } + }) + + t.Run("nobody accepts", func(t *testing.T) { + tr := dispatchTestTransport(t, reject, reject) + tr.dispatchPacket([]byte("packet"), from, local) + + got := tr.Metrics().Snapshot() + if got.PacketsFellThrough != 0 { + t.Errorf("PacketsFellThrough = %d, want 0", got.PacketsFellThrough) + } + if got.PacketsUnhandled != 1 { + t.Errorf("PacketsUnhandled = %d, want 1", got.PacketsUnhandled) + } + }) +} + +// Two discv5 identities share a socket, and the first rejects packets addressed +// to the second. That is identity demultiplexing, not other-protocol traffic, so +// it must not inflate the fallthrough counter. +func TestDispatchDoesNotCountSameProtocolDemux(t *testing.T) { + accept := func(_ []byte, _ *net.UDPAddr, _ *net.UDPAddr) bool { return true } + reject := func(_ []byte, _ *net.UDPAddr, _ *net.UDPAddr) bool { return false } + + from := &net.UDPAddr{IP: net.IPv4(10, 0, 0, 1), Port: 30303} + local := &net.UDPAddr{IP: net.IPv4(10, 0, 0, 2), Port: 9000} + + logger := logrus.New() + logger.SetLevel(logrus.ErrorLevel) + + t.Run("second discv5 identity accepts", func(t *testing.T) { + tr := &UDPTransport{logger: logger, metrics: NewMetrics()} + tr.AddHandlerFor("discv5", reject) + tr.AddHandlerFor("discv5", accept) + tr.AddHandlerFor("discv4", reject) + tr.dispatchPacket([]byte("packet"), from, local) + + if got := tr.Metrics().Snapshot().PacketsFellThrough; got != 0 { + t.Errorf("PacketsFellThrough = %d, want 0 for same-protocol demux", got) + } + }) + + t.Run("discv4 accepts after discv5 identities", func(t *testing.T) { + tr := &UDPTransport{logger: logger, metrics: NewMetrics()} + tr.AddHandlerFor("discv5", reject) + tr.AddHandlerFor("discv5", reject) + tr.AddHandlerFor("discv4", accept) + tr.dispatchPacket([]byte("packet"), from, local) + + if got := tr.Metrics().Snapshot().PacketsFellThrough; got != 1 { + t.Errorf("PacketsFellThrough = %d, want 1 when the protocol changed", got) + } + }) +} + +// Reset must clear the dispatch counters too, or a caller sees stale values. +func TestResetClearsDispatchCounters(t *testing.T) { + m := NewMetrics() + m.RecordFellThrough() + m.RecordUnhandled() + m.Reset() + + got := m.Snapshot() + if got.PacketsFellThrough != 0 || got.PacketsUnhandled != 0 { + t.Errorf("after Reset: fellThrough=%d unhandled=%d, want 0/0", got.PacketsFellThrough, got.PacketsUnhandled) + } +} diff --git a/transport/metrics.go b/transport/metrics.go index 2b35e5e..b2aeea4 100644 --- a/transport/metrics.go +++ b/transport/metrics.go @@ -21,6 +21,22 @@ type Metrics struct { sendErrors atomic.Uint64 receiveErrors atomic.Uint64 rateLimited atomic.Uint64 + + // Dispatch outcomes. A packet the first handler declines but a later one + // accepts is normal traffic for the other protocol on a shared socket; only + // a packet no handler accepts is unrecognised. + packetsFellThrough atomic.Uint64 + packetsUnhandled atomic.Uint64 +} + +// RecordFellThrough records a packet accepted by a handler other than the first. +func (m *Metrics) RecordFellThrough() { + m.packetsFellThrough.Add(1) +} + +// RecordUnhandled records a packet no handler accepted. +func (m *Metrics) RecordUnhandled() { + m.packetsUnhandled.Add(1) } // NewMetrics creates a new metrics tracker. @@ -62,14 +78,16 @@ func (m *Metrics) IncrementDropped() { // Snapshot returns a snapshot of the current metrics. type MetricsSnapshot struct { - PacketsSent uint64 - PacketsReceived uint64 - PacketsDropped uint64 - BytesSent uint64 - BytesReceived uint64 - SendErrors uint64 - ReceiveErrors uint64 - RateLimited uint64 + PacketsSent uint64 + PacketsReceived uint64 + PacketsDropped uint64 + BytesSent uint64 + BytesReceived uint64 + SendErrors uint64 + ReceiveErrors uint64 + RateLimited uint64 + PacketsFellThrough uint64 + PacketsUnhandled uint64 } // Snapshot returns a snapshot of the current metrics. @@ -81,14 +99,16 @@ type MetricsSnapshot struct { // snapshot.PacketsSent, snapshot.PacketsReceived) func (m *Metrics) Snapshot() MetricsSnapshot { return MetricsSnapshot{ - PacketsSent: m.packetsSent.Load(), - PacketsReceived: m.packetsReceived.Load(), - PacketsDropped: m.packetsDropped.Load(), - BytesSent: m.bytesSent.Load(), - BytesReceived: m.bytesReceived.Load(), - SendErrors: m.sendErrors.Load(), - ReceiveErrors: m.receiveErrors.Load(), - RateLimited: m.rateLimited.Load(), + PacketsSent: m.packetsSent.Load(), + PacketsReceived: m.packetsReceived.Load(), + PacketsDropped: m.packetsDropped.Load(), + BytesSent: m.bytesSent.Load(), + BytesReceived: m.bytesReceived.Load(), + SendErrors: m.sendErrors.Load(), + ReceiveErrors: m.receiveErrors.Load(), + RateLimited: m.rateLimited.Load(), + PacketsFellThrough: m.packetsFellThrough.Load(), + PacketsUnhandled: m.packetsUnhandled.Load(), } } @@ -102,6 +122,8 @@ func (m *Metrics) Reset() { m.sendErrors.Store(0) m.receiveErrors.Store(0) m.rateLimited.Store(0) + m.packetsFellThrough.Store(0) + m.packetsUnhandled.Store(0) } // PacketsSent returns the number of packets sent. diff --git a/transport/udp.go b/transport/udp.go index afbf34d..ec46240 100644 --- a/transport/udp.go +++ b/transport/udp.go @@ -58,8 +58,9 @@ type UDPTransport struct { ipv6Conn *ipv6.PacketConn // handlers is a list of packet handlers (tried in order) - handlers []PacketHandler - handlersMu sync.RWMutex + handlers []PacketHandler + handlerProtocols []string + handlersMu sync.RWMutex // logger for debug and error messages logger logrus.FieldLogger @@ -265,9 +266,20 @@ func (t *UDPTransport) Conn() *net.UDPConn { // return err == nil // }) func (t *UDPTransport) AddHandler(handler func(data []byte, from *net.UDPAddr, localAddr *net.UDPAddr) bool) { + t.AddHandlerFor("", handler) +} + +// AddHandlerFor registers a packet handler under a protocol label. +// +// The label only affects accounting: two discv5 identities share a socket and +// each rejects the other's packets, which is identity demultiplexing rather than +// a protocol mismatch. Without the label that rejection would be reported as +// other-protocol traffic. +func (t *UDPTransport) AddHandlerFor(protocol string, handler func(data []byte, from *net.UDPAddr, localAddr *net.UDPAddr) bool) { t.handlersMu.Lock() defer t.handlersMu.Unlock() t.handlers = append(t.handlers, PacketHandler(handler)) + t.handlerProtocols = append(t.handlerProtocols, protocol) } // SendTo sends a packet to the specified address. @@ -382,22 +394,46 @@ func (t *UDPTransport) sendWithSource(data []byte, to *net.UDPAddr, from *net.UD } } +// crossedProtocol reports whether any handler before accepted was registered +// under a different protocol label. +func crossedProtocol(protocols []string, accepted int) bool { + if accepted >= len(protocols) { + return accepted > 0 + } + for i := 0; i < accepted && i < len(protocols); i++ { + if protocols[i] != protocols[accepted] { + return true + } + } + return false +} + // dispatchPacket routes a packet to the registered handlers. // // Handlers are tried in order until one returns true. func (t *UDPTransport) dispatchPacket(data []byte, from *net.UDPAddr, localAddr *net.UDPAddr) { t.handlersMu.RLock() handlers := t.handlers + protocols := t.handlerProtocols t.handlersMu.RUnlock() // Try each handler in order - for _, handler := range handlers { + for i, handler := range handlers { if handler(data, from, localAddr) { - // Handler accepted the packet + // Only a handler for a different protocol counts as fallthrough. An + // earlier handler of the same protocol rejecting the packet is one + // identity declining another's traffic on a shared socket. + if t.metrics != nil && crossedProtocol(protocols, i) { + t.metrics.RecordFellThrough() + } return } } + if t.metrics != nil { + t.metrics.RecordUnhandled() + } + // No handler recognized the packet t.logger.WithFields(logrus.Fields{ "from": from, diff --git a/webui/handlers/cl-nodes.go b/webui/handlers/cl-nodes.go index a44b6f1..b638fba 100644 --- a/webui/handlers/cl-nodes.go +++ b/webui/handlers/cl-nodes.go @@ -103,7 +103,7 @@ func (fh *FrontendHandler) CLNodes(w http.ResponseWriter, r *http.Request) { pageData := CLNodesPageData{ TotalNodes: stats.TotalNodes, ActiveNodes: stats.ActiveNodes, - InactiveNodes: stats.TotalNodes - stats.ActiveNodes, + InactiveNodes: stats.InactiveNodes, AliveNodes: aliveCount, DeadNodes: deadCount, CurrentForkDigest: currentForkDigest, @@ -180,7 +180,7 @@ func (fh *FrontendHandler) CLNodesJSON(w http.ResponseWriter, r *http.Request) { pageData := CLNodesPageData{ TotalNodes: stats.TotalNodes, ActiveNodes: stats.ActiveNodes, - InactiveNodes: stats.TotalNodes - stats.ActiveNodes, + InactiveNodes: stats.InactiveNodes, AliveNodes: aliveCount, DeadNodes: deadCount, CurrentForkDigest: currentForkDigest, diff --git a/webui/handlers/el-nodes.go b/webui/handlers/el-nodes.go index 11eb7b1..bcd340a 100644 --- a/webui/handlers/el-nodes.go +++ b/webui/handlers/el-nodes.go @@ -120,7 +120,7 @@ func (fh *FrontendHandler) ELNodes(w http.ResponseWriter, r *http.Request) { pageData := ELNodesPageData{ TotalNodes: stats.TotalNodes, ActiveNodes: stats.ActiveNodes, - InactiveNodes: stats.TotalNodes - stats.ActiveNodes, + InactiveNodes: stats.InactiveNodes, AliveNodes: aliveCount, DeadNodes: deadCount, CurrentForkDigest: currentForkDigest, @@ -209,7 +209,7 @@ func (fh *FrontendHandler) ELNodesJSON(w http.ResponseWriter, r *http.Request) { pageData := ELNodesPageData{ TotalNodes: stats.TotalNodes, ActiveNodes: stats.ActiveNodes, - InactiveNodes: stats.TotalNodes - stats.ActiveNodes, + InactiveNodes: stats.InactiveNodes, AliveNodes: aliveCount, DeadNodes: deadCount, CurrentForkDigest: currentForkDigest, diff --git a/webui/handlers/overview.go b/webui/handlers/overview.go index 6564dea..1f2ce6e 100644 --- a/webui/handlers/overview.go +++ b/webui/handlers/overview.go @@ -105,11 +105,12 @@ type OverviewPageData struct { PendingChallenges int // Handler stats - PacketsReceived int - PacketsSent int - InvalidPackets int - FilteredResponses int - FindNodeReceived int + PacketsReceived int + PacketsSent int + InvalidPackets int + WrongProtocolPackets int + FilteredResponses int + FindNodeReceived int // CL fork digest filter stats FilterAcceptedCurrent int @@ -333,7 +334,7 @@ func (fh *FrontendHandler) getOverviewPageData() (*OverviewPageData, error) { // Get EL table stats if available if elTable := fh.bootnodeService.ELTable(); elTable != nil { elStats := elTable.GetStats() - elInactiveNodes := elStats.TotalNodes - elStats.ActiveNodes + elInactiveNodes := elStats.InactiveNodes pageData.ELActiveNodes = elStats.ActiveNodes pageData.ELTotalNodes = elStats.TotalNodes pageData.ELTableStats = TableStats{ @@ -350,7 +351,7 @@ func (fh *FrontendHandler) getOverviewPageData() (*OverviewPageData, error) { // Get CL table stats if available if clTable := fh.bootnodeService.CLTable(); clTable != nil { clStats := clTable.GetStats() - clInactiveNodes := clStats.TotalNodes - clStats.ActiveNodes + clInactiveNodes := clStats.InactiveNodes pageData.CLActiveNodes = clStats.ActiveNodes pageData.CLTotalNodes = clStats.TotalNodes pageData.CLTableStats = TableStats{ @@ -503,7 +504,13 @@ func (fh *FrontendHandler) getOverviewPageData() (*OverviewPageData, error) { // discv5-specific views stay on the handler counters. pageData.PacketsReceived = int(stats.Packets.PacketsReceived) pageData.PacketsSent = int(stats.Packets.PacketsSent) - pageData.InvalidPackets = stats.Discv5.InvalidPackets + int(stats.Discv4.InvalidPackets) + + // A packet the first handler declines but a later one accepts is ordinary + // traffic for the other protocol on the shared socket, so only the packets + // nothing accepted are unrecognised. Summing the per-handler "invalid" + // counters instead reported most normal discv4 load as invalid. + pageData.WrongProtocolPackets = int(stats.Packets.PacketsFellThrough) + pageData.InvalidPackets = int(stats.Packets.PacketsUnhandled) pageData.FilteredResponses = stats.Discv5.FilteredResponses pageData.FindNodeReceived = stats.Discv5.FindNodeReceived + int(stats.Discv4.FindnodeRequestsRecv) diff --git a/webui/templates/overview/overview.html b/webui/templates/overview/overview.html index 7df7972..c0656bc 100644 --- a/webui/templates/overview/overview.html +++ b/webui/templates/overview/overview.html @@ -463,6 +463,10 @@
Packet Statistics
FINDNODE Received {{ .FindNodeReceived }} + + Other Protocol + {{ .WrongProtocolPackets }} + Invalid Packets {{ .InvalidPackets }} @@ -892,6 +896,7 @@
Old Fork Digests (Grace Period)
updateValue('[data-stat="packets-sent"]', data.PacketsSent); updateValue('[data-stat="findnode-received"]', data.FindNodeReceived); updateValue('[data-stat="invalid-packets"]', data.InvalidPackets); + updateValue('[data-stat="wrong-protocol-packets"]', data.WrongProtocolPackets); updateValue('[data-stat="filtered-responses"]', data.FilteredResponses); /* Fork info (if present) */ diff --git a/webui/webui.go b/webui/webui.go index 9ba75f3..c35d9e6 100644 --- a/webui/webui.go +++ b/webui/webui.go @@ -48,8 +48,7 @@ func StartHttpServer(config *types.FrontendConfig, logger logrus.FieldLogger, bo // metrics endpoint router.Handle("/metrics", promhttp.Handler()).Methods("GET") - // add pprof handler - router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux) + registerPprofHandler(router, config.Pprof) router.PathPrefix("/").Handler(frontend) @@ -79,3 +78,9 @@ func StartHttpServer(config *types.FrontendConfig, logger logrus.FieldLogger, bo } }() } + +func registerPprofHandler(router *mux.Router, enabled bool) { + if enabled { + router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux) + } +} diff --git a/webui/webui_test.go b/webui/webui_test.go new file mode 100644 index 0000000..c7b9bdd --- /dev/null +++ b/webui/webui_test.go @@ -0,0 +1,43 @@ +package webui + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/gorilla/mux" +) + +func TestRegisterPprofHandler(t *testing.T) { + tests := []struct { + name string + enabled bool + statusCode int + }{ + { + name: "disabled", + enabled: false, + statusCode: http.StatusNotFound, + }, + { + name: "enabled", + enabled: true, + statusCode: http.StatusOK, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + router := mux.NewRouter() + registerPprofHandler(router, test.enabled) + + request := httptest.NewRequest(http.MethodGet, "/debug/pprof/", nil) + response := httptest.NewRecorder() + router.ServeHTTP(response, request) + + if response.Code != test.statusCode { + t.Fatalf("expected status code %d, got %d", test.statusCode, response.Code) + } + }) + } +}