From c53ebe47cbf5f8f355002d2deb9384bb12d155ab Mon Sep 17 00:00:00 2001 From: shiv Date: Sun, 9 Aug 2026 09:37:19 +0530 Subject: [PATCH 1/7] =?UTF-8?q?=E2=9A=A1=20Bolt:=20pre-parse=20IP=20addres?= =?UTF-8?q?ses=20before=20sorting=20discovery=20sweep=20results?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .jules/bolt.md | 7 ++++++ pkg/proxy/discovery/sweep.go | 35 ++++++++++++++++++---------- pkg/proxy/discovery/sweep_test.go | 38 +++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 12 deletions(-) create mode 100644 .jules/bolt.md diff --git a/.jules/bolt.md b/.jules/bolt.md new file mode 100644 index 0000000..8267c90 --- /dev/null +++ b/.jules/bolt.md @@ -0,0 +1,7 @@ +# Bolt's Performance Journal + +Critical learnings and performance patterns discovered in this codebase. + +## 2026-08-08 - Pre-parse IP Addresses for Comparator Functions +**Learning:** `sort.Slice` calls comparator functions $O(N \log N)$ times. Calling `netip.ParseAddr` or other parsing/conversion functions inside a sort comparator creates severe CPU overhead ($2 \cdot N \log_2 N$ string parses) during large discovery sweeps (up to 65,536 hosts). +**Action:** Always pre-parse IP strings into `netip.Addr` structs once into a temporary slice or wrapper struct before sorting. diff --git a/pkg/proxy/discovery/sweep.go b/pkg/proxy/discovery/sweep.go index 160e72f..d7f72cd 100644 --- a/pkg/proxy/discovery/sweep.go +++ b/pkg/proxy/discovery/sweep.go @@ -141,11 +141,31 @@ func runSweep(ctx context.Context, cfg sweepConfig) (*SweepResult, error) { enrichFromARP(hosts) enrichRDNS(ctx, hosts) - out := make([]SweepHost, 0, len(hosts)) + // Pre-parse IP addresses once to avoid O(N log N) string parses in sort comparator + type sweepHostAddr struct { + host SweepHost + addr netip.Addr + } + items := make([]sweepHostAddr, 0, len(hosts)) for _, h := range hosts { - out = append(out, *h) + addr, err := netip.ParseAddr(h.IP) + if err != nil { + items = append(items, sweepHostAddr{host: *h}) + continue + } + items = append(items, sweepHostAddr{host: *h, addr: addr}) + } + sort.Slice(items, func(i, j int) bool { + if !items[i].addr.IsValid() || !items[j].addr.IsValid() { + return items[i].host.IP < items[j].host.IP + } + return items[i].addr.Less(items[j].addr) + }) + + out := make([]SweepHost, len(items)) + for i, item := range items { + out[i] = item.host } - sort.Slice(out, func(i, j int) bool { return compareIPs(out[i].IP, out[j].IP) }) cidrStrings := make([]string, len(cfg.cidrs)) for i, c := range cfg.cidrs { @@ -272,15 +292,6 @@ func trimTrailingDot(s string) string { return s } -func compareIPs(a, b string) bool { - aa, errA := netip.ParseAddr(a) - bb, errB := netip.ParseAddr(b) - if errA != nil || errB != nil { - return a < b - } - return aa.Less(bb) -} - // parseSweepParams validates an action's parameters and clamps them to the // module's own limits. func parseSweepParams(params map[string]any, maxRate int) (sweepConfig, error) { diff --git a/pkg/proxy/discovery/sweep_test.go b/pkg/proxy/discovery/sweep_test.go index a4eba00..ac42cdf 100644 --- a/pkg/proxy/discovery/sweep_test.go +++ b/pkg/proxy/discovery/sweep_test.go @@ -3,8 +3,10 @@ package discovery import ( "context" "encoding/json" + "fmt" "net" "net/netip" + "sort" "sync/atomic" "testing" "time" @@ -453,3 +455,39 @@ func TestRunSweep_MoreWorkersDoNotExceedRateCap(t *testing.T) { cfg.workers, result.Scanned, elapsed, ratePPS, minDuration) } } + +func BenchmarkSweepResultSorting(b *testing.B) { + hosts := make(map[string]*SweepHost, 1000) + for i := 0; i < 1000; i++ { + ip := fmt.Sprintf("10.0.%d.%d", i/256, i%256) + hosts[ip] = &SweepHost{IP: ip} + } + + b.ResetTimer() + for n := 0; n < b.N; n++ { + type sweepHostAddr struct { + host SweepHost + addr netip.Addr + } + items := make([]sweepHostAddr, 0, len(hosts)) + for _, h := range hosts { + addr, err := netip.ParseAddr(h.IP) + if err != nil { + items = append(items, sweepHostAddr{host: *h}) + continue + } + items = append(items, sweepHostAddr{host: *h, addr: addr}) + } + sort.Slice(items, func(i, j int) bool { + if !items[i].addr.IsValid() || !items[j].addr.IsValid() { + return items[i].host.IP < items[j].host.IP + } + return items[i].addr.Less(items[j].addr) + }) + out := make([]SweepHost, len(items)) + for i, item := range items { + out[i] = item.host + } + _ = out + } +} From a563d2cf72261684c21595592a6f13ebbe68d4ba Mon Sep 17 00:00:00 2001 From: shiv Date: Sun, 9 Aug 2026 09:54:54 +0530 Subject: [PATCH 2/7] =?UTF-8?q?=E2=9A=A1=20Bolt:=20use=20*SweepHost=20poin?= =?UTF-8?q?ter=20in=20sweepHostAddr=20to=20avoid=20struct=20copy=20in=20so?= =?UTF-8?q?rt.Slice?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/proxy/discovery/sweep.go | 12 +++++++----- pkg/proxy/discovery/sweep_test.go | 8 ++++---- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/pkg/proxy/discovery/sweep.go b/pkg/proxy/discovery/sweep.go index d7f72cd..ffeef4f 100644 --- a/pkg/proxy/discovery/sweep.go +++ b/pkg/proxy/discovery/sweep.go @@ -141,19 +141,21 @@ func runSweep(ctx context.Context, cfg sweepConfig) (*SweepResult, error) { enrichFromARP(hosts) enrichRDNS(ctx, hosts) - // Pre-parse IP addresses once to avoid O(N log N) string parses in sort comparator + // Pre-parse IP addresses once to avoid O(N log N) string parses in sort comparator. + // Store pointers (*SweepHost) to keep sweepHostAddr small (32 bytes instead of 120 bytes) + // and eliminate struct copying overhead during sort.Slice swaps. type sweepHostAddr struct { - host SweepHost + host *SweepHost addr netip.Addr } items := make([]sweepHostAddr, 0, len(hosts)) for _, h := range hosts { addr, err := netip.ParseAddr(h.IP) if err != nil { - items = append(items, sweepHostAddr{host: *h}) + items = append(items, sweepHostAddr{host: h}) continue } - items = append(items, sweepHostAddr{host: *h, addr: addr}) + items = append(items, sweepHostAddr{host: h, addr: addr}) } sort.Slice(items, func(i, j int) bool { if !items[i].addr.IsValid() || !items[j].addr.IsValid() { @@ -164,7 +166,7 @@ func runSweep(ctx context.Context, cfg sweepConfig) (*SweepResult, error) { out := make([]SweepHost, len(items)) for i, item := range items { - out[i] = item.host + out[i] = *item.host } cidrStrings := make([]string, len(cfg.cidrs)) diff --git a/pkg/proxy/discovery/sweep_test.go b/pkg/proxy/discovery/sweep_test.go index ac42cdf..c255267 100644 --- a/pkg/proxy/discovery/sweep_test.go +++ b/pkg/proxy/discovery/sweep_test.go @@ -466,17 +466,17 @@ func BenchmarkSweepResultSorting(b *testing.B) { b.ResetTimer() for n := 0; n < b.N; n++ { type sweepHostAddr struct { - host SweepHost + host *SweepHost addr netip.Addr } items := make([]sweepHostAddr, 0, len(hosts)) for _, h := range hosts { addr, err := netip.ParseAddr(h.IP) if err != nil { - items = append(items, sweepHostAddr{host: *h}) + items = append(items, sweepHostAddr{host: h}) continue } - items = append(items, sweepHostAddr{host: *h, addr: addr}) + items = append(items, sweepHostAddr{host: h, addr: addr}) } sort.Slice(items, func(i, j int) bool { if !items[i].addr.IsValid() || !items[j].addr.IsValid() { @@ -486,7 +486,7 @@ func BenchmarkSweepResultSorting(b *testing.B) { }) out := make([]SweepHost, len(items)) for i, item := range items { - out[i] = item.host + out[i] = *item.host } _ = out } From 3cc29abcc0c70887df9cfcd71c15109ec053eade Mon Sep 17 00:00:00 2001 From: shiv Date: Sun, 9 Aug 2026 17:37:36 +0530 Subject: [PATCH 3/7] fix(discovery): enforce strict weak ordering in sweep result IP sorting --- pkg/proxy/discovery/sweep.go | 10 +++++++++- pkg/proxy/discovery/sweep_test.go | 10 +++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/pkg/proxy/discovery/sweep.go b/pkg/proxy/discovery/sweep.go index ffeef4f..810fbee 100644 --- a/pkg/proxy/discovery/sweep.go +++ b/pkg/proxy/discovery/sweep.go @@ -158,9 +158,17 @@ func runSweep(ctx context.Context, cfg sweepConfig) (*SweepResult, error) { items = append(items, sweepHostAddr{host: h, addr: addr}) } sort.Slice(items, func(i, j int) bool { - if !items[i].addr.IsValid() || !items[j].addr.IsValid() { + iValid := items[i].addr.IsValid() + jValid := items[j].addr.IsValid() + if !iValid && !jValid { return items[i].host.IP < items[j].host.IP } + if !iValid { + return false + } + if !jValid { + return true + } return items[i].addr.Less(items[j].addr) }) diff --git a/pkg/proxy/discovery/sweep_test.go b/pkg/proxy/discovery/sweep_test.go index c255267..b647e70 100644 --- a/pkg/proxy/discovery/sweep_test.go +++ b/pkg/proxy/discovery/sweep_test.go @@ -479,9 +479,17 @@ func BenchmarkSweepResultSorting(b *testing.B) { items = append(items, sweepHostAddr{host: h, addr: addr}) } sort.Slice(items, func(i, j int) bool { - if !items[i].addr.IsValid() || !items[j].addr.IsValid() { + iValid := items[i].addr.IsValid() + jValid := items[j].addr.IsValid() + if !iValid && !jValid { return items[i].host.IP < items[j].host.IP } + if !iValid { + return false + } + if !jValid { + return true + } return items[i].addr.Less(items[j].addr) }) out := make([]SweepHost, len(items)) From 339abdea45706a52e12e7c87fbf8f8ee0cdef293 Mon Sep 17 00:00:00 2001 From: shiv Date: Sun, 9 Aug 2026 21:55:01 +0530 Subject: [PATCH 4/7] refactor(discovery): extract sortHosts helper and reuse in benchmark --- pkg/proxy/discovery/sweep.go | 75 ++++++++++++++++--------------- pkg/proxy/discovery/sweep_test.go | 34 +------------- 2 files changed, 41 insertions(+), 68 deletions(-) diff --git a/pkg/proxy/discovery/sweep.go b/pkg/proxy/discovery/sweep.go index 810fbee..4ee0b9b 100644 --- a/pkg/proxy/discovery/sweep.go +++ b/pkg/proxy/discovery/sweep.go @@ -141,41 +141,7 @@ func runSweep(ctx context.Context, cfg sweepConfig) (*SweepResult, error) { enrichFromARP(hosts) enrichRDNS(ctx, hosts) - // Pre-parse IP addresses once to avoid O(N log N) string parses in sort comparator. - // Store pointers (*SweepHost) to keep sweepHostAddr small (32 bytes instead of 120 bytes) - // and eliminate struct copying overhead during sort.Slice swaps. - type sweepHostAddr struct { - host *SweepHost - addr netip.Addr - } - items := make([]sweepHostAddr, 0, len(hosts)) - for _, h := range hosts { - addr, err := netip.ParseAddr(h.IP) - if err != nil { - items = append(items, sweepHostAddr{host: h}) - continue - } - items = append(items, sweepHostAddr{host: h, addr: addr}) - } - sort.Slice(items, func(i, j int) bool { - iValid := items[i].addr.IsValid() - jValid := items[j].addr.IsValid() - if !iValid && !jValid { - return items[i].host.IP < items[j].host.IP - } - if !iValid { - return false - } - if !jValid { - return true - } - return items[i].addr.Less(items[j].addr) - }) - - out := make([]SweepHost, len(items)) - for i, item := range items { - out[i] = *item.host - } + out := sortHosts(hosts) cidrStrings := make([]string, len(cfg.cidrs)) for i, c := range cfg.cidrs { @@ -302,6 +268,45 @@ func trimTrailingDot(s string) string { return s } +// sortHosts converts the host map into an IP-sorted slice of SweepHost. +// Pre-parses IP addresses once into netip.Addr to avoid O(N log N) string parses in sort comparator. +// Stores pointers (*SweepHost) to keep sweepHostAddr small (32 bytes) and avoid struct copying overhead. +func sortHosts(hosts map[string]*SweepHost) []SweepHost { + type sweepHostAddr struct { + host *SweepHost + addr netip.Addr + } + items := make([]sweepHostAddr, 0, len(hosts)) + for _, h := range hosts { + addr, err := netip.ParseAddr(h.IP) + if err != nil { + items = append(items, sweepHostAddr{host: h}) + continue + } + items = append(items, sweepHostAddr{host: h, addr: addr}) + } + sort.Slice(items, func(i, j int) bool { + iValid := items[i].addr.IsValid() + jValid := items[j].addr.IsValid() + if !iValid && !jValid { + return items[i].host.IP < items[j].host.IP + } + if !iValid { + return false + } + if !jValid { + return true + } + return items[i].addr.Less(items[j].addr) + }) + + out := make([]SweepHost, len(items)) + for i, item := range items { + out[i] = *item.host + } + return out +} + // parseSweepParams validates an action's parameters and clamps them to the // module's own limits. func parseSweepParams(params map[string]any, maxRate int) (sweepConfig, error) { diff --git a/pkg/proxy/discovery/sweep_test.go b/pkg/proxy/discovery/sweep_test.go index b647e70..3cd8142 100644 --- a/pkg/proxy/discovery/sweep_test.go +++ b/pkg/proxy/discovery/sweep_test.go @@ -6,7 +6,6 @@ import ( "fmt" "net" "net/netip" - "sort" "sync/atomic" "testing" "time" @@ -465,37 +464,6 @@ func BenchmarkSweepResultSorting(b *testing.B) { b.ResetTimer() for n := 0; n < b.N; n++ { - type sweepHostAddr struct { - host *SweepHost - addr netip.Addr - } - items := make([]sweepHostAddr, 0, len(hosts)) - for _, h := range hosts { - addr, err := netip.ParseAddr(h.IP) - if err != nil { - items = append(items, sweepHostAddr{host: h}) - continue - } - items = append(items, sweepHostAddr{host: h, addr: addr}) - } - sort.Slice(items, func(i, j int) bool { - iValid := items[i].addr.IsValid() - jValid := items[j].addr.IsValid() - if !iValid && !jValid { - return items[i].host.IP < items[j].host.IP - } - if !iValid { - return false - } - if !jValid { - return true - } - return items[i].addr.Less(items[j].addr) - }) - out := make([]SweepHost, len(items)) - for i, item := range items { - out[i] = *item.host - } - _ = out + _ = sortHosts(hosts) } } From 6bb27f9f7392d79c0c28bd246ff968ee4d6f8288 Mon Sep 17 00:00:00 2001 From: shiv Date: Sun, 9 Aug 2026 22:00:17 +0530 Subject: [PATCH 5/7] fix(discovery): add defensive nil check in sortHosts loop --- pkg/proxy/discovery/sweep.go | 3 +++ pkg/proxy/discovery/sweep_test.go | 17 +++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/pkg/proxy/discovery/sweep.go b/pkg/proxy/discovery/sweep.go index 4ee0b9b..9f01902 100644 --- a/pkg/proxy/discovery/sweep.go +++ b/pkg/proxy/discovery/sweep.go @@ -278,6 +278,9 @@ func sortHosts(hosts map[string]*SweepHost) []SweepHost { } items := make([]sweepHostAddr, 0, len(hosts)) for _, h := range hosts { + if h == nil { + continue + } addr, err := netip.ParseAddr(h.IP) if err != nil { items = append(items, sweepHostAddr{host: h}) diff --git a/pkg/proxy/discovery/sweep_test.go b/pkg/proxy/discovery/sweep_test.go index 3cd8142..b0a3ecb 100644 --- a/pkg/proxy/discovery/sweep_test.go +++ b/pkg/proxy/discovery/sweep_test.go @@ -467,3 +467,20 @@ func BenchmarkSweepResultSorting(b *testing.B) { _ = sortHosts(hosts) } } + +func TestSortHosts_HandlesNilHostPointers(t *testing.T) { + hosts := map[string]*SweepHost{ + "10.0.0.2": {IP: "10.0.0.2"}, + "10.0.0.1": nil, + "10.0.0.3": {IP: "10.0.0.3"}, + } + + sorted := sortHosts(hosts) + if len(sorted) != 2 { + t.Fatalf("expected 2 non-nil hosts, got %d", len(sorted)) + } + if sorted[0].IP != "10.0.0.2" || sorted[1].IP != "10.0.0.3" { + t.Errorf("unexpected sorted IPs: %v", sorted) + } +} + From 875e3f2fa152fb76589fc9bb9955c5de52214233 Mon Sep 17 00:00:00 2001 From: shiv Date: Mon, 10 Aug 2026 08:35:27 +0530 Subject: [PATCH 6/7] fix(discovery): fix gofmt and simplify sortHosts comparator - Remove trailing blank line in sweep_test.go that failed the CI gofmt check - Drop the error branch in the pre-parse loop: netip.ParseAddr returns the zero Addr on failure, which is already invalid - Collapse the validity branches in the comparator to a single iValid != jValid check while preserving strict weak ordering Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01PzLvcfr58G3Gwhc52zQysd --- pkg/proxy/discovery/sweep.go | 18 +++++++----------- pkg/proxy/discovery/sweep_test.go | 1 - 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/pkg/proxy/discovery/sweep.go b/pkg/proxy/discovery/sweep.go index 9f01902..cd62840 100644 --- a/pkg/proxy/discovery/sweep.go +++ b/pkg/proxy/discovery/sweep.go @@ -281,24 +281,20 @@ func sortHosts(hosts map[string]*SweepHost) []SweepHost { if h == nil { continue } - addr, err := netip.ParseAddr(h.IP) - if err != nil { - items = append(items, sweepHostAddr{host: h}) - continue - } + // A parse failure yields the zero Addr, which reports IsValid() == false, + // so the comparator below can sort unparseable IPs to the end without + // branching on the error here. + addr, _ := netip.ParseAddr(h.IP) items = append(items, sweepHostAddr{host: h, addr: addr}) } sort.Slice(items, func(i, j int) bool { iValid := items[i].addr.IsValid() jValid := items[j].addr.IsValid() - if !iValid && !jValid { - return items[i].host.IP < items[j].host.IP + if iValid != jValid { + return iValid } if !iValid { - return false - } - if !jValid { - return true + return items[i].host.IP < items[j].host.IP } return items[i].addr.Less(items[j].addr) }) diff --git a/pkg/proxy/discovery/sweep_test.go b/pkg/proxy/discovery/sweep_test.go index b0a3ecb..0031ad7 100644 --- a/pkg/proxy/discovery/sweep_test.go +++ b/pkg/proxy/discovery/sweep_test.go @@ -483,4 +483,3 @@ func TestSortHosts_HandlesNilHostPointers(t *testing.T) { t.Errorf("unexpected sorted IPs: %v", sorted) } } - From f472f12eb6896f05862dc5f5c8aecdaef9f40393 Mon Sep 17 00:00:00 2001 From: shiv Date: Mon, 10 Aug 2026 08:43:26 +0530 Subject: [PATCH 7/7] perf(discovery): use slices.SortFunc in sortHosts and cover invalid IPs - Replace sort.Slice with slices.SortFunc to drop reflection and interface boxing on every comparison; use netip.Addr.Compare / cmp.Compare - Add TestSortHosts_UnparseableIPsSortLast asserting valid IPs sort numerically first and unparseable ones trail in lexicographic order Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01PzLvcfr58G3Gwhc52zQysd --- pkg/proxy/discovery/sweep.go | 25 ++++++++++++++++--------- pkg/proxy/discovery/sweep_test.go | 22 ++++++++++++++++++++++ 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/pkg/proxy/discovery/sweep.go b/pkg/proxy/discovery/sweep.go index cd62840..752ac75 100644 --- a/pkg/proxy/discovery/sweep.go +++ b/pkg/proxy/discovery/sweep.go @@ -1,11 +1,12 @@ package discovery import ( + "cmp" "context" "fmt" "net" "net/netip" - "sort" + "slices" "strconv" "sync" "time" @@ -287,16 +288,22 @@ func sortHosts(hosts map[string]*SweepHost) []SweepHost { addr, _ := netip.ParseAddr(h.IP) items = append(items, sweepHostAddr{host: h, addr: addr}) } - sort.Slice(items, func(i, j int) bool { - iValid := items[i].addr.IsValid() - jValid := items[j].addr.IsValid() - if iValid != jValid { - return iValid + // slices.SortFunc avoids the reflection and interface boxing that sort.Slice + // pays on every comparison. + slices.SortFunc(items, func(a, b sweepHostAddr) int { + aValid := a.addr.IsValid() + bValid := b.addr.IsValid() + if aValid != bValid { + // Unparseable IPs sort to the end. + if aValid { + return -1 + } + return 1 } - if !iValid { - return items[i].host.IP < items[j].host.IP + if !aValid { + return cmp.Compare(a.host.IP, b.host.IP) } - return items[i].addr.Less(items[j].addr) + return a.addr.Compare(b.addr) }) out := make([]SweepHost, len(items)) diff --git a/pkg/proxy/discovery/sweep_test.go b/pkg/proxy/discovery/sweep_test.go index 0031ad7..2e47f2e 100644 --- a/pkg/proxy/discovery/sweep_test.go +++ b/pkg/proxy/discovery/sweep_test.go @@ -483,3 +483,25 @@ func TestSortHosts_HandlesNilHostPointers(t *testing.T) { t.Errorf("unexpected sorted IPs: %v", sorted) } } + +func TestSortHosts_UnparseableIPsSortLast(t *testing.T) { + hosts := map[string]*SweepHost{ + "10.0.0.2": {IP: "10.0.0.2"}, + "10.0.0.1": nil, + "invalid-ip-b": {IP: "invalid-ip-b"}, + "10.0.0.10": {IP: "10.0.0.10"}, + "invalid-ip-a": {IP: "invalid-ip-a"}, + } + + sorted := sortHosts(hosts) + // Valid IPs first in numeric order, then unparseable ones lexicographically. + want := []string{"10.0.0.2", "10.0.0.10", "invalid-ip-a", "invalid-ip-b"} + if len(sorted) != len(want) { + t.Fatalf("expected %d non-nil hosts, got %d", len(want), len(sorted)) + } + for i, exp := range want { + if sorted[i].IP != exp { + t.Errorf("at index %d: want %q, got %q", i, exp, sorted[i].IP) + } + } +}