diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index f1ab0ab..1eaa7ea 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -22,6 +22,14 @@ jobs: - name: run tests run: go test -json ./... > test.json + - name: run bounded-memory acceptance tests + run: | + go test -tags acceptance ./pkg/beacon/api -run Acceptance -count=1 -timeout=20m + go test -tags "acceptance acceptance_http2" ./pkg/beacon/api -run TestAcceptanceConcurrentHTTP2BeaconStateStreams -count=1 -timeout=20m + + - name: run race tests + run: go test -race ./... + - name: Annotate tests if: always() uses: guyarb/golang-test-annotations@96fc379b171c49932041d6c789e73331a7bdeec1 # v0.9.0 diff --git a/.golangci.yml b/.golangci.yml index 2898513..02d20a2 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,5 +1,9 @@ version: "2" run: + build-tags: + - acceptance + - acceptance_full_buffered + - acceptance_http2 issues-exit-code: 1 linters: default: none diff --git a/README.md b/README.md index e2407f4..c8cc8da 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,45 @@ go get github.com/ethpandaops/beacon ## Usage +### Streaming raw responses + +The `OpenRaw*` methods return after the node accepts the request and sends response headers. The caller must close the response and treat any later read error as a failed transfer. + +```go +func archiveState(ctx context.Context, node beacon.Node, dst io.Writer) error { + response, err := node.OpenRawBeaconState(ctx, "finalized", "application/octet-stream") + if err != nil { + return err + } + defer response.Close() + + mediaType, _, err := mime.ParseMediaType(response.ContentType) + if err != nil || mediaType != "application/octet-stream" { + return fmt.Errorf("unexpected content type %q", response.ContentType) + } + + written, err := io.Copy(dst, response) + if err != nil { + return err + } + if response.ContentLength >= 0 && written != response.ContentLength { + return fmt.Errorf("copied %d bytes, expected %d", written, response.ContentLength) + } + + return nil +} +``` + +The request context and configured raw API client timeout both govern body reads. The raw client settings apply to both `FetchRaw*` and `OpenRaw*` calls; set `SetAPIClientTimeout(0)` only when every call has a suitable context deadline. By default the raw client permits 32 active response bodies and 32 connections per host; `MaxConcurrentRequests` also caps multiplexed HTTP/2 streams. + +Go may transparently decompress gzip responses, reported by `RawResponse.Uncompressed`. Call `Options.SetAPIClientAcceptEncoding("gzip")` when the caller needs the encoded wire bytes; the returned headers then retain `Content-Encoding`. A `ContentLength` of `-1` means the caller cannot verify an expected byte count from that field. Slow readers keep upstream resources open, so applications should bound concurrency and finish downstream writes before committing their output. + +Maintainers can run the streaming memory gates with `go test -tags acceptance ./pkg/beacon/api -run Acceptance -count=1`. The full 50-response buffered comparison requires at least 10 GiB of available memory and is intentionally manual: + +```bash +go test -tags "acceptance acceptance_full_buffered" ./pkg/beacon/api -run TestAcceptanceFullBufferedBaseline -count=1 -timeout=20m +``` + ### Simple example ```go diff --git a/pkg/beacon/api/api.go b/pkg/beacon/api/api.go index 881478e..5594860 100644 --- a/pkg/beacon/api/api.go +++ b/pkg/beacon/api/api.go @@ -8,7 +8,6 @@ import ( "fmt" "io" "net/http" - "net/url" "github.com/ethpandaops/beacon/pkg/beacon/api/types" "github.com/sirupsen/logrus" @@ -24,27 +23,52 @@ type ConsensusClient interface { NodePeer(ctx context.Context, peerID string) (types.Peer, error) NodePeers(ctx context.Context) (types.Peers, error) NodePeerCount(ctx context.Context) (types.PeerCount, error) - RawBlock(ctx context.Context, stateID string, contentType string) ([]byte, error) + RawBlock(ctx context.Context, blockID string, contentType string) ([]byte, error) RawExecutionPayloadEnvelope(ctx context.Context, blockID string, contentType string) ([]byte, error) RawDebugBeaconState(ctx context.Context, stateID string, contentType string) ([]byte, error) + // OpenRawBlock opens a successful raw block response without consuming its body. + OpenRawBlock(ctx context.Context, blockID string, contentType string) (*RawResponse, error) + // OpenRawExecutionPayloadEnvelope opens a successful raw envelope response without consuming its body. + OpenRawExecutionPayloadEnvelope(ctx context.Context, blockID string, contentType string) (*RawResponse, error) + // OpenRawDebugBeaconState opens a successful raw beacon state response without consuming its body. + OpenRawDebugBeaconState(ctx context.Context, stateID string, contentType string) (*RawResponse, error) DepositSnapshot(ctx context.Context) (*types.DepositSnapshot, error) NodeIdentity(ctx context.Context) (*types.Identity, error) } type consensusClient struct { - url string - log logrus.FieldLogger - client http.Client - headers map[string]string + url string + log logrus.FieldLogger + client *http.Client + rawClient *http.Client + headers map[string]string + rawResponseObserver RawResponseObserver } -// NewConsensusClient creates a new ConsensusClient. -func NewConsensusClient(ctx context.Context, log logrus.FieldLogger, url string, client http.Client, headers map[string]string) ConsensusClient { +// NewConsensusClient creates a ConsensusClient. Both HTTP clients must be non-nil. +func NewConsensusClient( + log logrus.FieldLogger, + url string, + client *http.Client, + rawClient *http.Client, + headers map[string]string, + rawResponseObserver RawResponseObserver, +) ConsensusClient { + if client == nil { + panic("api: nil HTTP client") + } + + if rawClient == nil { + panic("api: nil raw HTTP client") + } + return &consensusClient{ - url: url, - log: log, - client: client, - headers: headers, + url: url, + log: log, + client: client, + rawClient: rawClient, + headers: headers, + rawResponseObserver: rawResponseObserver, } } @@ -130,43 +154,19 @@ func (c *consensusClient) get(ctx context.Context, path string) (json.RawMessage } func (c *consensusClient) getRaw(ctx context.Context, path string, contentType string) ([]byte, error) { - if contentType == "" { - contentType = "application/json" - } - - u, err := url.Parse(c.url + path) + rsp, err := c.doRaw(ctx, path, contentType) if err != nil { return nil, err } - req, err := http.NewRequestWithContext(ctx, "GET", u.String(), nil) - if err != nil { - return nil, err - } - - // Set headers from c.headers - for k, v := range c.headers { - req.Header.Set(k, v) - } - - req.Header.Set("Accept", contentType) + defer rsp.Body.Close() - rsp, err := c.client.Do(req) + data, err := io.ReadAll(rsp.Body) if err != nil { return nil, err } - defer rsp.Body.Close() - - if rsp.StatusCode != http.StatusOK { - if rsp.StatusCode == http.StatusNotFound { - return nil, fmt.Errorf("status code: %d: %w", rsp.StatusCode, ErrNotFound) - } - - return nil, fmt.Errorf("status code: %d", rsp.StatusCode) - } - - return io.ReadAll(rsp.Body) + return data, nil } // NodePeers returns the list of peers connected to the node. @@ -225,8 +225,8 @@ func (c *consensusClient) RawDebugBeaconState(ctx context.Context, stateID strin } // RawBlock returns the block in the requested format. -func (c *consensusClient) RawBlock(ctx context.Context, stateID string, contentType string) ([]byte, error) { - data, err := c.getRaw(ctx, fmt.Sprintf("/eth/v2/beacon/blocks/%s", stateID), contentType) +func (c *consensusClient) RawBlock(ctx context.Context, blockID string, contentType string) ([]byte, error) { + data, err := c.getRaw(ctx, fmt.Sprintf("/eth/v2/beacon/blocks/%s", blockID), contentType) if err != nil { return nil, err } diff --git a/pkg/beacon/api/raw.go b/pkg/beacon/api/raw.go new file mode 100644 index 0000000..57aada8 --- /dev/null +++ b/pkg/beacon/api/raw.go @@ -0,0 +1,277 @@ +package api + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "runtime" + "sync" + "time" + + "github.com/sirupsen/logrus" +) + +const ( + maxErrorBodyBytes = 8 << 10 + errorBodyReadTimeout = 250 * time.Millisecond + defaultRawContentType = "application/json" +) + +// RawResponseObserver receives raw response body lifecycle events. +type RawResponseObserver interface { + RawResponseOpened() + RawResponseClosed() + RawResponseLeaked() +} + +// HTTPStatusError describes a non-200 response from a raw endpoint. +type HTTPStatusError struct { + StatusCode int + RequestPath string + RetryAfter string + BodyPrefix string +} + +func (e *HTTPStatusError) Error() string { + if e.BodyPrefix == "" { + return fmt.Sprintf("status code: %d for %s", e.StatusCode, e.RequestPath) + } + + return fmt.Sprintf("status code: %d for %s: %q", e.StatusCode, e.RequestPath, e.BodyPrefix) +} + +// Unwrap preserves ErrNotFound classification for HTTP 404 responses. +func (e *HTTPStatusError) Unwrap() error { + if e.StatusCode == http.StatusNotFound { + return ErrNotFound + } + + return nil +} + +// RawResponse streams a successful raw endpoint response. Opening the response +// only validates its status; callers must treat any read error as fatal and +// avoid committing partial output. The caller must close the response, +// including when a read fails or the body is only partially read. +type RawResponse struct { + // Body is the managed response stream. + Body io.ReadCloser + // ContentType is the response's actual Content-Type. + ContentType string + // ContentLength is -1 when the response length is unknown, so length-based + // completeness cannot be verified. + ContentLength int64 + // Uncompressed reports whether net/http transparently decompressed the body. + // Use Options.SetAPIClientAcceptEncoding to receive an encoded body without + // automatic decompression. + Uncompressed bool + // Header is a clone of the response headers. + Header http.Header +} + +// Read reads from the response body. +func (r *RawResponse) Read(p []byte) (int, error) { + return r.Body.Read(p) +} + +// Close releases the response body. Repeated calls return the first close +// result. +func (r *RawResponse) Close() error { + return r.Body.Close() +} + +type rawResponseCloseState struct { + once sync.Once + body io.ReadCloser + log logrus.FieldLogger + observer RawResponseObserver + path string + closeErr error +} + +type managedRawBody struct { + state *rawResponseCloseState + cleanup runtime.Cleanup +} + +func (b *managedRawBody) Read(p []byte) (int, error) { + n, err := b.state.body.Read(p) + runtime.KeepAlive(b) + + return n, err +} + +func (b *managedRawBody) Close() error { + b.cleanup.Stop() + err := b.state.close(false) + runtime.KeepAlive(b) + + return err +} + +func (s *rawResponseCloseState) close(leaked bool) error { + s.once.Do(func() { + s.closeErr = s.body.Close() + + if leaked { + if s.log != nil { + entry := s.log.WithField("request_path", s.path) + if s.closeErr != nil { + entry.WithError(s.closeErr).Error("raw response body was not closed") + } else { + entry.Error("raw response body was not closed") + } + } + + if s.observer != nil { + s.observer.RawResponseLeaked() + } + } + + if s.observer != nil { + s.observer.RawResponseClosed() + } + }) + + return s.closeErr +} + +func cleanupLeakedRawResponse(state *rawResponseCloseState) { + _ = state.close(true) +} + +func (c *consensusClient) doRaw(ctx context.Context, path string, contentType string) (*http.Response, error) { + if contentType == "" { + contentType = defaultRawContentType + } + + u, err := url.Parse(c.url + path) + if err != nil { + return nil, err + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, err + } + + for k, v := range c.headers { + req.Header.Set(k, v) + } + + req.Header.Set("Accept", contentType) + + rsp, err := c.rawClient.Do(req) + if err != nil { + return nil, err + } + + if rsp.StatusCode == http.StatusOK { + return rsp, nil + } + + bodyPrefix := readAndCloseErrorBody(rsp.Body) + + return nil, &HTTPStatusError{ + StatusCode: rsp.StatusCode, + RequestPath: path, + RetryAfter: rsp.Header.Get("Retry-After"), + BodyPrefix: bodyPrefix, + } +} + +func readAndCloseErrorBody(body io.ReadCloser) string { + timerDone := make(chan struct{}) + + var closeOnce sync.Once + + closeBody := func() { + closeOnce.Do(func() { + _ = body.Close() + }) + } + timer := time.AfterFunc(errorBodyReadTimeout, func() { + closeBody() + + close(timerDone) + }) + + data, _ := io.ReadAll(io.LimitReader(body, maxErrorBodyBytes+1)) + if len(data) > maxErrorBodyBytes { + data = data[:maxErrorBodyBytes] + } + + if timer.Stop() { + close(timerDone) + } + + <-timerDone + + closeBody() + + return string(data) +} + +func (c *consensusClient) openRaw(ctx context.Context, path string, contentType string) (*RawResponse, error) { + rsp, err := c.doRaw(ctx, path, contentType) //nolint:bodyclose // successful responses transfer body ownership + if err != nil { + return nil, err + } + + logger := c.log + if logger == nil { + logger = logrus.StandardLogger() + } + + state := &rawResponseCloseState{ + body: rsp.Body, + log: logger, + observer: c.rawResponseObserver, + path: path, + } + body := &managedRawBody{state: state} + body.cleanup = runtime.AddCleanup(body, cleanupLeakedRawResponse, state) + + transferred := false + defer func() { + if !transferred { + _ = body.Close() + } + }() + + response := &RawResponse{ + Body: body, + ContentType: rsp.Header.Get("Content-Type"), + ContentLength: rsp.ContentLength, + Uncompressed: rsp.Uncompressed, + Header: rsp.Header.Clone(), + } + + if c.rawResponseObserver != nil { + c.rawResponseObserver.RawResponseOpened() + } + + transferred = true + + runtime.KeepAlive(body) + + return response, nil +} + +// OpenRawDebugBeaconState opens the beacon state response for streaming. +func (c *consensusClient) OpenRawDebugBeaconState(ctx context.Context, stateID string, contentType string) (*RawResponse, error) { + return c.openRaw(ctx, fmt.Sprintf("/eth/v2/debug/beacon/states/%s", stateID), contentType) +} + +// OpenRawBlock opens the block response for streaming. +func (c *consensusClient) OpenRawBlock(ctx context.Context, blockID string, contentType string) (*RawResponse, error) { + return c.openRaw(ctx, fmt.Sprintf("/eth/v2/beacon/blocks/%s", blockID), contentType) +} + +// OpenRawExecutionPayloadEnvelope opens the signed execution payload envelope +// response for streaming (gloas onwards). +func (c *consensusClient) OpenRawExecutionPayloadEnvelope(ctx context.Context, blockID string, contentType string) (*RawResponse, error) { + return c.openRaw(ctx, fmt.Sprintf("/eth/v1/beacon/execution_payload_envelopes/%s", blockID), contentType) +} diff --git a/pkg/beacon/api/raw_acceptance_full_buffered_test.go b/pkg/beacon/api/raw_acceptance_full_buffered_test.go new file mode 100644 index 0000000..ef62caa --- /dev/null +++ b/pkg/beacon/api/raw_acceptance_full_buffered_test.go @@ -0,0 +1,31 @@ +//go:build acceptance && acceptance_full_buffered && linux + +package api + +import ( + "context" + "testing" +) + +// Run this manual baseline on a host with at least 10 GiB available: +// +// go test -tags "acceptance acceptance_full_buffered" ./pkg/beacon/api -run TestAcceptanceFullBufferedBaseline -count=1 -timeout=20m +// +// It is excluded from CI because all fifty payloads remain live simultaneously. +func TestAcceptanceFullBufferedBaseline(t *testing.T) { + pinGCSettings(t) + + ctx, cancel := context.WithTimeout(t.Context(), acceptanceTimeout) + defer cancel() + + report := measureBuffered(ctx, t, streamConcurrency, beaconStatePayloadBytes) + report.log(t, "50 concurrent buffered beacon states") + + liveFloor := uint64(beaconStatePayloadBytes) * streamConcurrency * 9 / 10 + if report.peakDelta() < liveFloor { + t.Errorf( + "buffered peak heap delta %s is below live payload floor %s", + mib(report.peakDelta()), mib(liveFloor), + ) + } +} diff --git a/pkg/beacon/api/raw_acceptance_http2_test.go b/pkg/beacon/api/raw_acceptance_http2_test.go new file mode 100644 index 0000000..9279469 --- /dev/null +++ b/pkg/beacon/api/raw_acceptance_http2_test.go @@ -0,0 +1,80 @@ +//go:build acceptance && acceptance_http2 && linux + +package api + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strconv" + "sync/atomic" + "testing" + + "github.com/sirupsen/logrus" +) + +func TestAcceptanceConcurrentHTTP2BeaconStateStreams(t *testing.T) { + const ( + transportHeapBudget uint64 = 128 << 20 + transportRSSBudget uint64 = 128 << 20 + ) + + pinGCSettings(t) + + ctx, cancel := context.WithTimeout(t.Context(), acceptanceTimeout) + defer cancel() + + var requests atomic.Int64 + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.ProtoMajor != 2 { + t.Errorf("protocol = %s", req.Proto) + } + if req.URL.Path != beaconStatePath { + http.NotFound(w, req) + + return + } + if got := req.Header.Get("Accept"); got != rawContentType { + t.Errorf("Accept = %q", got) + } + + requests.Add(1) + w.Header().Set("Content-Type", rawContentType) + w.Header().Set("Content-Length", strconv.FormatInt(beaconStatePayloadBytes, 10)) + w.Header().Set("Eth-Consensus-Version", "gloas") + w.WriteHeader(http.StatusOK) + if err := http.NewResponseController(w).Flush(); err != nil { + t.Errorf("flush response headers: %v", err) + + return + } + + _, _ = io.Copy(w, &generatedBody{remaining: beaconStatePayloadBytes}) + })) + server.EnableHTTP2 = true + server.StartTLS() + t.Cleanup(server.Close) + + httpClient := server.Client() + httpClient.Timeout = 0 + t.Cleanup(httpClient.CloseIdleConnections) + + observer := &countingObserver{} + log := logrus.New() + log.SetOutput(io.Discard) + client := NewConsensusClient(log, server.URL, httpClient, httpClient, nil, observer) + + report := measureConcurrentStreams(ctx, t, client, observer, streamConcurrency, beaconStatePayloadBytes) + report.log(t, "50 concurrent HTTP/2 beacon state streams") + + if got := requests.Load(); got != int64(streamConcurrency) { + t.Errorf("requests = %d, want %d", got, streamConcurrency) + } + if report.peakDelta() > transportHeapBudget { + t.Errorf("peak heap delta %s exceeds budget %s", mib(report.peakDelta()), mib(transportHeapBudget)) + } + if report.peakRSSDelta() > transportRSSBudget { + t.Errorf("peak RSS delta %s exceeds budget %s", mib(report.peakRSSDelta()), mib(transportRSSBudget)) + } +} diff --git a/pkg/beacon/api/raw_acceptance_test.go b/pkg/beacon/api/raw_acceptance_test.go new file mode 100644 index 0000000..686844c --- /dev/null +++ b/pkg/beacon/api/raw_acceptance_test.go @@ -0,0 +1,934 @@ +//go:build acceptance && linux + +package api + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "math" + "net/http" + "os" + "runtime" + "runtime/debug" + "strconv" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/sirupsen/logrus" +) + +// Run the bounded-memory checks with: +// +// go test -tags acceptance ./pkg/beacon/api -run Acceptance -count=1 +// +// The generated transport produces payloads without allocating. Reads use an +// explicit loop so io.Copy fast paths cannot bypass the path under test. +// Workers rendezvous before and midway through reads for deterministic samples. +// GOGC is fixed and GOMEMLIMIT is disabled for the duration of each test. +const ( + beaconStatePayloadBytes int64 = 73_216_552 + streamConcurrency = 50 + copyBufferSize = 32 << 10 + peakHeapBudget uint64 = 64 << 20 + peakRSSBudget uint64 = 64 << 20 + + comparisonConcurrency = 8 + comparisonPayloadBytes int64 = 8 << 20 + comparisonStreamBudget uint64 = 16 << 20 + + acceptanceGCPercent = 100 + heapSampleInterval = 5 * time.Millisecond + acceptanceTimeout = 10 * time.Minute + + rawContentType = "application/octet-stream" + beaconStateID = "head" + beaconStatePath = "/eth/v2/debug/beacon/states/" + beaconStateID +) + +// TestAcceptanceConcurrentBeaconStateStreamsAreBoundedMemory streams 50 +// concurrent 73,216,552-byte beacon states, records peak heap and RSS, and +// asserts that both stay bounded and every byte arrives intact. +func TestAcceptanceConcurrentBeaconStateStreamsAreBoundedMemory(t *testing.T) { + pinGCSettings(t) + + ctx, cancel := context.WithTimeout(context.Background(), acceptanceTimeout) + defer cancel() + + client, observer, transport := newAcceptanceClient(beaconStatePayloadBytes) + report := measureConcurrentStreams(ctx, t, client, observer, streamConcurrency, beaconStatePayloadBytes) + report.log(t, "50 concurrent streamed beacon states") + + if got, want := transport.requests.Load(), int64(streamConcurrency); got != want { + t.Errorf("transport served %d requests, want %d", got, want) + } + + if report.peakDelta() > peakHeapBudget { + t.Errorf( + "peak heap delta %s exceeds budget %s; a payload-sized buffer is being held somewhere", + mib(report.peakDelta()), mib(peakHeapBudget), + ) + } + + if report.peakRSSDelta() > peakRSSBudget { + t.Errorf( + "peak RSS delta %s exceeds budget %s; resident memory is not bounded", + mib(report.peakRSSDelta()), mib(peakRSSBudget), + ) + } + + // A run that streams 3.4 GiB while allocating less than one beacon state + // cannot be buffering payloads anywhere along the path. + if report.totalAlloc > uint64(beaconStatePayloadBytes) { + t.Errorf( + "total allocated %s exceeds a single payload %s; the path is allocating per-payload", + mib(report.totalAlloc), mib(uint64(beaconStatePayloadBytes)), + ) + } +} + +// TestAcceptanceStreamedVersusBufferedPeakHeap contrasts the streaming and +// buffered paths at capped, reliably measurable parameters. The buffered arm's +// floor is a liveness guarantee rather than an estimate: every worker holds its +// []byte at the gate, so the bytes are unambiguously resident when sampled. +func TestAcceptanceStreamedVersusBufferedPeakHeap(t *testing.T) { + pinGCSettings(t) + + ctx, cancel := context.WithTimeout(context.Background(), acceptanceTimeout) + defer cancel() + + streamed := measureStreamedArm(ctx, t) + streamed.log(t, "streamed arm (capped comparison)") + + buffered := measureBufferedArm(ctx, t) + buffered.log(t, "buffered arm (capped comparison)") + + liveFloor := uint64(comparisonPayloadBytes) * comparisonConcurrency * 9 / 10 + + if buffered.peakDelta() < liveFloor { + t.Errorf( + "buffered arm peak delta %s is below the %s that %d live payloads require; the arm did not measure what it claims", + mib(buffered.peakDelta()), mib(liveFloor), comparisonConcurrency, + ) + } + + if streamed.peakDelta() > comparisonStreamBudget { + t.Errorf("streamed arm peak delta %s exceeds budget %s", mib(streamed.peakDelta()), mib(comparisonStreamBudget)) + } + + if streamed.peakDelta()*4 > buffered.peakDelta() { + t.Errorf( + "streamed peak delta %s is not decisively below buffered peak delta %s", + mib(streamed.peakDelta()), mib(buffered.peakDelta()), + ) + } +} + +func measureStreamedArm(ctx context.Context, t *testing.T) memReport { + t.Helper() + + client, observer, _ := newAcceptanceClient(comparisonPayloadBytes) + + return measureConcurrentStreams(ctx, t, client, observer, comparisonConcurrency, comparisonPayloadBytes) +} + +func measureConcurrentStreams( + ctx context.Context, + t *testing.T, + client ConsensusClient, + observer *countingObserver, + concurrency int, + payloadBytes int64, +) memReport { + t.Helper() + + openGate := newGate(concurrency) + midGate := newGate(concurrency) + results := make([]streamResult, concurrency) + + var wg sync.WaitGroup + + baseline := settleAndReadMemStats() + baselineRSS := mustReadProcessRSS(t) + sampler := startMemorySampler(heapSampleInterval) + started := time.Now() + + for i := range concurrency { + wg.Add(1) + + go func(idx int) { + defer wg.Done() + + results[idx] = streamOneState(ctx, client, payloadBytes, openGate, midGate) + }(i) + } + + if err := openGate.awaitReached(ctx); err != nil { + t.Fatalf("waiting for streams to open: %v", err) + } + + atOpen := sampler.sample() + + openGate.release() + + if err := midGate.awaitReached(ctx); err != nil { + t.Fatalf("waiting for streams to reach mid-body: %v", err) + } + + atMidpoint := sampler.sample() + + midGate.release() + + wg.Wait() + + elapsed := time.Since(started) + peak, err := sampler.stopAndWait() + if err != nil { + t.Fatalf("sample process memory: %v", err) + } + final := readMemStats() + + var totalBytes int64 + + for i, result := range results { + if result.err != nil { + t.Errorf("stream %d: %v", i, result.err) + } + + totalBytes += result.copied + } + + if want := payloadBytes * int64(concurrency); totalBytes != want { + t.Errorf("copied %d bytes, want %d", totalBytes, want) + } + + observer.assert(t, int64(concurrency)) + + return memReport{ + streams: concurrency, + payloadBytes: payloadBytes, + totalBytes: totalBytes, + duration: elapsed, + baselineHeap: baseline.HeapAlloc, + peakHeap: peak.heap, + atOpen: atOpen.heap, + atMidpoint: atMidpoint.heap, + baselineRSS: baselineRSS, + peakRSS: peak.rss, + atOpenRSS: atOpen.rss, + atMidpointRSS: atMidpoint.rss, + totalAlloc: final.TotalAlloc - baseline.TotalAlloc, + mallocs: final.Mallocs - baseline.Mallocs, + numGC: final.NumGC - baseline.NumGC, + pauseTotal: time.Duration(final.PauseTotalNs - baseline.PauseTotalNs), + } +} + +func measureBufferedArm(ctx context.Context, t *testing.T) memReport { + t.Helper() + + return measureBuffered(ctx, t, comparisonConcurrency, comparisonPayloadBytes) +} + +func measureBuffered(ctx context.Context, t *testing.T, concurrency int, payloadBytes int64) memReport { + t.Helper() + + client, _, _ := newAcceptanceClient(payloadBytes) + + liveGate := newGate(concurrency) + results := make([]streamResult, concurrency) + held := make([][]byte, concurrency) + + var wg sync.WaitGroup + + baseline := settleAndReadMemStats() + baselineRSS := mustReadProcessRSS(t) + sampler := startMemorySampler(heapSampleInterval) + started := time.Now() + + for i := range concurrency { + wg.Add(1) + + go func(idx int) { + defer wg.Done() + + results[idx] = bufferOneState(ctx, client, payloadBytes, &held[idx], liveGate) + }(i) + } + + if err := liveGate.awaitReached(ctx); err != nil { + t.Fatalf("buffered arm: waiting for buffers to become live: %v", err) + } + + atLive := sampler.sample() + + liveGate.release() + + wg.Wait() + + elapsed := time.Since(started) + peak, err := sampler.stopAndWait() + if err != nil { + t.Fatalf("buffered arm: sample process memory: %v", err) + } + final := readMemStats() + + runtime.KeepAlive(held) + + var totalBytes int64 + + for i, res := range results { + if res.err != nil { + t.Errorf("buffered arm: fetch %d: %v", i, res.err) + } + + totalBytes += res.copied + } + + if want := payloadBytes * int64(concurrency); totalBytes != want { + t.Errorf("buffered arm: read %d bytes, want %d", totalBytes, want) + } + + return memReport{ + streams: concurrency, + payloadBytes: payloadBytes, + totalBytes: totalBytes, + duration: elapsed, + baselineHeap: baseline.HeapAlloc, + peakHeap: peak.heap, + atOpen: atLive.heap, + atMidpoint: atLive.heap, + baselineRSS: baselineRSS, + peakRSS: peak.rss, + atOpenRSS: atLive.rss, + atMidpointRSS: atLive.rss, + totalAlloc: final.TotalAlloc - baseline.TotalAlloc, + mallocs: final.Mallocs - baseline.Mallocs, + numGC: final.NumGC - baseline.NumGC, + pauseTotal: time.Duration(final.PauseTotalNs - baseline.PauseTotalNs), + } +} + +type streamResult struct { + copied int64 + err error +} + +// streamOneState opens a beacon state, rendezvouses at openGate before reading +// a byte, rendezvouses again at midGate half way through the body, and verifies +// every byte against the generator as it goes. +func streamOneState( + ctx context.Context, + client ConsensusClient, + size int64, + openGate, midGate *gate, +) (res streamResult) { + // Any early return must release both gates, or the peers block forever. + // Releasing early skews the sample, but only on a path that already fails + // the test. + defer openGate.abort() + defer midGate.abort() + + rsp, err := client.OpenRawDebugBeaconState(ctx, beaconStateID, rawContentType) + if err != nil { + res.err = fmt.Errorf("open: %w", err) + + return res + } + + defer func() { + if cerr := rsp.Close(); cerr != nil && res.err == nil { + res.err = fmt.Errorf("close: %w", cerr) + } + }() + + if rsp.ContentType != rawContentType { + res.err = fmt.Errorf("content type %q, want %q", rsp.ContentType, rawContentType) + + return res + } + + if rsp.ContentLength != size { + res.err = fmt.Errorf("content length %d, want %d", rsp.ContentLength, size) + + return res + } + + if err := openGate.arrive(ctx); err != nil { + res.err = fmt.Errorf("open gate: %w", err) + + return res + } + + var ( + buf = make([]byte, copyBufferSize) + sink = newVerifyingSink() + midpoint = size / 2 + atMid bool + ) + + for { + // Explicit Read loop: io.Copy could take a WriterTo or ReaderFrom fast + // path and stop measuring the path under test. + n, readErr := rsp.Read(buf) + if n > 0 { + if _, werr := sink.Write(buf[:n]); werr != nil { + res.err = werr + res.copied = sink.total + + return res + } + + if !atMid && sink.total >= midpoint { + atMid = true + + if err := midGate.arrive(ctx); err != nil { + res.err = fmt.Errorf("mid gate: %w", err) + res.copied = sink.total + + return res + } + } + } + + if errors.Is(readErr, io.EOF) { + break + } + + if readErr != nil { + res.err = fmt.Errorf("read at offset %d: %w", sink.total, readErr) + res.copied = sink.total + + return res + } + } + + if !atMid { + res.err = fmt.Errorf("stream ended at %d bytes without reaching the midpoint %d", sink.total, midpoint) + + return res + } + + if sink.total != size { + res.err = fmt.Errorf("streamed %d bytes, want %d", sink.total, size) + } + + res.copied = sink.total + + return res +} + +// bufferOneState fetches through the []byte path and holds the result live at +// the gate, so the comparison arm samples a heap that provably contains every +// payload at once. +func bufferOneState( + ctx context.Context, + client ConsensusClient, + size int64, + hold *[]byte, + liveGate *gate, +) (res streamResult) { + defer liveGate.abort() + + data, err := client.RawDebugBeaconState(ctx, beaconStateID, rawContentType) + if err != nil { + res.err = fmt.Errorf("fetch: %w", err) + + return res + } + + *hold = data + res.copied = int64(len(data)) + + if int64(len(data)) != size { + res.err = fmt.Errorf("buffered %d bytes, want %d", len(data), size) + + return res + } + + // Verify in fixed chunks so the verifier's own scratch stays at + // copyBufferSize; handing it the whole payload would grow a second + // payload-sized buffer and inflate the very number this arm reports. + sink := newVerifyingSink() + + for off := 0; off < len(data); off += copyBufferSize { + if _, err := sink.Write(data[off:min(off+copyBufferSize, len(data))]); err != nil { + res.err = err + + return res + } + } + + if err := liveGate.arrive(ctx); err != nil { + res.err = fmt.Errorf("live gate: %w", err) + } + + runtime.KeepAlive(data) + + return res +} + +func newAcceptanceClient(size int64) (ConsensusClient, *countingObserver, *generatedTransport) { + transport := &generatedTransport{size: size} + observer := &countingObserver{} + + log := logrus.New() + log.SetOutput(io.Discard) + log.SetLevel(logrus.ErrorLevel) + + // Timeout is left at zero deliberately: a non-zero Client.Timeout wraps the + // body in a cancellation shim, and this measurement should see the + // library's own plumbing and nothing else. + rawClient := &http.Client{Transport: transport} + + client := NewConsensusClient(log, "http://acceptance.invalid", rawClient, rawClient, nil, observer) + + return client, observer, transport +} + +// generatedTransport answers every beacon state request with a synthetic body +// of the configured size. No server, no socket, no buffer. +type generatedTransport struct { + size int64 + requests atomic.Int64 +} + +func (t *generatedTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if req.URL.Path != beaconStatePath { + return nil, fmt.Errorf("unexpected request path %q", req.URL.Path) + } + + if got := req.Header.Get("Accept"); got != rawContentType { + return nil, fmt.Errorf("unexpected Accept header %q", got) + } + + t.requests.Add(1) + + header := make(http.Header, 2) + header.Set("Content-Type", rawContentType) + header.Set("Eth-Consensus-Version", "gloas") + + return &http.Response{ + Status: "200 OK", + StatusCode: http.StatusOK, + Proto: "HTTP/2.0", + ProtoMajor: 2, + ProtoMinor: 0, + Header: header, + Body: &generatedBody{remaining: t.size}, + ContentLength: t.size, + Request: req, + }, nil +} + +var errBodyClosed = errors.New("read on closed generated body") + +// patternBlockSize is the size of the shared, read-only block every generated +// body copies from. One block for the whole process keeps body reads free of +// allocation. +const patternBlockSize = 64 << 10 + +var patternBlock = newPatternBlock() + +func newPatternBlock() []byte { + b := make([]byte, patternBlockSize) + + x := uint32(0x9E3779B9) + for i := range b { + x ^= x << 13 + x ^= x >> 17 + x ^= x << 5 + b[i] = byte(x) + } + + return b +} + +// fillFromPattern writes the bytes the generator owes at the given absolute +// offset. It allocates nothing and is a pure function of the offset, so the +// reader and the verifier agree without sharing state. +func fillFromPattern(p []byte, offset int64) { + for n := 0; n < len(p); { + start := int((offset + int64(n)) % patternBlockSize) + n += copy(p[n:], patternBlock[start:]) + } +} + +// generatedBody is an allocation-free io.ReadCloser. Each body is read by +// exactly one goroutine, so it needs no synchronisation of its own. +type generatedBody struct { + offset int64 + remaining int64 + closed bool +} + +func (b *generatedBody) Read(p []byte) (int, error) { + if b.closed { + return 0, errBodyClosed + } + + if b.remaining == 0 { + return 0, io.EOF + } + + if int64(len(p)) > b.remaining { + p = p[:b.remaining] + } + + fillFromPattern(p, b.offset) + + b.offset += int64(len(p)) + b.remaining -= int64(len(p)) + + return len(p), nil +} + +func (b *generatedBody) Close() error { + b.closed = true + + return nil +} + +// verifyingSink checks each chunk against the generator at its absolute offset, +// which catches dropped, duplicated or reordered bytes that a plain length +// check would miss. It implements only Write, so no io.ReaderFrom fast path can +// bypass it. +type verifyingSink struct { + total int64 + scratch []byte +} + +func newVerifyingSink() *verifyingSink { + return &verifyingSink{scratch: make([]byte, copyBufferSize)} +} + +func (s *verifyingSink) Write(p []byte) (int, error) { + if len(p) > len(s.scratch) { + s.scratch = make([]byte, len(p)) + } + + want := s.scratch[:len(p)] + fillFromPattern(want, s.total) + + if !bytes.Equal(p, want) { + return 0, fmt.Errorf("payload mismatch in the %d bytes at offset %d", len(p), s.total) + } + + s.total += int64(len(p)) + + return len(p), nil +} + +// countingObserver records the RawResponse lifecycle so the run can assert that +// ownership was transferred and returned exactly once per stream. +type countingObserver struct { + opened atomic.Int64 + closed atomic.Int64 + leaked atomic.Int64 +} + +func (o *countingObserver) RawResponseOpened() { o.opened.Add(1) } +func (o *countingObserver) RawResponseClosed() { o.closed.Add(1) } +func (o *countingObserver) RawResponseLeaked() { o.leaked.Add(1) } + +func (o *countingObserver) assert(t *testing.T, want int64) { + t.Helper() + + if got := o.opened.Load(); got != want { + t.Errorf("observer saw %d opens, want %d", got, want) + } + + if got := o.closed.Load(); got != want { + t.Errorf("observer saw %d closes, want %d", got, want) + } + + if got := o.leaked.Load(); got != 0 { + t.Errorf("observer saw %d leaked bodies, want 0", got) + } +} + +// gate is a single-use rendezvous whose release is driven by the test rather +// than by the last arriving worker, so the heap can be sampled at the instant +// every participant is known to be in position. +type gate struct { + target int + + mu sync.Mutex + arrived int + + reachedOnce sync.Once + reached chan struct{} + + proceedOnce sync.Once + proceed chan struct{} +} + +func newGate(target int) *gate { + return &gate{ + target: target, + reached: make(chan struct{}), + proceed: make(chan struct{}), + } +} + +// arrive reports the caller as in position and blocks until the test releases +// the gate or ctx is done. +func (g *gate) arrive(ctx context.Context) error { + g.mu.Lock() + g.arrived++ + full := g.arrived >= g.target + g.mu.Unlock() + + if full { + g.markReached() + } + + select { + case <-g.proceed: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// awaitReached blocks until every participant has arrived. +func (g *gate) awaitReached(ctx context.Context) error { + select { + case <-g.reached: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (g *gate) release() { + g.proceedOnce.Do(func() { close(g.proceed) }) +} + +// abort unblocks everyone regardless of how many arrived. It exists so a failing +// worker cannot deadlock its peers; the failure is reported through the worker's +// own result. +func (g *gate) abort() { + g.markReached() + g.release() +} + +func (g *gate) markReached() { + g.reachedOnce.Do(func() { close(g.reached) }) +} + +type memoryReading struct { + heap uint64 + rss uint64 +} + +// memorySampler tracks heap-object bytes and resident set size across a run. +type memorySampler struct { + mu sync.Mutex + ms runtime.MemStats + peakHeap atomic.Uint64 + peakRSS atomic.Uint64 + err error + + stop chan struct{} + done chan struct{} +} + +func startMemorySampler(interval time.Duration) *memorySampler { + s := &memorySampler{ + stop: make(chan struct{}), + done: make(chan struct{}), + } + + s.sample() + + go func() { + defer close(s.done) + + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + s.sample() + case <-s.stop: + return + } + } + }() + + return s +} + +// sample records heap-object bytes and resident set size. +func (s *memorySampler) sample() memoryReading { + s.mu.Lock() + runtime.ReadMemStats(&s.ms) + heap := s.ms.HeapAlloc + rss, err := readProcessRSS() + if err != nil && s.err == nil { + s.err = err + } + s.mu.Unlock() + + recordPeak(&s.peakHeap, heap) + recordPeak(&s.peakRSS, rss) + + return memoryReading{heap: heap, rss: rss} +} + +func recordPeak(peak *atomic.Uint64, value uint64) { + for { + current := peak.Load() + if value <= current || peak.CompareAndSwap(current, value) { + return + } + } +} + +func (s *memorySampler) stopAndWait() (memoryReading, error) { + close(s.stop) + <-s.done + + s.sample() + + s.mu.Lock() + err := s.err + s.mu.Unlock() + + return memoryReading{ + heap: s.peakHeap.Load(), + rss: s.peakRSS.Load(), + }, err +} + +func mustReadProcessRSS(t *testing.T) uint64 { + t.Helper() + + rss, err := readProcessRSS() + if err != nil { + t.Fatalf("read process RSS: %v", err) + } + + return rss +} + +func readProcessRSS() (uint64, error) { + data, err := os.ReadFile("/proc/self/statm") + if err != nil { + return 0, err + } + + fields := bytes.Fields(data) + if len(fields) < 2 { + return 0, fmt.Errorf("unexpected /proc/self/statm value %q", data) + } + + residentPages, err := strconv.ParseUint(string(fields[1]), 10, 64) + if err != nil { + return 0, fmt.Errorf("parse resident pages: %w", err) + } + + return residentPages * uint64(os.Getpagesize()), nil +} + +// settleAndReadMemStats collects twice before reading, so cleanups queued by +// previous work are done and the baseline is live heap rather than residue. +func settleAndReadMemStats() runtime.MemStats { + runtime.GC() + runtime.GC() + + return readMemStats() +} + +func readMemStats() runtime.MemStats { + var ms runtime.MemStats + + runtime.ReadMemStats(&ms) + + return ms +} + +// pinGCSettings fixes GOGC and GOMEMLIMIT for the test and restores whatever +// the process had before. GOMEMLIMIT is pinned off: a limit would push the GC +// to hold the heap down and would flatter the measurement. +func pinGCSettings(t *testing.T) { + t.Helper() + + prevGCPercent := debug.SetGCPercent(acceptanceGCPercent) + prevMemoryLimit := debug.SetMemoryLimit(-1) + + debug.SetMemoryLimit(math.MaxInt64) + + t.Cleanup(func() { + debug.SetGCPercent(prevGCPercent) + debug.SetMemoryLimit(prevMemoryLimit) + }) + + t.Logf("GOGC pinned to %d, GOMEMLIMIT pinned off (was %d bytes), GOMAXPROCS=%d", + acceptanceGCPercent, prevMemoryLimit, runtime.GOMAXPROCS(0)) +} + +type memReport struct { + streams int + payloadBytes int64 + totalBytes int64 + duration time.Duration + baselineHeap uint64 + peakHeap uint64 + atOpen uint64 + atMidpoint uint64 + baselineRSS uint64 + peakRSS uint64 + atOpenRSS uint64 + atMidpointRSS uint64 + totalAlloc uint64 + mallocs uint64 + numGC uint32 + pauseTotal time.Duration +} + +func (r memReport) peakDelta() uint64 { + if r.peakHeap < r.baselineHeap { + return 0 + } + + return r.peakHeap - r.baselineHeap +} + +func (r memReport) peakRSSDelta() uint64 { + if r.peakRSS < r.baselineRSS { + return 0 + } + + return r.peakRSS - r.baselineRSS +} + +func (r memReport) log(t *testing.T, name string) { + t.Helper() + + throughput := float64(r.totalBytes) / (1 << 20) / r.duration.Seconds() + + t.Logf("%s: %d x %d bytes", name, r.streams, r.payloadBytes) + t.Logf(" total bytes copied %d (%s)", r.totalBytes, mib(uint64(r.totalBytes))) + t.Logf(" duration %s (%.0f MiB/s)", r.duration.Round(time.Millisecond), throughput) + t.Logf(" heap baseline %s", mib(r.baselineHeap)) + t.Logf(" heap all-open %s", mib(r.atOpen)) + t.Logf(" heap mid-stream %s", mib(r.atMidpoint)) + t.Logf(" heap peak %s (delta %s)", mib(r.peakHeap), mib(r.peakDelta())) + t.Logf(" RSS baseline %s", mib(r.baselineRSS)) + t.Logf(" RSS all-open %s", mib(r.atOpenRSS)) + t.Logf(" RSS mid-stream %s", mib(r.atMidpointRSS)) + t.Logf(" RSS peak %s (delta %s)", mib(r.peakRSS), mib(r.peakRSSDelta())) + t.Logf(" total allocated %s in %d allocations", mib(r.totalAlloc), r.mallocs) + t.Logf(" GC cycles %d (%s total pause)", r.numGC, r.pauseTotal.Round(time.Microsecond)) +} + +func mib(b uint64) string { + return fmt.Sprintf("%.2f MiB", float64(b)/(1<<20)) +} diff --git a/pkg/beacon/api/raw_acceptance_unsupported_test.go b/pkg/beacon/api/raw_acceptance_unsupported_test.go new file mode 100644 index 0000000..3912d9f --- /dev/null +++ b/pkg/beacon/api/raw_acceptance_unsupported_test.go @@ -0,0 +1,9 @@ +//go:build acceptance && !linux + +package api + +import "testing" + +func TestAcceptanceRequiresLinux(t *testing.T) { + t.Skip("acceptance RSS measurement requires Linux") +} diff --git a/pkg/beacon/api/raw_http_test.go b/pkg/beacon/api/raw_http_test.go new file mode 100644 index 0000000..01bd969 --- /dev/null +++ b/pkg/beacon/api/raw_http_test.go @@ -0,0 +1,1064 @@ +package api + +import ( + "bufio" + "bytes" + "compress/gzip" + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "net/http/httptrace" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" +) + +const ( + testHTTP1Protocol = "HTTP/1.1" + testHTTP2Name = "HTTP/2" + testHTTP2Protocol = "HTTP/2.0" + testBodyABC = "abc" + testGzipEncoding = "gzip" +) + +type protocolServer struct { + url string + client *http.Client +} + +func newProtocolServer(t *testing.T, http2 bool, handler http.Handler) protocolServer { + t.Helper() + + server := httptest.NewUnstartedServer(handler) + server.EnableHTTP2 = http2 + if http2 { + server.StartTLS() + } else { + server.Start() + } + t.Cleanup(server.Close) + + client := server.Client() + client.Timeout = 0 + t.Cleanup(client.CloseIdleConnections) + + return protocolServer{ + url: server.URL, + client: client, + } +} + +func TestOpenRawReturnsAfterHeadersAndContextCancelsRead(t *testing.T) { + for _, http2 := range []bool{false, true} { + name := testHTTP1Protocol + if http2 { + name = testHTTP2Name + } + + t.Run(name, func(t *testing.T) { + headersSent := make(chan struct{}) + handlerDone := make(chan struct{}) + server := newProtocolServer(t, http2, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + defer close(handlerDone) + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("X-Protocol", req.Proto) + w.WriteHeader(http.StatusOK) + if err := http.NewResponseController(w).Flush(); err != nil { + t.Errorf("flush response headers: %v", err) + + return + } + close(headersSent) + <-req.Context().Done() + })) + client := &consensusClient{ + url: server.url, + client: server.client, + rawClient: server.client, + } + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + type openResult struct { + response *RawResponse + err error + } + opened := make(chan openResult, 1) + go func() { + response, err := client.OpenRawDebugBeaconState(ctx, "head", "application/octet-stream") + opened <- openResult{response: response, err: err} + }() + + select { + case <-headersSent: + case <-time.After(2 * time.Second): + t.Fatal("server did not send response headers") + } + + var result openResult + select { + case result = <-opened: + case <-time.After(2 * time.Second): + t.Fatal("open did not return while the response body was unavailable") + } + if result.err != nil { + t.Fatalf("open response: %v", result.err) + } + if result.response == nil { + t.Fatal("open response returned nil") + } + defer result.response.Close() + + wantProtocol := testHTTP1Protocol + if http2 { + wantProtocol = testHTTP2Protocol + } + if got := result.response.Header.Get("X-Protocol"); got != wantProtocol { + t.Fatalf("protocol = %q, want %q", got, wantProtocol) + } + + cancel() + _, err := io.ReadAll(result.response) + if !errors.Is(err, context.Canceled) { + t.Fatalf("read error = %v, want context.Canceled", err) + } + + select { + case <-handlerDone: + case <-time.After(2 * time.Second): + t.Fatal("handler did not observe context cancellation") + } + }) + } +} + +func TestRawClientTimeoutDuringBodyRead(t *testing.T) { + for _, http2 := range []bool{false, true} { + name := testHTTP1Protocol + if http2 { + name = testHTTP2Name + } + + t.Run(name, func(t *testing.T) { + handlerDone := make(chan struct{}) + server := newProtocolServer(t, http2, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + defer close(handlerDone) + w.Header().Set("X-Protocol", req.Proto) + w.WriteHeader(http.StatusOK) + if err := http.NewResponseController(w).Flush(); err != nil { + t.Errorf("flush response headers: %v", err) + + return + } + <-req.Context().Done() + })) + server.client.Timeout = 100 * time.Millisecond + + client := &consensusClient{ + url: server.url, + client: server.client, + rawClient: server.client, + } + response, err := client.OpenRawBlock(t.Context(), "head", "") + if err != nil { + t.Fatalf("open response: %v", err) + } + defer response.Close() + + type readResult struct { + err error + } + readDone := make(chan readResult, 1) + go func() { + _, readErr := io.ReadAll(response) + readDone <- readResult{err: readErr} + }() + + select { + case result := <-readDone: + if !errors.Is(result.err, context.DeadlineExceeded) { + t.Fatalf("read error = %v, want context.DeadlineExceeded", result.err) + } + case <-time.After(2 * time.Second): + t.Fatal("client timeout did not abort body read") + } + + select { + case <-handlerDone: + case <-time.After(2 * time.Second): + t.Fatal("handler did not observe client timeout") + } + }) + } +} + +func TestRawResponseHeaderTimeout(t *testing.T) { + for _, http2 := range []bool{false, true} { + name := testHTTP1Protocol + if http2 { + name = testHTTP2Name + } + + t.Run(name, func(t *testing.T) { + handlerStarted := make(chan struct{}) + handlerDone := make(chan struct{}) + server := newProtocolServer(t, http2, http.HandlerFunc(func(_ http.ResponseWriter, req *http.Request) { + close(handlerStarted) + <-req.Context().Done() + close(handlerDone) + })) + + baseTransport, ok := server.client.Transport.(*http.Transport) + if !ok { + t.Fatalf("transport type = %T", server.client.Transport) + } + transport := baseTransport.Clone() + transport.ResponseHeaderTimeout = 100 * time.Millisecond + timeoutClient := &http.Client{Transport: transport} + t.Cleanup(timeoutClient.CloseIdleConnections) + client := &consensusClient{ + url: server.url, + client: timeoutClient, + rawClient: timeoutClient, + } + + type openResult struct { + response *RawResponse + err error + } + opened := make(chan openResult, 1) + go func() { + response, err := client.OpenRawBlock(t.Context(), "head", "") + opened <- openResult{response: response, err: err} + }() + + select { + case <-handlerStarted: + case <-time.After(2 * time.Second): + t.Fatal("handler did not receive request") + } + + select { + case result := <-opened: + if result.response != nil { + t.Fatal("header timeout returned a response") + } + if result.err == nil { + t.Fatal("header timeout returned nil error") + } + var netErr net.Error + if !errors.As(result.err, &netErr) || !netErr.Timeout() { + t.Fatalf("error = %v, want timeout", result.err) + } + case <-time.After(2 * time.Second): + t.Fatal("response header timeout did not fire") + } + + select { + case <-handlerDone: + case <-time.After(2 * time.Second): + t.Fatal("handler did not observe response header timeout") + } + }) + } +} + +type rawHTTPServer struct { + url string + done <-chan struct{} +} + +func newRawHTTPServer(t *testing.T, rawResponse string) rawHTTPServer { + t.Helper() + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + done := make(chan struct{}) + go func() { + defer close(done) + connection, acceptErr := listener.Accept() + if acceptErr != nil { + return + } + defer connection.Close() + + request, readErr := http.ReadRequest(bufio.NewReader(connection)) + if readErr != nil { + return + } + _ = request.Body.Close() + _, _ = io.WriteString(connection, rawResponse) + }() + + t.Cleanup(func() { + _ = listener.Close() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Error("raw HTTP server did not stop") + } + }) + + return rawHTTPServer{ + url: "http://" + listener.Addr().String(), + done: done, + } +} + +func TestHTTP1ResponseFramingReadErrors(t *testing.T) { + tests := []struct { + name string + rawResponse string + wantContentLength int64 + wantBody string + wantUnexpectedEOF bool + }{ + { + name: "declared length truncation", + rawResponse: testHTTP1Protocol + " 200 OK\r\n" + + "Content-Length: 10\r\n" + + "Connection: close\r\n\r\n" + + testBodyABC, + wantContentLength: 10, + wantBody: testBodyABC, + wantUnexpectedEOF: true, + }, + { + name: "truncated chunked response", + rawResponse: testHTTP1Protocol + " 200 OK\r\n" + + "Transfer-Encoding: chunked\r\n" + + "Connection: close\r\n\r\n" + + "3\r\n" + testBodyABC + "\r\n5\r\nxy", + wantContentLength: -1, + wantBody: testBodyABC + "xy", + wantUnexpectedEOF: true, + }, + { + name: "close delimited response", + rawResponse: testHTTP1Protocol + " 200 OK\r\n" + + "Connection: close\r\n\r\n" + + testBodyABC, + wantContentLength: -1, + wantBody: testBodyABC, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + server := newRawHTTPServer(t, test.rawResponse) + transport := &http.Transport{DisableKeepAlives: true} + httpClient := &http.Client{Transport: transport} + t.Cleanup(httpClient.CloseIdleConnections) + client := &consensusClient{ + url: server.url, + client: httpClient, + rawClient: httpClient, + } + + response, err := client.OpenRawBlock(t.Context(), "head", "") + if err != nil { + t.Fatalf("open response: %v", err) + } + data, readErr := io.ReadAll(response) + closeErr := response.Close() + if closeErr != nil { + t.Fatalf("close response: %v", closeErr) + } + if string(data) != test.wantBody { + t.Fatalf("body = %q, want %q", data, test.wantBody) + } + if response.ContentLength != test.wantContentLength { + t.Fatalf("ContentLength = %d, want %d", response.ContentLength, test.wantContentLength) + } + if test.wantUnexpectedEOF { + if !errors.Is(readErr, io.ErrUnexpectedEOF) { + t.Fatalf("read error = %v, want io.ErrUnexpectedEOF", readErr) + } + } else if readErr != nil { + t.Fatalf("read error = %v, want nil", readErr) + } + + select { + case <-server.done: + case <-time.After(2 * time.Second): + t.Fatal("raw HTTP server did not finish") + } + }) + } +} + +func TestHTTP2DeclaredLengthTruncation(t *testing.T) { + server := newProtocolServer(t, true, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + w.Header().Set("Content-Length", "10") + w.Header().Set("X-Protocol", req.Proto) + _, _ = io.WriteString(w, testBodyABC) + })) + client := &consensusClient{ + url: server.url, + client: server.client, + rawClient: server.client, + } + + response, err := client.OpenRawBlock(t.Context(), "head", "") + if err != nil { + t.Fatalf("open response: %v", err) + } + defer response.Close() + + data, err := io.ReadAll(response) + if string(data) != testBodyABC { + t.Fatalf("body = %q", data) + } + if !errors.Is(err, io.ErrUnexpectedEOF) { + t.Fatalf("read error = %v, want io.ErrUnexpectedEOF", err) + } + if got := response.Header.Get("X-Protocol"); got != testHTTP2Protocol { + t.Fatalf("protocol = %q, want %s", got, testHTTP2Protocol) + } + if response.ContentLength != 10 { + t.Fatalf("ContentLength = %d, want 10", response.ContentLength) + } +} + +func TestHTTP2ResetDuringBodyRead(t *testing.T) { + server := newProtocolServer(t, true, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + w.Header().Set("Content-Length", "10") + w.Header().Set("X-Protocol", req.Proto) + _, _ = io.WriteString(w, testBodyABC) + if err := http.NewResponseController(w).Flush(); err != nil { + t.Errorf("flush response body: %v", err) + + return + } + + panic(http.ErrAbortHandler) + })) + client := &consensusClient{ + url: server.url, + client: server.client, + rawClient: server.client, + } + + response, err := client.OpenRawBlock(t.Context(), "head", "") + if err != nil { + t.Fatalf("open response: %v", err) + } + defer response.Close() + + data, readErr := io.ReadAll(response) + if string(data) != testBodyABC { + t.Fatalf("body = %q, want %q", data, testBodyABC) + } + if readErr == nil { + t.Fatal("HTTP/2 reset returned a nil read error") + } + if errors.Is(readErr, io.ErrUnexpectedEOF) { + t.Fatalf("HTTP/2 reset error = %v, unexpectedly classified as io.ErrUnexpectedEOF", readErr) + } + if got := response.Header.Get("X-Protocol"); got != testHTTP2Protocol { + t.Fatalf("protocol = %q, want %s", got, testHTTP2Protocol) + } +} + +func gzipBytes(t *testing.T, data []byte) []byte { + t.Helper() + + var compressed bytes.Buffer + writer := gzip.NewWriter(&compressed) + if _, err := writer.Write(data); err != nil { + t.Fatalf("gzip write: %v", err) + } + if err := writer.Close(); err != nil { + t.Fatalf("gzip close: %v", err) + } + + return compressed.Bytes() +} + +func TestRawGzipTransportSemantics(t *testing.T) { + const contentType = "application/octet-stream;charset=utf-8" + plain := []byte(strings.Repeat("beacon-state-", 32)) + compressed := gzipBytes(t, plain) + + tests := []struct { + name string + headers map[string]string + wantUncompressed bool + wantContentLength int64 + wantEncoding string + wantBody []byte + }{ + { + name: "transparent gzip", + wantUncompressed: true, + wantContentLength: -1, + wantBody: plain, + }, + { + name: "explicit gzip", + headers: map[string]string{"Accept-Encoding": testGzipEncoding}, + wantContentLength: int64(len(compressed)), + wantEncoding: testGzipEncoding, + wantBody: compressed, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + acceptEncoding := make(chan string, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + acceptEncoding <- req.Header.Get("Accept-Encoding") + w.Header().Set("Content-Type", contentType) + w.Header().Set("Content-Encoding", testGzipEncoding) + w.Header().Set("Content-Length", strconv.Itoa(len(compressed))) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(compressed) + })) + t.Cleanup(server.Close) + + transport := &http.Transport{} + httpClient := &http.Client{Transport: transport} + t.Cleanup(httpClient.CloseIdleConnections) + client := &consensusClient{ + url: server.URL, + client: httpClient, + rawClient: httpClient, + headers: test.headers, + } + + response, err := client.OpenRawDebugBeaconState(t.Context(), "head", "application/octet-stream") + if err != nil { + t.Fatalf("open response: %v", err) + } + data, readErr := io.ReadAll(response) + closeErr := response.Close() + if readErr != nil { + t.Fatalf("read response: %v", readErr) + } + if closeErr != nil { + t.Fatalf("close response: %v", closeErr) + } + if got := <-acceptEncoding; got != testGzipEncoding { + t.Fatalf("Accept-Encoding = %q, want %s", got, testGzipEncoding) + } + if response.Uncompressed != test.wantUncompressed { + t.Fatalf("Uncompressed = %v, want %v", response.Uncompressed, test.wantUncompressed) + } + if response.ContentLength != test.wantContentLength { + t.Fatalf("ContentLength = %d, want %d", response.ContentLength, test.wantContentLength) + } + if response.ContentType != contentType { + t.Fatalf("ContentType = %q", response.ContentType) + } + if got := response.Header.Get("Content-Encoding"); got != test.wantEncoding { + t.Fatalf("Content-Encoding = %q, want %q", got, test.wantEncoding) + } + if !bytes.Equal(data, test.wantBody) { + t.Fatalf("body differs: got %d bytes, want %d", len(data), len(test.wantBody)) + } + }) + } +} + +func contextWithReuseTrace(ctx context.Context) (context.Context, *atomic.Bool) { + reused := new(atomic.Bool) + trace := &httptrace.ClientTrace{ + GotConn: func(info httptrace.GotConnInfo) { + reused.Store(info.Reused) + }, + } + + return httptrace.WithClientTrace(ctx, trace), reused +} + +func TestHTTP1ConnectionReuseAfterFullSuccess(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Length", "7") + _, _ = io.WriteString(w, "payload") + })) + t.Cleanup(server.Close) + + transport := &http.Transport{MaxIdleConnsPerHost: 1} + httpClient := &http.Client{Transport: transport} + t.Cleanup(httpClient.CloseIdleConnections) + client := &consensusClient{url: server.URL, client: httpClient, rawClient: httpClient} + + for requestNumber := 1; requestNumber <= 2; requestNumber++ { + ctx, reused := contextWithReuseTrace(t.Context()) + response, err := client.OpenRawBlock(ctx, "head", "") + if err != nil { + t.Fatalf("request %d: open response: %v", requestNumber, err) + } + if _, err := io.Copy(io.Discard, response); err != nil { + t.Fatalf("request %d: drain response: %v", requestNumber, err) + } + if err := response.Close(); err != nil { + t.Fatalf("request %d: close response: %v", requestNumber, err) + } + + wantReused := requestNumber == 2 + if got := reused.Load(); got != wantReused { + t.Fatalf("request %d: reused = %v, want %v", requestNumber, got, wantReused) + } + } +} + +func TestHTTP1ConnectionReuseAfterSmallStatusError(t *testing.T) { + const errorBody = `{"message":"not found"}` + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Length", strconv.Itoa(len(errorBody))) + w.WriteHeader(http.StatusNotFound) + _, _ = io.WriteString(w, errorBody) + })) + t.Cleanup(server.Close) + + transport := &http.Transport{MaxIdleConnsPerHost: 1} + httpClient := &http.Client{Transport: transport} + t.Cleanup(httpClient.CloseIdleConnections) + client := &consensusClient{url: server.URL, client: httpClient, rawClient: httpClient} + + for requestNumber := 1; requestNumber <= 2; requestNumber++ { + ctx, reused := contextWithReuseTrace(t.Context()) + response, err := client.OpenRawExecutionPayloadEnvelope(ctx, "head", "") + if response != nil { + t.Fatalf("request %d returned a response", requestNumber) + } + if !errors.Is(err, ErrNotFound) { + t.Fatalf("request %d error = %v, want ErrNotFound", requestNumber, err) + } + + wantReused := requestNumber == 2 + if got := reused.Load(); got != wantReused { + t.Fatalf("request %d: reused = %v, want %v", requestNumber, got, wantReused) + } + } +} + +func TestHTTP1ConnectionReuseAfterRateLimit(t *testing.T) { + const ( + errorBody = `{"message":"rate limited"}` + retryAfter = "12" + ) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Length", strconv.Itoa(len(errorBody))) + w.Header().Set("Retry-After", retryAfter) + w.WriteHeader(http.StatusTooManyRequests) + _, _ = io.WriteString(w, errorBody) + })) + t.Cleanup(server.Close) + + transport := &http.Transport{MaxIdleConnsPerHost: 1} + httpClient := &http.Client{Transport: transport} + t.Cleanup(httpClient.CloseIdleConnections) + client := &consensusClient{url: server.URL, client: httpClient, rawClient: httpClient} + + for requestNumber := 1; requestNumber <= 2; requestNumber++ { + ctx, reused := contextWithReuseTrace(t.Context()) + response, err := client.OpenRawBlock(ctx, "head", "") + if response != nil { + t.Fatalf("request %d returned a response", requestNumber) + } + + var statusErr *HTTPStatusError + if !errors.As(err, &statusErr) { + t.Fatalf("request %d error = %v", requestNumber, err) + } + if statusErr.StatusCode != http.StatusTooManyRequests { + t.Fatalf("request %d status = %d", requestNumber, statusErr.StatusCode) + } + if statusErr.RetryAfter != retryAfter { + t.Fatalf("request %d Retry-After = %q", requestNumber, statusErr.RetryAfter) + } + + wantReused := requestNumber == 2 + if got := reused.Load(); got != wantReused { + t.Fatalf("request %d reused = %v, want %v", requestNumber, got, wantReused) + } + } +} + +func TestHTTP1ConnectionReuseAfterExactCapStatusError(t *testing.T) { + errorBody := strings.Repeat("x", maxErrorBodyBytes) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + if err := http.NewResponseController(w).Flush(); err != nil { + t.Errorf("flush status headers: %v", err) + + return + } + + _, _ = io.WriteString(w, errorBody) + })) + t.Cleanup(server.Close) + + transport := &http.Transport{MaxIdleConnsPerHost: 1} + httpClient := &http.Client{Transport: transport} + t.Cleanup(httpClient.CloseIdleConnections) + client := &consensusClient{url: server.URL, client: httpClient, rawClient: httpClient} + + for requestNumber := 1; requestNumber <= 2; requestNumber++ { + ctx, reused := contextWithReuseTrace(t.Context()) + response, err := client.OpenRawBlock(ctx, "head", "") + if response != nil { + t.Fatalf("request %d returned a response", requestNumber) + } + var statusErr *HTTPStatusError + if !errors.As(err, &statusErr) { + t.Fatalf("request %d error = %v", requestNumber, err) + } + if len(statusErr.BodyPrefix) != maxErrorBodyBytes { + t.Fatalf("request %d prefix length = %d", requestNumber, len(statusErr.BodyPrefix)) + } + + wantReused := requestNumber == 2 + if got := reused.Load(); got != wantReused { + t.Fatalf("request %d reused = %v, want %v", requestNumber, got, wantReused) + } + } +} + +func TestHTTPConnectionLifecycleLeavesNoOpenConnections(t *testing.T) { + var openConnections atomic.Int64 + stateChanged := make(chan struct{}, 16) + + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + switch { + case strings.HasSuffix(req.URL.Path, "/missing"): + w.WriteHeader(http.StatusNotFound) + _, _ = io.WriteString(w, "missing") + case strings.HasSuffix(req.URL.Path, "/partial"): + w.Header().Set("Content-Length", "1024") + _, _ = io.WriteString(w, "x") + default: + _, _ = io.WriteString(w, "complete") + } + })) + server.Config.ConnState = func(_ net.Conn, state http.ConnState) { + switch state { + case http.StateNew: + openConnections.Add(1) + case http.StateHijacked, http.StateClosed: + openConnections.Add(-1) + default: + } + + select { + case stateChanged <- struct{}{}: + default: + } + } + server.Start() + t.Cleanup(server.Close) + + transport := &http.Transport{MaxIdleConnsPerHost: 1} + httpClient := &http.Client{Transport: transport} + client := &consensusClient{url: server.URL, client: httpClient, rawClient: httpClient} + + response, err := client.OpenRawBlock(t.Context(), "complete", "") + if err != nil { + t.Fatalf("open complete response: %v", err) + } + if _, err = io.Copy(io.Discard, response); err != nil { + t.Fatalf("read complete response: %v", err) + } + if err = response.Close(); err != nil { + t.Fatalf("close complete response: %v", err) + } + + response, err = client.OpenRawBlock(t.Context(), "missing", "") + if response != nil { + t.Fatal("missing response returned a body") + } + if !errors.Is(err, ErrNotFound) { + t.Fatalf("missing response error = %v", err) + } + + response, err = client.OpenRawBlock(t.Context(), "partial", "") + if err != nil { + t.Fatalf("open partial response: %v", err) + } + if _, err := io.Copy(io.Discard, response); !errors.Is(err, io.ErrUnexpectedEOF) { + t.Fatalf("partial response read error = %v", err) + } + if err := response.Close(); err != nil { + t.Fatalf("close partial response: %v", err) + } + + httpClient.CloseIdleConnections() + + timer := time.NewTimer(2 * time.Second) + defer timer.Stop() + + for openConnections.Load() != 0 { + select { + case <-stateChanged: + case <-timer.C: + t.Fatalf("%d HTTP connections remain open", openConnections.Load()) + } + } +} + +func TestHTTP1ConnectionNotReusedAfterPartialSuccess(t *testing.T) { + var requests atomic.Int32 + firstWritten := make(chan struct{}) + firstDone := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if requests.Add(1) == 1 { + w.Header().Set("Content-Length", strconv.Itoa(1<<20)) + w.WriteHeader(http.StatusOK) + _, _ = io.WriteString(w, "x") + if err := http.NewResponseController(w).Flush(); err != nil { + t.Errorf("flush partial response: %v", err) + + return + } + close(firstWritten) + <-req.Context().Done() + close(firstDone) + + return + } + + _, _ = io.WriteString(w, "ok") + })) + t.Cleanup(server.Close) + + transport := &http.Transport{MaxIdleConnsPerHost: 1} + httpClient := &http.Client{Transport: transport} + t.Cleanup(httpClient.CloseIdleConnections) + client := &consensusClient{url: server.URL, client: httpClient, rawClient: httpClient} + + ctx, firstReused := contextWithReuseTrace(t.Context()) + response, err := client.OpenRawBlock(ctx, "head", "") + if err != nil { + t.Fatalf("open first response: %v", err) + } + select { + case <-firstWritten: + case <-time.After(2 * time.Second): + t.Fatal("first handler did not write body prefix") + } + buffer := make([]byte, 1) + if _, readErr := io.ReadFull(response, buffer); readErr != nil { + t.Fatalf("read first response prefix: %v", readErr) + } + if closeErr := response.Close(); closeErr != nil { + t.Fatalf("close first response: %v", closeErr) + } + if firstReused.Load() { + t.Fatal("first request unexpectedly reused a connection") + } + + select { + case <-firstDone: + case <-time.After(2 * time.Second): + t.Fatal("first handler did not observe the abandoned response") + } + + ctx, secondReused := contextWithReuseTrace(t.Context()) + response, err = client.OpenRawBlock(ctx, "head", "") + if err != nil { + t.Fatalf("open second response: %v", err) + } + _, readErr := io.Copy(io.Discard, response) + closeErr := response.Close() + if readErr != nil { + t.Fatalf("read second response: %v", readErr) + } + if closeErr != nil { + t.Fatalf("close second response: %v", closeErr) + } + if secondReused.Load() { + t.Fatal("connection was reused after a partially consumed HTTP/1.1 response") + } +} + +func TestHTTP1ConnectionNotReusedAfterOverCapStatusError(t *testing.T) { + errorBody := strings.Repeat("x", maxErrorBodyBytes+2) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Length", strconv.Itoa(len(errorBody))) + w.WriteHeader(http.StatusInternalServerError) + _, _ = io.WriteString(w, errorBody) + })) + t.Cleanup(server.Close) + + transport := &http.Transport{MaxIdleConnsPerHost: 1} + httpClient := &http.Client{Transport: transport} + t.Cleanup(httpClient.CloseIdleConnections) + client := &consensusClient{url: server.URL, client: httpClient, rawClient: httpClient} + + for requestNumber := 1; requestNumber <= 2; requestNumber++ { + ctx, reused := contextWithReuseTrace(t.Context()) + response, err := client.OpenRawBlock(ctx, "head", "") + if response != nil { + t.Fatalf("request %d returned a response", requestNumber) + } + var statusErr *HTTPStatusError + if !errors.As(err, &statusErr) { + t.Fatalf("request %d error = %v", requestNumber, err) + } + if len(statusErr.BodyPrefix) != maxErrorBodyBytes { + t.Fatalf("request %d prefix length = %d", requestNumber, len(statusErr.BodyPrefix)) + } + if reused.Load() { + t.Fatalf("request %d reused a connection after an over-cap response", requestNumber) + } + } +} + +func TestHTTP2ConnectionReusedAfterPartialSuccess(t *testing.T) { + var requests atomic.Int32 + firstDone := make(chan struct{}) + server := newProtocolServer(t, true, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if requests.Add(1) == 1 { + w.Header().Set("Content-Length", strconv.Itoa(1<<20)) + w.Header().Set("X-Protocol", req.Proto) + w.WriteHeader(http.StatusOK) + _, _ = io.WriteString(w, "x") + if err := http.NewResponseController(w).Flush(); err != nil { + t.Errorf("flush partial response: %v", err) + + return + } + <-req.Context().Done() + close(firstDone) + + return + } + + w.Header().Set("X-Protocol", req.Proto) + _, _ = io.WriteString(w, "ok") + })) + client := &consensusClient{url: server.url, client: server.client, rawClient: server.client} + + ctx, firstReused := contextWithReuseTrace(t.Context()) + response, err := client.OpenRawBlock(ctx, "head", "") + if err != nil { + t.Fatalf("open first response: %v", err) + } + buffer := make([]byte, 1) + if _, readErr := io.ReadFull(response, buffer); readErr != nil { + t.Fatalf("read first response prefix: %v", readErr) + } + if closeErr := response.Close(); closeErr != nil { + t.Fatalf("close first response: %v", closeErr) + } + if firstReused.Load() { + t.Fatal("first request unexpectedly reused a connection") + } + + select { + case <-firstDone: + case <-time.After(2 * time.Second): + t.Fatal("HTTP/2 handler did not observe stream cancellation") + } + + ctx, secondReused := contextWithReuseTrace(t.Context()) + response, err = client.OpenRawBlock(ctx, "head", "") + if err != nil { + t.Fatalf("open second response: %v", err) + } + data, readErr := io.ReadAll(response) + closeErr := response.Close() + if readErr != nil { + t.Fatalf("read second response: %v", readErr) + } + if closeErr != nil { + t.Fatalf("close second response: %v", closeErr) + } + if string(data) != "ok" { + t.Fatalf("second body = %q", data) + } + if got := response.Header.Get("X-Protocol"); got != testHTTP2Protocol { + t.Fatalf("protocol = %q, want %s", got, testHTTP2Protocol) + } + if !secondReused.Load() { + t.Fatal("HTTP/2 connection was not reused after cancelling one stream") + } +} + +func TestHTTP2ConnectionReuseAfterStatusErrors(t *testing.T) { + const errorBody = `{"message":"not found"}` + server := newProtocolServer(t, true, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + w.Header().Set("Content-Length", strconv.Itoa(len(errorBody))) + w.Header().Set("X-Protocol", req.Proto) + w.WriteHeader(http.StatusNotFound) + _, _ = io.WriteString(w, errorBody) + })) + client := &consensusClient{url: server.url, client: server.client, rawClient: server.client} + + for requestNumber := 1; requestNumber <= 2; requestNumber++ { + ctx, reused := contextWithReuseTrace(t.Context()) + response, err := client.OpenRawExecutionPayloadEnvelope(ctx, "head", "") + if response != nil { + t.Fatalf("request %d returned a response", requestNumber) + } + if !errors.Is(err, ErrNotFound) { + t.Fatalf("request %d error = %v", requestNumber, err) + } + + wantReused := requestNumber == 2 + if got := reused.Load(); got != wantReused { + t.Fatalf("request %d reused = %v, want %v", requestNumber, got, wantReused) + } + } +} + +func TestHTTP2ConnectionReuseAfterOverCapStatusError(t *testing.T) { + errorBody := strings.Repeat("x", maxErrorBodyBytes+2) + server := newProtocolServer(t, true, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + w.Header().Set("Content-Length", strconv.Itoa(len(errorBody))) + w.Header().Set("X-Protocol", req.Proto) + w.WriteHeader(http.StatusInternalServerError) + _, _ = io.WriteString(w, errorBody) + })) + client := &consensusClient{url: server.url, client: server.client, rawClient: server.client} + + for requestNumber := 1; requestNumber <= 2; requestNumber++ { + ctx, reused := contextWithReuseTrace(t.Context()) + response, err := client.OpenRawBlock(ctx, "head", "") + if response != nil { + t.Fatalf("request %d returned a response", requestNumber) + } + var statusErr *HTTPStatusError + if !errors.As(err, &statusErr) { + t.Fatalf("request %d error = %v", requestNumber, err) + } + if len(statusErr.BodyPrefix) != maxErrorBodyBytes { + t.Fatalf("request %d prefix length = %d", requestNumber, len(statusErr.BodyPrefix)) + } + + wantReused := requestNumber == 2 + if got := reused.Load(); got != wantReused { + t.Fatalf("request %d reused = %v, want %v", requestNumber, got, wantReused) + } + } +} + +func TestRawStatusErrorFormattingWithoutBody(t *testing.T) { + t.Parallel() + + statusErr := &HTTPStatusError{ + StatusCode: http.StatusBadGateway, + RequestPath: "/eth/v2/beacon/blocks/head", + } + want := fmt.Sprintf( + "status code: %d for /eth/v2/beacon/blocks/head", + http.StatusBadGateway, + ) + if got := statusErr.Error(); got != want { + t.Fatalf("Error() = %q, want %q", got, want) + } + if statusErr.Unwrap() != nil { + t.Fatalf("Unwrap() = %v, want nil", statusErr.Unwrap()) + } +} diff --git a/pkg/beacon/api/raw_test.go b/pkg/beacon/api/raw_test.go new file mode 100644 index 0000000..964701f --- /dev/null +++ b/pkg/beacon/api/raw_test.go @@ -0,0 +1,998 @@ +package api + +import ( + "bytes" + "errors" + "io" + "net/http" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/sirupsen/logrus" +) + +const ( + observerEventOpened = "opened" + observerEventClosed = "closed" + observerEventLeaked = "leaked" +) + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req) +} + +type trackingBody struct { + reader io.Reader + closeErr error + reads atomic.Int32 + closes atomic.Int32 +} + +func (b *trackingBody) Read(p []byte) (int, error) { + b.reads.Add(1) + + return b.reader.Read(p) +} + +func (b *trackingBody) Close() error { + b.closes.Add(1) + + return b.closeErr +} + +type panicBody struct { + closes atomic.Int32 +} + +func (*panicBody) Read([]byte) (int, error) { + panic("read panic") +} + +func (b *panicBody) Close() error { + b.closes.Add(1) + + return nil +} + +type blockingBody struct { + closed chan struct{} + closeOnce sync.Once + closes atomic.Int32 +} + +func newBlockingBody() *blockingBody { + return &blockingBody{closed: make(chan struct{})} +} + +func (b *blockingBody) Read([]byte) (int, error) { + <-b.closed + + return 0, errors.New("body closed") +} + +func (b *blockingBody) Close() error { + b.closes.Add(1) + b.closeOnce.Do(func() { + close(b.closed) + }) + + return nil +} + +type recordingObserver struct { + mu sync.Mutex + events []string +} + +type panicOpenedObserver struct { + closed atomic.Int32 +} + +func (*panicOpenedObserver) RawResponseOpened() { + panic("opened callback failed") +} + +func (o *panicOpenedObserver) RawResponseClosed() { + o.closed.Add(1) +} + +func (*panicOpenedObserver) RawResponseLeaked() {} + +func (o *recordingObserver) RawResponseOpened() { + o.record(observerEventOpened) +} + +func (o *recordingObserver) RawResponseClosed() { + o.record(observerEventClosed) +} + +func (o *recordingObserver) RawResponseLeaked() { + o.record(observerEventLeaked) +} + +func (o *recordingObserver) record(event string) { + o.mu.Lock() + defer o.mu.Unlock() + + o.events = append(o.events, event) +} + +func (o *recordingObserver) snapshot() []string { + o.mu.Lock() + defer o.mu.Unlock() + + return append([]string(nil), o.events...) +} + +func newRawTestClient(rawClient *http.Client) *consensusClient { + return &consensusClient{ + url: "http://beacon.test", + client: rawClient, + rawClient: rawClient, + } +} + +func TestNewConsensusClientRequiresHTTPClients(t *testing.T) { + t.Parallel() + + validClient := &http.Client{} + tests := []struct { + name string + client *http.Client + rawClient *http.Client + wantPanic string + }{ + { + name: "regular client", + rawClient: validClient, + wantPanic: "api: nil HTTP client", + }, + { + name: "raw client", + client: validClient, + wantPanic: "api: nil raw HTTP client", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + defer func() { + if recovered := recover(); recovered != test.wantPanic { + t.Fatalf("panic = %v, want %q", recovered, test.wantPanic) + } + }() + + NewConsensusClient(nil, "", test.client, test.rawClient, nil, nil) + }) + } +} + +func TestOpenRawEndpointsRequestAndMetadataWithoutEagerRead(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + path string + open func(*consensusClient) (*RawResponse, error) + }{ + { + name: "debug beacon state", + path: "/eth/v2/debug/beacon/states/finalized", + open: func(client *consensusClient) (*RawResponse, error) { + return client.OpenRawDebugBeaconState(t.Context(), "finalized", "application/octet-stream") + }, + }, + { + name: "block", + path: "/eth/v2/beacon/blocks/0x1234", + open: func(client *consensusClient) (*RawResponse, error) { + return client.OpenRawBlock(t.Context(), "0x1234", "application/octet-stream") + }, + }, + { + name: "execution payload envelope", + path: "/eth/v1/beacon/execution_payload_envelopes/head", + open: func(client *consensusClient) (*RawResponse, error) { + return client.OpenRawExecutionPayloadEnvelope(t.Context(), "head", "application/octet-stream") + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + body := &trackingBody{reader: strings.NewReader("payload")} + responseHeader := http.Header{ + "Content-Type": {"application/octet-stream;charset=utf-8"}, + "Eth-Consensus-Version": {"gloas"}, + "X-Mutable": {"before"}, + } + + var request *http.Request + transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { + request = req.Clone(req.Context()) + + return &http.Response{ + StatusCode: http.StatusOK, + Header: responseHeader, + Body: body, + ContentLength: 7, + }, nil + }) + client := newRawTestClient(&http.Client{Transport: transport}) + client.headers = map[string]string{ + "Authorization": "Bearer token", + "X-Request-ID": "request-id", + } + + response, err := test.open(client) + if err != nil { + t.Fatalf("open response: %v", err) + } + if response == nil { + t.Fatal("open response returned nil") + } + t.Cleanup(func() { + _ = response.Close() + }) + + if request.Method != http.MethodGet { + t.Fatalf("method = %q, want GET", request.Method) + } + if request.URL.Path != test.path { + t.Fatalf("path = %q, want %q", request.URL.Path, test.path) + } + if got := request.Header.Get("Accept"); got != "application/octet-stream" { + t.Fatalf("Accept = %q", got) + } + if got := request.Header.Get("Authorization"); got != "Bearer token" { + t.Fatalf("Authorization = %q", got) + } + if got := request.Header.Get("X-Request-ID"); got != "request-id" { + t.Fatalf("X-Request-ID = %q", got) + } + if got := body.reads.Load(); got != 0 { + t.Fatalf("body reads before return = %d, want 0", got) + } + if response.ContentType != "application/octet-stream;charset=utf-8" { + t.Fatalf("ContentType = %q", response.ContentType) + } + if response.ContentLength != 7 { + t.Fatalf("ContentLength = %d", response.ContentLength) + } + if response.Uncompressed { + t.Fatal("Uncompressed = true") + } + if got := response.Header.Get("Eth-Consensus-Version"); got != "gloas" { + t.Fatalf("Eth-Consensus-Version = %q", got) + } + + responseHeader.Set("X-Mutable", "after") + if got := response.Header.Get("X-Mutable"); got != "before" { + t.Fatalf("response header was not cloned: %q", got) + } + }) + } +} + +func TestOpenRawDefaultsAcceptToJSON(t *testing.T) { + t.Parallel() + + var accept string + body := &trackingBody{reader: http.NoBody} + client := newRawTestClient(&http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + accept = req.Header.Get("Accept") + + return &http.Response{ + StatusCode: http.StatusOK, + Header: make(http.Header), + Body: body, + }, nil + })}) + + response, err := client.OpenRawBlock(t.Context(), "head", "") + if err != nil { + t.Fatalf("open response: %v", err) + } + defer response.Close() + + if accept != defaultRawContentType { + t.Fatalf("Accept = %q, want application/json", accept) + } + if got := body.reads.Load(); got != 0 { + t.Fatalf("body reads before return = %d, want 0", got) + } +} + +func TestBufferedRawDefaultsAcceptToJSON(t *testing.T) { + t.Parallel() + + var accept string + body := &trackingBody{reader: http.NoBody} + client := newRawTestClient(&http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + accept = req.Header.Get("Accept") + + return &http.Response{ + StatusCode: http.StatusOK, + Header: make(http.Header), + Body: body, + }, nil + })}) + + data, err := client.RawBlock(t.Context(), "head", "") + if err != nil { + t.Fatalf("fetch response: %v", err) + } + if len(data) != 0 { + t.Fatalf("body length = %d", len(data)) + } + if accept != defaultRawContentType { + t.Fatalf("Accept = %q, want application/json", accept) + } + if got := body.closes.Load(); got != 1 { + t.Fatalf("body closes = %d, want 1", got) + } +} + +func TestRawBufferedAndStreamingParity(t *testing.T) { + t.Parallel() + + const payload = "raw-payload-\x00\x01" + + tests := []struct { + name string + path string + buffered func(*consensusClient) ([]byte, error) + streamed func(*consensusClient) (*RawResponse, error) + }{ + { + name: "debug beacon state", + path: "/eth/v2/debug/beacon/states/finalized", + buffered: func(client *consensusClient) ([]byte, error) { + return client.RawDebugBeaconState(t.Context(), "finalized", "application/octet-stream") + }, + streamed: func(client *consensusClient) (*RawResponse, error) { + return client.OpenRawDebugBeaconState(t.Context(), "finalized", "application/octet-stream") + }, + }, + { + name: "block", + path: "/eth/v2/beacon/blocks/head", + buffered: func(client *consensusClient) ([]byte, error) { + return client.RawBlock(t.Context(), "head", "application/octet-stream") + }, + streamed: func(client *consensusClient) (*RawResponse, error) { + return client.OpenRawBlock(t.Context(), "head", "application/octet-stream") + }, + }, + { + name: "execution payload envelope", + path: "/eth/v1/beacon/execution_payload_envelopes/0xabcd", + buffered: func(client *consensusClient) ([]byte, error) { + return client.RawExecutionPayloadEnvelope(t.Context(), "0xabcd", "application/octet-stream") + }, + streamed: func(client *consensusClient) (*RawResponse, error) { + return client.OpenRawExecutionPayloadEnvelope(t.Context(), "0xabcd", "application/octet-stream") + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + var calls atomic.Int32 + client := newRawTestClient(&http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + if req.URL.Path != test.path { + t.Errorf("path = %q, want %q", req.URL.Path, test.path) + } + calls.Add(1) + + return &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": {"application/octet-stream"}}, + Body: io.NopCloser(strings.NewReader(payload)), + ContentLength: int64(len(payload)), + }, nil + })}) + + buffered, err := test.buffered(client) + if err != nil { + t.Fatalf("buffered response: %v", err) + } + response, err := test.streamed(client) + if err != nil { + t.Fatalf("streamed response: %v", err) + } + streamed, readErr := io.ReadAll(response) + closeErr := response.Close() + if readErr != nil { + t.Fatalf("read streamed response: %v", readErr) + } + if closeErr != nil { + t.Fatalf("close streamed response: %v", closeErr) + } + if !bytes.Equal(buffered, streamed) { + t.Fatalf("buffered %q != streamed %q", buffered, streamed) + } + if string(streamed) != payload { + t.Fatalf("payload = %q", streamed) + } + if got := calls.Load(); got != 2 { + t.Fatalf("calls = %d, want 2", got) + } + }) + } +} + +func TestDoRawRequiresExactStatusOK(t *testing.T) { + t.Parallel() + + for _, status := range []int{ + http.StatusContinue, + http.StatusOK, + http.StatusCreated, + http.StatusNoContent, + http.StatusPartialContent, + } { + t.Run(http.StatusText(status), func(t *testing.T) { + t.Parallel() + + body := &trackingBody{reader: strings.NewReader("body")} + client := newRawTestClient(&http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: status, + Header: make(http.Header), + Body: body, + }, nil + })}) + + response, err := client.OpenRawBlock(t.Context(), "head", "") + if status == http.StatusOK { + if err != nil { + t.Fatalf("status 200: %v", err) + } + if response == nil { + t.Fatal("status 200 returned nil response") + } + if got := body.closes.Load(); got != 0 { + t.Fatalf("status 200 close count before ownership transfer = %d", got) + } + if closeErr := response.Close(); closeErr != nil { + t.Fatalf("close status 200 response: %v", closeErr) + } + + return + } + + if response != nil { + t.Fatal("non-200 returned a response") + } + var statusErr *HTTPStatusError + if !errors.As(err, &statusErr) { + t.Fatalf("error %T = %v, want *HTTPStatusError", err, err) + } + if statusErr.StatusCode != status { + t.Fatalf("status = %d, want %d", statusErr.StatusCode, status) + } + if got := body.closes.Load(); got != 1 { + t.Fatalf("close count = %d, want 1", got) + } + }) + } +} + +func TestRawHTTPStatusErrors(t *testing.T) { + t.Parallel() + + tests := []struct { + status int + retryAfter string + body string + wantNotFound bool + }{ + { + status: http.StatusNotFound, + body: `{"message":"missing"}`, + wantNotFound: true, + }, + { + status: http.StatusTooManyRequests, + retryAfter: "17", + body: "slow\ndown\tplease", + }, + { + status: http.StatusInternalServerError, + body: `"disk failed"`, + }, + } + + for _, test := range tests { + t.Run(http.StatusText(test.status), func(t *testing.T) { + t.Parallel() + + body := &trackingBody{reader: strings.NewReader(test.body)} + client := newRawTestClient(&http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: test.status, + Header: http.Header{"Retry-After": {test.retryAfter}}, + Body: body, + }, nil + })}) + + response, err := client.OpenRawBlock(t.Context(), "0xfeed", "application/octet-stream") + if response != nil { + t.Fatal("error returned a non-nil response") + } + + var statusErr *HTTPStatusError + if !errors.As(err, &statusErr) { + t.Fatalf("error %T = %v, want *HTTPStatusError", err, err) + } + if statusErr.StatusCode != test.status { + t.Fatalf("StatusCode = %d", statusErr.StatusCode) + } + if statusErr.RequestPath != "/eth/v2/beacon/blocks/0xfeed" { + t.Fatalf("RequestPath = %q", statusErr.RequestPath) + } + if statusErr.RetryAfter != test.retryAfter { + t.Fatalf("RetryAfter = %q", statusErr.RetryAfter) + } + if statusErr.BodyPrefix != test.body { + t.Fatalf("BodyPrefix = %q", statusErr.BodyPrefix) + } + if got := err.Error(); !strings.Contains(got, strconv.Quote(test.body)) { + t.Fatalf("error %q does not contain quoted body %q", got, strconv.Quote(test.body)) + } + if got := errors.Is(err, ErrNotFound); got != test.wantNotFound { + t.Fatalf("errors.Is(ErrNotFound) = %v", got) + } + if got := body.closes.Load(); got != 1 { + t.Fatalf("close count = %d, want 1", got) + } + }) + } +} + +func TestRawHTTPStatusErrorBodyPrefixBounded(t *testing.T) { + t.Parallel() + + const suffix = "must-not-be-read" + payload := strings.Repeat("x", maxErrorBodyBytes) + suffix + body := &trackingBody{reader: strings.NewReader(payload)} + client := newRawTestClient(&http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusInternalServerError, + Header: make(http.Header), + Body: body, + }, nil + })}) + + response, err := client.OpenRawBlock(t.Context(), "head", "") + if response != nil { + t.Fatal("error returned a non-nil response") + } + var statusErr *HTTPStatusError + if !errors.As(err, &statusErr) { + t.Fatalf("error %T = %v, want *HTTPStatusError", err, err) + } + if len(statusErr.BodyPrefix) != maxErrorBodyBytes { + t.Fatalf("BodyPrefix length = %d, want %d", len(statusErr.BodyPrefix), maxErrorBodyBytes) + } + if statusErr.BodyPrefix != strings.Repeat("x", maxErrorBodyBytes) { + t.Fatal("BodyPrefix was not the first 8 KiB") + } + if strings.Contains(err.Error(), suffix) { + t.Fatal("error contains bytes beyond the prefix bound") + } + if got := body.closes.Load(); got != 1 { + t.Fatalf("close count = %d, want 1", got) + } +} + +func TestRawHTTPStatusErrorReadTimeBound(t *testing.T) { + t.Parallel() + + body := newBlockingBody() + client := newRawTestClient(&http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusServiceUnavailable, + Header: make(http.Header), + Body: body, + }, nil + })}) + + type result struct { + response *RawResponse + err error + elapsed time.Duration + } + resultCh := make(chan result, 1) + go func() { + start := time.Now() + response, err := client.OpenRawBlock(t.Context(), "head", "") + resultCh <- result{response: response, err: err, elapsed: time.Since(start)} + }() + + select { + case got := <-resultCh: + if got.response != nil { + t.Fatal("error returned a non-nil response") + } + var statusErr *HTTPStatusError + if !errors.As(got.err, &statusErr) { + t.Fatalf("error %T = %v, want *HTTPStatusError", got.err, got.err) + } + if got.elapsed < errorBodyReadTimeout-(50*time.Millisecond) { + t.Fatalf("returned too early after %s", got.elapsed) + } + if got.elapsed > time.Second { + t.Fatalf("read cap took %s, want approximately %s", got.elapsed, errorBodyReadTimeout) + } + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for stalled error body to be closed") + } + + if got := body.closes.Load(); got != 1 { + t.Fatalf("close count = %d, want 1", got) + } +} + +func TestRawTransportErrors(t *testing.T) { + t.Parallel() + + transportErr := errors.New("transport failed") + tests := []struct { + name string + transport roundTripFunc + wantIs error + }{ + { + name: "round trip error", + transport: func(*http.Request) (*http.Response, error) { + return nil, transportErr + }, + wantIs: transportErr, + }, + { + name: "nil response", + transport: func(*http.Request) (*http.Response, error) { + return nil, nil //nolint:nilnil // verifies net/http rejects an invalid transport result + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + client := newRawTestClient(&http.Client{Transport: test.transport}) + response, err := client.OpenRawBlock(t.Context(), "head", "") + if response != nil { + t.Fatal("transport error returned a non-nil response") + } + if err == nil { + t.Fatal("transport error returned nil error") + } + if test.wantIs != nil && !errors.Is(err, test.wantIs) { + t.Fatalf("error %v does not wrap %v", err, test.wantIs) + } + }) + } +} + +func TestBufferedRawReadFailureReturnsNilAndCloses(t *testing.T) { + t.Parallel() + + readErr := errors.New("read failed") + body := &trackingBody{reader: io.MultiReader(strings.NewReader("partial"), errorReader{err: readErr})} + client := newRawTestClient(&http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusOK, + Header: make(http.Header), + Body: body, + }, nil + })}) + + data, err := client.RawBlock(t.Context(), "head", "") + if data != nil { + t.Fatalf("data = %q, want nil", data) + } + if !errors.Is(err, readErr) { + t.Fatalf("error = %v, want %v", err, readErr) + } + if got := body.closes.Load(); got != 1 { + t.Fatalf("close count = %d, want 1", got) + } +} + +type errorReader struct { + err error +} + +func (r errorReader) Read([]byte) (int, error) { + return 0, r.err +} + +func TestBufferedRawClosesBodyWhenReadPanics(t *testing.T) { + t.Parallel() + + body := &panicBody{} + client := newRawTestClient(&http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusOK, + Header: make(http.Header), + Body: body, + }, nil + })}) + + func() { + defer func() { + if recovered := recover(); recovered != "read panic" { + t.Fatalf("recovered = %v", recovered) + } + }() + + _, _ = client.RawBlock(t.Context(), "head", "") + }() + + if got := body.closes.Load(); got != 1 { + t.Fatalf("close count = %d, want 1", got) + } +} + +func TestRawResponseCloseIsIdempotentAndStable(t *testing.T) { + t.Parallel() + + closeErr := errors.New("close failed") + body := &trackingBody{reader: strings.NewReader("payload"), closeErr: closeErr} + client := newRawTestClient(&http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusOK, + Header: make(http.Header), + Body: body, + }, nil + })}) + + response, err := client.OpenRawBlock(t.Context(), "head", "") + if err != nil { + t.Fatalf("open response: %v", err) + } + if err := response.Body.Close(); !errors.Is(err, closeErr) { + t.Fatalf("Body.Close() = %v", err) + } + if err := response.Close(); !errors.Is(err, closeErr) { + t.Fatalf("RawResponse.Close() = %v", err) + } + if err := response.Body.Close(); !errors.Is(err, closeErr) { + t.Fatalf("second Body.Close() = %v", err) + } + if got := body.closes.Load(); got != 1 { + t.Fatalf("underlying close count = %d, want 1", got) + } +} + +func TestRawResponseDelegatesToBody(t *testing.T) { + t.Parallel() + + body := &trackingBody{reader: strings.NewReader("payload")} + response := &RawResponse{Body: body} + + data, err := io.ReadAll(response) + if err != nil { + t.Fatalf("read response: %v", err) + } + if string(data) != "payload" { + t.Fatalf("body = %q, want payload", data) + } + if err := response.Close(); err != nil { + t.Fatalf("close response: %v", err) + } + if got := body.closes.Load(); got != 1 { + t.Fatalf("underlying close count = %d, want 1", got) + } +} + +func TestRawResponseConcurrentClose(t *testing.T) { + t.Parallel() + + closeErr := errors.New("close failed") + body := &trackingBody{reader: strings.NewReader("payload"), closeErr: closeErr} + client := newRawTestClient(&http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusOK, + Header: make(http.Header), + Body: body, + }, nil + })}) + + response, err := client.OpenRawBlock(t.Context(), "head", "") + if err != nil { + t.Fatalf("open response: %v", err) + } + + const goroutines = 32 + start := make(chan struct{}) + errs := make(chan error, goroutines) + var waitGroup sync.WaitGroup + for i := 0; i < goroutines; i++ { + waitGroup.Add(1) + go func(useBody bool) { + defer waitGroup.Done() + <-start + if useBody { + errs <- response.Body.Close() + } else { + errs <- response.Close() + } + }(i%2 == 0) + } + close(start) + waitGroup.Wait() + close(errs) + + for err := range errs { + if !errors.Is(err, closeErr) { + t.Errorf("Close() = %v, want %v", err, closeErr) + } + } + if got := body.closes.Load(); got != 1 { + t.Fatalf("underlying close count = %d, want 1", got) + } +} + +func TestRawResponseObserverNormalClose(t *testing.T) { + t.Parallel() + + observer := &recordingObserver{} + body := &trackingBody{reader: strings.NewReader("payload")} + client := newRawTestClient(&http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusOK, + Header: make(http.Header), + Body: body, + }, nil + })}) + client.rawResponseObserver = observer + + response, err := client.OpenRawBlock(t.Context(), "head", "") + if err != nil { + t.Fatalf("open response: %v", err) + } + if got := observer.snapshot(); !stringSlicesEqual(got, []string{observerEventOpened}) { + t.Fatalf("events after open = %v", got) + } + if err := response.Close(); err != nil { + t.Fatalf("close response: %v", err) + } + if got := observer.snapshot(); !stringSlicesEqual(got, []string{observerEventOpened, observerEventClosed}) { + t.Fatalf("events after close = %v", got) + } +} + +func TestRawResponseObserverPanicClosesBodyBeforeTransfer(t *testing.T) { + t.Parallel() + + body := &trackingBody{reader: strings.NewReader("payload")} + observer := &panicOpenedObserver{} + client := newRawTestClient(&http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusOK, + Header: make(http.Header), + Body: body, + }, nil + })}) + client.rawResponseObserver = observer + + var recovered any + func() { + defer func() { + recovered = recover() + }() + + _, _ = client.OpenRawBlock(t.Context(), "head", "") + }() + + if recovered == nil { + t.Fatal("observer panic was not propagated") + } + if got := body.closes.Load(); got != 1 { + t.Fatalf("underlying close count = %d, want 1", got) + } + if got := observer.closed.Load(); got != 1 { + t.Fatalf("closed callbacks = %d, want 1", got) + } +} + +func TestRawResponseCleanupCallback(t *testing.T) { + t.Parallel() + + closeErr := errors.New("socket close failed") + body := &trackingBody{reader: strings.NewReader("payload"), closeErr: closeErr} + observer := &recordingObserver{} + var logOutput bytes.Buffer + logger := logrus.New() + logger.SetOutput(&logOutput) + logger.SetFormatter(&logrus.TextFormatter{ + DisableColors: true, + DisableTimestamp: true, + }) + + client := newRawTestClient(&http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusOK, + Header: make(http.Header), + Body: body, + }, nil + })}) + client.log = logger + client.rawResponseObserver = observer + + response, err := client.OpenRawExecutionPayloadEnvelope(t.Context(), "0xfeed", "") + if err != nil { + t.Fatalf("open response: %v", err) + } + managedBody, ok := response.Body.(*managedRawBody) + if !ok { + t.Fatalf("body type = %T", response.Body) + } + + cleanupLeakedRawResponse(managedBody.state) + + if got := body.closes.Load(); got != 1 { + t.Fatalf("underlying close count = %d, want 1", got) + } + if got := observer.snapshot(); !stringSlicesEqual(got, []string{ + observerEventOpened, + observerEventLeaked, + observerEventClosed, + }) { + t.Fatalf("events = %v", got) + } + logged := logOutput.String() + for _, want := range []string{ + `level=error`, + `msg="raw response body was not closed"`, + `request_path=/eth/v1/beacon/execution_payload_envelopes/0xfeed`, + `error="socket close failed"`, + } { + if !strings.Contains(logged, want) { + t.Fatalf("log %q does not contain %q", logged, want) + } + } + if err := response.Close(); !errors.Is(err, closeErr) { + t.Fatalf("close after cleanup = %v", err) + } + if got := body.closes.Load(); got != 1 { + t.Fatalf("underlying close count after explicit close = %d, want 1", got) + } + if got := observer.snapshot(); !stringSlicesEqual(got, []string{ + observerEventOpened, + observerEventLeaked, + observerEventClosed, + }) { + t.Fatalf("events after explicit close = %v", got) + } +} + +func stringSlicesEqual(left, right []string) bool { + if len(left) != len(right) { + return false + } + for index := range left { + if left[index] != right[index] { + return false + } + } + + return true +} diff --git a/pkg/beacon/beacon.go b/pkg/beacon/beacon.go index b797ce1..f2a6aa2 100644 --- a/pkg/beacon/beacon.go +++ b/pkg/beacon/beacon.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "net/http" "strings" "sync" "time" @@ -65,8 +66,8 @@ type Node interface { // Fetchers - these are not cached and will always fetch from the node. // FetchBlock fetches the block for the given state id. FetchBlock(ctx context.Context, stateID string) (*spec.VersionedSignedBeaconBlock, error) - // FetchRawBlock fetches the raw, unparsed block for the given state id. - FetchRawBlock(ctx context.Context, stateID string, contentType string) ([]byte, error) + // FetchRawBlock fetches the raw, unparsed block for the given block id. + FetchRawBlock(ctx context.Context, blockID string, contentType string) ([]byte, error) // FetchRawExecutionPayloadEnvelope fetches the raw, unparsed signed execution // payload envelope for the given block id (gloas onwards). FetchRawExecutionPayloadEnvelope(ctx context.Context, blockID string, contentType string) ([]byte, error) @@ -78,6 +79,24 @@ type Node interface { FetchBeaconStateRoot(ctx context.Context, stateID string) (phase0.Root, error) // FetchRawBeaconState fetches the raw, unparsed beacon state for the given state id. FetchRawBeaconState(ctx context.Context, stateID string, contentType string) ([]byte, error) + // OpenRawBeaconState opens a streaming beacon state response. A successful + // return only establishes the response status and body; reads can still fail + // and are governed by ctx and the configured raw API client timeout. The caller + // must always close the response. Slow or partial reads retain transport + // resources, so callers should bound concurrency. + OpenRawBeaconState(ctx context.Context, stateID string, contentType string) (*api.RawResponse, error) + // OpenRawBlock opens a streaming block response. A successful return only + // establishes the response status and body; reads can still fail and remain + // governed by ctx and the configured raw API client timeout. The caller must + // always close the response. Slow or partial reads retain transport + // resources, so callers should bound concurrency. + OpenRawBlock(ctx context.Context, blockID string, contentType string) (*api.RawResponse, error) + // OpenRawExecutionPayloadEnvelope opens a streaming signed execution payload + // envelope response. A successful return only establishes the response status + // and body; reads can still fail and are governed by ctx and the configured raw + // API client timeout. The caller must always close the response. Slow or + // partial reads retain transport resources, so callers should bound concurrency. + OpenRawExecutionPayloadEnvelope(ctx context.Context, blockID string, contentType string) (*api.RawResponse, error) // FetchValidators fetches the validators for the given state id and validator ids. FetchValidators(ctx context.Context, state string, indices []phase0.ValidatorIndex, pubKeys []phase0.BLSPubKey) (map[phase0.ValidatorIndex]*v1.Validator, error) // FetchFinality fetches the finality checkpoint for the state id. @@ -188,7 +207,7 @@ type node struct { log logrus.FieldLogger ctx context.Context //nolint:containedctx // existing. cancel context.CancelFunc - lifecycleMu sync.Mutex // protects ctx and cancel + lifecycleMu sync.Mutex // serializes cancellation and client installation // Configuration // Config should roughly be driven by end users. @@ -197,9 +216,10 @@ type node struct { options *Options // Clients - api api.ConsensusClient - client eth2client.Service - broker *emission.Emitter + api api.ConsensusClient + client eth2client.Service + rawHTTPClient *http.Client + broker *emission.Emitter // Internal data stores genesis *v1.Genesis @@ -226,8 +246,15 @@ type node struct { crons *gocron.Scheduler } +var _ api.RawResponseObserver = (*node)(nil) + // NewNode creates a new beacon node. func NewNode(log logrus.FieldLogger, config *Config, namespace string, options Options) Node { + if options.APIClient != nil { + apiClientOptions := *options.APIClient + options.APIClient = &apiClientOptions + } + n := &node{ log: log.WithField("module", "consensus/beacon"), @@ -334,9 +361,11 @@ func (n *node) StartAsync(ctx context.Context) { } func (n *node) Stop(ctx context.Context) error { - if n.options.PrometheusMetrics { + var stopErr error + + if n.metrics != nil { if err := n.metrics.Stop(); err != nil { - return err + stopErr = err } } @@ -350,9 +379,33 @@ func (n *node) Stop(ctx context.Context) error { n.cancel() } + rawHTTPClient := n.rawHTTPClient + n.lifecycleMu.Unlock() - return nil + if rawHTTPClient != nil { + rawHTTPClient.CloseIdleConnections() + } + + return stopErr +} + +func (n *node) RawResponseOpened() { + if n.metrics != nil { + n.metrics.RawResponses().OpenStreams.Inc() + } +} + +func (n *node) RawResponseClosed() { + if n.metrics != nil { + n.metrics.RawResponses().OpenStreams.Dec() + } +} + +func (n *node) RawResponseLeaked() { + if n.metrics != nil { + n.metrics.RawResponses().Leaks.Inc() + } } func (n *node) Options() *Options { diff --git a/pkg/beacon/bootstrap.go b/pkg/beacon/bootstrap.go index 2c8eb0a..c6bf9c2 100644 --- a/pkg/beacon/bootstrap.go +++ b/pkg/beacon/bootstrap.go @@ -57,17 +57,47 @@ func (n *node) ensureClients(ctx context.Context) error { continue } - n.client = client + regularHTTPClient := &http.Client{Timeout: timeout} + rawHTTPClient := n.options.apiClientOptions().httpClient() - httpClient := http.Client{ - Timeout: timeout, + if err := n.installClients(ctx, client, regularHTTPClient, rawHTTPClient); err != nil { + return err } - n.api = api.NewConsensusClient(ctx, n.log, n.config.Addr, httpClient, n.config.Headers) - break } } return nil } + +func (n *node) installClients( + ctx context.Context, + client eth2client.Service, + regularHTTPClient *http.Client, + rawHTTPClient *http.Client, +) error { + n.lifecycleMu.Lock() + + if err := ctx.Err(); err != nil { + n.lifecycleMu.Unlock() + rawHTTPClient.CloseIdleConnections() + + return err + } + + n.client = client + n.rawHTTPClient = rawHTTPClient + n.api = api.NewConsensusClient( + n.log, + n.config.Addr, + regularHTTPClient, + rawHTTPClient, + n.config.Headers, + n, + ) + + n.lifecycleMu.Unlock() + + return nil +} diff --git a/pkg/beacon/fetch.go b/pkg/beacon/fetch.go index 1dc1824..5073e99 100644 --- a/pkg/beacon/fetch.go +++ b/pkg/beacon/fetch.go @@ -4,10 +4,11 @@ import ( "context" "errors" + beaconapi "github.com/ethpandaops/beacon/pkg/beacon/api" "github.com/ethpandaops/beacon/pkg/beacon/api/types" "github.com/ethpandaops/beacon/pkg/beacon/state" eth2client "github.com/ethpandaops/go-eth2-client" - "github.com/ethpandaops/go-eth2-client/api" + eapi "github.com/ethpandaops/go-eth2-client/api" v1 "github.com/ethpandaops/go-eth2-client/api/v1" "github.com/ethpandaops/go-eth2-client/spec" "github.com/ethpandaops/go-eth2-client/spec/deneb" @@ -20,7 +21,7 @@ func (n *node) FetchSyncStatus(ctx context.Context) (*v1.SyncState, error) { return nil, errors.New("client does not implement eth2client.NodeSyncingProvider") } - status, err := provider.NodeSyncing(ctx, &api.NodeSyncingOpts{}) + status, err := provider.NodeSyncing(ctx, &eapi.NodeSyncingOpts{}) if err != nil { return nil, err } @@ -51,7 +52,7 @@ func (n *node) FetchNodeVersion(ctx context.Context) (string, error) { return "", errors.New("client does not implement eth2client.NodeVersionProvider") } - rsp, err := provider.NodeVersion(ctx, &api.NodeVersionOpts{}) + rsp, err := provider.NodeVersion(ctx, &eapi.NodeVersionOpts{}) if err != nil { return "", err } @@ -69,14 +70,26 @@ func (n *node) FetchBlock(ctx context.Context, stateID string) (*spec.VersionedS return n.getBlock(ctx, stateID) } -func (n *node) FetchRawBlock(ctx context.Context, stateID string, contentType string) ([]byte, error) { - return n.api.RawBlock(ctx, stateID, contentType) +func (n *node) FetchRawBlock(ctx context.Context, blockID string, contentType string) ([]byte, error) { + return n.api.RawBlock(ctx, blockID, contentType) } func (n *node) FetchRawExecutionPayloadEnvelope(ctx context.Context, blockID string, contentType string) ([]byte, error) { return n.api.RawExecutionPayloadEnvelope(ctx, blockID, contentType) } +func (n *node) OpenRawBeaconState(ctx context.Context, stateID string, contentType string) (*beaconapi.RawResponse, error) { + return n.api.OpenRawDebugBeaconState(ctx, stateID, contentType) +} + +func (n *node) OpenRawBlock(ctx context.Context, blockID string, contentType string) (*beaconapi.RawResponse, error) { + return n.api.OpenRawBlock(ctx, blockID, contentType) +} + +func (n *node) OpenRawExecutionPayloadEnvelope(ctx context.Context, blockID string, contentType string) (*beaconapi.RawResponse, error) { + return n.api.OpenRawExecutionPayloadEnvelope(ctx, blockID, contentType) +} + func (n *node) FetchBlockRoot(ctx context.Context, stateID string) (*phase0.Root, error) { return n.getBlockRoot(ctx, stateID) } @@ -87,7 +100,7 @@ func (n *node) FetchBeaconState(ctx context.Context, stateID string) (*spec.Vers return nil, errors.New("client does not implement eth2client.NodeVersionProvider") } - rsp, err := provider.BeaconState(ctx, &api.BeaconStateOpts{ + rsp, err := provider.BeaconState(ctx, &eapi.BeaconStateOpts{ State: stateID, }) if err != nil { @@ -107,7 +120,7 @@ func (n *node) FetchFinality(ctx context.Context, stateID string) (*v1.Finality, return nil, errors.New("client does not implement eth2client.FinalityProvider") } - rsp, err := provider.Finality(ctx, &api.FinalityOpts{ + rsp, err := provider.Finality(ctx, &eapi.FinalityOpts{ State: stateID, }) if err != nil { @@ -144,7 +157,7 @@ func (n *node) FetchRawSpec(ctx context.Context) (map[string]any, error) { return nil, errors.New("client does not implement eth2client.SpecProvider") } - rsp, err := provider.Spec(ctx, &api.SpecOpts{}) + rsp, err := provider.Spec(ctx, &eapi.SpecOpts{}) if err != nil { return nil, err } @@ -158,7 +171,7 @@ func (n *node) FetchSpec(ctx context.Context) (*state.Spec, error) { return nil, errors.New("client does not implement eth2client.SpecProvider") } - rsp, err := provider.Spec(ctx, &api.SpecOpts{}) + rsp, err := provider.Spec(ctx, &eapi.SpecOpts{}) if err != nil { return nil, err } @@ -180,7 +193,7 @@ func (n *node) FetchBeaconBlockBlobs(ctx context.Context, blockID string) ([]*de return nil, errors.New("client does not implement eth2client.BlobSidecarsProvider") } - rsp, err := provider.BlobSidecars(ctx, &api.BlobSidecarsOpts{ + rsp, err := provider.BlobSidecars(ctx, &eapi.BlobSidecarsOpts{ Block: blockID, }) if err != nil { @@ -198,7 +211,7 @@ func (n *node) FetchProposerDuties(ctx context.Context, epoch phase0.Epoch) ([]* return nil, errors.New("client does not implement eth2client.ProposerDutiesProvider") } - rsp, err := provider.ProposerDuties(ctx, &api.ProposerDutiesOpts{ + rsp, err := provider.ProposerDuties(ctx, &eapi.ProposerDutiesOpts{ Epoch: epoch, }) if err != nil { @@ -214,7 +227,7 @@ func (n *node) FetchForkChoice(ctx context.Context) (*v1.ForkChoice, error) { return nil, errors.New("client does not implement eth2client.ForkChoiceProvider") } - rsp, err := provider.ForkChoice(ctx, &api.ForkChoiceOpts{}) + rsp, err := provider.ForkChoice(ctx, &eapi.ForkChoiceOpts{}) if err != nil { return nil, err } @@ -236,7 +249,7 @@ func (n *node) FetchBeaconStateRoot(ctx context.Context, state string) (phase0.R return phase0.Root{}, errors.New("client does not implement eth2client.StateRootProvider") } - rsp, err := provider.BeaconStateRoot(ctx, &api.BeaconStateRootOpts{ + rsp, err := provider.BeaconStateRoot(ctx, &eapi.BeaconStateRootOpts{ State: state, }) if err != nil { @@ -252,7 +265,7 @@ func (n *node) FetchValidators(ctx context.Context, state string, indices []phas return nil, errors.New("client does not implement eth2client.ValidatorsProvider") } - rsp, err := provider.Validators(ctx, &api.ValidatorsOpts{ + rsp, err := provider.Validators(ctx, &eapi.ValidatorsOpts{ State: state, Indices: indices, PubKeys: pubKeys, @@ -270,7 +283,7 @@ func (n *node) FetchBeaconCommittees(ctx context.Context, state string, epoch *p return nil, errors.New("client does not implement eth2client.BeaconCommitteesProvider") } - opts := &api.BeaconCommitteesOpts{ + opts := &eapi.BeaconCommitteesOpts{ State: state, } @@ -292,7 +305,7 @@ func (n *node) FetchAttestationData(ctx context.Context, slot phase0.Slot, commi return nil, errors.New("client does not implement eth2client.AttestationDataProvider") } - rsp, err := provider.AttestationData(ctx, &api.AttestationDataOpts{ + rsp, err := provider.AttestationData(ctx, &eapi.AttestationDataOpts{ Slot: slot, CommitteeIndex: committeeIndex, }) @@ -303,7 +316,7 @@ func (n *node) FetchAttestationData(ctx context.Context, slot phase0.Slot, commi return rsp.Data, nil } -func (n *node) FetchBeaconBlockHeader(ctx context.Context, opts *api.BeaconBlockHeaderOpts) (*v1.BeaconBlockHeader, error) { +func (n *node) FetchBeaconBlockHeader(ctx context.Context, opts *eapi.BeaconBlockHeaderOpts) (*v1.BeaconBlockHeader, error) { provider, isProvider := n.client.(eth2client.BeaconBlockHeadersProvider) if !isProvider { return nil, errors.New("client does not implement eth2client.BeaconBlockHeadersProvider") diff --git a/pkg/beacon/metrics.go b/pkg/beacon/metrics.go index c33fe7e..89ec1ba 100644 --- a/pkg/beacon/metrics.go +++ b/pkg/beacon/metrics.go @@ -13,6 +13,7 @@ const ( metricLabelVersion = "version" metricLabelEvent = "event" metricLabelFork = "fork" + metricLabelNode = "node" ) // Metrics contains all the metrics jobs. @@ -31,7 +32,7 @@ type MetricsJob interface { // NewMetrics returns a new Metrics instance. func NewMetrics(log logrus.FieldLogger, namespace, nodeName string, beacon Node) *Metrics { constLabels := prometheus.Labels{ - "node": nodeName, + metricLabelNode: nodeName, } beac := NewBeaconMetrics(beacon, log, namespace, constLabels) @@ -41,15 +42,17 @@ func NewMetrics(log logrus.FieldLogger, namespace, nodeName string, beacon Node) spec := NewSpecJob(beacon, log, namespace, constLabels) sync := NewSyncMetrics(beacon, log, namespace, constLabels) health := NewHealthMetrics(beacon, log, namespace, constLabels) + rawResponses := NewRawResponseMetrics(namespace, constLabels) jobs := map[string]MetricsJob{ - sync.Name(): sync, - general.Name(): general, - event.Name(): event, - forks.Name(): forks, - spec.Name(): spec, - health.Name(): health, - beac.Name(): beac, + sync.Name(): sync, + general.Name(): general, + event.Name(): event, + forks.Name(): forks, + spec.Name(): spec, + health.Name(): health, + beac.Name(): beac, + rawResponses.Name(): rawResponses, } m := &Metrics{ @@ -116,3 +119,8 @@ func (m *Metrics) Health() *HealthMetrics { func (m *Metrics) Beacon() *BeaconMetrics { return m.jobs[metricsJobNameBeacon].(*BeaconMetrics) //nolint:errcheck // existing. } + +// RawResponses returns the raw response metrics job. +func (m *Metrics) RawResponses() *RawResponseMetrics { + return m.jobs[metricsJobNameRawResponse].(*RawResponseMetrics) //nolint:errcheck // Constructed with this concrete type. +} diff --git a/pkg/beacon/metrics_raw_response.go b/pkg/beacon/metrics_raw_response.go new file mode 100644 index 0000000..ecf965d --- /dev/null +++ b/pkg/beacon/metrics_raw_response.go @@ -0,0 +1,60 @@ +package beacon + +import ( + "context" + + "github.com/prometheus/client_golang/prometheus" +) + +const metricsJobNameRawResponse = "raw_response" + +// RawResponseMetrics reports the lifecycle of streaming raw responses. +type RawResponseMetrics struct { + OpenStreams prometheus.Gauge + Leaks prometheus.Counter +} + +// NewRawResponseMetrics creates and registers raw response metrics. +func NewRawResponseMetrics(namespace string, constLabels map[string]string) *RawResponseMetrics { + labels := make(prometheus.Labels, len(constLabels)+1) + for name, value := range constLabels { + labels[name] = value + } + + labels["module"] = metricsJobNameRawResponse + + metrics := &RawResponseMetrics{ + OpenStreams: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Name: "raw_response_open_streams", + Help: "Number of raw response streams currently open.", + ConstLabels: labels, + }), + Leaks: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Name: "raw_response_leaks_total", + Help: "Total number of raw response streams closed by cleanup.", + ConstLabels: labels, + }), + } + + prometheus.MustRegister(metrics.OpenStreams) + prometheus.MustRegister(metrics.Leaks) + + return metrics +} + +// Name returns the name of the job. +func (m *RawResponseMetrics) Name() string { + return metricsJobNameRawResponse +} + +// Start starts the job. +func (m *RawResponseMetrics) Start(context.Context) error { + return nil +} + +// Stop stops the job. +func (m *RawResponseMetrics) Stop() error { + return nil +} diff --git a/pkg/beacon/metrics_raw_response_test.go b/pkg/beacon/metrics_raw_response_test.go new file mode 100644 index 0000000..4ac25ab --- /dev/null +++ b/pkg/beacon/metrics_raw_response_test.go @@ -0,0 +1,231 @@ +package beacon + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" +) + +func TestNewRawResponseMetricsRegistersExpectedCollectors(t *testing.T) { + registry := prometheus.NewRegistry() + previousRegisterer := prometheus.DefaultRegisterer + prometheus.DefaultRegisterer = registry + t.Cleanup(func() { + prometheus.DefaultRegisterer = previousRegisterer + }) + + nodeName := "node-a" + metrics := NewRawResponseMetrics("test", prometheus.Labels{metricLabelNode: nodeName}) + metrics.OpenStreams.Inc() + metrics.Leaks.Inc() + + families, err := registry.Gather() + require.NoError(t, err) + + byName := make(map[string]*dto.MetricFamily, len(families)) + for _, family := range families { + byName[family.GetName()] = family + } + + openStreams := byName["test_raw_response_open_streams"] + require.NotNil(t, openStreams) + require.Equal(t, dto.MetricType_GAUGE, openStreams.GetType()) + require.Equal(t, map[string]string{ + "module": metricsJobNameRawResponse, + metricLabelNode: nodeName, + }, metricLabels(openStreams.Metric[0])) + + leaks := byName["test_raw_response_leaks_total"] + require.NotNil(t, leaks) + require.Equal(t, dto.MetricType_COUNTER, leaks.GetType()) + require.Equal(t, map[string]string{ + "module": metricsJobNameRawResponse, + metricLabelNode: nodeName, + }, metricLabels(leaks.Metric[0])) +} + +func TestNodeRawResponseObserverMetrics(t *testing.T) { + rawResponses := &RawResponseMetrics{ + OpenStreams: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "test_raw_response_open_streams", + Help: "Test open streams.", + }), + Leaks: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "test_raw_response_leaks_total", + Help: "Test leaked streams.", + }), + } + n := &node{ + metrics: &Metrics{ + jobs: map[string]MetricsJob{ + metricsJobNameRawResponse: rawResponses, + }, + }, + } + + n.RawResponseOpened() + n.RawResponseOpened() + require.Equal(t, float64(2), metricValue(t, rawResponses.OpenStreams)) + require.Zero(t, metricValue(t, rawResponses.Leaks)) + + n.RawResponseLeaked() + require.Equal(t, float64(2), metricValue(t, rawResponses.OpenStreams)) + require.Equal(t, float64(1), metricValue(t, rawResponses.Leaks)) + + n.RawResponseClosed() + require.Equal(t, float64(1), metricValue(t, rawResponses.OpenStreams)) + + n.RawResponseClosed() + require.Zero(t, metricValue(t, rawResponses.OpenStreams)) + + disabled := &node{} + disabled.RawResponseOpened() + disabled.RawResponseLeaked() + disabled.RawResponseClosed() +} + +func TestNewNodeCopiesRawHTTPClientOptions(t *testing.T) { + apiClientOptions := APIClientOptions{MaxConnsPerHost: 4} + options := Options{APIClient: &apiClientOptions} + + n, ok := NewNode(logrus.New(), &Config{}, "", options).(*node) + require.True(t, ok) + apiClientOptions.MaxConnsPerHost = 8 + + require.Equal(t, 4, n.options.APIClient.MaxConnsPerHost) +} + +func TestInstallClientsClosesUnpublishedRawClientAfterCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + transport := newTrackingTransport(t) + rawHTTPClient := &http.Client{Transport: transport} + n := &node{config: &Config{}} + + err := n.installClients(ctx, nil, &http.Client{}, rawHTTPClient) + require.ErrorIs(t, err, context.Canceled) + require.Nil(t, n.client) + require.Nil(t, n.api) + require.Nil(t, n.rawHTTPClient) + require.Equal(t, int64(1), transport.closeIdleCalls.Load()) +} + +func TestStopClosesPublishedRawClient(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + transport := newTrackingTransport(t) + rawHTTPClient := &http.Client{Transport: transport} + n := &node{ + config: &Config{}, + cancel: cancel, + } + + require.NoError(t, n.installClients(ctx, nil, &http.Client{}, rawHTTPClient)) + require.Same(t, rawHTTPClient, n.rawHTTPClient) + require.NoError(t, n.Stop(t.Context())) + require.Equal(t, int64(1), transport.closeIdleCalls.Load()) + require.ErrorIs(t, ctx.Err(), context.Canceled) +} + +func TestNodeStopClosesOnlyIdleRawHTTPConnections(t *testing.T) { + release := make(chan struct{}) + var releaseOnce sync.Once + releaseStream := func() { + releaseOnce.Do(func() { + close(release) + }) + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + flusher, ok := w.(http.Flusher) + if !ok { + return + } + + _, _ = w.Write([]byte("before")) + flusher.Flush() + + <-release + + _, _ = w.Write([]byte("after")) + })) + t.Cleanup(server.Close) + t.Cleanup(releaseStream) + + transport := newTrackingTransport(t) + client := &http.Client{Transport: transport} + + response, err := client.Get(server.URL) + require.NoError(t, err) + t.Cleanup(func() { + _ = response.Body.Close() + }) + + prefix := make([]byte, len("before")) + _, err = io.ReadFull(response.Body, prefix) + require.NoError(t, err) + require.Equal(t, "before", string(prefix)) + + n := &node{ + options: &Options{}, + rawHTTPClient: client, + } + require.NoError(t, n.Stop(context.Background())) + require.Equal(t, int64(1), transport.closeIdleCalls.Load()) + + releaseStream() + + suffix, err := io.ReadAll(response.Body) + require.NoError(t, err) + require.Equal(t, "after", string(suffix)) +} + +type trackingTransport struct { + *http.Transport + closeIdleCalls atomic.Int64 +} + +func newTrackingTransport(t *testing.T) *trackingTransport { + t.Helper() + + defaultTransport, ok := http.DefaultTransport.(*http.Transport) + require.True(t, ok) + + return &trackingTransport{Transport: defaultTransport.Clone()} +} + +func (t *trackingTransport) CloseIdleConnections() { + t.closeIdleCalls.Add(1) + t.Transport.CloseIdleConnections() +} + +func metricValue(t *testing.T, metric prometheus.Metric) float64 { + t.Helper() + + value := &dto.Metric{} + require.NoError(t, metric.Write(value)) + + if value.Gauge != nil { + return value.Gauge.GetValue() + } + + return value.Counter.GetValue() +} + +func metricLabels(metric *dto.Metric) map[string]string { + labels := make(map[string]string, len(metric.Label)) + for _, label := range metric.Label { + labels[label.GetName()] = label.GetValue() + } + + return labels +} diff --git a/pkg/beacon/options.go b/pkg/beacon/options.go index 60a3f50..e947cc0 100644 --- a/pkg/beacon/options.go +++ b/pkg/beacon/options.go @@ -1,6 +1,11 @@ package beacon import ( + "errors" + "io" + "net" + "net/http" + "sync" "time" "github.com/ethpandaops/beacon/pkg/human" @@ -14,6 +19,7 @@ type Options struct { PrometheusMetrics bool DetectEmptySlots bool GoEth2ClientParams []ehttp.Parameter + APIClient *APIClientOptions } // EnablePrometheusMetrics enables Prometheus metrics. @@ -46,11 +52,14 @@ func (o *Options) DisableEmptySlotDetection() *Options { // DefaultOptions returns the default options. func DefaultOptions() *Options { + apiClient := DefaultAPIClientOptions() + return &Options{ BeaconSubscription: DefaultDisabledBeaconSubscriptionOptions(), HealthCheck: DefaultHealthCheckOptions(), PrometheusMetrics: true, DetectEmptySlots: false, + APIClient: &apiClient, } } @@ -132,6 +141,233 @@ func DefaultHealthCheckOptions() HealthCheckOptions { } } +// APIClientOptions configures buffered and streaming requests to the raw block, +// beacon state, and execution payload envelope endpoints. +// +// It does not configure the go-eth2-client used for typed endpoints or the +// internal JSON client used for peer, identity, and deposit snapshot requests. +// Use AddGoEth2ClientParams for the typed client. Use DefaultAPIClientOptions +// as the base when overriding individual fields. +type APIClientOptions struct { + // AcceptEncoding requests a specific wire encoding for raw responses. Empty + // leaves negotiation to net/http, which may transparently decompress gzip. + AcceptEncoding string + // Timeout bounds the entire request, including reading the response body. + // + // http.Client.Timeout keeps running after the response headers arrive, so it + // also bounds any read the caller performs on a response body it owns. Set it + // to zero to leave the body read governed by the request context. + Timeout human.Duration + // ResponseHeaderTimeout bounds the wait for response headers after the request + // has been written. It never bounds reading the response body. Zero disables + // it. + ResponseHeaderTimeout human.Duration + // DialTimeout bounds establishing the TCP connection. Zero disables it. + DialTimeout human.Duration + // TLSHandshakeTimeout bounds the TLS handshake. Zero disables it. + TLSHandshakeTimeout human.Duration + // MaxIdleConnsPerHost caps idle connections kept per host. Zero uses + // http.DefaultMaxIdleConnsPerHost. + MaxIdleConnsPerHost int + // MaxConnsPerHost limits total connections per host, including connections + // that are dialing, active, or idle. Excess HTTP/1 requests wait for a + // connection; HTTP/2 streams can multiplex on one connection. Zero removes + // the limit. + MaxConnsPerHost int + // MaxConcurrentRequests limits raw requests from transport start until the + // response body is closed. Excess requests wait for a slot, including when + // HTTP/2 multiplexes them on one connection. With no client timeout or + // context deadline, that wait is unbounded. Zero removes the limit. + MaxConcurrentRequests int +} + +// DefaultAPIClientOptions returns bounded defaults for the raw API client. +func DefaultAPIClientOptions() APIClientOptions { + return APIClientOptions{ + Timeout: human.Duration{Duration: 10 * time.Minute}, + DialTimeout: human.Duration{Duration: 30 * time.Second}, + TLSHandshakeTimeout: human.Duration{Duration: 10 * time.Second}, + MaxConnsPerHost: 32, + MaxConcurrentRequests: 32, + } +} + +// SetAPIClientOptions sets the raw API client options. +func (o *Options) SetAPIClientOptions(opts APIClientOptions) *Options { + o.APIClient = &opts + + return o +} + +// SetAPIClientAcceptEncoding sets the wire encoding requested by the raw API +// client while preserving its other options. +func (o *Options) SetAPIClientAcceptEncoding(encoding string) *Options { + if o.APIClient == nil { + opts := DefaultAPIClientOptions() + o.APIClient = &opts + } + + o.APIClient.AcceptEncoding = encoding + + return o +} + +// SetAPIClientTimeout sets the total request timeout for the raw API client. +// +// Pass zero to remove the bound, which is what a caller streaming a large +// response body wants: the read is then governed by the request context rather +// than by a fixed ceiling it cannot see. +func (o *Options) SetAPIClientTimeout(timeout time.Duration) *Options { + if o.APIClient == nil { + opts := DefaultAPIClientOptions() + o.APIClient = &opts + } + + o.APIClient.Timeout = human.Duration{Duration: timeout} + + return o +} + +func (o *Options) apiClientOptions() APIClientOptions { + if o.APIClient == nil { + return DefaultAPIClientOptions() + } + + return *o.APIClient +} + +func (a APIClientOptions) transport() *http.Transport { + return &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: a.DialTimeout.Duration, + KeepAlive: 30 * time.Second, + }).DialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + MaxIdleConnsPerHost: a.MaxIdleConnsPerHost, + MaxConnsPerHost: a.MaxConnsPerHost, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: a.TLSHandshakeTimeout.Duration, + ExpectContinueTimeout: 1 * time.Second, + ResponseHeaderTimeout: a.ResponseHeaderTimeout.Duration, + } +} + +func (a APIClientOptions) httpClient() *http.Client { + transport := a.transport() + + var roundTripper http.RoundTripper = transport + if a.AcceptEncoding != "" { + roundTripper = &acceptEncodingRoundTripper{ + next: roundTripper, + value: a.AcceptEncoding, + } + } + + if a.MaxConcurrentRequests > 0 { + roundTripper = newConcurrencyLimitedRoundTripper(roundTripper, a.MaxConcurrentRequests) + } + + return &http.Client{ + Timeout: a.Timeout.Duration, + Transport: roundTripper, + } +} + +type acceptEncodingRoundTripper struct { + next http.RoundTripper + value string +} + +func (t *acceptEncodingRoundTripper) RoundTrip(request *http.Request) (*http.Response, error) { + request = request.Clone(request.Context()) + request.Header.Set("Accept-Encoding", t.value) + + return t.next.RoundTrip(request) +} + +func (t *acceptEncodingRoundTripper) CloseIdleConnections() { + closeIdleConnections(t.next) +} + +type concurrencyLimitedRoundTripper struct { + next http.RoundTripper + slots chan struct{} +} + +func newConcurrencyLimitedRoundTripper(next http.RoundTripper, limit int) *concurrencyLimitedRoundTripper { + return &concurrencyLimitedRoundTripper{ + next: next, + slots: make(chan struct{}, limit), + } +} + +func (t *concurrencyLimitedRoundTripper) RoundTrip(request *http.Request) (_ *http.Response, err error) { + select { + case t.slots <- struct{}{}: + case <-request.Context().Done(): + return nil, request.Context().Err() + } + + transferred := false + defer func() { + if !transferred { + <-t.slots + } + }() + + response, err := t.next.RoundTrip(request) + if err != nil { + if response != nil && response.Body != nil { + _ = response.Body.Close() + } + + return nil, err + } + + if response == nil || response.Body == nil { + return nil, errors.New("raw HTTP transport returned a nil response body") + } + + response.Body = &concurrencyLimitedBody{ + ReadCloser: response.Body, + release: func() { + <-t.slots + }, + } + transferred = true + + return response, nil +} + +func (t *concurrencyLimitedRoundTripper) CloseIdleConnections() { + closeIdleConnections(t.next) +} + +type concurrencyLimitedBody struct { + io.ReadCloser + once sync.Once + closeErr error + release func() +} + +func (b *concurrencyLimitedBody) Close() error { + b.once.Do(func() { + defer b.release() + + b.closeErr = b.ReadCloser.Close() + }) + + return b.closeErr +} + +func closeIdleConnections(roundTripper http.RoundTripper) { + if transport, ok := roundTripper.(interface{ CloseIdleConnections() }); ok { + transport.CloseIdleConnections() + } +} + // AddGoEth2ClientParams adds the given parameters to the options. func (o *Options) AddGoEth2ClientParams(params ...ehttp.Parameter) *Options { o.GoEth2ClientParams = append(o.GoEth2ClientParams, params...) diff --git a/pkg/beacon/options_test.go b/pkg/beacon/options_test.go new file mode 100644 index 0000000..a6af04f --- /dev/null +++ b/pkg/beacon/options_test.go @@ -0,0 +1,708 @@ +package beacon + +import ( + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/ethpandaops/beacon/pkg/human" +) + +const testAcceptEncodingGzip = "gzip" + +type optionsRoundTripFunc func(*http.Request) (*http.Response, error) + +func (f optionsRoundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) { + return f(request) +} + +func TestDefaultAPIClientOptions(t *testing.T) { + opts := DefaultAPIClientOptions() + client := opts.httpClient() + transport := opts.transport() + + if client.Timeout != 10*time.Minute { + t.Fatalf("timeout = %s, want 10m", client.Timeout) + } + + if transport.DialContext == nil { + t.Fatal("DialContext is nil") + } + + if transport.TLSHandshakeTimeout != 10*time.Second { + t.Fatalf("TLSHandshakeTimeout = %s, want 10s", transport.TLSHandshakeTimeout) + } + + if transport.MaxIdleConnsPerHost != 0 { + t.Fatalf("MaxIdleConnsPerHost = %d, want 0", transport.MaxIdleConnsPerHost) + } + + if transport.MaxConnsPerHost != 32 { + t.Fatalf("MaxConnsPerHost = %d, want 32", transport.MaxConnsPerHost) + } + + limited, ok := client.Transport.(*concurrencyLimitedRoundTripper) + if !ok { + t.Fatalf("Transport type = %T", client.Transport) + } + if cap(limited.slots) != 32 { + t.Fatalf("MaxConcurrentRequests = %d, want 32", cap(limited.slots)) + } +} + +func TestSetAPIClientTimeout(t *testing.T) { + tests := []struct { + name string + set time.Duration + want time.Duration + }{ + {name: "custom", set: 30 * time.Second, want: 30 * time.Second}, + {name: "zero removes the bound so the context governs the read", set: 0, want: 0}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + opts := (&Options{}).SetAPIClientTimeout(tt.set) + + if got := opts.APIClient.httpClient().Timeout; got != tt.want { + t.Fatalf("timeout = %s, want %s", got, tt.want) + } + + if got := opts.APIClient.MaxConnsPerHost; got != 32 { + t.Fatalf("MaxConnsPerHost = %d, want default 32", got) + } + if got := opts.APIClient.MaxConcurrentRequests; got != 32 { + t.Fatalf("MaxConcurrentRequests = %d, want default 32", got) + } + }) + } +} + +func TestSetAPIClientTimeoutPreservesExplicitOptions(t *testing.T) { + opts := (&Options{}). + SetAPIClientOptions(APIClientOptions{}). + SetAPIClientTimeout(time.Second) + + if got := opts.APIClient.Timeout.Duration; got != time.Second { + t.Fatalf("Timeout = %s, want 1s", got) + } + + if got := opts.APIClient.MaxConnsPerHost; got != 0 { + t.Fatalf("MaxConnsPerHost = %d, want explicit zero", got) + } + if got := opts.APIClient.MaxConcurrentRequests; got != 0 { + t.Fatalf("MaxConcurrentRequests = %d, want explicit zero", got) + } +} + +func TestTransportKeepsHTTP2(t *testing.T) { + transport := DefaultAPIClientOptions().transport() + + if !transport.ForceAttemptHTTP2 { + t.Fatal("ForceAttemptHTTP2 is false; a custom DialContext disables HTTP/2 without it") + } +} + +func TestAPIClientAcceptEncoding(t *testing.T) { + const encodedBody = "encoded" + + acceptEncoding := make(chan string, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + acceptEncoding <- req.Header.Get("Accept-Encoding") + w.Header().Set("Content-Encoding", testAcceptEncodingGzip) + _, _ = io.WriteString(w, encodedBody) + })) + t.Cleanup(server.Close) + + client := (APIClientOptions{AcceptEncoding: testAcceptEncodingGzip}).httpClient() + t.Cleanup(client.CloseIdleConnections) + + response, err := client.Get(server.URL) + if err != nil { + t.Fatalf("get response: %v", err) + } + data, readErr := io.ReadAll(response.Body) + closeErr := response.Body.Close() + if readErr != nil { + t.Fatalf("read response: %v", readErr) + } + if closeErr != nil { + t.Fatalf("close response: %v", closeErr) + } + + if got := <-acceptEncoding; got != testAcceptEncodingGzip { + t.Fatalf("Accept-Encoding = %q", got) + } + if response.Uncompressed { + t.Fatal("Uncompressed = true") + } + if got := response.Header.Get("Content-Encoding"); got != testAcceptEncodingGzip { + t.Fatalf("Content-Encoding = %q", got) + } + if string(data) != encodedBody { + t.Fatalf("body = %q", data) + } +} + +func TestSetAPIClientAcceptEncodingPreservesDefaults(t *testing.T) { + opts := (&Options{}).SetAPIClientAcceptEncoding(testAcceptEncodingGzip) + + if got := opts.APIClient.AcceptEncoding; got != testAcceptEncodingGzip { + t.Fatalf("AcceptEncoding = %q", got) + } + if got := opts.APIClient.Timeout.Duration; got != 10*time.Minute { + t.Fatalf("Timeout = %s, want default 10m", got) + } + if got := opts.APIClient.MaxConnsPerHost; got != 32 { + t.Fatalf("MaxConnsPerHost = %d, want default 32", got) + } + if got := opts.APIClient.MaxConcurrentRequests; got != 32 { + t.Fatalf("MaxConcurrentRequests = %d, want default 32", got) + } +} + +func TestSetAPIClientAcceptEncodingPreservesExplicitOptions(t *testing.T) { + opts := (&Options{}). + SetAPIClientOptions(APIClientOptions{}). + SetAPIClientAcceptEncoding(testAcceptEncodingGzip) + + if got := opts.APIClient.AcceptEncoding; got != testAcceptEncodingGzip { + t.Fatalf("AcceptEncoding = %q", got) + } + if got := opts.APIClient.Timeout.Duration; got != 0 { + t.Fatalf("Timeout = %s, want explicit zero", got) + } + if got := opts.APIClient.MaxConnsPerHost; got != 0 { + t.Fatalf("MaxConnsPerHost = %d, want explicit zero", got) + } + if got := opts.APIClient.MaxConcurrentRequests; got != 0 { + t.Fatalf("MaxConcurrentRequests = %d, want explicit zero", got) + } +} + +func TestTransportIsNotShared(t *testing.T) { + opts := DefaultAPIClientOptions() + + if opts.httpClient().Transport == nil { + t.Fatal("transport is nil, so the client would share http.DefaultTransport") + } + + first, second := opts.transport(), opts.transport() + if first == second { + t.Fatal("transport() returned the same instance twice") + } +} + +func TestResponseHeaderTimeoutDoesNotBoundBodyRead(t *testing.T) { + opts := DefaultAPIClientOptions() + opts.ResponseHeaderTimeout = human.Duration{Duration: 15 * time.Second} + opts.Timeout = human.Duration{} + + client := opts.httpClient() + transport := opts.transport() + + if client.Timeout != 0 { + t.Fatalf("client timeout = %s, want 0 so the body read is unbounded", client.Timeout) + } + + if transport.ResponseHeaderTimeout != 15*time.Second { + t.Fatalf("ResponseHeaderTimeout = %s, want 15s", transport.ResponseHeaderTimeout) + } +} + +func TestAPIClientOptionsFieldPropagation(t *testing.T) { + opts := APIClientOptions{ + Timeout: human.Duration{Duration: time.Second}, + ResponseHeaderTimeout: human.Duration{Duration: 2 * time.Second}, + TLSHandshakeTimeout: human.Duration{Duration: 3 * time.Second}, + MaxIdleConnsPerHost: 4, + MaxConnsPerHost: 5, + MaxConcurrentRequests: 6, + } + client := opts.httpClient() + limited, ok := client.Transport.(*concurrencyLimitedRoundTripper) + if !ok { + t.Fatalf("Transport type = %T", client.Transport) + } + transport, ok := limited.next.(*http.Transport) + if !ok { + t.Fatalf("nested Transport type = %T", limited.next) + } + + if client.Timeout != time.Second { + t.Fatalf("Timeout = %s, want 1s", client.Timeout) + } + + if transport.ResponseHeaderTimeout != 2*time.Second { + t.Fatalf("ResponseHeaderTimeout = %s, want 2s", transport.ResponseHeaderTimeout) + } + + if transport.TLSHandshakeTimeout != 3*time.Second { + t.Fatalf("TLSHandshakeTimeout = %s, want 3s", transport.TLSHandshakeTimeout) + } + + if transport.MaxIdleConnsPerHost != 4 { + t.Fatalf("MaxIdleConnsPerHost = %d, want 4", transport.MaxIdleConnsPerHost) + } + + if transport.MaxConnsPerHost != 5 { + t.Fatalf("MaxConnsPerHost = %d, want 5", transport.MaxConnsPerHost) + } + + if got := cap(limited.slots); got != 6 { + t.Fatalf("MaxConcurrentRequests = %d, want 6", got) + } +} + +func TestDefaultOptionsStoresAPIClientDefaults(t *testing.T) { + opts := DefaultOptions() + + if opts.APIClient == nil { + t.Fatal("APIClient is nil") + } + + if *opts.APIClient != DefaultAPIClientOptions() { + t.Fatalf("APIClient = %+v, want %+v", *opts.APIClient, DefaultAPIClientOptions()) + } +} + +func TestSetAPIClientOptionsStoresCopy(t *testing.T) { + apiOpts := APIClientOptions{ + AcceptEncoding: testAcceptEncodingGzip, + Timeout: human.Duration{Duration: time.Second}, + ResponseHeaderTimeout: human.Duration{Duration: 2 * time.Second}, + DialTimeout: human.Duration{Duration: 3 * time.Second}, + TLSHandshakeTimeout: human.Duration{Duration: 4 * time.Second}, + MaxIdleConnsPerHost: 5, + MaxConnsPerHost: 6, + MaxConcurrentRequests: 7, + } + opts := (&Options{}).SetAPIClientOptions(apiOpts) + apiOpts.AcceptEncoding = "" + apiOpts.Timeout = human.Duration{} + apiOpts.MaxConnsPerHost = 0 + apiOpts.MaxConcurrentRequests = 0 + + if opts.APIClient == nil { + t.Fatal("APIClient is nil") + } + + if got := opts.APIClient.AcceptEncoding; got != testAcceptEncodingGzip { + t.Fatalf("AcceptEncoding = %q", got) + } + + if got := opts.APIClient.Timeout.Duration; got != time.Second { + t.Fatalf("Timeout = %s, want 1s", got) + } + + if got := opts.APIClient.ResponseHeaderTimeout.Duration; got != 2*time.Second { + t.Fatalf("ResponseHeaderTimeout = %s, want 2s", got) + } + + if got := opts.APIClient.DialTimeout.Duration; got != 3*time.Second { + t.Fatalf("DialTimeout = %s, want 3s", got) + } + + if got := opts.APIClient.TLSHandshakeTimeout.Duration; got != 4*time.Second { + t.Fatalf("TLSHandshakeTimeout = %s, want 4s", got) + } + + if got := opts.APIClient.MaxIdleConnsPerHost; got != 5 { + t.Fatalf("MaxIdleConnsPerHost = %d, want 5", got) + } + + if got := opts.APIClient.MaxConnsPerHost; got != 6 { + t.Fatalf("MaxConnsPerHost = %d, want 6", got) + } + + if got := opts.APIClient.MaxConcurrentRequests; got != 7 { + t.Fatalf("MaxConcurrentRequests = %d, want 7", got) + } +} + +func TestNilAPIClientOptionsUseDefaults(t *testing.T) { + opts := (&Options{}).apiClientOptions() + + if opts != DefaultAPIClientOptions() { + t.Fatalf("options = %+v, want %+v", opts, DefaultAPIClientOptions()) + } +} + +func TestExplicitZeroAPIClientOptionsSurvive(t *testing.T) { + options := (&Options{}).SetAPIClientOptions(APIClientOptions{}) + opts := options.apiClientOptions() + client := opts.httpClient() + transport := opts.transport() + + if client.Timeout != 0 { + t.Fatalf("Timeout = %s, want 0", client.Timeout) + } + + if transport.DialContext == nil { + t.Fatal("DialContext is nil") + } + + if transport.TLSHandshakeTimeout != 0 { + t.Fatalf("TLSHandshakeTimeout = %s, want 0", transport.TLSHandshakeTimeout) + } + + if transport.MaxIdleConnsPerHost != 0 { + t.Fatalf("MaxIdleConnsPerHost = %d, want 0", transport.MaxIdleConnsPerHost) + } + + if transport.MaxConnsPerHost != 0 { + t.Fatalf("MaxConnsPerHost = %d, want 0", transport.MaxConnsPerHost) + } + + if opts.MaxConcurrentRequests != 0 { + t.Fatalf("MaxConcurrentRequests = %d, want 0", opts.MaxConcurrentRequests) + } + + if _, ok := client.Transport.(*http.Transport); !ok { + t.Fatalf("Transport type = %T, want *http.Transport", client.Transport) + } +} + +func TestMaxConnsPerHostCapsHTTP1Connections(t *testing.T) { + const maxConns = 2 + + entered := make(chan struct{}, 3) + release := make(chan struct{}) + var releaseOnce sync.Once + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + entered <- struct{}{} + <-release + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + defer releaseOnce.Do(func() { close(release) }) + + opts := APIClientOptions{MaxConnsPerHost: maxConns} + client := opts.httpClient() + defer client.CloseIdleConnections() + + results := make(chan error, 3) + for range 3 { + go func() { + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, server.URL, nil) + if err != nil { + results <- err + + return + } + + response, err := client.Do(req) + if err == nil { + err = response.Body.Close() + } + + results <- err + }() + } + + for range maxConns { + select { + case <-entered: + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for request to enter handler") + } + } + + select { + case <-entered: + t.Fatalf("more than %d requests entered before a connection was released", maxConns) + case <-time.After(100 * time.Millisecond): + } + + releaseOnce.Do(func() { close(release) }) + + select { + case <-entered: + case <-time.After(2 * time.Second): + t.Fatal("queued request did not enter after connections were released") + } + + for range 3 { + select { + case err := <-results: + if err != nil { + t.Fatalf("request failed: %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for request result") + } + } +} + +func TestMaxConcurrentRequestsCapsHTTP2Streams(t *testing.T) { + const ( + maxRequests = 2 + totalRequests = 3 + ) + + entered := make(chan struct{}, totalRequests) + release := make(chan struct{}) + var releaseOnce sync.Once + + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.ProtoMajor != 2 { + t.Errorf("protocol = %s", req.Proto) + } + + w.WriteHeader(http.StatusOK) + if err := http.NewResponseController(w).Flush(); err != nil { + t.Errorf("flush response: %v", err) + + return + } + + entered <- struct{}{} + <-release + })) + server.EnableHTTP2 = true + server.StartTLS() + t.Cleanup(server.Close) + t.Cleanup(func() { + releaseOnce.Do(func() { close(release) }) + }) + + transport := newConcurrencyLimitedRoundTripper(server.Client().Transport, maxRequests) + client := &http.Client{Transport: transport} + t.Cleanup(client.CloseIdleConnections) + + type result struct { + response *http.Response + err error + } + results := make(chan result, totalRequests) + + for range totalRequests { + go func() { + response, err := client.Get(server.URL) //nolint:bodyclose // The receiving goroutine closes every successful response. + results <- result{response: response, err: err} + }() + } + + for range maxRequests { + select { + case <-entered: + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for request to enter handler") + } + } + + responses := make([]*http.Response, 0, totalRequests) + trackResponse := func(response *http.Response) { + responses = append(responses, response) //nolint:bodyclose // Registered for cleanup immediately below. + t.Cleanup(func() { + _ = response.Body.Close() + }) + } + + for range maxRequests { + select { + case result := <-results: + if result.err != nil { + t.Fatalf("request failed: %v", result.err) + } + trackResponse(result.response) + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for response headers") + } + } + + select { + case <-entered: + t.Fatalf("more than %d HTTP/2 streams entered before a body was closed", maxRequests) + case <-time.After(100 * time.Millisecond): + } + + if err := responses[0].Body.Close(); err != nil { + t.Fatalf("close first response: %v", err) + } + + select { + case <-entered: + case <-time.After(2 * time.Second): + t.Fatal("queued HTTP/2 request did not enter after a body was closed") + } + + select { + case result := <-results: + if result.err != nil { + t.Fatalf("queued request failed: %v", result.err) + } + trackResponse(result.response) + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for queued response") + } + + releaseOnce.Do(func() { close(release) }) + + for _, response := range responses[1:] { + if err := response.Body.Close(); err != nil { + t.Fatalf("close response: %v", err) + } + } +} + +func TestConfiguredClientTimeoutReleasesConcurrentRequestSlot(t *testing.T) { + var requests atomic.Int32 + firstDone := make(chan struct{}) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if got := req.Header.Get("Accept-Encoding"); got != "identity" { + t.Errorf("Accept-Encoding = %q", got) + } + + if requests.Add(1) == 1 { + w.WriteHeader(http.StatusOK) + if err := http.NewResponseController(w).Flush(); err != nil { + t.Errorf("flush response: %v", err) + + return + } + + <-req.Context().Done() + close(firstDone) + + return + } + + _, _ = io.WriteString(w, "ok") + })) + t.Cleanup(server.Close) + + opts := DefaultAPIClientOptions() + opts.AcceptEncoding = "identity" + opts.Timeout = human.Duration{Duration: 100 * time.Millisecond} + opts.MaxConcurrentRequests = 1 + + client := opts.httpClient() + t.Cleanup(client.CloseIdleConnections) + + limited, ok := client.Transport.(*concurrencyLimitedRoundTripper) + if !ok { + t.Fatalf("Transport type = %T", client.Transport) + } + + response, err := client.Get(server.URL) + if err != nil { + t.Fatalf("get first response: %v", err) + } + + _, readErr := io.ReadAll(response.Body) + closeErr := response.Body.Close() + if !errors.Is(readErr, context.DeadlineExceeded) { + t.Fatalf("read error = %v, want context.DeadlineExceeded", readErr) + } + if closeErr != nil { + t.Fatalf("close first response: %v", closeErr) + } + if got := len(limited.slots); got != 0 { + t.Fatalf("occupied slots after timeout = %d, want 0", got) + } + + select { + case <-firstDone: + case <-time.After(2 * time.Second): + t.Fatal("first handler did not observe timeout") + } + + response, err = client.Get(server.URL) + if err != nil { + t.Fatalf("get second response: %v", err) + } + data, readErr := io.ReadAll(response.Body) + closeErr = response.Body.Close() + if readErr != nil { + t.Fatalf("read second response: %v", readErr) + } + if closeErr != nil { + t.Fatalf("close second response: %v", closeErr) + } + if string(data) != "ok" { + t.Fatalf("second response = %q", data) + } +} + +func TestConcurrentRequestLimitHonorsQueuedContext(t *testing.T) { + transport := newConcurrencyLimitedRoundTripper(optionsRoundTripFunc(func(*http.Request) (*http.Response, error) { + return nil, errors.New("nested transport was called") + }), 1) + transport.slots <- struct{}{} + t.Cleanup(func() { <-transport.slots }) + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + request, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://beacon.test", nil) + if err != nil { + t.Fatalf("create request: %v", err) + } + + response, err := transport.RoundTrip(request) //nolint:bodyclose // A canceled queued request cannot return a response. + if response != nil { + t.Fatal("response is non-nil") + } + if !errors.Is(err, context.Canceled) { + t.Fatalf("error = %v, want context.Canceled", err) + } + if got := len(transport.slots); got != 1 { + t.Fatalf("occupied slots = %d, want 1", got) + } +} + +func TestConcurrentRequestLimitReleasesSlotOnTransportFailure(t *testing.T) { + transportErr := errors.New("transport failed") + transport := newConcurrencyLimitedRoundTripper(optionsRoundTripFunc(func(*http.Request) (*http.Response, error) { + return nil, transportErr + }), 1) + request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "http://beacon.test", nil) + if err != nil { + t.Fatalf("create request: %v", err) + } + + response, err := transport.RoundTrip(request) //nolint:bodyclose // The failing transport returns no response. + if response != nil { + t.Fatal("response is non-nil") + } + if !errors.Is(err, transportErr) { + t.Fatalf("error = %v", err) + } + if got := len(transport.slots); got != 0 { + t.Fatalf("occupied slots = %d, want 0", got) + } +} + +func TestConcurrentRequestLimitReleasesSlotOnTransportPanic(t *testing.T) { + transport := newConcurrencyLimitedRoundTripper(optionsRoundTripFunc(func(*http.Request) (*http.Response, error) { + panic("transport panic") + }), 1) + request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "http://beacon.test", nil) + if err != nil { + t.Fatalf("create request: %v", err) + } + + var recovered any + func() { + defer func() { + recovered = recover() + }() + + _, _ = transport.RoundTrip(request) //nolint:bodyclose // The transport panics before returning a response. + }() + + if recovered == nil { + t.Fatal("transport panic was not propagated") + } + if got := len(transport.slots); got != 0 { + t.Fatalf("occupied slots = %d, want 0", got) + } +} diff --git a/pkg/beacon/raw_response_test.go b/pkg/beacon/raw_response_test.go new file mode 100644 index 0000000..4edcf39 --- /dev/null +++ b/pkg/beacon/raw_response_test.go @@ -0,0 +1,135 @@ +package beacon + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strconv" + "sync/atomic" + "testing" + + beaconapi "github.com/ethpandaops/beacon/pkg/beacon/api" + ehttp "github.com/ethpandaops/go-eth2-client/http" + "github.com/sirupsen/logrus" +) + +func TestNodeOpenRawEndpointsThroughBootstrapClient(t *testing.T) { + const ( + stateBody = "state-response" + blockBody = "block-response" + envelopeBody = "envelope-response" + ) + + responses := map[string]string{ + "/eth/v2/debug/beacon/states/finalized": stateBody, + "/eth/v2/beacon/blocks/head": blockBody, + "/eth/v1/beacon/execution_payload_envelopes/0x12345678": envelopeBody, + } + var requests atomic.Int32 + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + body, ok := responses[req.URL.Path] + if !ok { + http.NotFound(w, req) + + return + } + if got := req.Header.Get("Accept"); got != "application/octet-stream" { + t.Errorf("Accept = %q", got) + } + if got := req.Header.Get("X-Test-Header"); got != "present" { + t.Errorf("X-Test-Header = %q", got) + } + + requests.Add(1) + w.Header().Set("Content-Type", "application/octet-stream;charset=utf-8") + w.Header().Set("Content-Length", strconv.Itoa(len(body))) + w.Header().Set("Eth-Consensus-Version", "gloas") + _, _ = io.WriteString(w, body) + })) + t.Cleanup(server.Close) + + log := logrus.New() + log.SetOutput(io.Discard) + publicNode := NewNode(log, &Config{ + Addr: server.URL, + Headers: map[string]string{ + "X-Test-Header": "present", + }, + }, "", Options{ + GoEth2ClientParams: []ehttp.Parameter{ehttp.WithAllowDelayedStart(true)}, + }) + implementation, ok := publicNode.(*node) + if !ok { + t.Fatalf("node type = %T", publicNode) + } + + if err := implementation.ensureClients(t.Context()); err != nil { + t.Fatalf("ensure clients: %v", err) + } + t.Cleanup(func() { + if err := implementation.Stop(t.Context()); err != nil { + t.Errorf("stop node: %v", err) + } + }) + + tests := []struct { + name string + body string + open func(context.Context, Node) (*beaconapi.RawResponse, error) + }{ + { + name: "beacon state", + body: stateBody, + open: func(ctx context.Context, node Node) (*beaconapi.RawResponse, error) { + return node.OpenRawBeaconState(ctx, "finalized", "application/octet-stream") + }, + }, + { + name: "block", + body: blockBody, + open: func(ctx context.Context, node Node) (*beaconapi.RawResponse, error) { + return node.OpenRawBlock(ctx, "head", "application/octet-stream") + }, + }, + { + name: "execution payload envelope", + body: envelopeBody, + open: func(ctx context.Context, node Node) (*beaconapi.RawResponse, error) { + return node.OpenRawExecutionPayloadEnvelope(ctx, "0x12345678", "application/octet-stream") + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + response, err := test.open(t.Context(), publicNode) + if err != nil { + t.Fatalf("open response: %v", err) + } + + body, readErr := io.ReadAll(response) + closeErr := response.Close() + if readErr != nil { + t.Fatalf("read response: %v", readErr) + } + if closeErr != nil { + t.Fatalf("close response: %v", closeErr) + } + if string(body) != test.body { + t.Fatalf("body = %q, want %q", body, test.body) + } + if response.ContentLength != int64(len(test.body)) { + t.Fatalf("ContentLength = %d", response.ContentLength) + } + if got := response.Header.Get("Eth-Consensus-Version"); got != "gloas" { + t.Fatalf("Eth-Consensus-Version = %q", got) + } + }) + } + + if got := requests.Load(); got != int32(len(tests)) { + t.Fatalf("requests = %d, want %d", got, len(tests)) + } +}