From ee65a4c35aca25a610501622763d668cf9320e41 Mon Sep 17 00:00:00 2001 From: Andrew Davis <1709934+Savid@users.noreply.github.com> Date: Thu, 22 Jan 2026 15:25:31 +1000 Subject: [PATCH 1/5] refactor(clickhouse): migrate from HTTP to ch-go native protocol Switch ClickHouse client from HTTP JSON interface to ch-go native binary protocol for improved performance and memory efficiency. ClickHouse Client Changes: - Replace http.Client with ch-go connection pool (chpool.Pool) - Add Do() and Acquire() methods for native query execution - Update config to support native protocol options (compression, pool settings) - Change connectivity test from HTTP query to native Ping() - Add retry logic with exponential backoff for transient failures - Use chpool.Dial() for fail-fast connection validation - Add Start() idempotency guard to prevent goroutine leaks - Add configurable MaxRetries and RetryBaseDelay settings Structlog Processor Simplification: - Remove batch_manager.go - channel-based batching no longer needed - Remove big_transaction_manager.go - handled by ch-go streaming - Remove clickhouse_time.go - use standard time handling - Add columns.go for ch-go columnar data preparation - Rewrite transaction processing to use ch-go OnInput streaming - Simplify from producer/consumer pattern to direct columnar inserts Testing Infrastructure: - Add miniredis for in-memory Redis unit tests (no Docker required) - Add testcontainers for Redis/ClickHouse integration tests - Create internal/testutil package with test helpers - Replace skipIfRedisUnavailable() pattern with miniredis - Split CI workflow into unit-tests and integration-tests jobs Benefits: - Native binary protocol is faster than HTTP JSON - Columnar streaming reduces memory allocations - Simplified code with fewer goroutines and channels - Tests always run (no skip patterns) with proper isolation - Improved resilience with automatic retry for transient errors Co-Authored-By: Claude Opus 4.5 --- .github/workflows/go-test.yml | 22 +- go.mod | 71 +- go.sum | 186 ++++- internal/testutil/testutil.go | 125 ++++ pkg/clickhouse/client.go | 703 +++++++++++------- pkg/clickhouse/client_integration_test.go | 149 ++++ pkg/clickhouse/client_test.go | 629 ++++++++++------ pkg/clickhouse/config.go | 87 ++- pkg/clickhouse/interface.go | 17 +- pkg/clickhouse/mock.go | 97 ++- pkg/common/metrics.go | 53 ++ pkg/leaderelection/redis_election_test.go | 90 +-- pkg/processor/config.go | 4 +- pkg/processor/manager_test.go | 94 +-- pkg/processor/transaction/simple/columns.go | 196 +++++ pkg/processor/transaction/simple/handlers.go | 76 +- pkg/processor/transaction/simple/processor.go | 2 +- .../transaction/structlog/batch_manager.go | 225 ------ .../structlog/batch_manager_test.go | 377 ---------- .../structlog/big_transaction_manager.go | 69 -- .../transaction/structlog/clickhouse_time.go | 43 -- .../transaction/structlog/columns.go | 155 ++++ pkg/processor/transaction/structlog/config.go | 38 +- .../transaction/structlog/handlers.go | 60 +- .../transaction/structlog/processor.go | 117 +-- .../transaction/structlog/processor_test.go | 247 +++--- .../structlog/transaction_processing.go | 301 ++------ pkg/state/manager.go | 4 +- pkg/state/manager_test.go | 13 +- 29 files changed, 2235 insertions(+), 2015 deletions(-) create mode 100644 internal/testutil/testutil.go create mode 100644 pkg/clickhouse/client_integration_test.go create mode 100644 pkg/processor/transaction/simple/columns.go delete mode 100644 pkg/processor/transaction/structlog/batch_manager.go delete mode 100644 pkg/processor/transaction/structlog/batch_manager_test.go delete mode 100644 pkg/processor/transaction/structlog/big_transaction_manager.go delete mode 100644 pkg/processor/transaction/structlog/clickhouse_time.go create mode 100644 pkg/processor/transaction/structlog/columns.go diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml index bd32b43..4547fb2 100644 --- a/.github/workflows/go-test.yml +++ b/.github/workflows/go-test.yml @@ -11,14 +11,15 @@ permissions: contents: read jobs: - test: + unit-tests: + name: Unit Tests runs-on: ubuntu-latest steps: - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - uses: ./.github/workflows/go-setup - - name: run tests + - name: Run unit tests run: go test -race -json ./... > test.json - name: Annotate tests @@ -26,3 +27,20 @@ jobs: uses: guyarb/golang-test-annotations@9ab2ea84a399d03ffd114bf49dd23ffadc794541 # v0.6.0 with: test-results: test.json + + integration-tests: + name: Integration Tests + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + - uses: ./.github/workflows/go-setup + + - name: Run integration tests + run: go test -race -tags=integration -json ./... > test.json + + - name: Annotate tests + if: always() + uses: guyarb/golang-test-annotations@9ab2ea84a399d03ffd114bf49dd23ffadc794541 # v0.6.0 + with: + test-results: test.json diff --git a/go.mod b/go.mod index bf07bc0..dc38a0d 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,8 @@ module github.com/ethpandaops/execution-processor go 1.25.4 require ( + github.com/ClickHouse/ch-go v0.69.0 + github.com/alicebob/miniredis/v2 v2.36.0 github.com/cenkalti/backoff/v4 v4.3.0 github.com/creasty/defaults v1.8.0 github.com/ethereum/go-ethereum v1.16.7 @@ -14,50 +16,105 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.10.1 github.com/stretchr/testify v1.11.1 - golang.org/x/sync v0.17.0 + github.com/testcontainers/testcontainers-go v0.40.0 + github.com/testcontainers/testcontainers-go/modules/clickhouse v0.40.0 + github.com/testcontainers/testcontainers-go/modules/redis v0.40.0 + golang.org/x/sync v0.18.0 gopkg.in/yaml.v3 v3.0.1 ) require ( + dario.cat/mergo v1.0.2 // indirect + github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/ProjectZKM/Ziren/crates/go-runtime/zkvm_runtime v0.0.0-20251001021608-1fe7b43fc4d6 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.24.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/consensys/gnark-crypto v0.19.0 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/platforms v0.2.1 // indirect + github.com/cpuguy83/dockercfg v0.3.2 // indirect github.com/crate-crypto/go-eth-kzg v1.4.0 // indirect github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/deckarep/golang-set/v2 v2.8.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/dmarkham/enumer v1.6.1 // indirect + github.com/docker/docker v28.5.1+incompatible // indirect + github.com/docker/go-connections v0.6.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/ebitengine/purego v0.8.4 // indirect github.com/ethereum/c-kzg-4844/v2 v2.1.5 // indirect github.com/ethereum/go-verkle v0.2.2 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-faster/city v1.0.1 // indirect + github.com/go-faster/errors v0.7.1 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.5.3 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.5 // indirect + github.com/hashicorp/go-version v1.7.0 // indirect github.com/holiman/uint256 v1.3.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/klauspost/compress v1.17.11 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/klauspost/compress v1.18.0 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 // indirect + github.com/magiconair/properties v1.8.10 // indirect + github.com/mdelapenya/tlscert v0.2.0 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/go-archive v0.1.0 // indirect + github.com/moby/patternmatcher v0.6.0 // indirect + github.com/moby/sys/sequential v0.6.0 // indirect + github.com/moby/sys/user v0.4.0 // indirect + github.com/moby/sys/userns v0.1.0 // indirect + github.com/moby/term v0.5.2 // indirect + github.com/morikuni/aec v1.0.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect + github.com/pascaldekloe/name v1.0.1 // indirect + github.com/pierrec/lz4/v4 v4.1.22 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.64.0 // indirect github.com/prometheus/procfs v0.16.1 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect + github.com/segmentio/asm v1.2.1 // indirect github.com/shirou/gopsutil v3.21.11+incompatible // indirect + github.com/shirou/gopsutil/v4 v4.25.6 // indirect github.com/spf13/cast v1.7.0 // indirect github.com/spf13/pflag v1.0.10 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/supranational/blst v0.3.16 // indirect - github.com/tklauser/go-sysconf v0.3.12 // indirect - github.com/tklauser/numcpus v0.6.1 // indirect - github.com/yusufpapurcu/wmi v1.2.3 // indirect + github.com/tklauser/go-sysconf v0.3.15 // indirect + github.com/tklauser/numcpus v0.10.0 // indirect + github.com/yuin/gopher-lua v1.1.1 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect + go.opentelemetry.io/otel v1.38.0 // indirect + go.opentelemetry.io/otel/metric v1.38.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.38.0 // indirect + go.opentelemetry.io/otel/trace v1.38.0 // indirect go.uber.org/atomic v1.11.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect golang.org/x/crypto v0.43.0 // indirect golang.org/x/exp v0.0.0-20251009144603-d2f985daa21b // indirect - golang.org/x/sys v0.37.0 // indirect + golang.org/x/mod v0.29.0 // indirect + golang.org/x/sys v0.38.0 // indirect golang.org/x/time v0.9.0 // indirect - google.golang.org/protobuf v1.36.6 // indirect + golang.org/x/tools v0.38.0 // indirect + google.golang.org/grpc v1.78.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect ) diff --git a/go.sum b/go.sum index 101195b..c9e71e0 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,13 @@ +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/ClickHouse/ch-go v0.69.0 h1:nO0OJkpxOlN/eaXFj0KzjTz5p7vwP1/y3GN4qc5z/iM= +github.com/ClickHouse/ch-go v0.69.0/go.mod h1:9XeZpSAT4S0kVjOpaJ5186b7PY/NH/hhF8R6u0WIjwg= +github.com/ClickHouse/clickhouse-go/v2 v2.34.0 h1:Y4rqkdrRHgExvC4o/NTbLdY5LFQ3LHS77/RNFxFX3Co= +github.com/ClickHouse/clickhouse-go/v2 v2.34.0/go.mod h1:yioSINoRLVZkLyDzdMXPLRIqhDvel8iLBlwh6Iefso8= github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= @@ -6,6 +16,10 @@ github.com/ProjectZKM/Ziren/crates/go-runtime/zkvm_runtime v0.0.0-20251001021608 github.com/ProjectZKM/Ziren/crates/go-runtime/zkvm_runtime v0.0.0-20251001021608-1fe7b43fc4d6/go.mod h1:ioLG6R+5bUSO1oeGSDxOV3FADARuMoytZCSX6MEMQkI= github.com/VictoriaMetrics/fastcache v1.13.0 h1:AW4mheMR5Vd9FkAPUv+NH6Nhw+fmbTMGMsNAoA/+4G0= github.com/VictoriaMetrics/fastcache v1.13.0/go.mod h1:hHXhl4DA2fTL2HTZDJFXWgW0LNjo6B+4aj2Wmng3TjU= +github.com/alicebob/miniredis/v2 v2.36.0 h1:yKczg+ez0bQYsG/PrgqtMMmCfl820RPu27kVGjP53eY= +github.com/alicebob/miniredis/v2 v2.36.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bits-and-blooms/bitset v1.24.0 h1:H4x4TuulnokZKvHLfzVRTHJfFfnHEeSYJizujEZvmAM= @@ -32,6 +46,16 @@ github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAK github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= github.com/consensys/gnark-crypto v0.19.0 h1:zXCqeY2txSaMl6G5wFpZzMWJU9HPNh8qxPnYJ1BL9vA= github.com/consensys/gnark-crypto v0.19.0/go.mod h1:rT23F0XSZqE0mUA0+pRtnL56IbPxs6gp4CeRsBk4XS0= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= +github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/crate-crypto/go-eth-kzg v1.4.0 h1:WzDGjHk4gFg6YzV0rJOAsTK4z3Qkz5jd4RE3DAvPFkg= @@ -39,6 +63,8 @@ github.com/crate-crypto/go-eth-kzg v1.4.0/go.mod h1:J9/u5sWfznSObptgfa92Jq8rTswn github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a h1:W8mUrRp6NOVl3J+MYp5kPMoUZPp7aOYHtaua31lwRHg= github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a/go.mod h1:sTwzHBvIzm2RfVCGNEBZgRyjwK40bVoun3ZnGOCafNM= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= +github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/creasty/defaults v1.8.0 h1:z27FJxCAa0JKt3utc0sCImAEb+spPucmKoOdLHvHYKk= github.com/creasty/defaults v1.8.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -54,6 +80,18 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 h1:NMZiJj8QnKe1LgsbDayM4UoHwbvw github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/dmarkham/enumer v1.6.1 h1:aSc9awYtZL07TUueWs40QcHtxTvHTAwG0EqrNsK45w4= +github.com/dmarkham/enumer v1.6.1/go.mod h1:yixql+kDDQRYqcuBM2n9Vlt7NoT9ixgXhaXry8vmRg8= +github.com/docker/docker v28.5.1+incompatible h1:Bm8DchhSD2J6PsFzxC35TZo4TLGR2PdW/E69rU45NhM= +github.com/docker/docker v28.5.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= +github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/ebitengine/purego v0.8.4 h1:CF7LEKg5FFOsASUj0+QwaXf8Ht6TlFxg09+S9wz0omw= +github.com/ebitengine/purego v0.8.4/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/emicklei/dot v1.6.2 h1:08GN+DD79cy/tzN6uLCT84+2Wk9u+wvqP+Hkx/dIR8A= github.com/emicklei/dot v1.6.2/go.mod h1:DeV7GvQtIw4h2u73RKBkkFdvVAz0D9fzeJrgPW6gy/s= github.com/ethereum/c-kzg-4844/v2 v2.1.5 h1:aVtoLK5xwJ6c5RiqO8g8ptJ5KU+2Hdquf6G3aXiHh5s= @@ -64,6 +102,8 @@ github.com/ethereum/go-ethereum v1.16.7 h1:qeM4TvbrWK0UC0tgkZ7NiRsmBGwsjqc64BHo2 github.com/ethereum/go-ethereum v1.16.7/go.mod h1:Fs6QebQbavneQTYcA39PEKv2+zIjX7rPUZ14DER46wk= github.com/ethereum/go-verkle v0.2.2 h1:I2W0WjnrFUIzzVPwm8ykY+7pL2d4VhlsePn4j7cnFk8= github.com/ethereum/go-verkle v0.2.2/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/ferranbt/fastssz v0.1.4 h1:OCDB+dYDEQDvAgtAGnTSidK1Pe2tW3nFV40XyMkTeDY= github.com/ferranbt/fastssz v0.1.4/go.mod h1:Ea3+oeoRGGLGm5shYAeDgu6PGUlcvQhE2fILyD9+tGg= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= @@ -72,6 +112,15 @@ github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/go-co-op/gocron v1.37.0 h1:ZYDJGtQ4OMhTLKOKMIch+/CY70Brbb1dGdooLEhh7b0= github.com/go-co-op/gocron v1.37.0/go.mod h1:3L/n6BkO7ABj+TrfSVXLRzsP26zmikL4ISkLQ0O8iNY= +github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= +github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw= +github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg= +github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= @@ -90,8 +139,12 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.5 h1:jP1RStw811EvUDzsUQ9oESqw2e4RqCjSAD9qIL8eMns= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.5/go.mod h1:WXNBZ64q3+ZUemCMXD9kYnr56H7CgZxDBHCVwstfl3s= github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE= github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0= +github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= +github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hibiken/asynq v0.25.1 h1:phj028N0nm15n8O2ims+IvJ2gz4k2auvermngh9JhTw= github.com/hibiken/asynq v0.25.1/go.mod h1:pazWNOLBu0FEynQRBvHA26qdIKRSmfdIfUm4HdsLmXg= github.com/holiman/billy v0.0.0-20250707135307-f2f9b9aae7db h1:IZUYC/xb3giYwBLMnr8d0TGTzPKFGNTCGgGLoyeX330= @@ -104,10 +157,12 @@ github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= -github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= -github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -125,22 +180,56 @@ github.com/leanovate/gopter v0.2.11 h1:vRjThO1EKPb/1NsDXuDrzldR28RLkBflWYcU9CvzW github.com/leanovate/gopter v0.2.11/go.mod h1:aK3tzZP/C+p1m3SPRE4SYZFGP7jjkuSI4f7Xvpt0S9c= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 h1:PpXWgLPs+Fqr325bN2FD2ISlRRztXibcX6e8f5FR5Dc= +github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= +github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= +github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5LJHI= +github.com/mdelapenya/tlscert v0.2.0/go.mod h1:O4njj3ELLnJjGdkN7M/vIVCpZ+Cf0L6muqOG4tLSl8o= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/pointerstructure v1.2.0 h1:O+i9nHnXS3l/9Wu7r4NrEdwA2VFTicjUEN1uBnDo34A= github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/go-archive v0.1.0 h1:Kk/5rdW/g+H8NHdJW2gsXyZ7UnzvJNOy6VKJqueWdcQ= +github.com/moby/go-archive v0.1.0/go.mod h1:G9B+YoujNohJmrIYFBpSd54GTUB4lt9S+xVQvsJyFuo= +github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= +github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= +github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= +github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= +github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= +github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= +github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= +github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= +github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/pascaldekloe/name v1.0.1 h1:9lnXOHeqeHHnWLbKfH6X98+4+ETVqFqxN09UXSjcMb0= +github.com/pascaldekloe/name v1.0.1/go.mod h1:Z//MfYJnH4jVpQ9wkclwu2I2MkHmXTlT9wR5UZScttM= +github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU= +github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= +github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= +github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pion/dtls/v2 v2.2.7 h1:cSUBsETxepsCSFSxC3mc/aDo14qQLMSL+O6IjG28yV8= github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= @@ -156,6 +245,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= @@ -172,14 +263,21 @@ github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/russross/blackfriday v1.6.0 h1:KqfZb0pUVN2lYqZUYRddxF4OR8ZMURnJIG5Y3VRLtww= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/segmentio/asm v1.2.1 h1:DTNbBqs57ioxAD4PrArqftgypG4/qNpXoJx8TVXxPR0= +github.com/segmentio/asm v1.2.1/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/gopsutil/v4 v4.25.6 h1:kLysI2JsKorfaFPcYmcJqbzROzsBWEOAtw6A7dIfqXs= +github.com/shirou/gopsutil/v4 v4.25.6/go.mod h1:PfybzyydfZcN+JMMjkF6Zb8Mq1A/VcogFFg7hj50W9c= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= @@ -205,40 +303,86 @@ github.com/supranational/blst v0.3.16 h1:bTDadT+3fK497EvLdWRQEjiGnUtzJ7jjIUMF0jq github.com/supranational/blst v0.3.16/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= -github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= -github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= -github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= -github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +github.com/testcontainers/testcontainers-go v0.40.0 h1:pSdJYLOVgLE8YdUY2FHQ1Fxu+aMnb6JfVz1mxk7OeMU= +github.com/testcontainers/testcontainers-go v0.40.0/go.mod h1:FSXV5KQtX2HAMlm7U3APNyLkkap35zNLxukw9oBi/MY= +github.com/testcontainers/testcontainers-go/modules/clickhouse v0.40.0 h1:JhYAFtoTCEpzB5jF+wcEP5mL01+JChUUpaaX8sWuEzo= +github.com/testcontainers/testcontainers-go/modules/clickhouse v0.40.0/go.mod h1:UoMHEYTzGmwKyeCQaKfcQHSVs/kQwimfzX+y1gVSRIk= +github.com/testcontainers/testcontainers-go/modules/redis v0.40.0 h1:OG4qwcxp2O0re7V7M9lY9w0v6wWgWf7j7rtkpAnGMd0= +github.com/testcontainers/testcontainers-go/modules/redis v0.40.0/go.mod h1:Bc+EDhKMo5zI5V5zdBkHiMVzeAXbtI4n5isS/nzf6zw= +github.com/tklauser/go-sysconf v0.3.15 h1:VE89k0criAymJ/Os65CSn1IXaol+1wrsFHEB8Ol49K4= +github.com/tklauser/go-sysconf v0.3.15/go.mod h1:Dmjwr6tYFIseJw7a3dRLJfsHAMXZ3nEnL/aZY+0IuI4= +github.com/tklauser/numcpus v0.10.0 h1:18njr6LDBk1zuna922MgdjQuJFjrdppsZG60sHGfjso= +github.com/tklauser/numcpus v0.10.0/go.mod h1:BiTKazU708GQTYF4mB+cmlpT2Is1gLk7XVuEeem8LsQ= github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w= github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= -github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= -github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= +github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= golang.org/x/exp v0.0.0-20251009144603-d2f985daa21b h1:18qgiDvlvH7kk8Ioa8Ov+K6xCi0GMvmGfGW0sgd/SYA= golang.org/x/exp v0.0.0-20251009144603-d2f985daa21b/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70= -golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= -golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= +golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= -golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= -golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q= +golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss= +golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= +golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= -google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= -google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= +golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= +google.golang.org/genproto/googleapis/api v0.0.0-20260120221211-b8f7ae30c516 h1:vmC/ws+pLzWjj/gzApyoZuSVrDtF1aod4u/+bbj8hgM= +google.golang.org/genproto/googleapis/api v0.0.0-20260120221211-b8f7ae30c516/go.mod h1:p3MLuOwURrGBRoEyFHBT3GjUwaCQVKeNqqWxlcISGdw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260120221211-b8f7ae30c516 h1:sNrWoksmOyF5bvJUcnmbeAmQi8baNhqg5IWaI3llQqU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260120221211-b8f7ae30c516/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= +google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= +google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= @@ -251,3 +395,5 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= diff --git a/internal/testutil/testutil.go b/internal/testutil/testutil.go new file mode 100644 index 0000000..aaf1ece --- /dev/null +++ b/internal/testutil/testutil.go @@ -0,0 +1,125 @@ +// Package testutil provides test helper utilities for unit and integration tests. +package testutil + +import ( + "context" + "fmt" + "testing" + + "github.com/alicebob/miniredis/v2" + "github.com/redis/go-redis/v9" + "github.com/testcontainers/testcontainers-go" + tcclickhouse "github.com/testcontainers/testcontainers-go/modules/clickhouse" + tcredis "github.com/testcontainers/testcontainers-go/modules/redis" +) + +// NewMiniredis creates an in-memory Redis server for unit tests. +// The server is automatically cleaned up when the test completes. +func NewMiniredis(t *testing.T) *miniredis.Miniredis { + t.Helper() + + s := miniredis.RunT(t) + + return s +} + +// NewMiniredisClient creates a Redis client connected to an in-memory miniredis server. +// Both the server and client are automatically cleaned up when the test completes. +func NewMiniredisClient(t *testing.T) (*redis.Client, *miniredis.Miniredis) { + t.Helper() + + s := miniredis.RunT(t) + client := redis.NewClient(&redis.Options{Addr: s.Addr()}) + + t.Cleanup(func() { + _ = client.Close() + }) + + return client, s +} + +// NewRedisContainer creates a real Redis container for integration tests. +// The container is automatically cleaned up when the test completes. +func NewRedisContainer(t *testing.T) (*redis.Client, string) { + t.Helper() + + ctx := context.Background() + + c, err := tcredis.Run(ctx, "redis:7-alpine") + if err != nil { + t.Fatalf("failed to start redis container: %v", err) + } + + testcontainers.CleanupContainer(t, c) + + connStr, err := c.ConnectionString(ctx) + if err != nil { + t.Fatalf("failed to get redis connection string: %v", err) + } + + // Parse the connection string to get the address + // The connection string is in the format "redis://host:port" + opts, err := redis.ParseURL(connStr) + if err != nil { + t.Fatalf("failed to parse redis connection string: %v", err) + } + + client := redis.NewClient(opts) + + t.Cleanup(func() { + _ = client.Close() + }) + + return client, connStr +} + +// ClickHouseConnection holds ClickHouse connection details. +type ClickHouseConnection struct { + Host string + Port int + Database string + Username string + Password string +} + +// Addr returns the ClickHouse address in host:port format. +func (c ClickHouseConnection) Addr() string { + return fmt.Sprintf("%s:%d", c.Host, c.Port) +} + +// NewClickHouseContainer creates a real ClickHouse container for integration tests. +// The container is automatically cleaned up when the test completes. +func NewClickHouseContainer(t *testing.T) ClickHouseConnection { + t.Helper() + + ctx := context.Background() + + c, err := tcclickhouse.Run(ctx, "clickhouse/clickhouse-server:latest", + tcclickhouse.WithUsername("default"), + tcclickhouse.WithPassword(""), + tcclickhouse.WithDatabase("default"), + ) + if err != nil { + t.Fatalf("failed to start clickhouse container: %v", err) + } + + testcontainers.CleanupContainer(t, c) + + host, err := c.Host(ctx) + if err != nil { + t.Fatalf("failed to get clickhouse host: %v", err) + } + + port, err := c.MappedPort(ctx, "9000/tcp") + if err != nil { + t.Fatalf("failed to get clickhouse port: %v", err) + } + + return ClickHouseConnection{ + Host: host, + Port: port.Int(), + Database: "default", + Username: "default", + Password: "", + } +} diff --git a/pkg/clickhouse/client.go b/pkg/clickhouse/client.go index 24a2fbf..142bd03 100644 --- a/pkg/clickhouse/client.go +++ b/pkg/clickhouse/client.go @@ -1,17 +1,23 @@ package clickhouse import ( - "bytes" "context" "encoding/json" + "errors" "fmt" "io" - "net/http" + "net" "reflect" "strings" "sync" + "sync/atomic" + "syscall" "time" + "github.com/ClickHouse/ch-go" + "github.com/ClickHouse/ch-go/chpool" + "github.com/ClickHouse/ch-go/compress" + "github.com/ClickHouse/ch-go/proto" "github.com/sirupsen/logrus" "github.com/ethpandaops/execution-processor/pkg/common" @@ -22,94 +28,314 @@ const ( statusFailed = "failed" ) -// client implements the ClientInterface using HTTP. -type client struct { - log logrus.FieldLogger - httpClient *http.Client - baseURL string - network string - processor string - debug bool - lock sync.RWMutex +// Client implements the ClientInterface using ch-go native protocol. +type Client struct { + pool *chpool.Pool + config *Config + network string + processor string + log logrus.FieldLogger + lock sync.RWMutex + started atomic.Bool + + // Metrics collection + metricsDone chan struct{} + metricsWg sync.WaitGroup } -// New creates a new HTTP-based ClickHouse client. -func New(cfg *Config) (ClientInterface, error) { +// isRetryableError checks if an error is transient and can be retried. +func isRetryableError(err error) bool { + if err == nil { + return false + } + + // Check for context errors - these should not be retried + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return false + } + + // Check for ch-go sentinel errors - client permanently closed + if errors.Is(err, ch.ErrClosed) { + return false + } + + // Check for ch-go server exceptions + if exc, ok := ch.AsException(err); ok { + // Retryable server-side errors + if exc.IsCode( + proto.ErrTimeoutExceeded, // 159 - server timeout + proto.ErrNoFreeConnection, // 203 - server pool exhausted + proto.ErrTooManySimultaneousQueries, // 202 - rate limited + proto.ErrSocketTimeout, // 209 - network timeout + proto.ErrNetworkError, // 210 - generic network + ) { + return true + } + // All other server exceptions are non-retryable (syntax, data errors, etc.) + return false + } + + // Check for data corruption - never retry + var corruptedErr *compress.CorruptedDataErr + if errors.As(err, &corruptedErr) { + return false + } + + // Check for connection reset/refused errors (before net.Error since syscall.Errno implements net.Error) + if errors.Is(err, syscall.ECONNRESET) || + errors.Is(err, syscall.ECONNREFUSED) || + errors.Is(err, syscall.EPIPE) || + errors.Is(err, io.EOF) || + errors.Is(err, io.ErrUnexpectedEOF) { + return true + } + + // Check for network timeout errors + var netErr net.Error + if errors.As(err, &netErr) && netErr.Timeout() { + return true + } + + // Check for common transient error messages (fallback for edge cases) + errStr := err.Error() + transientPatterns := []string{ + "connection reset", + "connection refused", + "broken pipe", + "EOF", + "timeout", + "temporary failure", + "server is overloaded", + "too many connections", + } + + for _, pattern := range transientPatterns { + if strings.Contains(strings.ToLower(errStr), strings.ToLower(pattern)) { + return true + } + } + + return false +} + +// withQueryTimeout returns a context with the configured query timeout applied. +// If the context already has a deadline, the original context is returned unchanged. +func (c *Client) withQueryTimeout(ctx context.Context) (context.Context, context.CancelFunc) { + if c.config.QueryTimeout == 0 { + return ctx, func() {} + } + + if _, hasDeadline := ctx.Deadline(); hasDeadline { + return ctx, func() {} + } + + return context.WithTimeout(ctx, c.config.QueryTimeout) +} + +// doWithRetry executes a function with exponential backoff retry logic. +// The function receives a context with the configured query timeout applied per attempt. +func (c *Client) doWithRetry(ctx context.Context, operation string, fn func(ctx context.Context) error) error { + var lastErr error + + for attempt := 0; attempt <= c.config.MaxRetries; attempt++ { + if attempt > 0 { + // Calculate backoff delay with exponential increase + delay := c.config.RetryBaseDelay * time.Duration(1<<(attempt-1)) + + c.log.WithFields(logrus.Fields{ + "attempt": attempt, + "max": c.config.MaxRetries, + "delay": delay, + "operation": operation, + "error": lastErr, + }).Debug("Retrying after transient error") + + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(delay): + } + } + + attemptCtx, cancel := c.withQueryTimeout(ctx) + err := fn(attemptCtx) + + cancel() + + if err == nil { + return nil + } + + lastErr = err + + if !isRetryableError(err) { + return err + } + } + + return fmt.Errorf("max retries (%d) exceeded: %w", c.config.MaxRetries, lastErr) +} + +// New creates a new ch-go native ClickHouse client. +func New(ctx context.Context, cfg *Config) (ClientInterface, error) { if err := cfg.Validate(); err != nil { return nil, fmt.Errorf("invalid config: %w", err) } - // Set defaults cfg.SetDefaults() - // Create HTTP client with keep-alive settings - transport := &http.Transport{ - MaxIdleConns: 100, - MaxIdleConnsPerHost: 10, - IdleConnTimeout: cfg.KeepAlive, - DisableKeepAlives: false, + compression := ch.CompressionLZ4 + + switch cfg.Compression { + case "zstd": + compression = ch.CompressionZSTD + case "none": + compression = ch.CompressionDisabled + } + + log := logrus.WithField("component", "clickhouse-native") + + // Dial with startup retry logic for transient connection failures + var pool *chpool.Pool + + err := dialWithRetry(ctx, log, cfg, func() error { + var dialErr error + + pool, dialErr = chpool.Dial(ctx, chpool.Options{ + ClientOptions: ch.Options{ + Address: cfg.Addr, + Database: cfg.Database, + User: cfg.Username, + Password: cfg.Password, + Compression: compression, + DialTimeout: cfg.DialTimeout, + }, + MaxConns: cfg.MaxConns, + MinConns: cfg.MinConns, + MaxConnLifetime: cfg.ConnMaxLifetime, + MaxConnIdleTime: cfg.ConnMaxIdleTime, + HealthCheckPeriod: cfg.HealthCheckPeriod, + }) + + return dialErr + }) + if err != nil { + return nil, fmt.Errorf("failed to dial clickhouse: %w", err) } - httpClient := &http.Client{ - Transport: transport, - Timeout: 0, // We'll set per-request timeouts - } + return &Client{ + pool: pool, + config: cfg, + network: cfg.Network, + processor: cfg.Processor, + log: log, + }, nil +} + +// dialWithRetry wraps the dial operation with exponential backoff retry logic. +func dialWithRetry(ctx context.Context, log logrus.FieldLogger, cfg *Config, fn func() error) error { + var lastErr error + + for attempt := 0; attempt <= cfg.MaxRetries; attempt++ { + if attempt > 0 { + delay := cfg.RetryBaseDelay * time.Duration(1<<(attempt-1)) + + log.WithFields(logrus.Fields{ + "attempt": attempt, + "max": cfg.MaxRetries, + "delay": delay, + "operation": "dial", + "error": lastErr, + }).Debug("Retrying dial after transient error") + + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(delay): + } + } + + err := fn() + if err == nil { + return nil + } - c := &client{ - log: logrus.WithField("component", "clickhouse-http"), - httpClient: httpClient, - baseURL: strings.TrimRight(cfg.URL, "/"), - network: cfg.Network, - processor: cfg.Processor, - debug: cfg.Debug, + lastErr = err + + if !isRetryableError(err) { + return err + } } - return c, nil + return fmt.Errorf("max retries (%d) exceeded: %w", cfg.MaxRetries, lastErr) } -func (c *client) Start() error { - // Skip connectivity test if network is not set yet (e.g., during state manager initialization) - // The network will be set later when it's determined from the chain ID +// Start initializes the client and tests connectivity. +func (c *Client) Start() error { + // Idempotency guard - prevent multiple Start() calls from leaking goroutines. + // Once Start() is called, the client is considered "started" regardless of outcome. + // On failure, the client is effectively dead and Stop() should be called. + if c.started.Swap(true) { + c.log.Debug("Start() already called, skipping") + + return nil + } + if c.network == "" { c.log.Debug("Skipping ClickHouse connectivity test - network not yet determined") return nil } - // Test connectivity ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - if err := c.Execute(ctx, "SELECT 1"); err != nil { - return fmt.Errorf("failed to connect to ClickHouse: %w", err) + if err := c.pool.Ping(ctx); err != nil { + // Don't reset started - the client is now in a failed state. + // Caller should call Stop() to clean up. + return fmt.Errorf("failed to ping ClickHouse: %w", err) } - c.log.Info("Connected to ClickHouse HTTP interface") + c.log.Info("Connected to ClickHouse native interface") + + // Start pool metrics collection goroutine + c.metricsDone = make(chan struct{}) + c.metricsWg.Add(1) + + go c.collectPoolMetrics() return nil } -func (c *client) SetNetwork(network string) { - c.lock.Lock() - defer c.lock.Unlock() +// Stop closes the connection pool. +func (c *Client) Stop() error { + // Stop metrics collection goroutine + if c.metricsDone != nil { + close(c.metricsDone) + c.metricsWg.Wait() + } - c.network = network + c.pool.Close() + c.log.Info("Closed ClickHouse connection pool") + + return nil } -func (c *client) Stop() error { +// SetNetwork updates the network name for metrics labeling. +func (c *Client) SetNetwork(network string) { c.lock.Lock() defer c.lock.Unlock() - if c.httpClient != nil { - c.httpClient.CloseIdleConnections() - } - - c.log.Info("Closed ClickHouse HTTP client") + c.network = network +} - return nil +// Do executes a query using the pool. +func (c *Client) Do(ctx context.Context, query ch.Query) error { + return c.pool.Do(ctx, query) } -func (c *client) QueryOne(ctx context.Context, query string, dest interface{}) error { +// QueryOne executes a query and returns a single result. +func (c *Client) QueryOne(ctx context.Context, query string, dest any) error { start := time.Now() operation := "query_one" status := statusSuccess @@ -118,41 +344,38 @@ func (c *client) QueryOne(ctx context.Context, query string, dest interface{}) e c.recordMetrics(operation, status, time.Since(start), query) }() - // Add FORMAT JSON to query - formattedQuery := query + " FORMAT JSON" - - resp, err := c.executeHTTPRequest(ctx, formattedQuery, c.getTimeout(ctx, "query")) + var rows []string + + err := c.doWithRetry(ctx, operation, func(attemptCtx context.Context) error { + // Reset rows for each retry attempt + rows = nil + colStr := new(proto.ColStr) + + return c.pool.Do(attemptCtx, ch.Query{ + Body: query + " FORMAT JSONEachRow", + Result: proto.Results{ + {Name: "", Data: colStr}, + }, + OnResult: func(ctx context.Context, block proto.Block) error { + for i := 0; i < colStr.Rows(); i++ { + rows = append(rows, colStr.Row(i)) + } + + return nil + }, + }) + }) if err != nil { status = statusFailed return fmt.Errorf("query execution failed: %w", err) } - // Parse response - ClickHouse returns snake_case fields - //nolint:tagliatelle // ClickHouse JSON response format uses snake_case - var result struct { - Data []json.RawMessage `json:"data"` - Meta []struct { - Name string `json:"name"` - Type string `json:"type"` - } `json:"meta"` - Rows int `json:"rows"` - RowsRead int `json:"rows_read"` - } - - if err := json.Unmarshal(resp, &result); err != nil { - status = statusFailed - - return fmt.Errorf("failed to parse response: %w", err) - } - - if len(result.Data) == 0 { - // No rows found, return without error but don't unmarshal + if len(rows) == 0 { return nil } - // Unmarshal the first row into dest - if err := json.Unmarshal(result.Data[0], dest); err != nil { + if err := json.Unmarshal([]byte(rows[0]), dest); err != nil { status = statusFailed return fmt.Errorf("failed to unmarshal result: %w", err) @@ -161,7 +384,8 @@ func (c *client) QueryOne(ctx context.Context, query string, dest interface{}) e return nil } -func (c *client) QueryMany(ctx context.Context, query string, dest interface{}) error { +// QueryMany executes a query and returns multiple results. +func (c *Client) QueryMany(ctx context.Context, query string, dest any) error { start := time.Now() operation := "query_many" status := statusSuccess @@ -172,49 +396,48 @@ func (c *client) QueryMany(ctx context.Context, query string, dest interface{}) // Validate that dest is a pointer to a slice destValue := reflect.ValueOf(dest) - if destValue.Kind() != reflect.Ptr || destValue.Elem().Kind() != reflect.Slice { + if destValue.Kind() != reflect.Pointer || destValue.Elem().Kind() != reflect.Slice { status = statusFailed return fmt.Errorf("dest must be a pointer to a slice") } - // Add FORMAT JSON to query - formattedQuery := query + " FORMAT JSON" + var rows []string - resp, err := c.executeHTTPRequest(ctx, formattedQuery, c.getTimeout(ctx, "query")) - if err != nil { - status = statusFailed + err := c.doWithRetry(ctx, operation, func(attemptCtx context.Context) error { + // Reset rows for each retry attempt + rows = nil + colStr := new(proto.ColStr) - return fmt.Errorf("query execution failed: %w", err) - } - - // Parse response - ClickHouse returns snake_case fields - //nolint:tagliatelle // ClickHouse JSON response format uses snake_case - var result struct { - Data []json.RawMessage `json:"data"` - Meta []struct { - Name string `json:"name"` - Type string `json:"type"` - } `json:"meta"` - Rows int `json:"rows"` - RowsRead int `json:"rows_read"` - } + return c.pool.Do(attemptCtx, ch.Query{ + Body: query + " FORMAT JSONEachRow", + Result: proto.Results{ + {Name: "", Data: colStr}, + }, + OnResult: func(ctx context.Context, block proto.Block) error { + for i := 0; i < colStr.Rows(); i++ { + rows = append(rows, colStr.Row(i)) + } - if err := json.Unmarshal(resp, &result); err != nil { + return nil + }, + }) + }) + if err != nil { status = statusFailed - return fmt.Errorf("failed to parse response: %w", err) + return fmt.Errorf("query execution failed: %w", err) } // Create a slice of the appropriate type sliceType := destValue.Elem().Type() elemType := sliceType.Elem() - newSlice := reflect.MakeSlice(sliceType, len(result.Data), len(result.Data)) + newSlice := reflect.MakeSlice(sliceType, len(rows), len(rows)) // Unmarshal each row - for i, data := range result.Data { + for i, row := range rows { elem := reflect.New(elemType) - if err := json.Unmarshal(data, elem.Interface()); err != nil { + if err := json.Unmarshal([]byte(row), elem.Interface()); err != nil { status = statusFailed return fmt.Errorf("failed to unmarshal row %d: %w", i, err) @@ -229,7 +452,8 @@ func (c *client) QueryMany(ctx context.Context, query string, dest interface{}) return nil } -func (c *client) Execute(ctx context.Context, query string) error { +// Execute runs a query without expecting results. +func (c *Client) Execute(ctx context.Context, query string) error { start := time.Now() operation := "execute" status := statusSuccess @@ -238,7 +462,11 @@ func (c *client) Execute(ctx context.Context, query string) error { c.recordMetrics(operation, status, time.Since(start), query) }() - _, err := c.executeHTTPRequest(ctx, query, c.getTimeout(ctx, "query")) + err := c.doWithRetry(ctx, operation, func(attemptCtx context.Context) error { + return c.pool.Do(attemptCtx, ch.Query{ + Body: query, + }) + }) if err != nil { status = statusFailed @@ -248,141 +476,8 @@ func (c *client) Execute(ctx context.Context, query string) error { return nil } -func (c *client) BulkInsert(ctx context.Context, table string, data interface{}) error { - start := time.Now() - operation := "bulk_insert" - status := statusSuccess - - defer func() { - c.recordMetrics(operation, status, time.Since(start), table) - }() - - // Convert data to slice via reflection - dataValue := reflect.ValueOf(data) - if dataValue.Kind() != reflect.Slice { - status = statusFailed - - return fmt.Errorf("data must be a slice") - } - - if dataValue.Len() == 0 { - return nil // Nothing to insert - } - - // Build INSERT query with JSONEachRow format - var buf bytes.Buffer - - buf.WriteString(fmt.Sprintf("INSERT INTO %s FORMAT JSONEachRow\n", table)) - - // Marshal each item as JSON - for i := 0; i < dataValue.Len(); i++ { - item := dataValue.Index(i).Interface() - - jsonData, err := json.Marshal(item) - if err != nil { - status = statusFailed - - return fmt.Errorf("failed to marshal row %d: %w", i, err) - } - - buf.Write(jsonData) - buf.WriteByte('\n') - } - - // Execute the insert - _, err := c.executeHTTPRequest(ctx, buf.String(), c.getTimeout(ctx, "insert")) - if err != nil { - status = statusFailed - - return fmt.Errorf("bulk insert failed: %w", err) - } - - return nil -} - -func (c *client) executeHTTPRequest(ctx context.Context, query string, timeout time.Duration) ([]byte, error) { - // Create request with timeout - reqCtx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - - req, err := http.NewRequestWithContext(reqCtx, "POST", c.baseURL, strings.NewReader(query)) - if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) - } - - // Set headers - req.Header.Set("Content-Type", "text/plain") - req.Header.Set("X-ClickHouse-Format", "JSON") - - // Debug logging - if c.debug { - // For large inserts, truncate the query - logQuery := query - if len(query) > 1000 && strings.Contains(query, "INSERT") { - logQuery = query[:1000] + "... (truncated)" - } - - c.log.WithField("query", logQuery).Debug("Executing ClickHouse query") - } - - // Execute request - resp, err := c.httpClient.Do(req) - if err != nil { - return nil, fmt.Errorf("request failed: %w", err) - } - defer resp.Body.Close() - - // Read response body - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read response: %w", err) - } - - // Check status code - if resp.StatusCode != http.StatusOK { - bodyStr := string(body) - if bodyStr == "" { - bodyStr = "(empty response)" - } - - // Try to parse error message from JSON - var errorResp struct { - Exception string `json:"exception"` - } - - if jsonErr := json.Unmarshal(body, &errorResp); jsonErr == nil && errorResp.Exception != "" { - return nil, fmt.Errorf("ClickHouse error (status %d): %s", resp.StatusCode, errorResp.Exception) - } - - return nil, fmt.Errorf("ClickHouse error (status %d): %s", resp.StatusCode, bodyStr) - } - - // Debug logging - if c.debug && len(body) < 1000 { - c.log.WithField("response", string(body)).Debug("ClickHouse response") - } - - return body, nil -} - -func (c *client) getTimeout(ctx context.Context, operation string) time.Duration { - // Check if context already has a deadline - if deadline, ok := ctx.Deadline(); ok { - return time.Until(deadline) - } - - // Use default timeouts based on operation type - switch operation { - case "insert": - return 5 * time.Minute - case "query": - return 30 * time.Second - default: - return 30 * time.Second - } -} - -func (c *client) IsStorageEmpty(ctx context.Context, table string, conditions map[string]interface{}) (bool, error) { +// IsStorageEmpty checks if a table has any records matching the given conditions. +func (c *Client) IsStorageEmpty(ctx context.Context, table string, conditions map[string]any) (bool, error) { start := time.Now() operation := "is_storage_empty" status := statusSuccess @@ -392,15 +487,14 @@ func (c *client) IsStorageEmpty(ctx context.Context, table string, conditions ma }() // Build the query - query := fmt.Sprintf("SELECT COUNT(*) as count FROM %s FINAL", table) + query := fmt.Sprintf("SELECT count() as count FROM %s FINAL", table) if len(conditions) > 0 { query += " WHERE " - var conditionParts []string + conditionParts := make([]string, 0, len(conditions)) for key, value := range conditions { - // Handle different value types switch v := value.(type) { case string: conditionParts = append(conditionParts, fmt.Sprintf("%s = '%s'", key, v)) @@ -414,21 +508,31 @@ func (c *client) IsStorageEmpty(ctx context.Context, table string, conditions ma query += strings.Join(conditionParts, " AND ") } - var result struct { - Count json.Number `json:"count"` - } + // Execute and get count + var count uint64 - if err := c.QueryOne(ctx, query, &result); err != nil { - status = statusFailed + err := c.doWithRetry(ctx, operation, func(attemptCtx context.Context) error { + colCount := new(proto.ColUInt64) - return false, fmt.Errorf("failed to check if table is empty: %w", err) - } + if err := c.pool.Do(attemptCtx, ch.Query{ + Body: query, + Result: proto.Results{ + {Name: "count", Data: colCount}, + }, + }); err != nil { + return err + } - count, err := result.Count.Int64() + if colCount.Rows() > 0 { + count = colCount.Row(0) + } + + return nil + }) if err != nil { status = statusFailed - return false, fmt.Errorf("failed to parse count: %w", err) + return false, fmt.Errorf("failed to check if table is empty: %w", err) } return count == 0, nil @@ -436,7 +540,6 @@ func (c *client) IsStorageEmpty(ctx context.Context, table string, conditions ma // extractTableName attempts to extract the table name from various SQL query patterns. func extractTableName(query string) string { - // Work with original query to preserve case trimmedQuery := strings.TrimSpace(query) upperQuery := strings.ToUpper(trimmedQuery) @@ -444,23 +547,17 @@ func extractTableName(query string) string { if strings.HasPrefix(upperQuery, "INSERT INTO") { parts := strings.Fields(trimmedQuery) if len(parts) >= 3 { - // Return the third part which should be the table name - // Handle cases where table name might have backticks or quotes return strings.Trim(parts[2], "`'\"") } } // Handle SELECT ... FROM queries if idx := strings.Index(upperQuery, "FROM"); idx != -1 { - // Get the substring after FROM from the original query afterFrom := strings.TrimSpace(trimmedQuery[idx+4:]) parts := strings.Fields(afterFrom) if len(parts) > 0 { - // Return the first part which should be the table name - // Remove any trailing keywords like WHERE, ORDER BY, etc. tableName := parts[0] - // Handle FINAL keyword (e.g., "FROM table FINAL") if !strings.EqualFold(tableName, "FINAL") { return strings.Trim(tableName, "`'\"") } @@ -486,18 +583,82 @@ func extractTableName(query string) string { return "" } -func (c *client) recordMetrics(operation, status string, duration time.Duration, tableOrQuery string) { - var table string +func (c *Client) recordMetrics(operation, status string, duration time.Duration, tableOrQuery string) { + table := extractTableName(tableOrQuery) - // If it's a bulk_insert operation, tableOrQuery is already the table name - if operation == "bulk_insert" { - table = tableOrQuery - } else { - // Otherwise, try to extract table name from the query - table = extractTableName(tableOrQuery) - } + c.lock.RLock() + network := c.network + c.lock.RUnlock() + + common.ClickHouseOperationDuration.WithLabelValues(network, c.processor, operation, table, status, "").Observe(duration.Seconds()) + common.ClickHouseOperationTotal.WithLabelValues(network, c.processor, operation, table, status, "").Inc() +} + +// collectPoolMetrics periodically collects pool statistics and updates Prometheus metrics. +func (c *Client) collectPoolMetrics() { + defer c.metricsWg.Done() + + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + + // Track previous counter values for delta calculation + var prevAcquireCount, prevEmptyAcquireCount, prevCanceledAcquireCount int64 + + for { + select { + case <-c.metricsDone: + return + case <-ticker.C: + c.lock.RLock() + network := c.network + c.lock.RUnlock() - // Use existing metrics from common package - common.ClickHouseOperationDuration.WithLabelValues(c.network, c.processor, operation, table, status, "").Observe(duration.Seconds()) - common.ClickHouseOperationTotal.WithLabelValues(c.network, c.processor, operation, table, status, "").Inc() + if network == "" { + continue + } + + stat := c.pool.Stat() + + // Set gauge values (current state) + common.ClickHousePoolAcquiredResources.WithLabelValues(network, c.processor).Set(float64(stat.AcquiredResources())) + common.ClickHousePoolIdleResources.WithLabelValues(network, c.processor).Set(float64(stat.IdleResources())) + common.ClickHousePoolConstructingResources.WithLabelValues(network, c.processor).Set(float64(stat.ConstructingResources())) + common.ClickHousePoolTotalResources.WithLabelValues(network, c.processor).Set(float64(stat.TotalResources())) + common.ClickHousePoolMaxResources.WithLabelValues(network, c.processor).Set(float64(stat.MaxResources())) + + // Set cumulative duration gauges + common.ClickHousePoolAcquireDuration.WithLabelValues(network, c.processor).Set(stat.AcquireDuration().Seconds()) + common.ClickHousePoolEmptyAcquireWaitDuration.WithLabelValues(network, c.processor).Set(stat.EmptyAcquireWaitTime().Seconds()) + + // Calculate deltas and add to counters + currentAcquireCount := stat.AcquireCount() + currentEmptyAcquireCount := stat.EmptyAcquireCount() + currentCanceledAcquireCount := stat.CanceledAcquireCount() + + if prevAcquireCount > 0 { + delta := currentAcquireCount - prevAcquireCount + if delta > 0 { + common.ClickHousePoolAcquireTotal.WithLabelValues(network, c.processor).Add(float64(delta)) + } + } + + if prevEmptyAcquireCount > 0 { + delta := currentEmptyAcquireCount - prevEmptyAcquireCount + if delta > 0 { + common.ClickHousePoolEmptyAcquireTotal.WithLabelValues(network, c.processor).Add(float64(delta)) + } + } + + if prevCanceledAcquireCount > 0 { + delta := currentCanceledAcquireCount - prevCanceledAcquireCount + if delta > 0 { + common.ClickHousePoolCanceledAcquireTotal.WithLabelValues(network, c.processor).Add(float64(delta)) + } + } + + prevAcquireCount = currentAcquireCount + prevEmptyAcquireCount = currentEmptyAcquireCount + prevCanceledAcquireCount = currentCanceledAcquireCount + } + } } diff --git a/pkg/clickhouse/client_integration_test.go b/pkg/clickhouse/client_integration_test.go new file mode 100644 index 0000000..dfe7874 --- /dev/null +++ b/pkg/clickhouse/client_integration_test.go @@ -0,0 +1,149 @@ +//go:build integration + +package clickhouse + +import ( + "testing" + + "github.com/ethpandaops/execution-processor/internal/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Integration tests using testcontainers - run with: go test -tags=integration ./... + +func TestClient_Integration_Container_New(t *testing.T) { + conn := testutil.NewClickHouseContainer(t) + + cfg := &Config{ + Addr: conn.Addr(), + Database: conn.Database, + Username: conn.Username, + Password: conn.Password, + Compression: "lz4", + } + + client, err := New(cfg) + require.NoError(t, err) + require.NotNil(t, client) + + err = client.Stop() + require.NoError(t, err) +} + +func TestClient_Integration_Container_StartStop(t *testing.T) { + conn := testutil.NewClickHouseContainer(t) + + cfg := &Config{ + Addr: conn.Addr(), + Database: conn.Database, + Username: conn.Username, + Password: conn.Password, + Compression: "lz4", + Network: "test", + } + + client, err := New(cfg) + require.NoError(t, err) + + // Start should ping successfully + err = client.Start() + require.NoError(t, err) + + // Stop should close pool + err = client.Stop() + require.NoError(t, err) +} + +func TestClient_Integration_Container_Execute(t *testing.T) { + conn := testutil.NewClickHouseContainer(t) + + cfg := &Config{ + Addr: conn.Addr(), + Database: conn.Database, + Username: conn.Username, + Password: conn.Password, + Compression: "lz4", + Network: "test", + } + + client, err := New(cfg) + require.NoError(t, err) + + defer func() { _ = client.Stop() }() + + err = client.Start() + require.NoError(t, err) + + // Execute a simple query + err = client.Execute(t.Context(), "SELECT 1") + require.NoError(t, err) +} + +func TestClient_Integration_Container_QueryOne(t *testing.T) { + conn := testutil.NewClickHouseContainer(t) + + cfg := &Config{ + Addr: conn.Addr(), + Database: conn.Database, + Username: conn.Username, + Password: conn.Password, + Compression: "lz4", + Network: "test", + } + + client, err := New(cfg) + require.NoError(t, err) + + defer func() { _ = client.Stop() }() + + err = client.Start() + require.NoError(t, err) + + var result struct { + Value int64 `json:"value"` + } + + err = client.QueryOne(t.Context(), "SELECT 42 as value", &result) + require.NoError(t, err) + assert.Equal(t, int64(42), result.Value) +} + +func TestClient_Integration_Container_IsStorageEmpty(t *testing.T) { + conn := testutil.NewClickHouseContainer(t) + + cfg := &Config{ + Addr: conn.Addr(), + Database: conn.Database, + Username: conn.Username, + Password: conn.Password, + Compression: "lz4", + Network: "test", + } + + client, err := New(cfg) + require.NoError(t, err) + + defer func() { _ = client.Stop() }() + + err = client.Start() + require.NoError(t, err) + + // Create a temporary table + err = client.Execute(t.Context(), ` + CREATE TABLE IF NOT EXISTS test_empty_check ( + id UInt64, + name String + ) ENGINE = Memory + `) + require.NoError(t, err) + + // Should be empty + isEmpty, err := client.IsStorageEmpty(t.Context(), "test_empty_check", nil) + require.NoError(t, err) + assert.True(t, isEmpty) + + // Drop the table + err = client.Execute(t.Context(), "DROP TABLE IF EXISTS test_empty_check") + require.NoError(t, err) +} diff --git a/pkg/clickhouse/client_test.go b/pkg/clickhouse/client_test.go index 7b175c3..ec42c30 100644 --- a/pkg/clickhouse/client_test.go +++ b/pkg/clickhouse/client_test.go @@ -2,328 +2,475 @@ package clickhouse import ( "context" - "net/http" - "net/http/httptest" + "errors" + "io" + "net" + "os" + "syscall" "testing" "time" + "github.com/ClickHouse/ch-go" + "github.com/ClickHouse/ch-go/compress" + "github.com/ClickHouse/ch-go/proto" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestHTTPClient_QueryOne(t *testing.T) { - type testResult struct { - Count int `json:"count"` +func TestConfig_Validate(t *testing.T) { + tests := []struct { + name string + config Config + expectError bool + }{ + { + name: "valid config with addr", + config: Config{ + Addr: "localhost:9000", + }, + expectError: false, + }, + { + name: "missing addr", + config: Config{}, + expectError: true, + }, + { + name: "valid config with all fields", + config: Config{ + Addr: "localhost:9000", + Database: "test_db", + Username: "default", + Password: "secret", + MaxConns: 20, + MinConns: 5, + ConnMaxLifetime: 2 * time.Hour, + ConnMaxIdleTime: 1 * time.Hour, + HealthCheckPeriod: 30 * time.Second, + Compression: "zstd", + }, + expectError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.config.Validate() + if tt.expectError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) } +} +func TestConfig_SetDefaults(t *testing.T) { + config := Config{ + Addr: "localhost:9000", + } + + config.SetDefaults() + + assert.Equal(t, "default", config.Database) + assert.Equal(t, int32(10), config.MaxConns) + assert.Equal(t, int32(2), config.MinConns) + assert.Equal(t, time.Hour, config.ConnMaxLifetime) + assert.Equal(t, 30*time.Minute, config.ConnMaxIdleTime) + assert.Equal(t, time.Minute, config.HealthCheckPeriod) + assert.Equal(t, 10*time.Second, config.DialTimeout) + assert.Equal(t, "lz4", config.Compression) + assert.Equal(t, 60*time.Second, config.QueryTimeout) +} + +func TestConfig_SetDefaults_PreservesValues(t *testing.T) { + config := Config{ + Addr: "localhost:9000", + Database: "custom_db", + MaxConns: 50, + MinConns: 10, + ConnMaxLifetime: 3 * time.Hour, + ConnMaxIdleTime: 2 * time.Hour, + HealthCheckPeriod: 5 * time.Minute, + DialTimeout: 30 * time.Second, + Compression: "zstd", + QueryTimeout: 120 * time.Second, + } + + config.SetDefaults() + + // Should preserve custom values + assert.Equal(t, "custom_db", config.Database) + assert.Equal(t, int32(50), config.MaxConns) + assert.Equal(t, int32(10), config.MinConns) + assert.Equal(t, 3*time.Hour, config.ConnMaxLifetime) + assert.Equal(t, 2*time.Hour, config.ConnMaxIdleTime) + assert.Equal(t, 5*time.Minute, config.HealthCheckPeriod) + assert.Equal(t, 30*time.Second, config.DialTimeout) + assert.Equal(t, "zstd", config.Compression) + assert.Equal(t, 120*time.Second, config.QueryTimeout) +} + +func TestClient_withQueryTimeout(t *testing.T) { tests := []struct { - name string - query string - serverResponse string - serverStatus int - expectedResult *testResult - expectedError bool + name string + queryTimeout time.Duration + ctxHasDeadline bool + expectNewContext bool }{ { - name: "successful query with result", - query: "SELECT count FROM test_table", - serverResponse: `{ - "data": [{"count": 42}], - "meta": [{"name": "count", "type": "UInt64"}], - "rows": 1, - "rows_read": 1 - }`, - serverStatus: http.StatusOK, - expectedResult: &testResult{Count: 42}, - expectedError: false, + name: "applies timeout when no deadline exists", + queryTimeout: 5 * time.Second, + ctxHasDeadline: false, + expectNewContext: true, }, { - name: "query with no results", - query: "SELECT count FROM test_table WHERE id = 999", - serverResponse: `{ - "data": [], - "meta": [{"name": "count", "type": "UInt64"}], - "rows": 0, - "rows_read": 0 - }`, - serverStatus: http.StatusOK, - expectedResult: nil, - expectedError: false, + name: "preserves existing deadline", + queryTimeout: 5 * time.Second, + ctxHasDeadline: true, + expectNewContext: false, }, { - name: "server error", - query: "SELECT invalid", - serverResponse: `{"exception": "DB::Exception: Unknown identifier: invalid"}`, - serverStatus: http.StatusBadRequest, - expectedResult: nil, - expectedError: true, + name: "no-op when timeout is zero", + queryTimeout: 0, + ctxHasDeadline: false, + expectNewContext: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - // Create test server - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - assert.Equal(t, "POST", r.Method) - assert.Contains(t, r.Header.Get("Content-Type"), "text/plain") + client := &Client{ + config: &Config{ + QueryTimeout: tt.queryTimeout, + }, + } - w.WriteHeader(tt.serverStatus) - _, _ = w.Write([]byte(tt.serverResponse)) - })) - defer server.Close() + var ctx context.Context - // Create client - client, err := New(&Config{ - URL: server.URL, - }) - require.NoError(t, err) + var originalCancel context.CancelFunc - // Execute query - var result testResult + if tt.ctxHasDeadline { + ctx, originalCancel = context.WithTimeout(context.Background(), 10*time.Second) + defer originalCancel() + } else { + ctx = context.Background() + } - err = client.QueryOne(context.Background(), tt.query, &result) + newCtx, cancel := client.withQueryTimeout(ctx) + defer cancel() - if tt.expectedError { - assert.Error(t, err) - } else { - assert.NoError(t, err) + _, hasDeadline := newCtx.Deadline() - if tt.expectedResult != nil { - assert.Equal(t, *tt.expectedResult, result) - } + if tt.expectNewContext { + require.True(t, hasDeadline, "expected context to have deadline") + } else if tt.ctxHasDeadline { + require.True(t, hasDeadline, "expected original deadline to be preserved") + } else { + require.False(t, hasDeadline, "expected no deadline") } }) } } -func TestHTTPClient_QueryMany(t *testing.T) { - // Create test server - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - response := `{ - "data": [ - {"id": 1, "name": "first"}, - {"id": 2, "name": "second"}, - {"id": 3, "name": "third"} - ], - "meta": [ - {"name": "id", "type": "UInt64"}, - {"name": "name", "type": "String"} - ], - "rows": 3, - "rows_read": 3 - }` - - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte(response)) - })) - defer server.Close() - - // Create client - client, err := New(&Config{ - URL: server.URL, - }) - require.NoError(t, err) +// Integration tests - require CLICKHOUSE_ADDR environment variable - // Execute query - type testRow struct { - ID int `json:"id"` - Name string `json:"name"` +func TestClient_Integration_New(t *testing.T) { + addr := os.Getenv("CLICKHOUSE_ADDR") + if addr == "" { + t.Skip("CLICKHOUSE_ADDR not set, skipping integration test") } - var results []testRow + cfg := &Config{ + Addr: addr, + Database: "default", + Compression: "lz4", + } - err = client.QueryMany(context.Background(), "SELECT id, name FROM test_table", &results) + client, err := New(context.Background(), cfg) require.NoError(t, err) + require.NotNil(t, client) - // Verify results - assert.Len(t, results, 3) - assert.Equal(t, 1, results[0].ID) - assert.Equal(t, "first", results[0].Name) - assert.Equal(t, 2, results[1].ID) - assert.Equal(t, "second", results[1].Name) + err = client.Stop() + require.NoError(t, err) } -func TestHTTPClient_BulkInsert(t *testing.T) { - var receivedBody string +func TestClient_Integration_StartStop(t *testing.T) { + addr := os.Getenv("CLICKHOUSE_ADDR") + if addr == "" { + t.Skip("CLICKHOUSE_ADDR not set, skipping integration test") + } - // Create test server - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - assert.Equal(t, "POST", r.Method) + cfg := &Config{ + Addr: addr, + Database: "default", + Compression: "lz4", + Network: "test", + } - // Read body - body := make([]byte, r.ContentLength) - _, _ = r.Body.Read(body) - receivedBody = string(body) + client, err := New(context.Background(), cfg) + require.NoError(t, err) - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte("Ok.")) - })) - defer server.Close() + // Start should ping successfully + err = client.Start() + require.NoError(t, err) - // Create client - client, err := New(&Config{ - URL: server.URL, - }) + // Stop should close pool + err = client.Stop() require.NoError(t, err) +} - // Test data - type testData struct { - ID int `json:"id"` - Name string `json:"name"` - Timestamp int64 `json:"timestamp"` +func TestClient_Integration_Execute(t *testing.T) { + addr := os.Getenv("CLICKHOUSE_ADDR") + if addr == "" { + t.Skip("CLICKHOUSE_ADDR not set, skipping integration test") } - data := []testData{ - {ID: 1, Name: "first", Timestamp: 1000}, - {ID: 2, Name: "second", Timestamp: 2000}, - {ID: 3, Name: "third", Timestamp: 3000}, + cfg := &Config{ + Addr: addr, + Database: "default", + Compression: "lz4", + Network: "test", } - // Execute bulk insert - err = client.BulkInsert(context.Background(), "test_table", data) + client, err := New(context.Background(), cfg) require.NoError(t, err) - // Verify the request body - assert.Contains(t, receivedBody, "INSERT INTO test_table FORMAT JSONEachRow") - assert.Contains(t, receivedBody, `{"id":1,"name":"first","timestamp":1000}`) - assert.Contains(t, receivedBody, `{"id":2,"name":"second","timestamp":2000}`) - assert.Contains(t, receivedBody, `{"id":3,"name":"third","timestamp":3000}`) -} + defer func() { _ = client.Stop() }() -func TestHTTPClient_Execute(t *testing.T) { - // Create test server - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte("Ok.")) - })) - defer server.Close() - - // Create client - client, err := New(&Config{ - URL: server.URL, - }) + err = client.Start() require.NoError(t, err) - // Execute command - err = client.Execute(context.Background(), "CREATE TABLE test_table (id UInt64) ENGINE = Memory") + // Execute a simple query + err = client.Execute(t.Context(), "SELECT 1") require.NoError(t, err) } -func TestHTTPClient_Timeouts(t *testing.T) { - // Create a slow server - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - time.Sleep(2 * time.Second) - w.WriteHeader(http.StatusOK) - })) - defer server.Close() - - // Create client with short timeout - client, err := New(&Config{ - URL: server.URL, - QueryTimeout: 100 * time.Millisecond, - }) +func TestClient_Integration_QueryOne(t *testing.T) { + addr := os.Getenv("CLICKHOUSE_ADDR") + if addr == "" { + t.Skip("CLICKHOUSE_ADDR not set, skipping integration test") + } + + cfg := &Config{ + Addr: addr, + Database: "default", + Compression: "lz4", + Network: "test", + } + + client, err := New(context.Background(), cfg) require.NoError(t, err) - // Execute query - should timeout - ctx := context.Background() + defer func() { _ = client.Stop() }() + + err = client.Start() + require.NoError(t, err) - var result struct{} + var result struct { + Value int64 `json:"value"` + } - err = client.QueryOne(ctx, "SELECT 1", &result) - assert.Error(t, err) - // The timeout causes the request to fail - we just need to ensure an error occurred + err = client.QueryOne(t.Context(), "SELECT 42 as value", &result) + require.NoError(t, err) + assert.Equal(t, int64(42), result.Value) } -func TestHTTPClient_StartStop(t *testing.T) { - // Create test server - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte("Ok.")) - })) - defer server.Close() - - // Create client - client, err := New(&Config{ - URL: server.URL, - }) +func TestClient_Integration_IsStorageEmpty(t *testing.T) { + addr := os.Getenv("CLICKHOUSE_ADDR") + if addr == "" { + t.Skip("CLICKHOUSE_ADDR not set, skipping integration test") + } + + cfg := &Config{ + Addr: addr, + Database: "default", + Compression: "lz4", + Network: "test", + } + + client, err := New(context.Background(), cfg) require.NoError(t, err) - // Start should succeed + defer func() { _ = client.Stop() }() + err = client.Start() - assert.NoError(t, err) + require.NoError(t, err) - // Stop should succeed - err = client.Stop() - assert.NoError(t, err) + // Create a temporary table + err = client.Execute(t.Context(), ` + CREATE TABLE IF NOT EXISTS test_empty_check ( + id UInt64, + name String + ) ENGINE = Memory + `) + require.NoError(t, err) + + // Should be empty + isEmpty, err := client.IsStorageEmpty(t.Context(), "test_empty_check", nil) + require.NoError(t, err) + assert.True(t, isEmpty) + + // Drop the table + err = client.Execute(t.Context(), "DROP TABLE IF EXISTS test_empty_check") + require.NoError(t, err) } -func TestConfig_Validate(t *testing.T) { +// mockNetError implements net.Error for testing. +type mockNetError struct { + timeout bool + temporary bool +} + +func (e *mockNetError) Error() string { return "mock network error" } +func (e *mockNetError) Timeout() bool { return e.timeout } +func (e *mockNetError) Temporary() bool { return e.temporary } + +func TestIsRetryableError(t *testing.T) { tests := []struct { - name string - config Config - expectError bool + name string + err error + expected bool }{ + // Nil error { - name: "valid config", - config: Config{ - URL: "http://localhost:8123", - }, - expectError: false, + name: "nil error", + err: nil, + expected: false, }, + // Context errors - non-retryable { - name: "missing URL", - config: Config{}, - expectError: true, + name: "context canceled", + err: context.Canceled, + expected: false, + }, + { + name: "context deadline exceeded", + err: context.DeadlineExceeded, + expected: false, + }, + // ch-go sentinel errors - non-retryable + { + name: "ch.ErrClosed", + err: ch.ErrClosed, + expected: false, + }, + { + name: "wrapped ch.ErrClosed", + err: errors.Join(errors.New("operation failed"), ch.ErrClosed), + expected: false, + }, + // ch-go server exceptions - retryable codes (using ch.Exception which implements error) + { + name: "proto.ErrTimeoutExceeded", + err: &ch.Exception{Code: proto.ErrTimeoutExceeded, Message: "timeout"}, + expected: true, + }, + { + name: "proto.ErrNoFreeConnection", + err: &ch.Exception{Code: proto.ErrNoFreeConnection, Message: "no free connection"}, + expected: true, + }, + { + name: "proto.ErrTooManySimultaneousQueries", + err: &ch.Exception{Code: proto.ErrTooManySimultaneousQueries, Message: "rate limited"}, + expected: true, + }, + { + name: "proto.ErrSocketTimeout", + err: &ch.Exception{Code: proto.ErrSocketTimeout, Message: "socket timeout"}, + expected: true, + }, + { + name: "proto.ErrNetworkError", + err: &ch.Exception{Code: proto.ErrNetworkError, Message: "network error"}, + expected: true, + }, + // ch-go server exceptions - non-retryable codes + { + name: "proto.ErrBadArguments", + err: &ch.Exception{Code: proto.ErrBadArguments, Message: "bad arguments"}, + expected: false, + }, + { + name: "proto.ErrUnknownTable", + err: &ch.Exception{Code: proto.ErrUnknownTable, Message: "unknown table"}, + expected: false, + }, + // Data corruption - non-retryable + { + name: "compress.CorruptedDataErr", + err: &compress.CorruptedDataErr{}, + expected: false, + }, + // Network errors + { + name: "network timeout error", + err: &mockNetError{timeout: true}, + expected: true, + }, + { + name: "network non-timeout error", + err: &mockNetError{timeout: false}, + expected: false, + }, + // Syscall errors - retryable + { + name: "syscall.ECONNRESET", + err: syscall.ECONNRESET, + expected: true, + }, + { + name: "syscall.ECONNREFUSED", + err: syscall.ECONNREFUSED, + expected: true, + }, + { + name: "syscall.EPIPE", + err: syscall.EPIPE, + expected: true, + }, + { + name: "io.EOF", + err: io.EOF, + expected: true, + }, + { + name: "io.ErrUnexpectedEOF", + err: io.ErrUnexpectedEOF, + expected: true, + }, + // String pattern fallback - retryable + { + name: "connection reset string", + err: errors.New("connection reset by peer"), + expected: true, + }, + { + name: "server overloaded string", + err: errors.New("server is overloaded"), + expected: true, + }, + { + name: "too many connections string", + err: errors.New("too many connections"), + expected: true, + }, + // Unknown errors - non-retryable + { + name: "unknown error", + err: errors.New("some unknown error"), + expected: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := tt.config.Validate() - if tt.expectError { - assert.Error(t, err) - } else { - assert.NoError(t, err) - } + result := isRetryableError(tt.err) + assert.Equal(t, tt.expected, result) }) } } -func TestConfig_SetDefaults(t *testing.T) { - config := Config{ - URL: "http://localhost:8123", - } - - config.SetDefaults() - - assert.Equal(t, 30*time.Second, config.QueryTimeout) - assert.Equal(t, 5*time.Minute, config.InsertTimeout) - assert.Equal(t, 30*time.Second, config.KeepAlive) -} - -func TestHTTPClient_DebugLogging(t *testing.T) { - // Create test server - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - response := `{"data": [{"value": 1}], "meta": [], "rows": 1}` - - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte(response)) - })) - defer server.Close() - - // Create client with debug enabled - client, err := New(&Config{ - URL: server.URL, - Debug: true, - }) - require.NoError(t, err) - - // Execute query - debug logging should not cause errors - var result struct { - Value int `json:"value"` - } - - err = client.QueryOne(context.Background(), "SELECT 1 as value", &result) - assert.NoError(t, err) - assert.Equal(t, 1, result.Value) -} +// Ensure mockNetError implements net.Error. +var _ net.Error = (*mockNetError)(nil) diff --git a/pkg/clickhouse/config.go b/pkg/clickhouse/config.go index fe218b4..efb9204 100644 --- a/pkg/clickhouse/config.go +++ b/pkg/clickhouse/config.go @@ -5,35 +5,92 @@ import ( "time" ) +// Config holds configuration for ClickHouse ch-go native client. +// //nolint:tagliatelle // YAML config uses snake_case by convention type Config struct { - URL string `yaml:"url"` - QueryTimeout time.Duration `yaml:"query_timeout"` - InsertTimeout time.Duration `yaml:"insert_timeout"` - Network string `yaml:"network"` - Processor string `yaml:"processor"` - Debug bool `yaml:"debug"` - KeepAlive time.Duration `yaml:"keep_alive"` + // Connection + Addr string `yaml:"addr"` // Native protocol address, e.g., "localhost:9000" + Database string `yaml:"database"` // Database name, default: "default" + Username string `yaml:"username"` + Password string `yaml:"password"` + + // Pool settings + MaxConns int32 `yaml:"max_conns"` // Maximum connections in pool, default: 10 + MinConns int32 `yaml:"min_conns"` // Minimum connections in pool, default: 2 + ConnMaxLifetime time.Duration `yaml:"conn_max_lifetime"` // Maximum connection lifetime, default: 1h + ConnMaxIdleTime time.Duration `yaml:"conn_max_idle_time"` // Maximum idle time, default: 30m + HealthCheckPeriod time.Duration `yaml:"health_check_period"` // Health check period, default: 1m + DialTimeout time.Duration `yaml:"dial_timeout"` // Dial timeout, default: 10s + + // Performance + Compression string `yaml:"compression"` // Compression: lz4, zstd, none (default: lz4) + + // Retry settings + MaxRetries int `yaml:"max_retries"` // Maximum retry attempts, default: 3 + RetryBaseDelay time.Duration `yaml:"retry_base_delay"` // Base delay for exponential backoff, default: 100ms + + // Timeout settings + QueryTimeout time.Duration `yaml:"query_timeout"` // Query timeout per attempt, default: 60s + + // Metrics labels + Network string `yaml:"network"` + Processor string `yaml:"processor"` + Debug bool `yaml:"debug"` } +// Validate checks if the config is valid. func (c *Config) Validate() error { - if c.URL == "" { - return fmt.Errorf("URL is required") + if c.Addr == "" { + return fmt.Errorf("addr is required") } return nil } +// SetDefaults sets default values for unset fields. func (c *Config) SetDefaults() { - if c.QueryTimeout == 0 { - c.QueryTimeout = 30 * time.Second + if c.Database == "" { + c.Database = "default" + } + + if c.MaxConns == 0 { + c.MaxConns = 10 + } + + if c.DialTimeout == 0 { + c.DialTimeout = 10 * time.Second + } + + if c.MinConns == 0 { + c.MinConns = 2 } - if c.InsertTimeout == 0 { - c.InsertTimeout = 5 * time.Minute + if c.ConnMaxLifetime == 0 { + c.ConnMaxLifetime = time.Hour } - if c.KeepAlive == 0 { - c.KeepAlive = 30 * time.Second + if c.ConnMaxIdleTime == 0 { + c.ConnMaxIdleTime = 30 * time.Minute + } + + if c.HealthCheckPeriod == 0 { + c.HealthCheckPeriod = time.Minute + } + + if c.Compression == "" { + c.Compression = "lz4" + } + + if c.MaxRetries == 0 { + c.MaxRetries = 3 + } + + if c.RetryBaseDelay == 0 { + c.RetryBaseDelay = 100 * time.Millisecond + } + + if c.QueryTimeout == 0 { + c.QueryTimeout = 60 * time.Second } } diff --git a/pkg/clickhouse/interface.go b/pkg/clickhouse/interface.go index 2b56635..608f8fe 100644 --- a/pkg/clickhouse/interface.go +++ b/pkg/clickhouse/interface.go @@ -1,23 +1,28 @@ package clickhouse -import "context" +import ( + "context" + + "github.com/ClickHouse/ch-go" +) // ClientInterface defines the methods for interacting with ClickHouse. type ClientInterface interface { // QueryOne executes a query and returns a single result - QueryOne(ctx context.Context, query string, dest interface{}) error + QueryOne(ctx context.Context, query string, dest any) error // QueryMany executes a query and returns multiple results - QueryMany(ctx context.Context, query string, dest interface{}) error + QueryMany(ctx context.Context, query string, dest any) error // Execute runs a query without expecting results Execute(ctx context.Context, query string) error - // BulkInsert performs a bulk insert operation - BulkInsert(ctx context.Context, table string, data interface{}) error // IsStorageEmpty checks if a table has any records matching the given conditions - IsStorageEmpty(ctx context.Context, table string, conditions map[string]interface{}) (bool, error) + IsStorageEmpty(ctx context.Context, table string, conditions map[string]any) (bool, error) // SetNetwork updates the network name for metrics labeling SetNetwork(network string) // Start initializes the client Start() error // Stop closes the client Stop() error + + // Do executes a ch-go query directly for streaming operations + Do(ctx context.Context, query ch.Query) error } diff --git a/pkg/clickhouse/mock.go b/pkg/clickhouse/mock.go index 9473136..a835366 100644 --- a/pkg/clickhouse/mock.go +++ b/pkg/clickhouse/mock.go @@ -7,18 +7,21 @@ import ( "encoding/json" "fmt" "reflect" + + "github.com/ClickHouse/ch-go" ) // MockClient is a mock implementation of ClientInterface for testing. // It should only be used in test files, not in production code. type MockClient struct { // Function fields that can be set by tests - QueryOneFunc func(ctx context.Context, query string, dest interface{}) error - QueryManyFunc func(ctx context.Context, query string, dest interface{}) error - ExecuteFunc func(ctx context.Context, query string) error - BulkInsertFunc func(ctx context.Context, table string, data interface{}) error - StartFunc func() error - StopFunc func() error + QueryOneFunc func(ctx context.Context, query string, dest any) error + QueryManyFunc func(ctx context.Context, query string, dest any) error + ExecuteFunc func(ctx context.Context, query string) error + IsStorageEmptyFunc func(ctx context.Context, table string, conditions map[string]any) (bool, error) + StartFunc func() error + StopFunc func() error + DoFunc func(ctx context.Context, query ch.Query) error // Track calls for assertions Calls []MockCall @@ -27,23 +30,23 @@ type MockClient struct { // MockCall represents a method call made to the mock. type MockCall struct { Method string - Args []interface{} + Args []any } // NewMockClient creates a new mock client with default implementations. func NewMockClient() *MockClient { return &MockClient{ - QueryOneFunc: func(ctx context.Context, query string, dest interface{}) error { + QueryOneFunc: func(ctx context.Context, query string, dest any) error { return nil }, - QueryManyFunc: func(ctx context.Context, query string, dest interface{}) error { + QueryManyFunc: func(ctx context.Context, query string, dest any) error { return nil }, ExecuteFunc: func(ctx context.Context, query string) error { return nil }, - BulkInsertFunc: func(ctx context.Context, table string, data interface{}) error { - return nil + IsStorageEmptyFunc: func(ctx context.Context, table string, conditions map[string]any) (bool, error) { + return true, nil }, StartFunc: func() error { return nil @@ -51,15 +54,18 @@ func NewMockClient() *MockClient { StopFunc: func() error { return nil }, + DoFunc: func(ctx context.Context, query ch.Query) error { + return nil + }, Calls: make([]MockCall, 0), } } // QueryOne implements ClientInterface. -func (m *MockClient) QueryOne(ctx context.Context, query string, dest interface{}) error { +func (m *MockClient) QueryOne(ctx context.Context, query string, dest any) error { m.Calls = append(m.Calls, MockCall{ Method: "QueryOne", - Args: []interface{}{ctx, query, dest}, + Args: []any{ctx, query, dest}, }) if m.QueryOneFunc != nil { @@ -70,10 +76,10 @@ func (m *MockClient) QueryOne(ctx context.Context, query string, dest interface{ } // QueryMany implements ClientInterface. -func (m *MockClient) QueryMany(ctx context.Context, query string, dest interface{}) error { +func (m *MockClient) QueryMany(ctx context.Context, query string, dest any) error { m.Calls = append(m.Calls, MockCall{ Method: "QueryMany", - Args: []interface{}{ctx, query, dest}, + Args: []any{ctx, query, dest}, }) if m.QueryManyFunc != nil { @@ -87,7 +93,7 @@ func (m *MockClient) QueryMany(ctx context.Context, query string, dest interface func (m *MockClient) Execute(ctx context.Context, query string) error { m.Calls = append(m.Calls, MockCall{ Method: "Execute", - Args: []interface{}{ctx, query}, + Args: []any{ctx, query}, }) if m.ExecuteFunc != nil { @@ -97,25 +103,33 @@ func (m *MockClient) Execute(ctx context.Context, query string) error { return nil } -// BulkInsert implements ClientInterface. -func (m *MockClient) BulkInsert(ctx context.Context, table string, data interface{}) error { +// IsStorageEmpty implements ClientInterface. +func (m *MockClient) IsStorageEmpty(ctx context.Context, table string, conditions map[string]any) (bool, error) { m.Calls = append(m.Calls, MockCall{ - Method: "BulkInsert", - Args: []interface{}{ctx, table, data}, + Method: "IsStorageEmpty", + Args: []any{ctx, table, conditions}, }) - if m.BulkInsertFunc != nil { - return m.BulkInsertFunc(ctx, table, data) + if m.IsStorageEmptyFunc != nil { + return m.IsStorageEmptyFunc(ctx, table, conditions) } - return nil + return true, nil +} + +// SetNetwork implements ClientInterface. +func (m *MockClient) SetNetwork(network string) { + m.Calls = append(m.Calls, MockCall{ + Method: "SetNetwork", + Args: []any{network}, + }) } // Start implements ClientInterface. func (m *MockClient) Start() error { m.Calls = append(m.Calls, MockCall{ Method: "Start", - Args: []interface{}{}, + Args: []any{}, }) if m.StartFunc != nil { @@ -129,7 +143,7 @@ func (m *MockClient) Start() error { func (m *MockClient) Stop() error { m.Calls = append(m.Calls, MockCall{ Method: "Stop", - Args: []interface{}{}, + Args: []any{}, }) if m.StopFunc != nil { @@ -139,6 +153,20 @@ func (m *MockClient) Stop() error { return nil } +// Do implements ClientInterface. +func (m *MockClient) Do(ctx context.Context, query ch.Query) error { + m.Calls = append(m.Calls, MockCall{ + Method: "Do", + Args: []any{ctx, query}, + }) + + if m.DoFunc != nil { + return m.DoFunc(ctx, query) + } + + return nil +} + // GetCallCount returns the number of times a method was called. func (m *MockClient) GetCallCount(method string) int { count := 0 @@ -165,8 +193,8 @@ func (m *MockClient) Reset() { // Helper functions for common test scenarios // SetQueryOneResponse sets up the mock to return specific data for QueryOne. -func (m *MockClient) SetQueryOneResponse(data interface{}) { - m.QueryOneFunc = func(ctx context.Context, query string, dest interface{}) error { +func (m *MockClient) SetQueryOneResponse(data any) { + m.QueryOneFunc = func(ctx context.Context, query string, dest any) error { // Marshal the data to JSON and then unmarshal into dest jsonData, err := json.Marshal(data) if err != nil { @@ -178,8 +206,8 @@ func (m *MockClient) SetQueryOneResponse(data interface{}) { } // SetQueryManyResponse sets up the mock to return specific data for QueryMany. -func (m *MockClient) SetQueryManyResponse(data interface{}) { - m.QueryManyFunc = func(ctx context.Context, query string, dest interface{}) error { +func (m *MockClient) SetQueryManyResponse(data any) { + m.QueryManyFunc = func(ctx context.Context, query string, dest any) error { // Use reflection to set the slice destValue := reflect.ValueOf(dest).Elem() srcValue := reflect.ValueOf(data) @@ -196,17 +224,17 @@ func (m *MockClient) SetQueryManyResponse(data interface{}) { // SetError sets all functions to return the specified error. func (m *MockClient) SetError(err error) { - m.QueryOneFunc = func(ctx context.Context, query string, dest interface{}) error { + m.QueryOneFunc = func(ctx context.Context, query string, dest any) error { return err } - m.QueryManyFunc = func(ctx context.Context, query string, dest interface{}) error { + m.QueryManyFunc = func(ctx context.Context, query string, dest any) error { return err } m.ExecuteFunc = func(ctx context.Context, query string) error { return err } - m.BulkInsertFunc = func(ctx context.Context, table string, data interface{}) error { - return err + m.IsStorageEmptyFunc = func(ctx context.Context, table string, conditions map[string]any) (bool, error) { + return false, err } m.StartFunc = func() error { return err @@ -214,4 +242,7 @@ func (m *MockClient) SetError(err error) { m.StopFunc = func() error { return err } + m.DoFunc = func(ctx context.Context, query ch.Query) error { + return err + } } diff --git a/pkg/common/metrics.go b/pkg/common/metrics.go index 2552d6f..f452116 100644 --- a/pkg/common/metrics.go +++ b/pkg/common/metrics.go @@ -152,4 +152,57 @@ var ( Name: "execution_processor_retry_count_total", Help: "Total number of retry attempts", }, []string{"network", "processor", "reason"}) + + // ClickHouse pool metrics - gauges for current state. + ClickHousePoolAcquiredResources = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "execution_processor_clickhouse_pool_acquired_resources", + Help: "Number of currently acquired resources in the ClickHouse connection pool", + }, []string{"network", "processor"}) + + ClickHousePoolIdleResources = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "execution_processor_clickhouse_pool_idle_resources", + Help: "Number of currently idle resources in the ClickHouse connection pool", + }, []string{"network", "processor"}) + + ClickHousePoolConstructingResources = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "execution_processor_clickhouse_pool_constructing_resources", + Help: "Number of resources currently being constructed in the ClickHouse connection pool", + }, []string{"network", "processor"}) + + ClickHousePoolTotalResources = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "execution_processor_clickhouse_pool_total_resources", + Help: "Total number of resources in the ClickHouse connection pool", + }, []string{"network", "processor"}) + + ClickHousePoolMaxResources = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "execution_processor_clickhouse_pool_max_resources", + Help: "Maximum number of resources allowed in the ClickHouse connection pool", + }, []string{"network", "processor"}) + + // ClickHouse pool metrics - counters for cumulative values. + ClickHousePoolAcquireTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "execution_processor_clickhouse_pool_acquire_total", + Help: "Total number of successful resource acquisitions from the ClickHouse connection pool", + }, []string{"network", "processor"}) + + ClickHousePoolEmptyAcquireTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "execution_processor_clickhouse_pool_empty_acquire_total", + Help: "Total number of acquires that waited for a resource because the pool was empty", + }, []string{"network", "processor"}) + + ClickHousePoolCanceledAcquireTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "execution_processor_clickhouse_pool_canceled_acquire_total", + Help: "Total number of acquires that were canceled due to context cancellation", + }, []string{"network", "processor"}) + + // ClickHouse pool timing metrics - cumulative durations. + ClickHousePoolAcquireDuration = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "execution_processor_clickhouse_pool_acquire_duration_seconds", + Help: "Cumulative time spent acquiring resources from the ClickHouse connection pool", + }, []string{"network", "processor"}) + + ClickHousePoolEmptyAcquireWaitDuration = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "execution_processor_clickhouse_pool_empty_acquire_wait_duration_seconds", + Help: "Cumulative time spent waiting for a resource when pool was empty", + }, []string{"network", "processor"}) ) diff --git a/pkg/leaderelection/redis_election_test.go b/pkg/leaderelection/redis_election_test.go index 5c78e42..b772413 100644 --- a/pkg/leaderelection/redis_election_test.go +++ b/pkg/leaderelection/redis_election_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/alicebob/miniredis/v2" "github.com/ethpandaops/execution-processor/pkg/leaderelection" "github.com/redis/go-redis/v9" "github.com/sirupsen/logrus" @@ -14,24 +15,17 @@ import ( "github.com/stretchr/testify/require" ) -// skipIfRedisUnavailable checks if Redis is available and skips the test if not. -func skipIfRedisUnavailable(t *testing.T) *redis.Client { +// newTestRedis creates an in-memory Redis server for testing. +// The server and client are automatically cleaned up when the test completes. +func newTestRedis(t *testing.T) *redis.Client { t.Helper() - client := redis.NewClient(&redis.Options{ - Addr: "localhost:6379", - DB: 15, // Use high DB number for tests - }) - - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() + s := miniredis.RunT(t) + client := redis.NewClient(&redis.Options{Addr: s.Addr()}) - if err := client.Ping(ctx).Err(); err != nil { - client.Close() - t.Skipf("Redis unavailable at localhost:6379: %v", err) - - return nil - } + t.Cleanup(func() { + _ = client.Close() + }) return client } @@ -41,11 +35,7 @@ func skipIfRedisUnavailable(t *testing.T) *redis.Client { // ===================================== func TestNewRedisElector(t *testing.T) { - client := skipIfRedisUnavailable(t) - if client == nil { - return - } - defer client.Close() + client := newTestRedis(t) log := logrus.New() log.SetLevel(logrus.ErrorLevel) @@ -127,11 +117,7 @@ func TestNewRedisElector(t *testing.T) { } func TestRedisElector_GetLeaderID_NoLeader(t *testing.T) { - client := skipIfRedisUnavailable(t) - if client == nil { - return - } - defer client.Close() + client := newTestRedis(t) log := logrus.New() log.SetLevel(logrus.ErrorLevel) @@ -162,11 +148,7 @@ func TestRedisElector_GetLeaderID_NoLeader(t *testing.T) { } func TestRedisElector_StartStop(t *testing.T) { - client := skipIfRedisUnavailable(t) - if client == nil { - return - } - defer client.Close() + client := newTestRedis(t) log := logrus.New() log.SetLevel(logrus.ErrorLevel) @@ -212,11 +194,7 @@ func TestRedisElector_StartStop(t *testing.T) { } func TestRedisElector_LeadershipAcquisition(t *testing.T) { - client := skipIfRedisUnavailable(t) - if client == nil { - return - } - defer client.Close() + client := newTestRedis(t) log := logrus.New() log.SetLevel(logrus.ErrorLevel) @@ -285,11 +263,7 @@ func TestRedisElector_LeadershipAcquisition(t *testing.T) { } func TestRedisElector_MultipleNodes(t *testing.T) { - client := skipIfRedisUnavailable(t) - if client == nil { - return - } - defer client.Close() + client := newTestRedis(t) log := logrus.New() log.SetLevel(logrus.ErrorLevel) @@ -368,11 +342,7 @@ func TestRedisElector_MultipleNodes(t *testing.T) { } func TestRedisElector_LeadershipTransition(t *testing.T) { - client := skipIfRedisUnavailable(t) - if client == nil { - return - } - defer client.Close() + client := newTestRedis(t) log := logrus.New() log.SetLevel(logrus.ErrorLevel) @@ -479,11 +449,7 @@ func TestRedisElector_LeadershipTransition(t *testing.T) { // ===================================== func TestRedisElector_RenewalFailure(t *testing.T) { - client := skipIfRedisUnavailable(t) - if client == nil { - return - } - defer client.Close() + client := newTestRedis(t) log := logrus.New() log.SetLevel(logrus.ErrorLevel) @@ -545,11 +511,7 @@ func TestRedisElector_RenewalFailure(t *testing.T) { } func TestRedisElector_ConcurrentElectors(t *testing.T) { - client := skipIfRedisUnavailable(t) - if client == nil { - return - } - defer client.Close() + client := newTestRedis(t) log := logrus.New() log.SetLevel(logrus.ErrorLevel) @@ -638,11 +600,7 @@ func TestRedisElector_ConcurrentElectors(t *testing.T) { } func TestRedisElector_StopWithoutStart(t *testing.T) { - client := skipIfRedisUnavailable(t) - if client == nil { - return - } - defer client.Close() + client := newTestRedis(t) log := logrus.New() log.SetLevel(logrus.ErrorLevel) @@ -665,11 +623,7 @@ func TestRedisElector_StopWithoutStart(t *testing.T) { } func TestRedisElector_MultipleStops(t *testing.T) { - client := skipIfRedisUnavailable(t) - if client == nil { - return - } - defer client.Close() + client := newTestRedis(t) log := logrus.New() log.SetLevel(logrus.ErrorLevel) @@ -709,11 +663,7 @@ func TestRedisElector_MultipleStops(t *testing.T) { } func TestRedisElector_ContextCancellation(t *testing.T) { - client := skipIfRedisUnavailable(t) - if client == nil { - return - } - defer client.Close() + client := newTestRedis(t) log := logrus.New() log.SetLevel(logrus.ErrorLevel) diff --git a/pkg/processor/config.go b/pkg/processor/config.go index 71a9c77..d64fe57 100644 --- a/pkg/processor/config.go +++ b/pkg/processor/config.go @@ -99,8 +99,8 @@ func (c *Config) Validate() error { } if c.TransactionStructlog.Enabled { - if c.TransactionStructlog.URL == "" { - return fmt.Errorf("transaction structlog URL is required when enabled") + if c.TransactionStructlog.Addr == "" { + return fmt.Errorf("transaction structlog addr is required when enabled") } if c.TransactionStructlog.Table == "" { diff --git a/pkg/processor/manager_test.go b/pkg/processor/manager_test.go index ed34068..b267aff 100644 --- a/pkg/processor/manager_test.go +++ b/pkg/processor/manager_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "github.com/alicebob/miniredis/v2" "github.com/ethpandaops/execution-processor/pkg/clickhouse" "github.com/ethpandaops/execution-processor/pkg/ethereum" "github.com/ethpandaops/execution-processor/pkg/ethereum/execution" @@ -18,6 +19,20 @@ import ( "github.com/stretchr/testify/require" ) +// newTestRedis creates an in-memory Redis server for testing. +func newTestRedis(t *testing.T) *redis.Client { + t.Helper() + + s := miniredis.RunT(t) + client := redis.NewClient(&redis.Options{Addr: s.Addr()}) + + t.Cleanup(func() { + _ = client.Close() + }) + + return client +} + // Basic functionality tests func TestManager_Creation(t *testing.T) { @@ -39,11 +54,7 @@ func TestManager_Creation(t *testing.T) { }, } - redisClient := redis.NewClient(&redis.Options{ - Addr: "localhost:6379", - DB: 15, // Use high DB number for tests - }) - defer redisClient.Close() + redisClient := newTestRedis(t) // Create proper pool and state mocks poolConfig := ðereum.Config{ @@ -59,7 +70,7 @@ func TestManager_Creation(t *testing.T) { stateConfig := &state.Config{ Storage: state.StorageConfig{ Config: clickhouse.Config{ - URL: "http://localhost:8123", + Addr: "localhost:9000", }, Table: "test_manager_blocks", }, @@ -94,11 +105,7 @@ func TestManager_StartStop(t *testing.T) { }, } - redisClient := redis.NewClient(&redis.Options{ - Addr: "localhost:6379", - DB: 15, - }) - defer redisClient.Close() + redisClient := newTestRedis(t) // Create proper components poolConfig := ðereum.Config{ @@ -114,7 +121,7 @@ func TestManager_StartStop(t *testing.T) { stateConfig := &state.Config{ Storage: state.StorageConfig{ Config: clickhouse.Config{ - URL: "http://localhost:8123", + Addr: "localhost:9000", }, Table: "test_manager_start_stop_blocks", }, @@ -176,11 +183,7 @@ func TestManager_MultipleStops(t *testing.T) { }, } - redisClient := redis.NewClient(&redis.Options{ - Addr: "localhost:6379", - DB: 14, - }) - defer redisClient.Close() + redisClient := newTestRedis(t) // Create proper components poolConfig := ðereum.Config{ @@ -196,7 +199,7 @@ func TestManager_MultipleStops(t *testing.T) { stateConfig := &state.Config{ Storage: state.StorageConfig{ Config: clickhouse.Config{ - URL: "http://localhost:8123", + Addr: "localhost:9000", }, Table: "test_manager_multi_stop_blocks", }, @@ -263,12 +266,8 @@ func TestManager_ModeSpecificLeaderElection(t *testing.T) { Enabled: false, // Disable to avoid ClickHouse requirements Table: "test_structlog", Config: clickhouse.Config{ - URL: "http://localhost:8123", + Addr: "localhost:9000", }, - BigTransactionThreshold: 500000, - BatchInsertThreshold: 50000, - BatchFlushInterval: 5 * time.Second, - BatchMaxSize: 100000, }, } @@ -286,7 +285,7 @@ func TestManager_ModeSpecificLeaderElection(t *testing.T) { stateConfig := &state.Config{ Storage: state.StorageConfig{ Config: clickhouse.Config{ - URL: "http://localhost:8123", + Addr: "localhost:9000", }, Table: "test_mode_blocks", }, @@ -297,11 +296,7 @@ func TestManager_ModeSpecificLeaderElection(t *testing.T) { stateManager, err := state.NewManager(context.Background(), log.WithField("component", "state"), stateConfig) require.NoError(t, err) - redisClient := redis.NewClient(&redis.Options{ - Addr: "localhost:6379", - DB: 15, // Use test DB - }) - defer redisClient.Close() + redisClient := newTestRedis(t) // Create manager (leader election will be initialized) manager, err := processor.NewManager(log, config, pool, stateManager, redisClient, "test-prefix") @@ -363,7 +358,7 @@ func TestManager_ConcurrentModes(t *testing.T) { stateConfig := &state.Config{ Storage: state.StorageConfig{ Config: clickhouse.Config{ - URL: "http://localhost:8123", + Addr: "localhost:9000", }, Table: "test_concurrent_blocks", }, @@ -374,18 +369,9 @@ func TestManager_ConcurrentModes(t *testing.T) { stateManager, err := state.NewManager(context.Background(), log.WithField("component", "state"), stateConfig) require.NoError(t, err) - // Use different Redis DBs to simulate separate Redis instances - forwardsRedis := redis.NewClient(&redis.Options{ - Addr: "localhost:6379", - DB: 10, // Forwards Redis DB - }) - defer forwardsRedis.Close() - - backwardsRedis := redis.NewClient(&redis.Options{ - Addr: "localhost:6379", - DB: 11, // Backwards Redis DB - }) - defer backwardsRedis.Close() + // Each miniredis instance is isolated, simulating separate Redis servers + forwardsRedis := newTestRedis(t) + backwardsRedis := newTestRedis(t) // Create both managers - they should not conflict with mode-specific leader keys forwardsManager, err := processor.NewManager(log.WithField("mode", "forwards"), forwardsConfig, pool, stateManager, forwardsRedis, "test-prefix") @@ -427,7 +413,7 @@ func TestManager_LeaderElectionDisabled(t *testing.T) { stateConfig := &state.Config{ Storage: state.StorageConfig{ Config: clickhouse.Config{ - URL: "http://localhost:8123", + Addr: "localhost:9000", }, Table: "test_no_leader_blocks", }, @@ -438,11 +424,7 @@ func TestManager_LeaderElectionDisabled(t *testing.T) { stateManager, err := state.NewManager(context.Background(), log.WithField("component", "state"), stateConfig) require.NoError(t, err) - redisClient := redis.NewClient(&redis.Options{ - Addr: "localhost:6379", - DB: 15, - }) - defer redisClient.Close() + redisClient := newTestRedis(t) // Should work fine even with leader election disabled manager, err := processor.NewManager(log, config, pool, stateManager, redisClient, "test-prefix") @@ -487,7 +469,7 @@ func TestManager_RaceConditions(t *testing.T) { stateConfig := &state.Config{ Storage: state.StorageConfig{ Config: clickhouse.Config{ - URL: "http://localhost:8123", + Addr: "localhost:9000", }, Table: "test_race_blocks", }, @@ -498,11 +480,7 @@ func TestManager_RaceConditions(t *testing.T) { stateManager, err := state.NewManager(context.Background(), log.WithField("component", "state"), stateConfig) require.NoError(t, err) - redisClient := redis.NewClient(&redis.Options{ - Addr: "localhost:6379", - DB: 15, - }) - defer redisClient.Close() + redisClient := newTestRedis(t) manager, err := processor.NewManager(log, config, pool, stateManager, redisClient, "test-prefix") require.NoError(t, err) @@ -572,7 +550,7 @@ func TestManager_ConcurrentConfiguration(t *testing.T) { stateConfig := &state.Config{ Storage: state.StorageConfig{ Config: clickhouse.Config{ - URL: "http://localhost:8123", + Addr: "localhost:9000", }, Table: "test_config_blocks", }, @@ -583,11 +561,7 @@ func TestManager_ConcurrentConfiguration(t *testing.T) { stateManager, err := state.NewManager(context.Background(), log.WithField("component", "state"), stateConfig) require.NoError(t, err) - redisClient := redis.NewClient(&redis.Options{ - Addr: "localhost:6379", - DB: 14, - }) - defer redisClient.Close() + redisClient := newTestRedis(t) _, err = processor.NewManager(log, config, pool, stateManager, redisClient, "test-prefix") require.NoError(t, err) diff --git a/pkg/processor/transaction/simple/columns.go b/pkg/processor/transaction/simple/columns.go new file mode 100644 index 0000000..7a87ed1 --- /dev/null +++ b/pkg/processor/transaction/simple/columns.go @@ -0,0 +1,196 @@ +package simple + +import ( + "math/big" + "time" + + "github.com/ClickHouse/ch-go/proto" +) + +// Columns holds all columns for transaction batch insert using ch-go columnar protocol. +type Columns struct { + UpdatedDateTime proto.ColDateTime + BlockNumber proto.ColUInt64 + BlockHash proto.ColStr + ParentHash proto.ColStr + Position proto.ColUInt32 + Hash proto.ColStr + From proto.ColStr + To *proto.ColNullable[string] + Nonce proto.ColUInt64 + GasPrice proto.ColUInt128 + Gas proto.ColUInt64 + GasTipCap *proto.ColNullable[proto.UInt128] + GasFeeCap *proto.ColNullable[proto.UInt128] + Value proto.ColUInt128 + Type proto.ColUInt8 + Size proto.ColUInt32 + CallDataSize proto.ColUInt32 + BlobGas *proto.ColNullable[uint64] + BlobGasFeeCap *proto.ColNullable[proto.UInt128] + BlobHashes *proto.ColArr[string] + Success proto.ColBool + NInputBytes proto.ColUInt32 + NInputZeroBytes proto.ColUInt32 + NInputNonzeroBytes proto.ColUInt32 + MetaNetworkName proto.ColStr +} + +// NewColumns creates a new Columns instance with all nullable and array columns initialized. +func NewColumns() *Columns { + return &Columns{ + To: new(proto.ColStr).Nullable(), + GasTipCap: new(proto.ColUInt128).Nullable(), + GasFeeCap: new(proto.ColUInt128).Nullable(), + BlobGas: new(proto.ColUInt64).Nullable(), + BlobGasFeeCap: new(proto.ColUInt128).Nullable(), + BlobHashes: new(proto.ColStr).Array(), + } +} + +// Append adds a Transaction row to all columns. +func (c *Columns) Append(tx Transaction) { + c.UpdatedDateTime.Append(time.Time(tx.UpdatedDateTime)) + c.BlockNumber.Append(tx.BlockNumber) + c.BlockHash.Append(tx.BlockHash) + c.ParentHash.Append(tx.ParentHash) + c.Position.Append(tx.Position) + c.Hash.Append(tx.Hash) + c.From.Append(tx.From) + c.To.Append(nullableStr(tx.To)) + c.Nonce.Append(tx.Nonce) + c.GasPrice.Append(parseUInt128(tx.GasPrice)) + c.Gas.Append(tx.Gas) + c.GasTipCap.Append(nullableUInt128(tx.GasTipCap)) + c.GasFeeCap.Append(nullableUInt128(tx.GasFeeCap)) + c.Value.Append(parseUInt128(tx.Value)) + c.Type.Append(tx.Type) + c.Size.Append(tx.Size) + c.CallDataSize.Append(tx.CallDataSize) + c.BlobGas.Append(nullableUint64(tx.BlobGas)) + c.BlobGasFeeCap.Append(nullableUInt128(tx.BlobGasFeeCap)) + c.BlobHashes.Append(tx.BlobHashes) + c.Success.Append(tx.Success) + c.NInputBytes.Append(tx.NInputBytes) + c.NInputZeroBytes.Append(tx.NInputZeroBytes) + c.NInputNonzeroBytes.Append(tx.NInputNonzeroBytes) + c.MetaNetworkName.Append(tx.MetaNetworkName) +} + +// Reset clears all columns for reuse. +func (c *Columns) Reset() { + c.UpdatedDateTime.Reset() + c.BlockNumber.Reset() + c.BlockHash.Reset() + c.ParentHash.Reset() + c.Position.Reset() + c.Hash.Reset() + c.From.Reset() + c.To.Reset() + c.Nonce.Reset() + c.GasPrice.Reset() + c.Gas.Reset() + c.GasTipCap.Reset() + c.GasFeeCap.Reset() + c.Value.Reset() + c.Type.Reset() + c.Size.Reset() + c.CallDataSize.Reset() + c.BlobGas.Reset() + c.BlobGasFeeCap.Reset() + c.BlobHashes.Reset() + c.Success.Reset() + c.NInputBytes.Reset() + c.NInputZeroBytes.Reset() + c.NInputNonzeroBytes.Reset() + c.MetaNetworkName.Reset() +} + +// Input returns the proto.Input for inserting data. +func (c *Columns) Input() proto.Input { + return proto.Input{ + {Name: "updated_date_time", Data: &c.UpdatedDateTime}, + {Name: "block_number", Data: &c.BlockNumber}, + {Name: "block_hash", Data: &c.BlockHash}, + {Name: "parent_hash", Data: &c.ParentHash}, + {Name: "position", Data: &c.Position}, + {Name: "hash", Data: &c.Hash}, + {Name: "from", Data: &c.From}, + {Name: "to", Data: c.To}, + {Name: "nonce", Data: &c.Nonce}, + {Name: "gas_price", Data: &c.GasPrice}, + {Name: "gas", Data: &c.Gas}, + {Name: "gas_tip_cap", Data: c.GasTipCap}, + {Name: "gas_fee_cap", Data: c.GasFeeCap}, + {Name: "value", Data: &c.Value}, + {Name: "type", Data: &c.Type}, + {Name: "size", Data: &c.Size}, + {Name: "call_data_size", Data: &c.CallDataSize}, + {Name: "blob_gas", Data: c.BlobGas}, + {Name: "blob_gas_fee_cap", Data: c.BlobGasFeeCap}, + {Name: "blob_hashes", Data: c.BlobHashes}, + {Name: "success", Data: &c.Success}, + {Name: "n_input_bytes", Data: &c.NInputBytes}, + {Name: "n_input_zero_bytes", Data: &c.NInputZeroBytes}, + {Name: "n_input_nonzero_bytes", Data: &c.NInputNonzeroBytes}, + {Name: "meta_network_name", Data: &c.MetaNetworkName}, + } +} + +// Rows returns the number of rows in the columns. +func (c *Columns) Rows() int { + return c.BlockNumber.Rows() +} + +// parseUInt128 parses a decimal string to proto.UInt128. +// Returns zero if the string is empty or invalid. +func parseUInt128(s string) proto.UInt128 { + if s == "" { + return proto.UInt128{} + } + + bi := new(big.Int) + if _, ok := bi.SetString(s, 10); !ok { + return proto.UInt128{} + } + + // Extract low and high 64-bit words + // UInt128 = High * 2^64 + Low + maxUint64 := new(big.Int).SetUint64(^uint64(0)) + maxUint64.Add(maxUint64, big.NewInt(1)) // 2^64 + + low := new(big.Int).And(bi, new(big.Int).Sub(maxUint64, big.NewInt(1))) + high := new(big.Int).Rsh(bi, 64) + + return proto.UInt128{ + Low: low.Uint64(), + High: high.Uint64(), + } +} + +// nullableStr converts a *string to proto.Nullable[string]. +func nullableStr(s *string) proto.Nullable[string] { + if s == nil { + return proto.Null[string]() + } + + return proto.NewNullable(*s) +} + +// nullableUint64 converts a *uint64 to proto.Nullable[uint64]. +func nullableUint64(v *uint64) proto.Nullable[uint64] { + if v == nil { + return proto.Null[uint64]() + } + + return proto.NewNullable(*v) +} + +// nullableUInt128 converts a *string (decimal) to proto.Nullable[proto.UInt128]. +func nullableUInt128(s *string) proto.Nullable[proto.UInt128] { + if s == nil { + return proto.Null[proto.UInt128]() + } + + return proto.NewNullable(parseUInt128(*s)) +} diff --git a/pkg/processor/transaction/simple/handlers.go b/pkg/processor/transaction/simple/handlers.go index 2bbf6be..18d4ef3 100644 --- a/pkg/processor/transaction/simple/handlers.go +++ b/pkg/processor/transaction/simple/handlers.go @@ -6,44 +6,52 @@ import ( "math/big" "time" + "github.com/ClickHouse/ch-go" "github.com/ethereum/go-ethereum/core/types" "github.com/hibiken/asynq" "github.com/sirupsen/logrus" "github.com/ethpandaops/execution-processor/pkg/common" c "github.com/ethpandaops/execution-processor/pkg/processor/common" - "github.com/ethpandaops/execution-processor/pkg/processor/transaction/structlog" ) +// ClickHouseDateTime is a time.Time wrapper that formats correctly for ClickHouse JSON. +type ClickHouseDateTime time.Time + +// MarshalJSON formats time for ClickHouse DateTime column. +func (t ClickHouseDateTime) MarshalJSON() ([]byte, error) { + return []byte(fmt.Sprintf(`"%s"`, time.Time(t).UTC().Format("2006-01-02 15:04:05"))), nil +} + // Transaction represents a row in the execution_transaction table. // //nolint:tagliatelle // ClickHouse uses snake_case column names type Transaction struct { - UpdatedDateTime structlog.ClickHouseTime `json:"updated_date_time"` - BlockNumber uint64 `json:"block_number"` - BlockHash string `json:"block_hash"` - ParentHash string `json:"parent_hash"` - Position uint32 `json:"position"` - Hash string `json:"hash"` - From string `json:"from"` - To *string `json:"to"` - Nonce uint64 `json:"nonce"` - GasPrice string `json:"gas_price"` // Effective gas price as UInt128 string - Gas uint64 `json:"gas"` // Gas limit - GasTipCap *string `json:"gas_tip_cap"` // Nullable UInt128 string - GasFeeCap *string `json:"gas_fee_cap"` // Nullable UInt128 string - Value string `json:"value"` // UInt128 string - Type uint8 `json:"type"` // Transaction type - Size uint32 `json:"size"` // Transaction size in bytes - CallDataSize uint32 `json:"call_data_size"` // Size of call data - BlobGas *uint64 `json:"blob_gas"` // Nullable - for type 3 txs - BlobGasFeeCap *string `json:"blob_gas_fee_cap"` // Nullable UInt128 string - BlobHashes []string `json:"blob_hashes"` // Array of versioned hashes - Success bool `json:"success"` - NInputBytes uint32 `json:"n_input_bytes"` - NInputZeroBytes uint32 `json:"n_input_zero_bytes"` - NInputNonzeroBytes uint32 `json:"n_input_nonzero_bytes"` - MetaNetworkName string `json:"meta_network_name"` + UpdatedDateTime ClickHouseDateTime `json:"updated_date_time"` + BlockNumber uint64 `json:"block_number"` + BlockHash string `json:"block_hash"` + ParentHash string `json:"parent_hash"` + Position uint32 `json:"position"` + Hash string `json:"hash"` + From string `json:"from"` + To *string `json:"to"` + Nonce uint64 `json:"nonce"` + GasPrice string `json:"gas_price"` // Effective gas price as UInt128 string + Gas uint64 `json:"gas"` // Gas limit + GasTipCap *string `json:"gas_tip_cap"` // Nullable UInt128 string + GasFeeCap *string `json:"gas_fee_cap"` // Nullable UInt128 string + Value string `json:"value"` // UInt128 string + Type uint8 `json:"type"` // Transaction type + Size uint32 `json:"size"` // Transaction size in bytes + CallDataSize uint32 `json:"call_data_size"` // Size of call data + BlobGas *uint64 `json:"blob_gas"` // Nullable - for type 3 txs + BlobGasFeeCap *string `json:"blob_gas_fee_cap"` // Nullable UInt128 string + BlobHashes []string `json:"blob_hashes"` // Array of versioned hashes + Success bool `json:"success"` + NInputBytes uint32 `json:"n_input_bytes"` + NInputZeroBytes uint32 `json:"n_input_zero_bytes"` + NInputNonzeroBytes uint32 `json:"n_input_nonzero_bytes"` + MetaNetworkName string `json:"meta_network_name"` } // GetHandlers returns the task handlers for this processor. @@ -240,7 +248,7 @@ func (p *Processor) buildTransactionRow( // Build transaction row txRow := Transaction{ - UpdatedDateTime: structlog.NewClickHouseTime(time.Now()), + UpdatedDateTime: ClickHouseDateTime(time.Now()), BlockNumber: block.Number().Uint64(), BlockHash: block.Hash().String(), ParentHash: block.ParentHash().String(), @@ -322,13 +330,23 @@ func calculateEffectiveGasPrice(block *types.Block, tx *types.Transaction) *big. return effectiveGasPrice } -// insertTransactions inserts transactions into ClickHouse. +// insertTransactions inserts transactions into ClickHouse using columnar protocol. func (p *Processor) insertTransactions(ctx context.Context, transactions []Transaction) error { if len(transactions) == 0 { return nil } - if err := p.clickhouse.BulkInsert(ctx, p.config.Table, transactions); err != nil { + cols := NewColumns() + for _, tx := range transactions { + cols.Append(tx) + } + + input := cols.Input() + + if err := p.clickhouse.Do(ctx, ch.Query{ + Body: input.Into(p.config.Table), + Input: input, + }); err != nil { common.ClickHouseInsertsRows.WithLabelValues( p.network.Name, ProcessorName, diff --git a/pkg/processor/transaction/simple/processor.go b/pkg/processor/transaction/simple/processor.go index b417185..6a967b8 100644 --- a/pkg/processor/transaction/simple/processor.go +++ b/pkg/processor/transaction/simple/processor.go @@ -50,7 +50,7 @@ func New(ctx context.Context, deps *Dependencies, config *Config) (*Processor, e clickhouseConfig.Network = deps.Network.Name clickhouseConfig.Processor = ProcessorName - clickhouseClient, err := clickhouse.New(&clickhouseConfig) + clickhouseClient, err := clickhouse.New(ctx, &clickhouseConfig) if err != nil { return nil, fmt.Errorf("failed to create clickhouse client: %w", err) } diff --git a/pkg/processor/transaction/structlog/batch_manager.go b/pkg/processor/transaction/structlog/batch_manager.go deleted file mode 100644 index 8c4d083..0000000 --- a/pkg/processor/transaction/structlog/batch_manager.go +++ /dev/null @@ -1,225 +0,0 @@ -package structlog - -import ( - "context" - "sync" - "time" - - "github.com/ethpandaops/execution-processor/pkg/common" - "github.com/hibiken/asynq" - "github.com/sirupsen/logrus" -) - -// BatchItem represents a single transaction's data within a batch. -type BatchItem struct { - structlogs []Structlog - task *asynq.Task - payload *ProcessPayload -} - -// BatchManager accumulates small transactions and flushes them in batches. -type BatchManager struct { - mu sync.Mutex - items []BatchItem - currentSize int64 - flushThreshold int64 - maxSize int64 - flushInterval time.Duration - flushTimer *time.Timer - flushCh chan struct{} - processor *Processor - cancel context.CancelFunc - wg sync.WaitGroup - stopCh chan struct{} -} - -// NewBatchManager creates a new batch manager. -func NewBatchManager(processor *Processor, config *Config) *BatchManager { - _, cancel := context.WithCancel(context.Background()) - - return &BatchManager{ - items: make([]BatchItem, 0, 100), - flushThreshold: config.BatchInsertThreshold, - maxSize: config.BatchMaxSize, - flushInterval: config.BatchFlushInterval, - flushCh: make(chan struct{}, 1), - processor: processor, - cancel: cancel, - stopCh: make(chan struct{}), - } -} - -// Start begins the batch manager's flush goroutine. -func (bm *BatchManager) Start() error { - bm.wg.Add(1) - - go bm.flushLoop() - - bm.processor.log.WithFields(logrus.Fields{ - "flush_threshold": bm.flushThreshold, - "max_size": bm.maxSize, - "flush_interval": bm.flushInterval, - }).Info("batch manager started") - - return nil -} - -// Stop gracefully shuts down the batch manager. -func (bm *BatchManager) Stop() { - bm.cancel() - close(bm.stopCh) - - // Flush any remaining items - bm.Flush() - - // Wait for flush loop to exit - bm.wg.Wait() - - bm.processor.log.Info("batch manager stopped") -} - -// Add adds a transaction's structlogs to the batch. -func (bm *BatchManager) Add(structlogs []Structlog, task *asynq.Task, payload *ProcessPayload) error { - bm.mu.Lock() - defer bm.mu.Unlock() - - // Check if adding this would exceed max size - newSize := bm.currentSize + int64(len(structlogs)) - if newSize > bm.maxSize { - // Flush current batch first - bm.flushLocked() - } - - // Add to batch - bm.items = append(bm.items, BatchItem{ - structlogs: structlogs, - task: task, - payload: payload, - }) - bm.currentSize += int64(len(structlogs)) - - // Reset flush timer - if bm.flushTimer != nil { - bm.flushTimer.Stop() - } - - bm.flushTimer = time.AfterFunc(bm.flushInterval, func() { - select { - case bm.flushCh <- struct{}{}: - default: - } - }) - - // Check if we should flush - bm.FlushIfNeeded() - - return nil -} - -// FlushIfNeeded checks if the batch should be flushed based on size threshold. -func (bm *BatchManager) FlushIfNeeded() { - if bm.currentSize >= bm.flushThreshold { - bm.flushLocked() - } -} - -// Flush performs a batch flush (thread-safe). -func (bm *BatchManager) Flush() { - bm.mu.Lock() - defer bm.mu.Unlock() - - bm.flushLocked() -} - -// flushLocked performs the actual flush (must be called with lock held). -func (bm *BatchManager) flushLocked() { - if len(bm.items) == 0 { - return - } - - // Stop the flush timer - if bm.flushTimer != nil { - bm.flushTimer.Stop() - bm.flushTimer = nil - } - - // Collect all structlogs - var allStructlogs []Structlog - for _, item := range bm.items { - allStructlogs = append(allStructlogs, item.structlogs...) - } - - // Log batch details - bm.processor.log.WithFields(logrus.Fields{ - "batch_items": len(bm.items), - "batch_structlogs": len(allStructlogs), - }).Debug("flushing batch") - - // Perform bulk insert - startTime := time.Now() - - err := bm.processor.insertStructlogs(context.Background(), allStructlogs) - if err != nil { - // Failed - fail all tasks in batch - bm.processor.log.WithError(err).Error("batch insert failed") - - for _, item := range bm.items { - // Increment error metrics - common.TasksErrored.WithLabelValues( - bm.processor.network.Name, - ProcessorName, - item.payload.ProcessingMode, - ProcessForwardsTaskType, - "batch_insert_error", - ).Inc() - } - } else { - // Success - complete all tasks - for _, item := range bm.items { - // Increment success metrics - common.TasksProcessed.WithLabelValues( - bm.processor.network.Name, - ProcessorName, - item.payload.ProcessingMode, - ProcessForwardsTaskType, - "success", - ).Inc() - } - - // Record batch metrics using ClickHouse metrics - common.ClickHouseInsertsRows.WithLabelValues( - bm.processor.network.Name, - ProcessorName, - "structlogs_batch", - "success", - "", - ).Add(float64(len(allStructlogs))) - - common.ClickHouseOperationDuration.WithLabelValues( - bm.processor.network.Name, - ProcessorName, - "batch_insert", - "structlogs_batch", - "success", - "", - ).Observe(time.Since(startTime).Seconds()) - } - - // Clear the batch - bm.items = bm.items[:0] - bm.currentSize = 0 -} - -// flushLoop runs the time-based flush goroutine. -func (bm *BatchManager) flushLoop() { - defer bm.wg.Done() - - for { - select { - case <-bm.stopCh: - return - case <-bm.flushCh: - bm.Flush() - } - } -} diff --git a/pkg/processor/transaction/structlog/batch_manager_test.go b/pkg/processor/transaction/structlog/batch_manager_test.go deleted file mode 100644 index 26a94b4..0000000 --- a/pkg/processor/transaction/structlog/batch_manager_test.go +++ /dev/null @@ -1,377 +0,0 @@ -package structlog_test - -import ( - "context" - "sync" - "testing" - "time" - - "github.com/ethpandaops/execution-processor/pkg/clickhouse" - "github.com/ethpandaops/execution-processor/pkg/processor/transaction/structlog" - "github.com/hibiken/asynq" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" -) - -// MockClickHouseClient is a mock implementation of clickhouse.ClientInterface. -type MockClickHouseClient struct { - mock.Mock -} - -func (m *MockClickHouseClient) Start() error { - args := m.Called() - - return args.Error(0) -} - -func (m *MockClickHouseClient) Stop() error { - args := m.Called() - - return args.Error(0) -} - -func (m *MockClickHouseClient) BulkInsert(ctx context.Context, table string, data interface{}) error { - args := m.Called(ctx, table, data) - - return args.Error(0) -} - -func (m *MockClickHouseClient) QueryOne(ctx context.Context, query string, dest interface{}) error { - args := m.Called(ctx, query, dest) - - return args.Error(0) -} - -func (m *MockClickHouseClient) QueryMany(ctx context.Context, query string, dest interface{}) error { - args := m.Called(ctx, query, dest) - - return args.Error(0) -} - -func (m *MockClickHouseClient) Execute(ctx context.Context, query string) error { - args := m.Called(ctx, query) - - return args.Error(0) -} - -func (m *MockClickHouseClient) IsStorageEmpty(ctx context.Context, table string, conditions map[string]interface{}) (bool, error) { - args := m.Called(ctx, table, conditions) - - return args.Bool(0), args.Error(1) -} - -func (m *MockClickHouseClient) SetNetwork(network string) { - m.Called(network) -} - -// TestBatchManager_Accumulation tests that the batch manager accumulates transactions up to threshold. -func TestBatchManager_Accumulation(t *testing.T) { - t.Skip("Skipping due to inability to properly mock processor internals") -} - -// TestBatchManager_TimeBasedFlush tests that batches are flushed based on time. -func TestBatchManager_TimeBasedFlush(t *testing.T) { - t.Skip("Skipping due to inability to properly mock processor internals") -} - -// TestBatchManager_BigTransactionFlush tests that batches are flushed before big transactions. -func TestBatchManager_BigTransactionFlush(t *testing.T) { - t.Skip("Skipping due to inability to properly mock processor internals") -} - -// TestProcessor_RoutingLogic tests that transactions are correctly routed based on size. -func TestProcessor_RoutingLogic(t *testing.T) { - config := &structlog.Config{ - Enabled: true, - Table: "test_structlog", - BatchInsertThreshold: 100, - BatchFlushInterval: 5 * time.Second, - BatchMaxSize: 200, - BigTransactionThreshold: 500000, - Config: clickhouse.Config{ - URL: "http://localhost:8123", - }, - } - - // Test the ShouldBatch logic directly - // The logic is: count < BatchInsertThreshold - - // Test small transaction - should be batched - assert.True(t, 50 < int(config.BatchInsertThreshold)) - assert.True(t, 99 < int(config.BatchInsertThreshold)) - - // Test at threshold - should NOT be batched - assert.False(t, 100 < int(config.BatchInsertThreshold)) - - // Test large transaction - should NOT be batched - assert.False(t, 1000 < int(config.BatchInsertThreshold)) - assert.False(t, 500000 < int(config.BatchInsertThreshold)) - - // Test edge cases - assert.True(t, 0 < int(config.BatchInsertThreshold)) // Empty transaction should be less than threshold - assert.True(t, -1 < int(config.BatchInsertThreshold)) // Invalid count is still technically less than threshold -} - -// TestBatchManager_ErrorHandling tests batch failure handling. -func TestBatchManager_ErrorHandling(t *testing.T) { - t.Skip("Skipping due to inability to properly mock processor internals") -} - -// TestBatchManager_Shutdown tests graceful shutdown with pending batches. -func TestBatchManager_Shutdown(t *testing.T) { - t.Skip("Skipping due to inability to properly mock processor internals") -} - -// TestBatchManager_ConcurrentAdd tests thread-safe concurrent additions. -func TestBatchManager_ConcurrentAdd(t *testing.T) { - t.Skip("Skipping due to inability to properly mock processor internals") -} - -// TestBatchManager_ThresholdBoundaries tests behavior at exact threshold values. -func TestBatchManager_ThresholdBoundaries(t *testing.T) { - t.Skip("Skipping due to inability to properly mock processor internals") -} - -// TestBatchManager_PerformanceImprovement benchmarks batching benefits. -func TestBatchManager_PerformanceImprovement(t *testing.T) { - if testing.Short() { - t.Skip("Skipping performance test in short mode") - } - - // Test configuration - numTransactions := 1000 - - // Non-batched timing simulation - nonBatchedStart := time.Now() - - for i := 0; i < numTransactions; i++ { - // Simulate individual insert delay - time.Sleep(100 * time.Microsecond) - } - - nonBatchedDuration := time.Since(nonBatchedStart) - - // Batched timing simulation (batch size 100) - batchedStart := time.Now() - batchSize := 100 - numBatches := numTransactions / batchSize - - for i := 0; i < numBatches; i++ { - // Simulate batch insert delay (slightly longer per batch, but fewer operations) - time.Sleep(500 * time.Microsecond) - } - - batchedDuration := time.Since(batchedStart) - - // Calculate improvement - improvement := float64(nonBatchedDuration) / float64(batchedDuration) - - t.Logf("Non-batched duration: %v", nonBatchedDuration) - t.Logf("Batched duration: %v", batchedDuration) - t.Logf("Performance improvement: %.2fx", improvement) - - // Batching should be faster - assert.Less(t, batchedDuration, nonBatchedDuration) - assert.Greater(t, improvement, 1.0) -} - -// TestConfigValidation_BatchFields tests validation of batch configuration fields. -func TestConfigValidation_BatchFields(t *testing.T) { - testCases := []struct { - name string - config structlog.Config - expectError bool - errorMsg string - }{ - { - name: "valid batch config", - config: structlog.Config{ - Enabled: true, - Table: "test", - BatchInsertThreshold: 100, - BatchFlushInterval: 5 * time.Second, - BatchMaxSize: 200, - BigTransactionThreshold: 500000, - Config: clickhouse.Config{ - URL: "http://localhost:8123", - }, - }, - expectError: false, - }, - { - name: "batch threshold zero", - config: structlog.Config{ - Enabled: true, - Table: "test", - BatchInsertThreshold: 0, - BatchFlushInterval: 5 * time.Second, - BatchMaxSize: 200, - BigTransactionThreshold: 500000, - Config: clickhouse.Config{ - URL: "http://localhost:8123", - }, - }, - expectError: true, - errorMsg: "batchInsertThreshold must be greater than 0", - }, - { - name: "batch threshold exceeds big transaction threshold", - config: structlog.Config{ - Enabled: true, - Table: "test", - BatchInsertThreshold: 600000, - BatchFlushInterval: 5 * time.Second, - BatchMaxSize: 700000, - BigTransactionThreshold: 500000, - Config: clickhouse.Config{ - URL: "http://localhost:8123", - }, - }, - expectError: true, - errorMsg: "batchInsertThreshold must be less than bigTransactionThreshold", - }, - { - name: "batch flush interval zero", - config: structlog.Config{ - Enabled: true, - Table: "test", - BatchInsertThreshold: 100, - BatchFlushInterval: 0, - BatchMaxSize: 200, - BigTransactionThreshold: 500000, - Config: clickhouse.Config{ - URL: "http://localhost:8123", - }, - }, - expectError: true, - errorMsg: "batchFlushInterval must be greater than 0", - }, - { - name: "batch max size less than threshold", - config: structlog.Config{ - Enabled: true, - Table: "test", - BatchInsertThreshold: 200, - BatchFlushInterval: 5 * time.Second, - BatchMaxSize: 100, - BigTransactionThreshold: 500000, - Config: clickhouse.Config{ - URL: "http://localhost:8123", - }, - }, - expectError: true, - errorMsg: "batchMaxSize must be greater than or equal to batchInsertThreshold", - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - err := tc.config.Validate() - if tc.expectError { - assert.Error(t, err) - - if tc.errorMsg != "" { - assert.Contains(t, err.Error(), tc.errorMsg) - } - } else { - assert.NoError(t, err) - } - }) - } -} - -// TestBatchManagerConcepts tests the concepts behind batch management. -func TestBatchManagerConcepts(t *testing.T) { - // Test batch size calculations - t.Run("batch size tracking", func(t *testing.T) { - var currentSize int64 - - threshold := int64(100) - - // Simulate adding items - items := []int64{10, 20, 30, 40, 50} - - for _, size := range items { - currentSize += size - - if currentSize >= threshold { - // Would trigger flush - assert.GreaterOrEqual(t, currentSize, threshold) - - break - } - } - }) - - // Test timer-based flushing concept - t.Run("timer based flush", func(t *testing.T) { - flushInterval := 100 * time.Millisecond - timer := time.NewTimer(flushInterval) - - select { - case <-timer.C: - // Timer expired, would trigger flush - assert.True(t, true, "Timer expired as expected") - case <-time.After(200 * time.Millisecond): - t.Fatal("Timer did not expire in expected time") - } - }) - - // Test concurrent access patterns - t.Run("concurrent access", func(t *testing.T) { - var mu sync.Mutex - - var items []int - - var wg sync.WaitGroup - - for i := 0; i < 10; i++ { - wg.Add(1) - - go func(val int) { - defer wg.Done() - - mu.Lock() - defer mu.Unlock() - - items = append(items, val) - }(i) - } - - wg.Wait() - assert.Len(t, items, 10) - }) -} - -// TestBatchItemStructure tests the BatchItem structure. -func TestBatchItemStructure(t *testing.T) { - // Test creating batch items - structlogs := []structlog.Structlog{ - { - BlockNumber: 12345, - TransactionHash: "0xtest", - Index: 0, - }, - } - - task := asynq.NewTask("test_type", []byte("test_payload")) - payload := &structlog.ProcessPayload{ - TransactionHash: "0xtest", - } - - // Test that we can create a batch item with these components - batchItem := struct { - structlogs []structlog.Structlog - task *asynq.Task - payload *structlog.ProcessPayload - }{ - structlogs: structlogs, - task: task, - payload: payload, - } - - assert.NotNil(t, batchItem.structlogs) - assert.NotNil(t, batchItem.task) - assert.NotNil(t, batchItem.payload) - assert.Equal(t, "0xtest", batchItem.payload.TransactionHash) -} diff --git a/pkg/processor/transaction/structlog/big_transaction_manager.go b/pkg/processor/transaction/structlog/big_transaction_manager.go deleted file mode 100644 index 0945773..0000000 --- a/pkg/processor/transaction/structlog/big_transaction_manager.go +++ /dev/null @@ -1,69 +0,0 @@ -package structlog - -import ( - "sync/atomic" - - "github.com/sirupsen/logrus" -) - -// BigTransactionManager coordinates processing of large transactions to prevent OOM. -type BigTransactionManager struct { - // Active big transaction count - currentBigCount atomic.Int32 - pauseNewWork atomic.Bool - - // Configuration - threshold int // Default: 500k structlogs - log logrus.FieldLogger -} - -// NewBigTransactionManager creates a new manager for big transaction coordination. -func NewBigTransactionManager(threshold int, log logrus.FieldLogger) *BigTransactionManager { - return &BigTransactionManager{ - threshold: threshold, - log: log, - } -} - -// RegisterBigTransaction marks a big transaction as actively processing. -func (btm *BigTransactionManager) RegisterBigTransaction(txHash string, processor *Processor) { - // Flush any pending batches before blocking - if processor.batchManager != nil { - processor.batchManager.Flush() - } - - count := btm.currentBigCount.Add(1) - - // Set pause flag when ANY big transaction is active - btm.pauseNewWork.Store(true) - btm.log.WithFields(logrus.Fields{ - "tx_hash": txHash, - "total_big_txs": count, - }).Debug("Registered big transaction") -} - -// UnregisterBigTransaction marks a big transaction as complete. -func (btm *BigTransactionManager) UnregisterBigTransaction(txHash string) { - newCount := btm.currentBigCount.Add(-1) - - // Only clear pause when NO big transactions remain - if newCount == 0 { - btm.pauseNewWork.Store(false) - btm.log.Info("All big transactions complete, resuming normal processing") - } else { - btm.log.WithFields(logrus.Fields{ - "tx_hash": txHash, - "remaining_big_txs": newCount, - }).Debug("Big transaction complete") - } -} - -// ShouldPauseNewWork returns true if new work should be paused. -func (btm *BigTransactionManager) ShouldPauseNewWork() bool { - return btm.pauseNewWork.Load() -} - -// GetThreshold returns the threshold for big transactions. -func (btm *BigTransactionManager) GetThreshold() int { - return btm.threshold -} diff --git a/pkg/processor/transaction/structlog/clickhouse_time.go b/pkg/processor/transaction/structlog/clickhouse_time.go deleted file mode 100644 index 5e7fcd7..0000000 --- a/pkg/processor/transaction/structlog/clickhouse_time.go +++ /dev/null @@ -1,43 +0,0 @@ -package structlog - -import ( - "fmt" - "time" -) - -// ClickHouseTime wraps time.Time to provide custom JSON marshaling for ClickHouse. -type ClickHouseTime struct { - time.Time -} - -// MarshalJSON formats the time for ClickHouse DateTime format. -func (t ClickHouseTime) MarshalJSON() ([]byte, error) { - // ClickHouse expects DateTime in the format: "2006-01-02 15:04:05" - formatted := fmt.Sprintf("%q", t.Format("2006-01-02 15:04:05")) - - return []byte(formatted), nil -} - -// UnmarshalJSON parses the time from ClickHouse DateTime format. -func (t *ClickHouseTime) UnmarshalJSON(data []byte) error { - // Remove quotes - if len(data) < 2 || data[0] != '"' || data[len(data)-1] != '"' { - return fmt.Errorf("invalid time format") - } - - timeStr := string(data[1 : len(data)-1]) - - parsed, err := time.Parse("2006-01-02 15:04:05", timeStr) - if err != nil { - return err - } - - t.Time = parsed - - return nil -} - -// NewClickHouseTime creates a new ClickHouseTime from a time.Time. -func NewClickHouseTime(t time.Time) ClickHouseTime { - return ClickHouseTime{Time: t} -} diff --git a/pkg/processor/transaction/structlog/columns.go b/pkg/processor/transaction/structlog/columns.go new file mode 100644 index 0000000..a08f2af --- /dev/null +++ b/pkg/processor/transaction/structlog/columns.go @@ -0,0 +1,155 @@ +package structlog + +import ( + "time" + + "github.com/ClickHouse/ch-go/proto" +) + +// Columns holds all columns for structlog batch insert using ch-go columnar protocol. +type Columns struct { + UpdatedDateTime proto.ColDateTime + BlockNumber proto.ColUInt64 + TransactionHash proto.ColStr + TransactionIndex proto.ColUInt32 + TransactionGas proto.ColUInt64 + TransactionFailed proto.ColBool + TransactionReturnValue *proto.ColNullable[string] + Index proto.ColUInt32 + ProgramCounter proto.ColUInt32 + Operation proto.ColStr + Gas proto.ColUInt64 + GasCost proto.ColUInt64 + GasUsed proto.ColUInt64 + Depth proto.ColUInt64 + ReturnData *proto.ColNullable[string] + Refund *proto.ColNullable[uint64] + Error *proto.ColNullable[string] + CallToAddress *proto.ColNullable[string] + MetaNetworkName proto.ColStr +} + +// NewColumns creates a new Columns instance with all columns initialized. +func NewColumns() *Columns { + return &Columns{ + TransactionReturnValue: new(proto.ColStr).Nullable(), + ReturnData: new(proto.ColStr).Nullable(), + Refund: new(proto.ColUInt64).Nullable(), + Error: new(proto.ColStr).Nullable(), + CallToAddress: new(proto.ColStr).Nullable(), + } +} + +// Append adds a row to all columns. +func (c *Columns) Append( + updatedDateTime time.Time, + blockNumber uint64, + txHash string, + txIndex uint32, + txGas uint64, + txFailed bool, + txReturnValue *string, + index uint32, + pc uint32, + op string, + gas uint64, + gasCost uint64, + gasUsed uint64, + depth uint64, + returnData *string, + refund *uint64, + errStr *string, + callTo *string, + network string, +) { + c.UpdatedDateTime.Append(updatedDateTime) + c.BlockNumber.Append(blockNumber) + c.TransactionHash.Append(txHash) + c.TransactionIndex.Append(txIndex) + c.TransactionGas.Append(txGas) + c.TransactionFailed.Append(txFailed) + c.TransactionReturnValue.Append(nullableStr(txReturnValue)) + c.Index.Append(index) + c.ProgramCounter.Append(pc) + c.Operation.Append(op) + c.Gas.Append(gas) + c.GasCost.Append(gasCost) + c.GasUsed.Append(gasUsed) + c.Depth.Append(depth) + c.ReturnData.Append(nullableStr(returnData)) + c.Refund.Append(nullableUint64(refund)) + c.Error.Append(nullableStr(errStr)) + c.CallToAddress.Append(nullableStr(callTo)) + c.MetaNetworkName.Append(network) +} + +// Reset clears all columns for reuse. +func (c *Columns) Reset() { + c.UpdatedDateTime.Reset() + c.BlockNumber.Reset() + c.TransactionHash.Reset() + c.TransactionIndex.Reset() + c.TransactionGas.Reset() + c.TransactionFailed.Reset() + c.TransactionReturnValue.Reset() + c.Index.Reset() + c.ProgramCounter.Reset() + c.Operation.Reset() + c.Gas.Reset() + c.GasCost.Reset() + c.GasUsed.Reset() + c.Depth.Reset() + c.ReturnData.Reset() + c.Refund.Reset() + c.Error.Reset() + c.CallToAddress.Reset() + c.MetaNetworkName.Reset() +} + +// Input returns the proto.Input for inserting data. +func (c *Columns) Input() proto.Input { + return proto.Input{ + {Name: "updated_date_time", Data: &c.UpdatedDateTime}, + {Name: "block_number", Data: &c.BlockNumber}, + {Name: "transaction_hash", Data: &c.TransactionHash}, + {Name: "transaction_index", Data: &c.TransactionIndex}, + {Name: "transaction_gas", Data: &c.TransactionGas}, + {Name: "transaction_failed", Data: &c.TransactionFailed}, + {Name: "transaction_return_value", Data: c.TransactionReturnValue}, + {Name: "index", Data: &c.Index}, + {Name: "program_counter", Data: &c.ProgramCounter}, + {Name: "operation", Data: &c.Operation}, + {Name: "gas", Data: &c.Gas}, + {Name: "gas_cost", Data: &c.GasCost}, + {Name: "gas_used", Data: &c.GasUsed}, + {Name: "depth", Data: &c.Depth}, + {Name: "return_data", Data: c.ReturnData}, + {Name: "refund", Data: c.Refund}, + {Name: "error", Data: c.Error}, + {Name: "call_to_address", Data: c.CallToAddress}, + {Name: "meta_network_name", Data: &c.MetaNetworkName}, + } +} + +// Rows returns the number of rows in the columns. +func (c *Columns) Rows() int { + return c.BlockNumber.Rows() +} + +// nullableStr converts a *string to proto.Nullable[string]. +func nullableStr(s *string) proto.Nullable[string] { + if s == nil { + return proto.Null[string]() + } + + return proto.NewNullable(*s) +} + +// nullableUint64 converts a *uint64 to proto.Nullable[uint64]. +func nullableUint64(v *uint64) proto.Nullable[uint64] { + if v == nil { + return proto.Null[uint64]() + } + + return proto.NewNullable(*v) +} diff --git a/pkg/processor/transaction/structlog/config.go b/pkg/processor/transaction/structlog/config.go index 909f4a2..57d809f 100644 --- a/pkg/processor/transaction/structlog/config.go +++ b/pkg/processor/transaction/structlog/config.go @@ -2,7 +2,6 @@ package structlog import ( "fmt" - "time" "github.com/ethpandaops/execution-processor/pkg/clickhouse" ) @@ -13,24 +12,12 @@ type Config struct { Enabled bool `yaml:"enabled"` Table string `yaml:"table"` - // Big transaction handling - BigTransactionThreshold int `yaml:"bigTransactionThreshold"` // Default: 500,000 - ChunkSize int `yaml:"chunkSize"` // Default: 10,000 - ChannelBufferSize int `yaml:"channelBufferSize"` // Default: 2 - ProgressLogThreshold int `yaml:"progressLogThreshold"` // Default: 100,000 - - // Batch processing configuration - // BatchInsertThreshold is the minimum number of structlogs to accumulate before batch insert - // Transactions with more structlogs than this will bypass batching - BatchInsertThreshold int64 `yaml:"batchInsertThreshold" default:"50000"` - - // BatchFlushInterval is the maximum time to wait before flushing a batch - BatchFlushInterval time.Duration `yaml:"batchFlushInterval" default:"5s"` - - // BatchMaxSize is the maximum number of structlogs to accumulate in a batch - BatchMaxSize int64 `yaml:"batchMaxSize" default:"100000"` + // Streaming settings + ChunkSize int `yaml:"chunkSize"` // Default: 10,000 rows per OnInput iteration + ProgressLogThreshold int `yaml:"progressLogThreshold"` // Default: 100,000 - log progress for large txs } +// Validate validates the configuration. func (c *Config) Validate() error { if !c.Enabled { return nil @@ -45,22 +32,5 @@ func (c *Config) Validate() error { return fmt.Errorf("transaction structlog table is required when enabled") } - // Validate batch configuration - if c.BatchInsertThreshold <= 0 { - return fmt.Errorf("batchInsertThreshold must be greater than 0") - } - - if c.BatchInsertThreshold >= int64(c.BigTransactionThreshold) { - return fmt.Errorf("batchInsertThreshold must be less than bigTransactionThreshold to prevent routing conflicts") - } - - if c.BatchFlushInterval <= 0 { - return fmt.Errorf("batchFlushInterval must be greater than 0") - } - - if c.BatchMaxSize < c.BatchInsertThreshold { - return fmt.Errorf("batchMaxSize must be greater than or equal to batchInsertThreshold") - } - return nil } diff --git a/pkg/processor/transaction/structlog/handlers.go b/pkg/processor/transaction/structlog/handlers.go index d425038..e4cecbf 100644 --- a/pkg/processor/transaction/structlog/handlers.go +++ b/pkg/processor/transaction/structlog/handlers.go @@ -28,9 +28,6 @@ func (p *Processor) handleProcessForwardsTask(ctx context.Context, task *asynq.T return fmt.Errorf("failed to unmarshal process payload: %w", err) } - // Wait for any active big transactions before starting - p.waitForBigTransactions("process_forwards") - // Get healthy execution node node := p.pool.GetHealthyExecutionNode() if node == nil { @@ -55,30 +52,8 @@ func (p *Processor) handleProcessForwardsTask(ctx context.Context, task *asynq.T return fmt.Errorf("transaction hash mismatch: expected %s, got %s", payload.TransactionHash, tx.Hash().String()) } - // Extract structlogs from the transaction - structlogs, err := p.ExtractStructlogs(ctx, block, int(payload.TransactionIndex), tx) - if err != nil { - common.TasksErrored.WithLabelValues(p.network.Name, ProcessorName, c.ProcessForwardsQueue(ProcessorName), ProcessForwardsTaskType, "extraction_error").Inc() - - return fmt.Errorf("failed to extract structlogs: %w", err) - } - - structlogCount := int64(len(structlogs)) - - // Check if transaction should be batched - if p.ShouldBatch(structlogCount) { - // Route to batch manager - if addErr := p.batchManager.Add(structlogs, task, &payload); addErr != nil { - common.TasksErrored.WithLabelValues(p.network.Name, ProcessorName, c.ProcessForwardsQueue(ProcessorName), ProcessForwardsTaskType, "batch_add_error").Inc() - - return fmt.Errorf("failed to add to batch: %w", addErr) - } - // Note: Task completion handled by batch manager - return nil - } - - // Process large transaction using existing logic - _, err = p.ProcessTransaction(ctx, block, int(payload.TransactionIndex), tx) + // Process transaction using ch-go streaming + structlogCount, err := p.ProcessTransaction(ctx, block, int(payload.TransactionIndex), tx) if err != nil { common.TasksErrored.WithLabelValues(p.network.Name, ProcessorName, c.ProcessForwardsQueue(ProcessorName), ProcessForwardsTaskType, "processing_error").Inc() @@ -112,9 +87,6 @@ func (p *Processor) handleProcessBackwardsTask(ctx context.Context, task *asynq. return fmt.Errorf("failed to unmarshal process payload: %w", err) } - // Wait for any active big transactions before starting - p.waitForBigTransactions("process_backwards") - // Get healthy execution node node := p.pool.GetHealthyExecutionNode() if node == nil { @@ -139,34 +111,12 @@ func (p *Processor) handleProcessBackwardsTask(ctx context.Context, task *asynq. return fmt.Errorf("transaction hash mismatch: expected %s, got %s", payload.TransactionHash, tx.Hash().String()) } - // Extract structlogs from the transaction - structlogs, err := p.ExtractStructlogs(ctx, block, int(payload.TransactionIndex), tx) + // Process transaction using ch-go streaming + structlogCount, err := p.ProcessTransaction(ctx, block, int(payload.TransactionIndex), tx) if err != nil { - common.TasksErrored.WithLabelValues(p.network.Name, ProcessorName, c.ProcessBackwardsQueue(ProcessorName), ProcessBackwardsTaskType, "extraction_error").Inc() - - return fmt.Errorf("failed to extract structlogs: %w", err) - } - - structlogCount := int64(len(structlogs)) - - // Check if transaction should be batched - if p.ShouldBatch(structlogCount) { - // Route to batch manager - if addErr := p.batchManager.Add(structlogs, task, &payload); addErr != nil { - common.TasksErrored.WithLabelValues(p.network.Name, ProcessorName, c.ProcessBackwardsQueue(ProcessorName), ProcessBackwardsTaskType, "batch_add_error").Inc() - - return fmt.Errorf("failed to add to batch: %w", addErr) - } - // Note: Task completion handled by batch manager - return nil - } - - // Process large transaction using existing logic - _, processErr := p.ProcessTransaction(ctx, block, int(payload.TransactionIndex), tx) - if processErr != nil { common.TasksErrored.WithLabelValues(p.network.Name, ProcessorName, c.ProcessBackwardsQueue(ProcessorName), ProcessBackwardsTaskType, "processing_error").Inc() - return fmt.Errorf("failed to process transaction: %w", processErr) + return fmt.Errorf("failed to process transaction: %w", err) } // Record successful processing diff --git a/pkg/processor/transaction/structlog/processor.go b/pkg/processor/transaction/structlog/processor.go index 0b82d4b..7f6828f 100644 --- a/pkg/processor/transaction/structlog/processor.go +++ b/pkg/processor/transaction/structlog/processor.go @@ -3,11 +3,8 @@ package structlog import ( "context" "fmt" - "strings" - "time" "github.com/ethpandaops/execution-processor/pkg/clickhouse" - "github.com/ethpandaops/execution-processor/pkg/common" "github.com/ethpandaops/execution-processor/pkg/ethereum" c "github.com/ethpandaops/execution-processor/pkg/processor/common" "github.com/ethpandaops/execution-processor/pkg/state" @@ -36,8 +33,6 @@ type Processor struct { asynqClient *asynq.Client processingMode string redisPrefix string - bigTxManager *BigTransactionManager - batchManager *BatchManager } // New creates a new transaction structlog processor. @@ -47,7 +42,7 @@ func New(ctx context.Context, deps *Dependencies, config *Config) (*Processor, e clickhouseConfig.Network = deps.Network.Name clickhouseConfig.Processor = ProcessorName - clickhouseClient, err := clickhouse.New(&clickhouseConfig) + clickhouseClient, err := clickhouse.New(ctx, &clickhouseConfig) if err != nil { return nil, fmt.Errorf("failed to create clickhouse client for transaction_structlog: %w", err) } @@ -74,24 +69,6 @@ func New(ctx context.Context, deps *Dependencies, config *Config) (*Processor, e // Start starts the processor. func (p *Processor) Start(ctx context.Context) error { - // Use configured value or default - threshold := p.config.BigTransactionThreshold - if threshold == 0 { - threshold = 500_000 // Default - } - - p.bigTxManager = NewBigTransactionManager(threshold, p.log) - - p.log.WithFields(logrus.Fields{ - "big_transaction_threshold": threshold, - }).Info("Initialized big transaction manager") - - // Create and start batch manager - p.batchManager = NewBatchManager(p, p.config) - if err := p.batchManager.Start(); err != nil { - return fmt.Errorf("failed to start batch manager: %w", err) - } - // Start the ClickHouse client if err := p.clickhouse.Start(); err != nil { return fmt.Errorf("failed to start ClickHouse client: %w", err) @@ -106,11 +83,6 @@ func (p *Processor) Start(ctx context.Context) error { func (p *Processor) Stop(ctx context.Context) error { p.log.Info("Stopping transaction structlog processor") - // Stop the batch manager first to flush any pending batches - if p.batchManager != nil { - p.batchManager.Stop() - } - // Stop the ClickHouse client return p.clickhouse.Stop() } @@ -164,90 +136,3 @@ func (p *Processor) getProcessForwardsQueue() string { func (p *Processor) getProcessBackwardsQueue() string { return c.PrefixedProcessBackwardsQueue(ProcessorName, p.redisPrefix) } - -// insertStructlogs inserts structlog rows directly into ClickHouse. -func (p *Processor) insertStructlogs(ctx context.Context, structlogs []Structlog) error { - // Short-circuit for empty structlog arrays - if len(structlogs) == 0 { - return nil - } - - // Create timeout context for insert - timeout := 5 * time.Minute - insertCtx, cancel := context.WithTimeout(ctx, timeout) - - defer cancel() - - // Perform the insert with metrics tracking - start := time.Now() - err := p.clickhouse.BulkInsert(insertCtx, p.config.Table, structlogs) - duration := time.Since(start) - - if err != nil { - code := parseErrorCode(err) - common.ClickHouseOperationDuration.WithLabelValues(p.network.Name, ProcessorName, "bulk_insert", p.config.Table, "failed", code).Observe(duration.Seconds()) - common.ClickHouseOperationTotal.WithLabelValues(p.network.Name, ProcessorName, "bulk_insert", p.config.Table, "failed", code).Inc() - common.ClickHouseInsertsRows.WithLabelValues(p.network.Name, ProcessorName, p.config.Table, "failed", code).Add(float64(len(structlogs))) - - p.log.WithFields(logrus.Fields{ - "table": p.config.Table, - "error": err.Error(), - "structlog_count": len(structlogs), - }).Error("Failed to insert structlogs") - - return fmt.Errorf("failed to insert %d structlogs: %w", len(structlogs), err) - } - - common.ClickHouseOperationDuration.WithLabelValues(p.network.Name, ProcessorName, "bulk_insert", p.config.Table, "success", "").Observe(duration.Seconds()) - common.ClickHouseOperationTotal.WithLabelValues(p.network.Name, ProcessorName, "bulk_insert", p.config.Table, "success", "").Inc() - common.ClickHouseInsertsRows.WithLabelValues(p.network.Name, ProcessorName, p.config.Table, "success", "").Add(float64(len(structlogs))) - - return nil -} - -// waitForBigTransactions waits for big transactions to complete before proceeding. -func (p *Processor) waitForBigTransactions(taskType string) { - if p.bigTxManager.ShouldPauseNewWork() { - p.log.WithFields(logrus.Fields{ - "task_type": taskType, - "active_big_txs": p.bigTxManager.currentBigCount.Load(), - }).Debug("Task waiting for big transactions to complete") - - startWait := time.Now() - - for p.bigTxManager.ShouldPauseNewWork() { - time.Sleep(100 * time.Millisecond) - } - - waitDuration := time.Since(startWait) - p.log.WithFields(logrus.Fields{ - "task_type": taskType, - "wait_duration": waitDuration.String(), - }).Debug("Task resumed after big transactions completed") - } -} - -// parseErrorCode extracts error code from error message. -func parseErrorCode(err error) string { - if err == nil { - return "" - } - // For HTTP client errors, we don't have specific error codes - // Return a generic code based on error type - errStr := err.Error() - - if strings.Contains(errStr, "timeout") { - return "TIMEOUT" - } - - if strings.Contains(errStr, "connection") { - return "CONNECTION" - } - - return "UNKNOWN" -} - -// ShouldBatch determines if a transaction should be batched based on structlog count. -func (p *Processor) ShouldBatch(structlogCount int64) bool { - return structlogCount > 0 && structlogCount < p.config.BatchInsertThreshold -} diff --git a/pkg/processor/transaction/structlog/processor_test.go b/pkg/processor/transaction/structlog/processor_test.go index 033a642..add904d 100644 --- a/pkg/processor/transaction/structlog/processor_test.go +++ b/pkg/processor/transaction/structlog/processor_test.go @@ -19,12 +19,9 @@ func TestProcessor_Creation(t *testing.T) { Enabled: true, Table: "test_structlog", Config: clickhouse.Config{ - URL: "http://localhost:8123", + Addr: "localhost:9000", }, - BigTransactionThreshold: 500000, - BatchInsertThreshold: 50000, - BatchFlushInterval: 5 * time.Second, - BatchMaxSize: 100000, + ChunkSize: 10000, } // Test config validation @@ -44,12 +41,9 @@ func TestProcessor_ConfigValidation(t *testing.T) { Enabled: true, Table: "test_table", Config: clickhouse.Config{ - URL: "http://localhost:8123", + Addr: "localhost:9000", }, - BigTransactionThreshold: 500000, - BatchInsertThreshold: 50000, - BatchFlushInterval: 5 * time.Second, - BatchMaxSize: 100000, + ChunkSize: 10000, }, expectError: false, }, @@ -61,14 +55,11 @@ func TestProcessor_ConfigValidation(t *testing.T) { expectError: false, }, { - name: "missing URL", + name: "missing addr", config: transaction_structlog.Config{ - Enabled: true, - Table: "test_table", - BigTransactionThreshold: 500000, - BatchInsertThreshold: 50000, - BatchFlushInterval: 5 * time.Second, - BatchMaxSize: 100000, + Enabled: true, + Table: "test_table", + ChunkSize: 10000, }, expectError: true, }, @@ -77,12 +68,9 @@ func TestProcessor_ConfigValidation(t *testing.T) { config: transaction_structlog.Config{ Enabled: true, Config: clickhouse.Config{ - URL: "http://localhost:8123", + Addr: "localhost:9000", }, - BigTransactionThreshold: 500000, - BatchInsertThreshold: 50000, - BatchFlushInterval: 5 * time.Second, - BatchMaxSize: 100000, + ChunkSize: 10000, }, expectError: true, }, @@ -113,12 +101,9 @@ func TestProcessor_ConcurrentConfigValidation(t *testing.T) { Enabled: true, Table: "test_concurrent", Config: clickhouse.Config{ - URL: "http://localhost:8123", + Addr: "localhost:9000", }, - BigTransactionThreshold: 500000, - BatchInsertThreshold: 50000, - BatchFlushInterval: 5 * time.Second, - BatchMaxSize: 100000, + ChunkSize: 10000, } results <- cfg.Validate() }() @@ -146,60 +131,55 @@ func TestStructlogCountReturn(t *testing.T) { }, } - // Note: cannot directly create processor with private fields in _test package - // This test validates the counting logic conceptually - // Test the count calculation and return expectedCount := len(mockTrace.Structlogs) - // We can not easily test the full processTransaction method without a real execution node, - // but we can test the logic of creating structlogs and counting them - structlogs := make([]transaction_structlog.Structlog, 0, expectedCount) - + // Test using Columns type for counting + cols := transaction_structlog.NewColumns() now := time.Now() - // Simulate the structlog creation logic + // Simulate the structlog appending logic for i, structLog := range mockTrace.Structlogs { - row := transaction_structlog.Structlog{ - UpdatedDateTime: transaction_structlog.NewClickHouseTime(now), - BlockNumber: 12345, - TransactionHash: "0x1234567890abcdef", - TransactionIndex: 0, - TransactionGas: mockTrace.Gas, - TransactionFailed: mockTrace.Failed, - TransactionReturnValue: mockTrace.ReturnValue, - Index: uint32(i), - ProgramCounter: structLog.PC, - Operation: structLog.Op, - Gas: structLog.Gas, - GasCost: structLog.GasCost, - Depth: structLog.Depth, - ReturnData: structLog.ReturnData, - Refund: structLog.Refund, - Error: structLog.Error, - MetaNetworkName: "test", - } - - structlogs = append(structlogs, row) - } - - // Test the key fix: save count before clearing slice - structlogCount := len(structlogs) - - // Clear slice like the real code does - structlogs = nil + cols.Append( + now, + 12345, // blockNumber + "0x1234567890abcdef", // txHash + 0, // txIndex + mockTrace.Gas, // txGas + mockTrace.Failed, // txFailed + mockTrace.ReturnValue, // txReturnValue + uint32(i), // index + structLog.PC, // pc + structLog.Op, // op + structLog.Gas, // gas + structLog.GasCost, // gasCost + structLog.GasCost, // gasUsed (simplified) + structLog.Depth, // depth + structLog.ReturnData, // returnData + structLog.Refund, // refund + structLog.Error, // error + nil, // callTo + "test", // network + ) + } + + // Test the key fix: count is tracked by Columns + structlogCount := cols.Rows() // Verify the count was saved correctly if structlogCount != expectedCount { t.Errorf("Expected count %d, but got %d", expectedCount, structlogCount) } - // Verify that len(structlogs) is now 0 (which would be the bug) - if len(structlogs) != 0 { - t.Errorf("Expected cleared slice to have length 0, but got %d", len(structlogs)) + // Reset columns (like the real code does between chunks) + cols.Reset() + + // Verify that Rows() is now 0 (which would be the bug if we returned this after reset) + if cols.Rows() != 0 { + t.Errorf("Expected reset columns to have 0 rows, but got %d", cols.Rows()) } - // The fix ensures we return structlogCount, not len(structlogs) + // The fix ensures we track count before reset if structlogCount == 0 { t.Error("structlogCount should not be 0 after processing valid structlogs") } @@ -235,45 +215,52 @@ func TestMemoryManagement(t *testing.T) { runtime.ReadMemStats(&initialMemStats) - // Create a large slice of structlogs to simulate memory usage - largeStructlogs := make([]transaction_structlog.Structlog, 10000) + // Create columns and simulate large batch of structlogs + cols := transaction_structlog.NewColumns() now := time.Now() - for i := range largeStructlogs { - largeStructlogs[i] = transaction_structlog.Structlog{ - UpdatedDateTime: transaction_structlog.NewClickHouseTime(now), - BlockNumber: uint64(i), - TransactionHash: "0x1234567890abcdef1234567890abcdef12345678", - TransactionIndex: uint32(i % 100), - TransactionGas: 21000, - TransactionFailed: false, - TransactionReturnValue: nil, - Index: uint32(i), - ProgramCounter: uint32(i * 2), - Operation: "SSTORE", - Gas: uint64(21000 - i), - GasCost: 5000, - Depth: 1, - ReturnData: nil, - Refund: nil, - Error: nil, - MetaNetworkName: "mainnet", - } - } + const rowCount = 10000 + + for i := 0; i < rowCount; i++ { + cols.Append( + now, + uint64(i), // blockNumber + "0x1234567890abcdef1234567890abcdef12345678", // txHash + uint32(i%100), // txIndex + 21000, // txGas + false, // txFailed + nil, // txReturnValue + uint32(i), // index + uint32(i*2), // pc + "SSTORE", // op + uint64(21000-i), // gas + 5000, // gasCost + 5000, // gasUsed + 1, // depth + nil, // returnData + nil, // refund + nil, // error + nil, // callTo + "mainnet", // network + ) + } + + assert.Equal(t, rowCount, cols.Rows(), "Should have correct row count") // Test that chunking calculations work properly const chunkSize = 100 - expectedChunks := (len(largeStructlogs) + chunkSize - 1) / chunkSize + expectedChunks := (rowCount + chunkSize - 1) / chunkSize // Verify chunking logic actualChunks := 0 - for i := 0; i < len(largeStructlogs); i += chunkSize { + + for i := 0; i < rowCount; i += chunkSize { actualChunks++ end := i + chunkSize - if end > len(largeStructlogs) { - end = len(largeStructlogs) + if end > rowCount { + end = rowCount } // Verify chunk size constraints @@ -287,6 +274,10 @@ func TestMemoryManagement(t *testing.T) { t.Errorf("Expected %d chunks, got %d", expectedChunks, actualChunks) } + // Reset columns to free memory + cols.Reset() + assert.Equal(t, 0, cols.Rows(), "Reset should clear all rows") + runtime.GC() runtime.ReadMemStats(&finalMemStats) @@ -341,8 +332,20 @@ func TestChunkProcessing(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - // Create input data - structlogs := make([]transaction_structlog.Structlog, tt.inputSize) + // Test chunking logic using Columns + cols := transaction_structlog.NewColumns() + now := time.Now() + + // Fill columns with test data + for i := 0; i < tt.inputSize; i++ { + cols.Append( + now, uint64(i), "0xtest", 0, 21000, false, nil, + uint32(i), uint32(i), "PUSH1", 20000, 3, 3, 1, + nil, nil, nil, nil, "test", + ) + } + + assert.Equal(t, tt.inputSize, cols.Rows(), "Should have correct row count") // Calculate expected chunks expectedChunks := (tt.inputSize + tt.chunkSize - 1) / tt.chunkSize @@ -353,12 +356,13 @@ func TestChunkProcessing(t *testing.T) { // Test that the chunking logic would work correctly chunkCount := 0 - for i := 0; i < len(structlogs); i += tt.chunkSize { + + for i := 0; i < tt.inputSize; i += tt.chunkSize { chunkCount++ end := i + tt.chunkSize - if end > len(structlogs) { - end = len(structlogs) + if end > tt.inputSize { + end = tt.inputSize } // Verify chunk boundaries if end <= i { @@ -373,6 +377,51 @@ func TestChunkProcessing(t *testing.T) { } } +func TestColumnsAppendAndReset(t *testing.T) { + cols := transaction_structlog.NewColumns() + now := time.Now() + + // Initially empty + assert.Equal(t, 0, cols.Rows()) + + // Append a row + str := "test" + num := uint64(42) + + cols.Append( + now, 100, "0xabc", 0, 21000, false, &str, + 0, 100, "PUSH1", 20000, 3, 3, 1, + nil, &num, nil, nil, "mainnet", + ) + + assert.Equal(t, 1, cols.Rows()) + + // Append more rows + for i := 0; i < 99; i++ { + cols.Append( + now, 100, "0xabc", 0, 21000, false, nil, + uint32(i+1), 100, "PUSH1", 20000, 3, 3, 1, + nil, nil, nil, nil, "mainnet", + ) + } + + assert.Equal(t, 100, cols.Rows()) + + // Reset + cols.Reset() + assert.Equal(t, 0, cols.Rows()) +} + +func TestColumnsInput(t *testing.T) { + cols := transaction_structlog.NewColumns() + input := cols.Input() + + // Verify all 19 columns are present + assert.Len(t, input, 19) + assert.Equal(t, "updated_date_time", input[0].Name) + assert.Equal(t, "meta_network_name", input[18].Name) +} + // Tests from tasks_test.go func TestProcessPayload(t *testing.T) { diff --git a/pkg/processor/transaction/structlog/transaction_processing.go b/pkg/processor/transaction/structlog/transaction_processing.go index 1031ab8..ce0f54c 100644 --- a/pkg/processor/transaction/structlog/transaction_processing.go +++ b/pkg/processor/transaction/structlog/transaction_processing.go @@ -3,8 +3,10 @@ package structlog import ( "context" "fmt" + "io" "time" + "github.com/ClickHouse/ch-go" "github.com/ethereum/go-ethereum/core/types" "github.com/sirupsen/logrus" @@ -12,194 +14,106 @@ import ( "github.com/ethpandaops/execution-processor/pkg/ethereum/execution" ) -//nolint:tagliatelle // ClickHouse uses snake_case column names -type Structlog struct { - UpdatedDateTime ClickHouseTime `json:"updated_date_time"` - BlockNumber uint64 `json:"block_number"` - TransactionHash string `json:"transaction_hash"` - TransactionIndex uint32 `json:"transaction_index"` - TransactionGas uint64 `json:"transaction_gas"` - TransactionFailed bool `json:"transaction_failed"` - TransactionReturnValue *string `json:"transaction_return_value"` - Index uint32 `json:"index"` - ProgramCounter uint32 `json:"program_counter"` - Operation string `json:"operation"` - Gas uint64 `json:"gas"` - GasCost uint64 `json:"gas_cost"` - GasUsed uint64 `json:"gas_used"` - Depth uint64 `json:"depth"` - ReturnData *string `json:"return_data"` - Refund *uint64 `json:"refund"` - Error *string `json:"error"` - CallToAddress *string `json:"call_to_address"` - MetaNetworkName string `json:"meta_network_name"` -} - -// ProcessSingleTransaction processes a single transaction and inserts its structlogs directly to ClickHouse. -func (p *Processor) ProcessSingleTransaction(ctx context.Context, block *types.Block, index int, tx *types.Transaction) (int, error) { - // Extract structlog data - structlogs, err := p.ExtractStructlogs(ctx, block, index, tx) - if err != nil { - return 0, err - } - - // Store count before processing - structlogCount := len(structlogs) - - // Ensure we clear the slice on exit to allow GC - defer func() { - // Clear the slice to release memory - structlogs = nil - }() - - // Send for direct insertion - if err := p.insertStructlogs(ctx, structlogs); err != nil { - common.TransactionsProcessed.WithLabelValues(p.network.Name, "structlog", "failed").Inc() - - return 0, fmt.Errorf("failed to insert structlogs: %w", err) - } - - // Record success metrics - common.TransactionsProcessed.WithLabelValues(p.network.Name, "structlog", "success").Inc() - - return structlogCount, nil -} - -// ProcessTransaction processes a transaction using memory-efficient channel-based batching. +// ProcessTransaction processes a transaction using ch-go columnar streaming. func (p *Processor) ProcessTransaction(ctx context.Context, block *types.Block, index int, tx *types.Transaction) (int, error) { - // Get trace from execution node trace, err := p.getTransactionTrace(ctx, tx, block) if err != nil { return 0, fmt.Errorf("failed to get trace: %w", err) } totalCount := len(trace.Structlogs) + if totalCount == 0 { + return 0, nil + } // Compute actual gas used for each structlog gasUsed := ComputeGasUsed(trace.Structlogs) - // Check if this is a big transaction and register if needed - if totalCount >= p.bigTxManager.GetThreshold() { - p.bigTxManager.RegisterBigTransaction(tx.Hash().String(), p) - defer p.bigTxManager.UnregisterBigTransaction(tx.Hash().String()) - - p.log.WithFields(logrus.Fields{ - "tx_hash": tx.Hash().String(), - "structlog_count": totalCount, - "current_big_count": p.bigTxManager.currentBigCount.Load(), - }).Info("Processing big transaction") - } - chunkSize := p.config.ChunkSize if chunkSize == 0 { - chunkSize = 10_000 // Default - } - - // Buffered channel holds configured number of chunks - bufferSize := p.config.ChannelBufferSize - if bufferSize == 0 { - bufferSize = 2 // Default + chunkSize = 10_000 } - batchChan := make(chan []Structlog, bufferSize) - errChan := make(chan error, 1) - - // Consumer goroutine - inserts to ClickHouse - go func() { - inserted := 0 - - for batch := range batchChan { - if err := p.insertStructlogs(ctx, batch); err != nil { - errChan <- fmt.Errorf("failed to insert at %d: %w", inserted, err) + cols := NewColumns() + input := cols.Input() + + currentIdx := 0 + blockNum := block.Number().Uint64() + txHash := tx.Hash().String() + txIndex := uint32(index) //nolint:gosec // index is bounded by block.Transactions() length + now := time.Now() + + // Use Do with OnInput for streaming insert + err = p.clickhouse.Do(ctx, ch.Query{ + Body: input.Into(p.config.Table), + Input: input, + OnInput: func(ctx context.Context) error { + // Reset columns for next chunk + cols.Reset() + + if currentIdx >= totalCount { + return io.EOF + } - return + // Fill columns with next chunk + end := currentIdx + chunkSize + if end > totalCount { + end = totalCount } - inserted += len(batch) + for i := currentIdx; i < end; i++ { + sl := &trace.Structlogs[i] + cols.Append( + now, + blockNum, + txHash, + txIndex, + trace.Gas, + trace.Failed, + trace.ReturnValue, + uint32(i), //nolint:gosec // index is bounded by structlogs length + sl.PC, + sl.Op, + sl.Gas, + sl.GasCost, + gasUsed[i], + sl.Depth, + sl.ReturnData, + sl.Refund, + sl.Error, + p.extractCallAddress(sl), + p.network.Name, + ) + + // Free original trace data immediately to help GC + trace.Structlogs[i] = execution.StructLog{} + } // Log progress for large transactions progressThreshold := p.config.ProgressLogThreshold if progressThreshold == 0 { - progressThreshold = 100_000 // Default + progressThreshold = 100_000 } - if totalCount > progressThreshold && inserted%progressThreshold < chunkSize { + if totalCount > progressThreshold && end%progressThreshold < chunkSize { p.log.WithFields(logrus.Fields{ - "tx_hash": tx.Hash(), - "progress": fmt.Sprintf("%d/%d", inserted, totalCount), + "tx_hash": txHash, + "progress": fmt.Sprintf("%d/%d", end, totalCount), }).Debug("Processing large transaction") } - } - - errChan <- nil - }() - - // Producer - convert and send batches - batch := make([]Structlog, 0, chunkSize) - for i := 0; i < totalCount; i++ { - // Convert structlog - batch = append(batch, Structlog{ - UpdatedDateTime: NewClickHouseTime(time.Now()), - BlockNumber: block.Number().Uint64(), - TransactionHash: tx.Hash().String(), - TransactionIndex: uint32(index), //nolint:gosec // index is bounded by block.Transactions() length - TransactionGas: trace.Gas, - TransactionFailed: trace.Failed, - TransactionReturnValue: trace.ReturnValue, - Index: uint32(i), //nolint:gosec // index is bounded by structlogs length - ProgramCounter: trace.Structlogs[i].PC, - Operation: trace.Structlogs[i].Op, - Gas: trace.Structlogs[i].Gas, - GasCost: trace.Structlogs[i].GasCost, - GasUsed: gasUsed[i], - Depth: trace.Structlogs[i].Depth, - ReturnData: trace.Structlogs[i].ReturnData, - Refund: trace.Structlogs[i].Refund, - Error: trace.Structlogs[i].Error, - CallToAddress: p.extractCallAddress(&trace.Structlogs[i]), - MetaNetworkName: p.network.Name, - }) - // CRITICAL: Free original trace data immediately - trace.Structlogs[i] = execution.StructLog{} - - // Send full batch - if len(batch) == chunkSize { - select { - case batchChan <- batch: - batch = make([]Structlog, 0, chunkSize) - case <-ctx.Done(): - close(batchChan) - - return 0, ctx.Err() - } - } - } - - // Clear trace reference to help GC - trace = nil - - // Send final batch if any - if len(batch) > 0 { - select { - case batchChan <- batch: - case <-ctx.Done(): - close(batchChan) - - return 0, ctx.Err() - } - } + currentIdx = end - // Signal completion and wait - close(batchChan) - - // Wait for consumer to finish - if err := <-errChan; err != nil { - return 0, err + return nil + }, + }) + if err != nil { + return 0, fmt.Errorf("insert failed: %w", err) } // Record success metrics common.TransactionsProcessed.WithLabelValues(p.network.Name, "structlog", "success").Inc() + common.ClickHouseInsertsRows.WithLabelValues(p.network.Name, ProcessorName, p.config.Table, "success", "").Add(float64(totalCount)) return totalCount, nil } @@ -235,80 +149,3 @@ func (p *Processor) extractCallAddress(structLog *execution.StructLog) *string { return nil } - -// ExtractStructlogs extracts structlog data from a transaction without inserting to database. -func (p *Processor) ExtractStructlogs(ctx context.Context, block *types.Block, index int, tx *types.Transaction) ([]Structlog, error) { - start := time.Now() - - defer func() { - duration := time.Since(start) - common.TransactionProcessingDuration.WithLabelValues(p.network.Name, "structlog").Observe(duration.Seconds()) - }() - - // Get execution node - node := p.pool.GetHealthyExecutionNode() - if node == nil { - return nil, fmt.Errorf("no healthy execution node available") - } - - // Process transaction with timeout - processCtx, cancel := context.WithTimeout(ctx, 30*time.Second) - defer cancel() - - // Get transaction trace - trace, err := node.DebugTraceTransaction(processCtx, tx.Hash().String(), block.Number(), execution.StackTraceOptions()) - if err != nil { - return nil, fmt.Errorf("failed to trace transaction: %w", err) - } - - // Convert trace to structlog rows - var structlogs []Structlog - - uIndex := uint32(index) //nolint:gosec // index is bounded by block.Transactions() length - - if trace != nil { - // Compute actual gas used for each structlog - gasUsed := ComputeGasUsed(trace.Structlogs) - - // Pre-allocate slice for better memory efficiency - structlogs = make([]Structlog, 0, len(trace.Structlogs)) - - for i, structLog := range trace.Structlogs { - var callToAddress *string - - if structLog.Op == "CALL" && structLog.Stack != nil && len(*structLog.Stack) > 1 { - stackValue := (*structLog.Stack)[len(*structLog.Stack)-2] - callToAddress = &stackValue - } - - row := Structlog{ - UpdatedDateTime: NewClickHouseTime(time.Now()), - BlockNumber: block.Number().Uint64(), - TransactionHash: tx.Hash().String(), - TransactionIndex: uIndex, - TransactionGas: trace.Gas, - TransactionFailed: trace.Failed, - TransactionReturnValue: trace.ReturnValue, - Index: uint32(i), //nolint:gosec // index is bounded by structlogs length - ProgramCounter: structLog.PC, - Operation: structLog.Op, - Gas: structLog.Gas, - GasCost: structLog.GasCost, - GasUsed: gasUsed[i], - Depth: structLog.Depth, - ReturnData: structLog.ReturnData, - Refund: structLog.Refund, - Error: structLog.Error, - CallToAddress: callToAddress, - MetaNetworkName: p.network.Name, - } - - structlogs = append(structlogs, row) - } - - // Clear the original trace data to free memory - trace.Structlogs = nil - } - - return structlogs, nil -} diff --git a/pkg/state/manager.go b/pkg/state/manager.go index 3e5c0d5..59ea005 100644 --- a/pkg/state/manager.go +++ b/pkg/state/manager.go @@ -45,7 +45,7 @@ func NewManager(ctx context.Context, log logrus.FieldLogger, config *Config) (*M storageConfig := config.Storage.Config storageConfig.Processor = "state" - storageClient, err := clickhouse.New(&storageConfig) + storageClient, err := clickhouse.New(ctx, &storageConfig) if err != nil { return nil, fmt.Errorf("failed to create storage clickhouse client: %w", err) } @@ -62,7 +62,7 @@ func NewManager(ctx context.Context, log logrus.FieldLogger, config *Config) (*M limiterConfig := config.Limiter.Config limiterConfig.Processor = "state-limiter" - limiterClient, err := clickhouse.New(&limiterConfig) + limiterClient, err := clickhouse.New(ctx, &limiterConfig) if err != nil { return nil, fmt.Errorf("failed to create limiter clickhouse client: %w", err) } diff --git a/pkg/state/manager_test.go b/pkg/state/manager_test.go index 6e0b829..122ffd2 100644 --- a/pkg/state/manager_test.go +++ b/pkg/state/manager_test.go @@ -5,6 +5,7 @@ import ( "math/big" "testing" + "github.com/ClickHouse/ch-go" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -33,12 +34,6 @@ func (m *MockClickHouseClient) Execute(ctx context.Context, query string) error return args.Error(0) } -func (m *MockClickHouseClient) BulkInsert(ctx context.Context, table string, data interface{}) error { - args := m.Called(ctx, table, data) - - return args.Error(0) -} - func (m *MockClickHouseClient) Start() error { args := m.Called() @@ -61,6 +56,12 @@ func (m *MockClickHouseClient) SetNetwork(network string) { m.Called(network) } +func (m *MockClickHouseClient) Do(ctx context.Context, query ch.Query) error { + args := m.Called(ctx, query) + + return args.Error(0) +} + func TestNextBlock_NoResultsVsBlock0(t *testing.T) { ctx := context.Background() log := logrus.New() From 36b1c00de160a1a4c354580df5097e3f0bdc9272 Mon Sep 17 00:00:00 2001 From: Andrew Davis <1709934+Savid@users.noreply.github.com> Date: Thu, 22 Jan 2026 17:22:13 +1000 Subject: [PATCH 2/5] fix tests --- internal/testutil/testutil.go | 4 +-- pkg/clickhouse/client_integration_test.go | 31 +++++++++++++++-------- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/internal/testutil/testutil.go b/internal/testutil/testutil.go index aaf1ece..696c9ea 100644 --- a/internal/testutil/testutil.go +++ b/internal/testutil/testutil.go @@ -96,7 +96,7 @@ func NewClickHouseContainer(t *testing.T) ClickHouseConnection { c, err := tcclickhouse.Run(ctx, "clickhouse/clickhouse-server:latest", tcclickhouse.WithUsername("default"), - tcclickhouse.WithPassword(""), + tcclickhouse.WithPassword("test"), tcclickhouse.WithDatabase("default"), ) if err != nil { @@ -120,6 +120,6 @@ func NewClickHouseContainer(t *testing.T) ClickHouseConnection { Port: port.Int(), Database: "default", Username: "default", - Password: "", + Password: "test", } } diff --git a/pkg/clickhouse/client_integration_test.go b/pkg/clickhouse/client_integration_test.go index dfe7874..e9606b9 100644 --- a/pkg/clickhouse/client_integration_test.go +++ b/pkg/clickhouse/client_integration_test.go @@ -3,6 +3,7 @@ package clickhouse import ( + "context" "testing" "github.com/ethpandaops/execution-processor/internal/testutil" @@ -23,7 +24,7 @@ func TestClient_Integration_Container_New(t *testing.T) { Compression: "lz4", } - client, err := New(cfg) + client, err := New(context.Background(), cfg) require.NoError(t, err) require.NotNil(t, client) @@ -43,7 +44,7 @@ func TestClient_Integration_Container_StartStop(t *testing.T) { Network: "test", } - client, err := New(cfg) + client, err := New(context.Background(), cfg) require.NoError(t, err) // Start should ping successfully @@ -67,7 +68,7 @@ func TestClient_Integration_Container_Execute(t *testing.T) { Network: "test", } - client, err := New(cfg) + client, err := New(context.Background(), cfg) require.NoError(t, err) defer func() { _ = client.Stop() }() @@ -75,8 +76,15 @@ func TestClient_Integration_Container_Execute(t *testing.T) { err = client.Start() require.NoError(t, err) - // Execute a simple query - err = client.Execute(t.Context(), "SELECT 1") + // Execute DDL - Create and drop a table + err = client.Execute(t.Context(), ` + CREATE TABLE IF NOT EXISTS test_execute ( + id UInt64 + ) ENGINE = Memory + `) + require.NoError(t, err) + + err = client.Execute(t.Context(), "DROP TABLE IF EXISTS test_execute") require.NoError(t, err) } @@ -92,7 +100,7 @@ func TestClient_Integration_Container_QueryOne(t *testing.T) { Network: "test", } - client, err := New(cfg) + client, err := New(context.Background(), cfg) require.NoError(t, err) defer func() { _ = client.Stop() }() @@ -100,11 +108,13 @@ func TestClient_Integration_Container_QueryOne(t *testing.T) { err = client.Start() require.NoError(t, err) + // QueryOne expects JSON string result that gets deserialized var result struct { Value int64 `json:"value"` } - err = client.QueryOne(t.Context(), "SELECT 42 as value", &result) + // The query must return a single column with JSON string + err = client.QueryOne(t.Context(), "SELECT '{\"value\": 42}' as json_result", &result) require.NoError(t, err) assert.Equal(t, int64(42), result.Value) } @@ -121,7 +131,7 @@ func TestClient_Integration_Container_IsStorageEmpty(t *testing.T) { Network: "test", } - client, err := New(cfg) + client, err := New(context.Background(), cfg) require.NoError(t, err) defer func() { _ = client.Stop() }() @@ -129,12 +139,13 @@ func TestClient_Integration_Container_IsStorageEmpty(t *testing.T) { err = client.Start() require.NoError(t, err) - // Create a temporary table + // Create a table with ReplacingMergeTree engine (supports FINAL) err = client.Execute(t.Context(), ` CREATE TABLE IF NOT EXISTS test_empty_check ( id UInt64, name String - ) ENGINE = Memory + ) ENGINE = ReplacingMergeTree() + ORDER BY id `) require.NoError(t, err) From 52606de6c6d08166395b1e9e9f398ec9cda1fc7a Mon Sep 17 00:00:00 2001 From: Andrew Davis <1709934+Savid@users.noreply.github.com> Date: Fri, 23 Jan 2026 11:22:18 +1000 Subject: [PATCH 3/5] refactor(clickhouse): replace QueryOne/QueryMany with typed query methods Replace generic JSON-based QueryOne/QueryMany with type-specific native column binding methods to fix ch-go type validation errors. Changes: - Add QueryUInt64 for single UInt64 value queries - Add QueryMinMaxUInt64 for min/max aggregate queries - Update state manager to use new typed methods - Remove json_number.go (types no longer needed) - Update example_config.yaml for native protocol format - Update tests and mocks for new interface --- example_config.yaml | 12 ++- pkg/clickhouse/client.go | 108 ++++++++----------- pkg/clickhouse/client_integration_test.go | 39 +++++-- pkg/clickhouse/client_test.go | 11 +- pkg/clickhouse/interface.go | 25 +++-- pkg/clickhouse/mock.go | 101 +++++++---------- pkg/state/json_number.go | 126 ---------------------- pkg/state/manager.go | 67 ++++-------- pkg/state/manager_test.go | 102 ++++++++---------- 9 files changed, 205 insertions(+), 386 deletions(-) delete mode 100644 pkg/state/json_number.go diff --git a/example_config.yaml b/example_config.yaml index 7643099..51b21ff 100644 --- a/example_config.yaml +++ b/example_config.yaml @@ -17,13 +17,15 @@ redis: stateManager: storage: - url: "http://localhost:8123?database=admin" + addr: "localhost:9000" + database: "admin" table: "execution_block" limiter: # enable limiter to stop processing blocks up to the latest canonical beacon block table # if disabled, the state manager will process blocks up to head of the execution node enabled: false - url: "http://localhost:8123?database=default" + addr: "localhost:9000" + database: "default" table: "canonical_beacon_block" # Processor configuration (block discovery with leader election) @@ -45,7 +47,8 @@ processors: # Processor configuration transactionStructlog: enabled: true - url: "http://localhost:8123?database=default" + addr: "localhost:9000" + database: "default" table: "canonical_execution_transaction_structlog" # debug: false # Enable debug logging for ClickHouse queries # Channel-based batching configuration for memory-efficient processing @@ -62,7 +65,8 @@ processors: # Simple transaction processor (lightweight - no debug traces) transactionSimple: enabled: false - url: "http://localhost:8123?database=default" + addr: "localhost:9000" + database: "default" table: "execution_transaction" # debug: false # Enable debug logging for ClickHouse queries diff --git a/pkg/clickhouse/client.go b/pkg/clickhouse/client.go index 142bd03..2131af4 100644 --- a/pkg/clickhouse/client.go +++ b/pkg/clickhouse/client.go @@ -2,12 +2,10 @@ package clickhouse import ( "context" - "encoding/json" "errors" "fmt" "io" "net" - "reflect" "strings" "sync" "sync/atomic" @@ -334,31 +332,35 @@ func (c *Client) Do(ctx context.Context, query ch.Query) error { return c.pool.Do(ctx, query) } -// QueryOne executes a query and returns a single result. -func (c *Client) QueryOne(ctx context.Context, query string, dest any) error { +// QueryUInt64 executes a query and returns a single UInt64 value from the specified column. +// Returns nil if no rows are found. +func (c *Client) QueryUInt64(ctx context.Context, query string, columnName string) (*uint64, error) { start := time.Now() - operation := "query_one" + operation := "query_uint64" status := statusSuccess defer func() { c.recordMetrics(operation, status, time.Since(start), query) }() - var rows []string + var result *uint64 + + col := new(proto.ColUInt64) err := c.doWithRetry(ctx, operation, func(attemptCtx context.Context) error { - // Reset rows for each retry attempt - rows = nil - colStr := new(proto.ColStr) + col.Reset() + + result = nil return c.pool.Do(attemptCtx, ch.Query{ - Body: query + " FORMAT JSONEachRow", + Body: query, Result: proto.Results{ - {Name: "", Data: colStr}, + {Name: columnName, Data: col}, }, OnResult: func(ctx context.Context, block proto.Block) error { - for i := 0; i < colStr.Rows(); i++ { - rows = append(rows, colStr.Row(i)) + if col.Rows() > 0 { + val := col.Row(0) + result = &val } return nil @@ -368,55 +370,49 @@ func (c *Client) QueryOne(ctx context.Context, query string, dest any) error { if err != nil { status = statusFailed - return fmt.Errorf("query execution failed: %w", err) + return nil, fmt.Errorf("query failed: %w", err) } - if len(rows) == 0 { - return nil - } - - if err := json.Unmarshal([]byte(rows[0]), dest); err != nil { - status = statusFailed - - return fmt.Errorf("failed to unmarshal result: %w", err) - } - - return nil + return result, nil } -// QueryMany executes a query and returns multiple results. -func (c *Client) QueryMany(ctx context.Context, query string, dest any) error { +// QueryMinMaxUInt64 executes a query that returns min and max UInt64 values. +// The query must return columns named "min" and "max". +// Returns nil for both values if no rows are found. +func (c *Client) QueryMinMaxUInt64(ctx context.Context, query string) (minVal, maxVal *uint64, err error) { start := time.Now() - operation := "query_many" + operation := "query_min_max" status := statusSuccess defer func() { c.recordMetrics(operation, status, time.Since(start), query) }() - // Validate that dest is a pointer to a slice - destValue := reflect.ValueOf(dest) - if destValue.Kind() != reflect.Pointer || destValue.Elem().Kind() != reflect.Slice { - status = statusFailed - - return fmt.Errorf("dest must be a pointer to a slice") - } + colMin := new(proto.ColUInt64) + colMax := new(proto.ColUInt64) - var rows []string + err = c.doWithRetry(ctx, operation, func(attemptCtx context.Context) error { + colMin.Reset() + colMax.Reset() - err := c.doWithRetry(ctx, operation, func(attemptCtx context.Context) error { - // Reset rows for each retry attempt - rows = nil - colStr := new(proto.ColStr) + minVal = nil + maxVal = nil return c.pool.Do(attemptCtx, ch.Query{ - Body: query + " FORMAT JSONEachRow", + Body: query, Result: proto.Results{ - {Name: "", Data: colStr}, + {Name: "min", Data: colMin}, + {Name: "max", Data: colMax}, }, OnResult: func(ctx context.Context, block proto.Block) error { - for i := 0; i < colStr.Rows(); i++ { - rows = append(rows, colStr.Row(i)) + if colMin.Rows() > 0 { + v := colMin.Row(0) + minVal = &v + } + + if colMax.Rows() > 0 { + v := colMax.Row(0) + maxVal = &v } return nil @@ -426,30 +422,10 @@ func (c *Client) QueryMany(ctx context.Context, query string, dest any) error { if err != nil { status = statusFailed - return fmt.Errorf("query execution failed: %w", err) + return nil, nil, fmt.Errorf("query failed: %w", err) } - // Create a slice of the appropriate type - sliceType := destValue.Elem().Type() - elemType := sliceType.Elem() - newSlice := reflect.MakeSlice(sliceType, len(rows), len(rows)) - - // Unmarshal each row - for i, row := range rows { - elem := reflect.New(elemType) - if err := json.Unmarshal([]byte(row), elem.Interface()); err != nil { - status = statusFailed - - return fmt.Errorf("failed to unmarshal row %d: %w", i, err) - } - - newSlice.Index(i).Set(elem.Elem()) - } - - // Set the result - destValue.Elem().Set(newSlice) - - return nil + return minVal, maxVal, nil } // Execute runs a query without expecting results. diff --git a/pkg/clickhouse/client_integration_test.go b/pkg/clickhouse/client_integration_test.go index e9606b9..491b06b 100644 --- a/pkg/clickhouse/client_integration_test.go +++ b/pkg/clickhouse/client_integration_test.go @@ -88,7 +88,7 @@ func TestClient_Integration_Container_Execute(t *testing.T) { require.NoError(t, err) } -func TestClient_Integration_Container_QueryOne(t *testing.T) { +func TestClient_Integration_Container_QueryUInt64(t *testing.T) { conn := testutil.NewClickHouseContainer(t) cfg := &Config{ @@ -108,15 +108,40 @@ func TestClient_Integration_Container_QueryOne(t *testing.T) { err = client.Start() require.NoError(t, err) - // QueryOne expects JSON string result that gets deserialized - var result struct { - Value int64 `json:"value"` + // QueryUInt64 returns a single UInt64 value + result, err := client.QueryUInt64(t.Context(), "SELECT 42 as value", "value") + require.NoError(t, err) + require.NotNil(t, result) + assert.Equal(t, uint64(42), *result) +} + +func TestClient_Integration_Container_QueryMinMaxUInt64(t *testing.T) { + conn := testutil.NewClickHouseContainer(t) + + cfg := &Config{ + Addr: conn.Addr(), + Database: conn.Database, + Username: conn.Username, + Password: conn.Password, + Compression: "lz4", + Network: "test", } - // The query must return a single column with JSON string - err = client.QueryOne(t.Context(), "SELECT '{\"value\": 42}' as json_result", &result) + client, err := New(context.Background(), cfg) + require.NoError(t, err) + + defer func() { _ = client.Stop() }() + + err = client.Start() + require.NoError(t, err) + + // QueryMinMaxUInt64 returns min and max values + minVal, maxVal, err := client.QueryMinMaxUInt64(t.Context(), "SELECT 10 as min, 100 as max") require.NoError(t, err) - assert.Equal(t, int64(42), result.Value) + require.NotNil(t, minVal) + require.NotNil(t, maxVal) + assert.Equal(t, uint64(10), *minVal) + assert.Equal(t, uint64(100), *maxVal) } func TestClient_Integration_Container_IsStorageEmpty(t *testing.T) { diff --git a/pkg/clickhouse/client_test.go b/pkg/clickhouse/client_test.go index ec42c30..0b2ba93 100644 --- a/pkg/clickhouse/client_test.go +++ b/pkg/clickhouse/client_test.go @@ -246,7 +246,7 @@ func TestClient_Integration_Execute(t *testing.T) { require.NoError(t, err) } -func TestClient_Integration_QueryOne(t *testing.T) { +func TestClient_Integration_QueryUInt64(t *testing.T) { addr := os.Getenv("CLICKHOUSE_ADDR") if addr == "" { t.Skip("CLICKHOUSE_ADDR not set, skipping integration test") @@ -267,13 +267,10 @@ func TestClient_Integration_QueryOne(t *testing.T) { err = client.Start() require.NoError(t, err) - var result struct { - Value int64 `json:"value"` - } - - err = client.QueryOne(t.Context(), "SELECT 42 as value", &result) + result, err := client.QueryUInt64(t.Context(), "SELECT 42 as value", "value") require.NoError(t, err) - assert.Equal(t, int64(42), result.Value) + require.NotNil(t, result) + assert.Equal(t, uint64(42), *result) } func TestClient_Integration_IsStorageEmpty(t *testing.T) { diff --git a/pkg/clickhouse/interface.go b/pkg/clickhouse/interface.go index 608f8fe..ff506d6 100644 --- a/pkg/clickhouse/interface.go +++ b/pkg/clickhouse/interface.go @@ -8,21 +8,24 @@ import ( // ClientInterface defines the methods for interacting with ClickHouse. type ClientInterface interface { - // QueryOne executes a query and returns a single result - QueryOne(ctx context.Context, query string, dest any) error - // QueryMany executes a query and returns multiple results - QueryMany(ctx context.Context, query string, dest any) error - // Execute runs a query without expecting results - Execute(ctx context.Context, query string) error - // IsStorageEmpty checks if a table has any records matching the given conditions - IsStorageEmpty(ctx context.Context, table string, conditions map[string]any) (bool, error) - // SetNetwork updates the network name for metrics labeling - SetNetwork(network string) // Start initializes the client Start() error // Stop closes the client Stop() error - + // SetNetwork updates the network name for metrics labeling + SetNetwork(network string) // Do executes a ch-go query directly for streaming operations Do(ctx context.Context, query ch.Query) error + // Execute runs a query without expecting results + Execute(ctx context.Context, query string) error + // IsStorageEmpty checks if a table has any records matching the given conditions + IsStorageEmpty(ctx context.Context, table string, conditions map[string]any) (bool, error) + + // QueryUInt64 executes a query and returns a single UInt64 value from the specified column. + // Returns nil if no rows are found. + QueryUInt64(ctx context.Context, query string, columnName string) (*uint64, error) + // QueryMinMaxUInt64 executes a query that returns min and max UInt64 values. + // The query must return columns named "min" and "max". + // Returns nil for both values if no rows are found. + QueryMinMaxUInt64(ctx context.Context, query string) (minVal, maxVal *uint64, err error) } diff --git a/pkg/clickhouse/mock.go b/pkg/clickhouse/mock.go index a835366..ff1bc93 100644 --- a/pkg/clickhouse/mock.go +++ b/pkg/clickhouse/mock.go @@ -4,9 +4,6 @@ package clickhouse import ( "context" - "encoding/json" - "fmt" - "reflect" "github.com/ClickHouse/ch-go" ) @@ -15,13 +12,13 @@ import ( // It should only be used in test files, not in production code. type MockClient struct { // Function fields that can be set by tests - QueryOneFunc func(ctx context.Context, query string, dest any) error - QueryManyFunc func(ctx context.Context, query string, dest any) error - ExecuteFunc func(ctx context.Context, query string) error - IsStorageEmptyFunc func(ctx context.Context, table string, conditions map[string]any) (bool, error) - StartFunc func() error - StopFunc func() error - DoFunc func(ctx context.Context, query ch.Query) error + ExecuteFunc func(ctx context.Context, query string) error + IsStorageEmptyFunc func(ctx context.Context, table string, conditions map[string]any) (bool, error) + StartFunc func() error + StopFunc func() error + DoFunc func(ctx context.Context, query ch.Query) error + QueryUInt64Func func(ctx context.Context, query string, columnName string) (*uint64, error) + QueryMinMaxUInt64Func func(ctx context.Context, query string) (minVal, maxVal *uint64, err error) // Track calls for assertions Calls []MockCall @@ -36,12 +33,6 @@ type MockCall struct { // NewMockClient creates a new mock client with default implementations. func NewMockClient() *MockClient { return &MockClient{ - QueryOneFunc: func(ctx context.Context, query string, dest any) error { - return nil - }, - QueryManyFunc: func(ctx context.Context, query string, dest any) error { - return nil - }, ExecuteFunc: func(ctx context.Context, query string) error { return nil }, @@ -57,36 +48,42 @@ func NewMockClient() *MockClient { DoFunc: func(ctx context.Context, query ch.Query) error { return nil }, + QueryUInt64Func: func(ctx context.Context, query string, columnName string) (*uint64, error) { + return nil, nil + }, + QueryMinMaxUInt64Func: func(ctx context.Context, query string) (minVal, maxVal *uint64, err error) { + return nil, nil, nil + }, Calls: make([]MockCall, 0), } } -// QueryOne implements ClientInterface. -func (m *MockClient) QueryOne(ctx context.Context, query string, dest any) error { +// QueryUInt64 implements ClientInterface. +func (m *MockClient) QueryUInt64(ctx context.Context, query string, columnName string) (*uint64, error) { m.Calls = append(m.Calls, MockCall{ - Method: "QueryOne", - Args: []any{ctx, query, dest}, + Method: "QueryUInt64", + Args: []any{ctx, query, columnName}, }) - if m.QueryOneFunc != nil { - return m.QueryOneFunc(ctx, query, dest) + if m.QueryUInt64Func != nil { + return m.QueryUInt64Func(ctx, query, columnName) } - return nil + return nil, nil //nolint:nilnil // valid for mock: nil means no value found } -// QueryMany implements ClientInterface. -func (m *MockClient) QueryMany(ctx context.Context, query string, dest any) error { +// QueryMinMaxUInt64 implements ClientInterface. +func (m *MockClient) QueryMinMaxUInt64(ctx context.Context, query string) (minVal, maxVal *uint64, err error) { m.Calls = append(m.Calls, MockCall{ - Method: "QueryMany", - Args: []any{ctx, query, dest}, + Method: "QueryMinMaxUInt64", + Args: []any{ctx, query}, }) - if m.QueryManyFunc != nil { - return m.QueryManyFunc(ctx, query, dest) + if m.QueryMinMaxUInt64Func != nil { + return m.QueryMinMaxUInt64Func(ctx, query) } - return nil + return nil, nil, nil } // Execute implements ClientInterface. @@ -192,44 +189,22 @@ func (m *MockClient) Reset() { // Helper functions for common test scenarios -// SetQueryOneResponse sets up the mock to return specific data for QueryOne. -func (m *MockClient) SetQueryOneResponse(data any) { - m.QueryOneFunc = func(ctx context.Context, query string, dest any) error { - // Marshal the data to JSON and then unmarshal into dest - jsonData, err := json.Marshal(data) - if err != nil { - return err - } - - return json.Unmarshal(jsonData, dest) +// SetQueryUInt64Response sets up the mock to return a specific value for QueryUInt64. +func (m *MockClient) SetQueryUInt64Response(value *uint64) { + m.QueryUInt64Func = func(ctx context.Context, query string, columnName string) (*uint64, error) { + return value, nil } } -// SetQueryManyResponse sets up the mock to return specific data for QueryMany. -func (m *MockClient) SetQueryManyResponse(data any) { - m.QueryManyFunc = func(ctx context.Context, query string, dest any) error { - // Use reflection to set the slice - destValue := reflect.ValueOf(dest).Elem() - srcValue := reflect.ValueOf(data) - - if destValue.Kind() != reflect.Slice || srcValue.Kind() != reflect.Slice { - return fmt.Errorf("both dest and data must be slices") - } - - destValue.Set(srcValue) - - return nil +// SetQueryMinMaxUInt64Response sets up the mock to return specific values for QueryMinMaxUInt64. +func (m *MockClient) SetQueryMinMaxUInt64Response(minVal, maxVal *uint64) { + m.QueryMinMaxUInt64Func = func(ctx context.Context, query string) (*uint64, *uint64, error) { + return minVal, maxVal, nil } } // SetError sets all functions to return the specified error. func (m *MockClient) SetError(err error) { - m.QueryOneFunc = func(ctx context.Context, query string, dest any) error { - return err - } - m.QueryManyFunc = func(ctx context.Context, query string, dest any) error { - return err - } m.ExecuteFunc = func(ctx context.Context, query string) error { return err } @@ -245,4 +220,10 @@ func (m *MockClient) SetError(err error) { m.DoFunc = func(ctx context.Context, query ch.Query) error { return err } + m.QueryUInt64Func = func(ctx context.Context, query string, columnName string) (*uint64, error) { + return nil, err + } + m.QueryMinMaxUInt64Func = func(ctx context.Context, query string) (*uint64, *uint64, error) { + return nil, nil, err + } } diff --git a/pkg/state/json_number.go b/pkg/state/json_number.go deleted file mode 100644 index 6c5cf17..0000000 --- a/pkg/state/json_number.go +++ /dev/null @@ -1,126 +0,0 @@ -package state - -import ( - "encoding/json" - "strconv" -) - -// JSONInt64 handles JSON numbers that might be strings or numbers. -type JSONInt64 int64 - -// UnmarshalJSON implements json.Unmarshaler. -func (j *JSONInt64) UnmarshalJSON(data []byte) error { - // Try to unmarshal as number first - var num int64 - if err := json.Unmarshal(data, &num); err == nil { - *j = JSONInt64(num) - - return nil - } - - // Try to unmarshal as string - var str string - if err := json.Unmarshal(data, &str); err != nil { - return err - } - - // Parse string to int64 - num, err := strconv.ParseInt(str, 10, 64) - if err != nil { - return err - } - - *j = JSONInt64(num) - - return nil -} - -// MarshalJSON implements json.Marshaler. -func (j JSONInt64) MarshalJSON() ([]byte, error) { - return json.Marshal(int64(j)) -} - -// Int64 returns the value as int64. -func (j JSONInt64) Int64() int64 { - return int64(j) -} - -// JSONInt handles JSON numbers that might be strings or numbers. -type JSONInt int - -// UnmarshalJSON implements json.Unmarshaler. -func (j *JSONInt) UnmarshalJSON(data []byte) error { - // Try to unmarshal as number first - var num int - if err := json.Unmarshal(data, &num); err == nil { - *j = JSONInt(num) - - return nil - } - - // Try to unmarshal as string - var str string - if err := json.Unmarshal(data, &str); err != nil { - return err - } - - // Parse string to int - num, err := strconv.Atoi(str) - if err != nil { - return err - } - - *j = JSONInt(num) - - return nil -} - -// MarshalJSON implements json.Marshaler. -func (j JSONInt) MarshalJSON() ([]byte, error) { - return json.Marshal(int(j)) -} - -// Int returns the value as int. -func (j JSONInt) Int() int { - return int(j) -} - -// JSONUint64 handles JSON numbers that might be strings or numbers (unsigned). -type JSONUint64 uint64 - -// UnmarshalJSON implements json.Unmarshaler. -func (j *JSONUint64) UnmarshalJSON(data []byte) error { - // Try to unmarshal as number first - var num uint64 - if err := json.Unmarshal(data, &num); err == nil { - *j = JSONUint64(num) - - return nil - } - - // Try to unmarshal as string - var str string - if err := json.Unmarshal(data, &str); err != nil { - return err - } - - // Parse string to uint64 - num, err := strconv.ParseUint(str, 10, 64) - if err != nil { - return err - } - - *j = JSONUint64(num) - - return nil -} - -// MarshalJSON implements json.Marshaler. -func (j JSONUint64) MarshalJSON() ([]byte, error) { - return json.Marshal(uint64(j)) -} - -// Uint64 returns the value as uint64. -func (j JSONUint64) Uint64() uint64 { - return uint64(j) -} diff --git a/pkg/state/manager.go b/pkg/state/manager.go index 59ea005..004c5e0 100644 --- a/pkg/state/manager.go +++ b/pkg/state/manager.go @@ -17,19 +17,6 @@ var ( ErrNoMoreBlocks = errors.New("no more blocks to process") ) -// blockNumberResult is used for queries that return a single block number. -// -//nolint:tagliatelle // ClickHouse uses snake_case column names -type blockNumberResult struct { - BlockNumber *JSONInt64 `json:"block_number"` -} - -// minMaxResult is used for queries that return min and max values. -type minMaxResult struct { - Min *JSONInt64 `json:"min"` - Max *JSONInt64 `json:"max"` -} - type Manager struct { log logrus.FieldLogger storageClient clickhouse.ClientInterface @@ -237,15 +224,13 @@ func (s *Manager) getProgressiveNextBlock(ctx context.Context, processor, networ "table": s.storageTable, }).Debug("Querying for last processed block") - var result blockNumberResult - - err := s.storageClient.QueryOne(ctx, query, &result) + blockNumber, err := s.storageClient.QueryUInt64(ctx, query, "block_number") if err != nil { return nil, false, fmt.Errorf("failed to get next block from %s: %w", s.storageTable, err) } // Check if we got a result - if result.BlockNumber == nil { + if blockNumber == nil { // Double-check if this is actually empty or no data isEmpty, err := s.storageClient.IsStorageEmpty(ctx, s.storageTable, map[string]interface{}{ "processor": processor, @@ -277,20 +262,20 @@ func (s *Manager) getProgressiveNextBlock(ctx context.Context, processor, networ } } - // Block 0 was actually processed but BlockNumber is nil (shouldn't happen but be defensive) + // Block 0 was actually processed but blockNumber is nil (shouldn't happen but be defensive) s.log.WithFields(logrus.Fields{ "processor": processor, "network": network, - }).Warn("Unexpected state: BlockNumber is nil but storage is not empty") + }).Warn("Unexpected state: block_number is nil but storage is not empty") return big.NewInt(0), false, nil } - nextBlock := big.NewInt(result.BlockNumber.Int64() + 1) + nextBlock := new(big.Int).SetUint64(*blockNumber + 1) s.log.WithFields(logrus.Fields{ "processor": processor, "network": network, - "last_processed": result.BlockNumber.Int64(), + "last_processed": *blockNumber, "progressive_next": nextBlock.String(), }).Debug("Found last processed block, calculated progressive next block") @@ -313,15 +298,13 @@ func (s *Manager) getProgressiveNextBlockBackwards(ctx context.Context, processo "table": s.storageTable, }).Debug("Querying for earliest processed block (backwards mode)") - var result blockNumberResult - - err := s.storageClient.QueryOne(ctx, query, &result) + blockNumber, err := s.storageClient.QueryUInt64(ctx, query, "block_number") if err != nil { return nil, fmt.Errorf("failed to get earliest block from %s: %w", s.storageTable, err) } // Check if we got a result - if result.BlockNumber == nil { + if blockNumber == nil { // No entry in table, need to start from chain tip for backwards processing if chainHead != nil && chainHead.Int64() > 0 { s.log.WithFields(logrus.Fields{ @@ -343,7 +326,7 @@ func (s *Manager) getProgressiveNextBlockBackwards(ctx context.Context, processo } // Calculate previous block (go backwards) - if result.BlockNumber.Int64() <= 0 { + if *blockNumber == 0 { // Already at genesis, no more blocks to process backwards s.log.WithFields(logrus.Fields{ "processor": processor, @@ -353,11 +336,11 @@ func (s *Manager) getProgressiveNextBlockBackwards(ctx context.Context, processo return nil, ErrNoMoreBlocks } - prevBlock := big.NewInt(result.BlockNumber.Int64() - 1) + prevBlock := new(big.Int).SetUint64(*blockNumber - 1) s.log.WithFields(logrus.Fields{ "processor": processor, "network": network, - "earliest_processed": result.BlockNumber.Int64(), + "earliest_processed": *blockNumber, "progressive_prev": prevBlock.String(), }).Debug("Found earliest processed block, calculated previous block for backwards processing") @@ -376,15 +359,13 @@ func (s *Manager) getLimiterMaxBlock(ctx context.Context, network string) (*big. "table": s.limiterTable, }).Debug("Querying for maximum execution payload block number") - var result blockNumberResult - - err := s.limiterClient.QueryOne(ctx, query, &result) + blockNumber, err := s.limiterClient.QueryUInt64(ctx, query, "block_number") if err != nil { return nil, fmt.Errorf("failed to get max execution payload block from %s: %w", s.limiterTable, err) } // Check if we got a result - if result.BlockNumber == nil { + if blockNumber == nil { // No blocks in limiter table, return genesis s.log.WithFields(logrus.Fields{ "network": network, @@ -393,7 +374,7 @@ func (s *Manager) getLimiterMaxBlock(ctx context.Context, network string) (*big. return big.NewInt(0), nil } - maxBlock := big.NewInt(result.BlockNumber.Int64()) + maxBlock := new(big.Int).SetUint64(*blockNumber) s.log.WithFields(logrus.Fields{ "network": network, "limiter_max": maxBlock.String(), @@ -434,15 +415,13 @@ func (s *Manager) GetMinMaxStoredBlocks(ctx context.Context, network, processor "table": s.storageTable, }).Debug("Querying for min/max stored blocks") - var result minMaxResult - - err = s.storageClient.QueryOne(ctx, query, &result) + minResult, maxResult, err := s.storageClient.QueryMinMaxUInt64(ctx, query) if err != nil { return nil, nil, fmt.Errorf("failed to get min/max blocks: %w", err) } // Handle case where no blocks are stored - if result.Min == nil || result.Max == nil { + if minResult == nil || maxResult == nil { s.log.WithFields(logrus.Fields{ "network": network, "processor": processor, @@ -456,11 +435,11 @@ func (s *Manager) GetMinMaxStoredBlocks(ctx context.Context, network, processor "network": network, "processor": processor, "table": s.storageTable, - "min": result.Min.Int64(), - "max": result.Max.Int64(), + "min": *minResult, + "max": *maxResult, }).Debug("Found min/max blocks") - return big.NewInt(result.Min.Int64()), big.NewInt(result.Max.Int64()), nil + return new(big.Int).SetUint64(*minResult), new(big.Int).SetUint64(*maxResult), nil } // IsBlockRecentlyProcessed checks if a block was processed within the specified number of seconds. @@ -482,16 +461,12 @@ func (s *Manager) IsBlockRecentlyProcessed(ctx context.Context, blockNumber uint "table": s.storageTable, }).Debug("Checking if block was recently processed") - var result struct { - Count JSONInt64 `json:"count"` - } - - err := s.storageClient.QueryOne(ctx, query, &result) + count, err := s.storageClient.QueryUInt64(ctx, query, "count") if err != nil { return false, fmt.Errorf("failed to check recent block processing: %w", err) } - return result.Count.Int64() > 0, nil + return count != nil && *count > 0, nil } // GetHeadDistance calculates the distance between current processing block and the relevant head. diff --git a/pkg/state/manager_test.go b/pkg/state/manager_test.go index 122ffd2..da33f5b 100644 --- a/pkg/state/manager_test.go +++ b/pkg/state/manager_test.go @@ -16,16 +16,32 @@ type MockClickHouseClient struct { mock.Mock } -func (m *MockClickHouseClient) QueryOne(ctx context.Context, query string, dest interface{}) error { - args := m.Called(ctx, query, dest) +func (m *MockClickHouseClient) QueryUInt64(ctx context.Context, query string, columnName string) (*uint64, error) { + args := m.Called(ctx, query, columnName) - return args.Error(0) + if args.Get(0) == nil { + return nil, args.Error(1) + } + + val, _ := args.Get(0).(*uint64) + + return val, args.Error(1) } -func (m *MockClickHouseClient) QueryMany(ctx context.Context, query string, dest interface{}) error { - args := m.Called(ctx, query, dest) +func (m *MockClickHouseClient) QueryMinMaxUInt64(ctx context.Context, query string) (*uint64, *uint64, error) { + args := m.Called(ctx, query) - return args.Error(0) + var minResult, maxResult *uint64 + + if args.Get(0) != nil { + minResult = args.Get(0).(*uint64) //nolint:errcheck // test mock + } + + if args.Get(1) != nil { + maxResult = args.Get(1).(*uint64) //nolint:errcheck // test mock + } + + return minResult, maxResult, args.Error(2) } func (m *MockClickHouseClient) Execute(ctx context.Context, query string) error { @@ -62,6 +78,11 @@ func (m *MockClickHouseClient) Do(ctx context.Context, query ch.Query) error { return args.Error(0) } +// Helper to create uint64 pointer. +func uint64Ptr(v uint64) *uint64 { + return &v +} + func TestNextBlock_NoResultsVsBlock0(t *testing.T) { ctx := context.Background() log := logrus.New() @@ -76,12 +97,9 @@ func TestNextBlock_NoResultsVsBlock0(t *testing.T) { { name: "No results - should return chain head", setupMock: func(m *MockClickHouseClient) { - // QueryOne returns nil error but doesn't modify the dest struct - // This simulates the behavior when no rows are found - m.On("QueryOne", ctx, mock.AnythingOfType("string"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { - // Don't modify dest - this is what happens when no rows are found - }) - // IsStorageEmpty is called inside getProgressiveNextBlock when result is nil + // QueryUInt64 returns nil (no rows found) + m.On("QueryUInt64", ctx, mock.AnythingOfType("string"), "block_number").Return(nil, nil) + // IsStorageEmpty is called when result is nil m.On("IsStorageEmpty", ctx, "test_table", map[string]interface{}{ "processor": "test_processor", "meta_network_name": "mainnet", @@ -93,12 +111,7 @@ func TestNextBlock_NoResultsVsBlock0(t *testing.T) { { name: "Block 0 found - should return 1", setupMock: func(m *MockClickHouseClient) { - m.On("QueryOne", ctx, mock.AnythingOfType("string"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { - // Simulate finding block 0 - result := args.Get(2).(*blockNumberResult) //nolint:errcheck // type assertion in test - blockNum := JSONInt64(0) - result.BlockNumber = &blockNum - }) + m.On("QueryUInt64", ctx, mock.AnythingOfType("string"), "block_number").Return(uint64Ptr(0), nil) }, expectedNextBlock: big.NewInt(1), // Next block after 0 expectError: false, @@ -106,12 +119,7 @@ func TestNextBlock_NoResultsVsBlock0(t *testing.T) { { name: "Block 100 found - should return 101", setupMock: func(m *MockClickHouseClient) { - m.On("QueryOne", ctx, mock.AnythingOfType("string"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { - // Simulate finding block 100 - result := args.Get(2).(*blockNumberResult) //nolint:errcheck // type assertion in test - blockNum := JSONInt64(100) - result.BlockNumber = &blockNum - }) + m.On("QueryUInt64", ctx, mock.AnythingOfType("string"), "block_number").Return(uint64Ptr(100), nil) }, expectedNextBlock: big.NewInt(101), // Next block after 100 expectError: false, @@ -152,28 +160,28 @@ func TestNextBlock_EmptyStorageWithLimiter(t *testing.T) { tests := []struct { name string chainHead *big.Int - limiterMax *big.Int + limiterMax *uint64 limiterEnabled bool expectedNext *big.Int }{ { name: "empty storage with limiter and high chain head", chainHead: big.NewInt(5000), - limiterMax: big.NewInt(1000), + limiterMax: uint64Ptr(1000), limiterEnabled: true, expectedNext: big.NewInt(999), // limiterMax - 1 }, { name: "empty storage with limiter and nil chain head", chainHead: nil, - limiterMax: big.NewInt(1000), + limiterMax: uint64Ptr(1000), limiterEnabled: true, expectedNext: big.NewInt(999), // limiterMax - 1 }, { name: "empty storage with limiter and zero chain head", chainHead: big.NewInt(0), - limiterMax: big.NewInt(1000), + limiterMax: uint64Ptr(1000), limiterEnabled: true, expectedNext: big.NewInt(999), // limiterMax - 1 }, @@ -190,9 +198,7 @@ func TestNextBlock_EmptyStorageWithLimiter(t *testing.T) { t.Run(tt.name, func(t *testing.T) { // Create mock storage that reports empty mockStorage := new(MockClickHouseClient) - mockStorage.On("QueryOne", ctx, mock.AnythingOfType("string"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { - // Don't modify dest - storage is empty - }) + mockStorage.On("QueryUInt64", ctx, mock.AnythingOfType("string"), "block_number").Return(nil, nil) mockStorage.On("IsStorageEmpty", ctx, "test_table", map[string]interface{}{ "processor": "test-processor", "meta_network_name": "mainnet", @@ -202,15 +208,7 @@ func TestNextBlock_EmptyStorageWithLimiter(t *testing.T) { var mockLimiter *MockClickHouseClient if tt.limiterEnabled { mockLimiter = new(MockClickHouseClient) - mockLimiter.On("QueryOne", ctx, mock.AnythingOfType("string"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { - // Set the limiter max block - result := args.Get(2).(*blockNumberResult) //nolint:errcheck // type assertion in test - - if tt.limiterMax != nil { - blockNum := JSONInt64(tt.limiterMax.Int64()) - result.BlockNumber = &blockNum - } - }) + mockLimiter.On("QueryUInt64", ctx, mock.AnythingOfType("string"), "block_number").Return(tt.limiterMax, nil) } // Create manager @@ -291,20 +289,14 @@ func TestGetProgressiveNextBlock_EmptyStorage(t *testing.T) { if tt.storageEmpty { // Empty storage case - mockStorage.On("QueryOne", ctx, mock.AnythingOfType("string"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { - // Don't modify dest - storage is empty - }) + mockStorage.On("QueryUInt64", ctx, mock.AnythingOfType("string"), "block_number").Return(nil, nil) mockStorage.On("IsStorageEmpty", ctx, "test_table", map[string]interface{}{ "processor": "test-processor", "meta_network_name": "mainnet", }).Return(true, nil) } else { // Non-empty storage case - mockStorage.On("QueryOne", ctx, mock.AnythingOfType("string"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { - result := args.Get(2).(*blockNumberResult) //nolint:errcheck // type assertion in test - blockNum := JSONInt64(tt.storedBlock) - result.BlockNumber = &blockNum - }) + mockStorage.On("QueryUInt64", ctx, mock.AnythingOfType("string"), "block_number").Return(uint64Ptr(tt.storedBlock), nil) } manager := &Manager{ @@ -339,9 +331,7 @@ func TestGetProgressiveNextBlock_NoResultsVsBlock0(t *testing.T) { { name: "No results with chain head - should return chain head", setupMock: func(m *MockClickHouseClient) { - m.On("QueryOne", ctx, mock.AnythingOfType("string"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { - // Don't modify dest - this is what happens when no rows are found - }) + m.On("QueryUInt64", ctx, mock.AnythingOfType("string"), "block_number").Return(nil, nil) // IsStorageEmpty is called when result is nil m.On("IsStorageEmpty", ctx, "test_table", map[string]interface{}{ "processor": "test_processor", @@ -355,9 +345,7 @@ func TestGetProgressiveNextBlock_NoResultsVsBlock0(t *testing.T) { { name: "No results without chain head - should return 0", setupMock: func(m *MockClickHouseClient) { - m.On("QueryOne", ctx, mock.AnythingOfType("string"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { - // Don't modify dest - this is what happens when no rows are found - }) + m.On("QueryUInt64", ctx, mock.AnythingOfType("string"), "block_number").Return(nil, nil) // IsStorageEmpty is called when result is nil m.On("IsStorageEmpty", ctx, "test_table", map[string]interface{}{ "processor": "test_processor", @@ -371,11 +359,7 @@ func TestGetProgressiveNextBlock_NoResultsVsBlock0(t *testing.T) { { name: "Block 0 found - should return 1", setupMock: func(m *MockClickHouseClient) { - m.On("QueryOne", ctx, mock.AnythingOfType("string"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { - result := args.Get(2).(*blockNumberResult) //nolint:errcheck // type assertion in test - blockNum := JSONInt64(0) - result.BlockNumber = &blockNum - }) + m.On("QueryUInt64", ctx, mock.AnythingOfType("string"), "block_number").Return(uint64Ptr(0), nil) }, chainHead: big.NewInt(1000), expectedNextBlock: big.NewInt(1), From 79738dd5dbc6b70e39cc267aae24ef6708d51213 Mon Sep 17 00:00:00 2001 From: Andrew Davis <1709934+Savid@users.noreply.github.com> Date: Fri, 23 Jan 2026 12:20:30 +1000 Subject: [PATCH 4/5] feat(clickhouse): add infinite retry with capped backoff for startup This change ensures the application waits for ClickHouse to become available at startup instead of failing after a fixed number of retries. Changes: - Add RetryMaxDelay config option (default: 10s) to cap exponential backoff - Move dial from New() to Start() - clients are created without connecting - State manager retries infinitely with capped backoff when starting clients - Processor manager retries infinitely when starting processors - Server waits for state manager to be ready before starting processors - Fix idempotency bug where Start() would skip retry after failure The startup order is now: 1. State manager connects to ClickHouse (infinite retry) 2. Once ready, processors start (also with infinite retry) This allows the app to gracefully wait for ClickHouse during initial deployment or restarts without crashing. Co-Authored-By: Claude Opus 4.5 --- pkg/clickhouse/client.go | 137 ++++++++++-------- pkg/clickhouse/client_test.go | 15 +- pkg/clickhouse/config.go | 5 + pkg/processor/manager.go | 51 ++++++- pkg/processor/manager_test.go | 16 +- pkg/processor/transaction/simple/processor.go | 4 +- .../transaction/structlog/processor.go | 4 +- pkg/server/server.go | 16 +- pkg/state/manager.go | 55 ++++++- 9 files changed, 210 insertions(+), 93 deletions(-) diff --git a/pkg/clickhouse/client.go b/pkg/clickhouse/client.go index 2131af4..d36e78c 100644 --- a/pkg/clickhouse/client.go +++ b/pkg/clickhouse/client.go @@ -28,13 +28,14 @@ const ( // Client implements the ClientInterface using ch-go native protocol. type Client struct { - pool *chpool.Pool - config *Config - network string - processor string - log logrus.FieldLogger - lock sync.RWMutex - started atomic.Bool + pool *chpool.Pool + config *Config + compression ch.Compression + network string + processor string + log logrus.FieldLogger + lock sync.RWMutex + started atomic.Bool // Metrics collection metricsDone chan struct{} @@ -137,8 +138,11 @@ func (c *Client) doWithRetry(ctx context.Context, operation string, fn func(ctx for attempt := 0; attempt <= c.config.MaxRetries; attempt++ { if attempt > 0 { - // Calculate backoff delay with exponential increase + // Calculate backoff delay with exponential increase, capped at RetryMaxDelay delay := c.config.RetryBaseDelay * time.Duration(1<<(attempt-1)) + if c.config.RetryMaxDelay > 0 && delay > c.config.RetryMaxDelay { + delay = c.config.RetryMaxDelay + } c.log.WithFields(logrus.Fields{ "attempt": attempt, @@ -175,7 +179,8 @@ func (c *Client) doWithRetry(ctx context.Context, operation string, fn func(ctx } // New creates a new ch-go native ClickHouse client. -func New(ctx context.Context, cfg *Config) (ClientInterface, error) { +// The client is not connected until Start() is called. +func New(cfg *Config) (ClientInterface, error) { if err := cfg.Validate(); err != nil { return nil, fmt.Errorf("invalid config: %w", err) } @@ -193,40 +198,12 @@ func New(ctx context.Context, cfg *Config) (ClientInterface, error) { log := logrus.WithField("component", "clickhouse-native") - // Dial with startup retry logic for transient connection failures - var pool *chpool.Pool - - err := dialWithRetry(ctx, log, cfg, func() error { - var dialErr error - - pool, dialErr = chpool.Dial(ctx, chpool.Options{ - ClientOptions: ch.Options{ - Address: cfg.Addr, - Database: cfg.Database, - User: cfg.Username, - Password: cfg.Password, - Compression: compression, - DialTimeout: cfg.DialTimeout, - }, - MaxConns: cfg.MaxConns, - MinConns: cfg.MinConns, - MaxConnLifetime: cfg.ConnMaxLifetime, - MaxConnIdleTime: cfg.ConnMaxIdleTime, - HealthCheckPeriod: cfg.HealthCheckPeriod, - }) - - return dialErr - }) - if err != nil { - return nil, fmt.Errorf("failed to dial clickhouse: %w", err) - } - return &Client{ - pool: pool, - config: cfg, - network: cfg.Network, - processor: cfg.Processor, - log: log, + config: cfg, + compression: compression, + network: cfg.Network, + processor: cfg.Processor, + log: log, }, nil } @@ -236,7 +213,11 @@ func dialWithRetry(ctx context.Context, log logrus.FieldLogger, cfg *Config, fn for attempt := 0; attempt <= cfg.MaxRetries; attempt++ { if attempt > 0 { + // Calculate backoff delay with exponential increase, capped at RetryMaxDelay delay := cfg.RetryBaseDelay * time.Duration(1<<(attempt-1)) + if cfg.RetryMaxDelay > 0 && delay > cfg.RetryMaxDelay { + delay = cfg.RetryMaxDelay + } log.WithFields(logrus.Fields{ "attempt": attempt, @@ -268,39 +249,65 @@ func dialWithRetry(ctx context.Context, log logrus.FieldLogger, cfg *Config, fn return fmt.Errorf("max retries (%d) exceeded: %w", cfg.MaxRetries, lastErr) } -// Start initializes the client and tests connectivity. +// Start initializes the client by dialing ClickHouse with retry logic. func (c *Client) Start() error { - // Idempotency guard - prevent multiple Start() calls from leaking goroutines. - // Once Start() is called, the client is considered "started" regardless of outcome. - // On failure, the client is effectively dead and Stop() should be called. - if c.started.Swap(true) { - c.log.Debug("Start() already called, skipping") - - return nil - } + // Idempotency guard - prevent multiple successful Start() calls from leaking goroutines. + // We check if pool is already set to allow retries after failures. + c.lock.Lock() - if c.network == "" { - c.log.Debug("Skipping ClickHouse connectivity test - network not yet determined") + if c.pool != nil { + c.lock.Unlock() + c.log.Debug("Start() already completed successfully, skipping") return nil } - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + c.lock.Unlock() + + // Dial with startup retry logic for transient connection failures + ctx, cancel := context.WithTimeout(context.Background(), c.config.DialTimeout*time.Duration(c.config.MaxRetries+1)) defer cancel() - if err := c.pool.Ping(ctx); err != nil { - // Don't reset started - the client is now in a failed state. - // Caller should call Stop() to clean up. - return fmt.Errorf("failed to ping ClickHouse: %w", err) + var pool *chpool.Pool + + err := dialWithRetry(ctx, c.log, c.config, func() error { + var dialErr error + + pool, dialErr = chpool.Dial(ctx, chpool.Options{ + ClientOptions: ch.Options{ + Address: c.config.Addr, + Database: c.config.Database, + User: c.config.Username, + Password: c.config.Password, + Compression: c.compression, + DialTimeout: c.config.DialTimeout, + }, + MaxConns: c.config.MaxConns, + MinConns: c.config.MinConns, + MaxConnLifetime: c.config.ConnMaxLifetime, + MaxConnIdleTime: c.config.ConnMaxIdleTime, + HealthCheckPeriod: c.config.HealthCheckPeriod, + }) + + return dialErr + }) + if err != nil { + return fmt.Errorf("failed to dial clickhouse: %w", err) } + c.lock.Lock() + c.pool = pool + c.lock.Unlock() + c.log.Info("Connected to ClickHouse native interface") - // Start pool metrics collection goroutine - c.metricsDone = make(chan struct{}) - c.metricsWg.Add(1) + // Start pool metrics collection goroutine - use started flag to prevent duplicate goroutines + if !c.started.Swap(true) { + c.metricsDone = make(chan struct{}) + c.metricsWg.Add(1) - go c.collectPoolMetrics() + go c.collectPoolMetrics() + } return nil } @@ -313,8 +320,10 @@ func (c *Client) Stop() error { c.metricsWg.Wait() } - c.pool.Close() - c.log.Info("Closed ClickHouse connection pool") + if c.pool != nil { + c.pool.Close() + c.log.Info("Closed ClickHouse connection pool") + } return nil } diff --git a/pkg/clickhouse/client_test.go b/pkg/clickhouse/client_test.go index 0b2ba93..8338d5f 100644 --- a/pkg/clickhouse/client_test.go +++ b/pkg/clickhouse/client_test.go @@ -81,6 +81,7 @@ func TestConfig_SetDefaults(t *testing.T) { assert.Equal(t, 10*time.Second, config.DialTimeout) assert.Equal(t, "lz4", config.Compression) assert.Equal(t, 60*time.Second, config.QueryTimeout) + assert.Equal(t, 10*time.Second, config.RetryMaxDelay) } func TestConfig_SetDefaults_PreservesValues(t *testing.T) { @@ -95,6 +96,7 @@ func TestConfig_SetDefaults_PreservesValues(t *testing.T) { DialTimeout: 30 * time.Second, Compression: "zstd", QueryTimeout: 120 * time.Second, + RetryMaxDelay: 30 * time.Second, } config.SetDefaults() @@ -109,6 +111,7 @@ func TestConfig_SetDefaults_PreservesValues(t *testing.T) { assert.Equal(t, 30*time.Second, config.DialTimeout) assert.Equal(t, "zstd", config.Compression) assert.Equal(t, 120*time.Second, config.QueryTimeout) + assert.Equal(t, 30*time.Second, config.RetryMaxDelay) } func TestClient_withQueryTimeout(t *testing.T) { @@ -187,7 +190,7 @@ func TestClient_Integration_New(t *testing.T) { Compression: "lz4", } - client, err := New(context.Background(), cfg) + client, err := New(cfg) require.NoError(t, err) require.NotNil(t, client) @@ -208,10 +211,10 @@ func TestClient_Integration_StartStop(t *testing.T) { Network: "test", } - client, err := New(context.Background(), cfg) + client, err := New(cfg) require.NoError(t, err) - // Start should ping successfully + // Start should dial and connect successfully err = client.Start() require.NoError(t, err) @@ -233,7 +236,7 @@ func TestClient_Integration_Execute(t *testing.T) { Network: "test", } - client, err := New(context.Background(), cfg) + client, err := New(cfg) require.NoError(t, err) defer func() { _ = client.Stop() }() @@ -259,7 +262,7 @@ func TestClient_Integration_QueryUInt64(t *testing.T) { Network: "test", } - client, err := New(context.Background(), cfg) + client, err := New(cfg) require.NoError(t, err) defer func() { _ = client.Stop() }() @@ -286,7 +289,7 @@ func TestClient_Integration_IsStorageEmpty(t *testing.T) { Network: "test", } - client, err := New(context.Background(), cfg) + client, err := New(cfg) require.NoError(t, err) defer func() { _ = client.Stop() }() diff --git a/pkg/clickhouse/config.go b/pkg/clickhouse/config.go index efb9204..62410e0 100644 --- a/pkg/clickhouse/config.go +++ b/pkg/clickhouse/config.go @@ -29,6 +29,7 @@ type Config struct { // Retry settings MaxRetries int `yaml:"max_retries"` // Maximum retry attempts, default: 3 RetryBaseDelay time.Duration `yaml:"retry_base_delay"` // Base delay for exponential backoff, default: 100ms + RetryMaxDelay time.Duration `yaml:"retry_max_delay"` // Max delay between retries, default: 10s // Timeout settings QueryTimeout time.Duration `yaml:"query_timeout"` // Query timeout per attempt, default: 60s @@ -90,6 +91,10 @@ func (c *Config) SetDefaults() { c.RetryBaseDelay = 100 * time.Millisecond } + if c.RetryMaxDelay == 0 { + c.RetryMaxDelay = 10 * time.Second + } + if c.QueryTimeout == 0 { c.QueryTimeout = 60 * time.Second } diff --git a/pkg/processor/manager.go b/pkg/processor/manager.go index 5ce39b6..cf5e2ac 100644 --- a/pkg/processor/manager.go +++ b/pkg/processor/manager.go @@ -312,7 +312,7 @@ func (m *Manager) initializeProcessors(ctx context.Context) error { if m.config.TransactionStructlog.Enabled { m.log.Debug("Transaction structlog processor is enabled, initializing...") - processor, err := transaction_structlog.New(ctx, &transaction_structlog.Dependencies{ + processor, err := transaction_structlog.New(&transaction_structlog.Dependencies{ Log: m.log.WithField("processor", "transaction_structlog"), Pool: m.pool, State: m.state, @@ -331,7 +331,7 @@ func (m *Manager) initializeProcessors(ctx context.Context) error { m.log.WithField("processor", "transaction_structlog").Info("Initialized processor") - if err := processor.Start(ctx); err != nil { + if err := m.startProcessorWithRetry(ctx, processor, "transaction_structlog"); err != nil { return fmt.Errorf("failed to start transaction_structlog processor: %w", err) } } else { @@ -342,7 +342,7 @@ func (m *Manager) initializeProcessors(ctx context.Context) error { if m.config.TransactionSimple.Enabled { m.log.Debug("Transaction simple processor is enabled, initializing...") - processor, err := transaction_simple.New(ctx, &transaction_simple.Dependencies{ + processor, err := transaction_simple.New(&transaction_simple.Dependencies{ Log: m.log.WithField("processor", "transaction_simple"), Pool: m.pool, State: m.state, @@ -361,7 +361,7 @@ func (m *Manager) initializeProcessors(ctx context.Context) error { m.log.WithField("processor", "transaction_simple").Info("Initialized processor") - if err := processor.Start(ctx); err != nil { + if err := m.startProcessorWithRetry(ctx, processor, "transaction_simple"); err != nil { return fmt.Errorf("failed to start transaction_simple processor: %w", err) } } else { @@ -373,6 +373,49 @@ func (m *Manager) initializeProcessors(ctx context.Context) error { return nil } +// startProcessorWithRetry starts a processor with infinite retry and capped exponential backoff. +// This ensures processors can wait for their dependencies (like ClickHouse) to become available. +func (m *Manager) startProcessorWithRetry(ctx context.Context, processor c.BlockProcessor, name string) error { + const ( + baseDelay = 100 * time.Millisecond + maxDelay = 10 * time.Second + ) + + attempt := 0 + + for { + err := processor.Start(ctx) + if err == nil { + if attempt > 0 { + m.log.WithField("processor", name).Info("Processor started successfully after retries") + } + + return nil + } + + // Calculate delay with exponential backoff, capped at maxDelay + delay := baseDelay * time.Duration(1< maxDelay { + delay = maxDelay + } + + m.log.WithFields(logrus.Fields{ + "processor": name, + "attempt": attempt + 1, + "delay": delay, + "error": err, + }).Warn("Failed to start processor, retrying...") + + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(delay): + } + + attempt++ + } +} + func (m *Manager) processBlocks(ctx context.Context) { m.log.WithField("processor_count", len(m.processors)).Debug("Starting to process blocks") diff --git a/pkg/processor/manager_test.go b/pkg/processor/manager_test.go index b267aff..c21c97d 100644 --- a/pkg/processor/manager_test.go +++ b/pkg/processor/manager_test.go @@ -78,7 +78,7 @@ func TestManager_Creation(t *testing.T) { Enabled: false, }, } - stateManager, err := state.NewManager(context.Background(), log.WithField("component", "state"), stateConfig) + stateManager, err := state.NewManager(log.WithField("component", "state"), stateConfig) require.NoError(t, err) manager, err := processor.NewManager(log, config, pool, stateManager, redisClient, "test-prefix") @@ -129,7 +129,7 @@ func TestManager_StartStop(t *testing.T) { Enabled: false, }, } - stateManager, err := state.NewManager(context.Background(), log.WithField("component", "state"), stateConfig) + stateManager, err := state.NewManager(log.WithField("component", "state"), stateConfig) require.NoError(t, err) manager, err := processor.NewManager(log, config, pool, stateManager, redisClient, "test-prefix") @@ -207,7 +207,7 @@ func TestManager_MultipleStops(t *testing.T) { Enabled: false, }, } - stateManager, err := state.NewManager(context.Background(), log.WithField("component", "state"), stateConfig) + stateManager, err := state.NewManager(log.WithField("component", "state"), stateConfig) require.NoError(t, err) manager, err := processor.NewManager(log, config, pool, stateManager, redisClient, "test-prefix") @@ -293,7 +293,7 @@ func TestManager_ModeSpecificLeaderElection(t *testing.T) { Enabled: false, }, } - stateManager, err := state.NewManager(context.Background(), log.WithField("component", "state"), stateConfig) + stateManager, err := state.NewManager(log.WithField("component", "state"), stateConfig) require.NoError(t, err) redisClient := newTestRedis(t) @@ -366,7 +366,7 @@ func TestManager_ConcurrentModes(t *testing.T) { Enabled: false, }, } - stateManager, err := state.NewManager(context.Background(), log.WithField("component", "state"), stateConfig) + stateManager, err := state.NewManager(log.WithField("component", "state"), stateConfig) require.NoError(t, err) // Each miniredis instance is isolated, simulating separate Redis servers @@ -421,7 +421,7 @@ func TestManager_LeaderElectionDisabled(t *testing.T) { Enabled: false, }, } - stateManager, err := state.NewManager(context.Background(), log.WithField("component", "state"), stateConfig) + stateManager, err := state.NewManager(log.WithField("component", "state"), stateConfig) require.NoError(t, err) redisClient := newTestRedis(t) @@ -477,7 +477,7 @@ func TestManager_RaceConditions(t *testing.T) { Enabled: false, }, } - stateManager, err := state.NewManager(context.Background(), log.WithField("component", "state"), stateConfig) + stateManager, err := state.NewManager(log.WithField("component", "state"), stateConfig) require.NoError(t, err) redisClient := newTestRedis(t) @@ -558,7 +558,7 @@ func TestManager_ConcurrentConfiguration(t *testing.T) { Enabled: false, }, } - stateManager, err := state.NewManager(context.Background(), log.WithField("component", "state"), stateConfig) + stateManager, err := state.NewManager(log.WithField("component", "state"), stateConfig) require.NoError(t, err) redisClient := newTestRedis(t) diff --git a/pkg/processor/transaction/simple/processor.go b/pkg/processor/transaction/simple/processor.go index 6a967b8..aa6d9bd 100644 --- a/pkg/processor/transaction/simple/processor.go +++ b/pkg/processor/transaction/simple/processor.go @@ -40,7 +40,7 @@ type Processor struct { } // New creates a new simple transaction processor. -func New(ctx context.Context, deps *Dependencies, config *Config) (*Processor, error) { +func New(deps *Dependencies, config *Config) (*Processor, error) { if err := config.Validate(); err != nil { return nil, fmt.Errorf("invalid config: %w", err) } @@ -50,7 +50,7 @@ func New(ctx context.Context, deps *Dependencies, config *Config) (*Processor, e clickhouseConfig.Network = deps.Network.Name clickhouseConfig.Processor = ProcessorName - clickhouseClient, err := clickhouse.New(ctx, &clickhouseConfig) + clickhouseClient, err := clickhouse.New(&clickhouseConfig) if err != nil { return nil, fmt.Errorf("failed to create clickhouse client: %w", err) } diff --git a/pkg/processor/transaction/structlog/processor.go b/pkg/processor/transaction/structlog/processor.go index 7f6828f..65ba613 100644 --- a/pkg/processor/transaction/structlog/processor.go +++ b/pkg/processor/transaction/structlog/processor.go @@ -36,13 +36,13 @@ type Processor struct { } // New creates a new transaction structlog processor. -func New(ctx context.Context, deps *Dependencies, config *Config) (*Processor, error) { +func New(deps *Dependencies, config *Config) (*Processor, error) { // Create a copy of the embedded config and set processor-specific values clickhouseConfig := config.Config clickhouseConfig.Network = deps.Network.Name clickhouseConfig.Processor = ProcessorName - clickhouseClient, err := clickhouse.New(ctx, &clickhouseConfig) + clickhouseClient, err := clickhouse.New(&clickhouseConfig) if err != nil { return nil, fmt.Errorf("failed to create clickhouse client for transaction_structlog: %w", err) } diff --git a/pkg/server/server.go b/pkg/server/server.go index fa8485e..25a21d6 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -49,7 +49,7 @@ func NewServer(ctx context.Context, log logrus.FieldLogger, namespace string, co pool := ethereum.NewPool(log.WithField("component", "ethereum"), namespace, &config.Ethereum) - stateManager, err := state.NewManager(ctx, log.WithField("component", "state"), &config.StateManager) + stateManager, err := state.NewManager(log.WithField("component", "state"), &config.StateManager) if err != nil { return nil, fmt.Errorf("failed to create state manager: %w", err) } @@ -145,20 +145,32 @@ func (s *Server) Start(ctx context.Context) error { return nil }) + // Channel to signal when state manager is ready + stateReady := make(chan struct{}) + g.Go(func() error { if err := s.state.Start(ctx); err != nil { s.log.WithError(err).Error("State manager start failed") + close(stateReady) // Signal even on failure so processor doesn't hang return err } + close(stateReady) // Signal that state manager is ready <-ctx.Done() return nil }) - // Start processor + // Start processor - must wait for state manager to be ready g.Go(func() error { + // Wait for state manager to be ready before starting processor + select { + case <-stateReady: + case <-ctx.Done(): + return ctx.Err() + } + return s.processor.Start(ctx) }) diff --git a/pkg/state/manager.go b/pkg/state/manager.go index 004c5e0..dcaa4d9 100644 --- a/pkg/state/manager.go +++ b/pkg/state/manager.go @@ -27,12 +27,12 @@ type Manager struct { network string } -func NewManager(ctx context.Context, log logrus.FieldLogger, config *Config) (*Manager, error) { +func NewManager(log logrus.FieldLogger, config *Config) (*Manager, error) { // Create storage client storageConfig := config.Storage.Config storageConfig.Processor = "state" - storageClient, err := clickhouse.New(ctx, &storageConfig) + storageClient, err := clickhouse.New(&storageConfig) if err != nil { return nil, fmt.Errorf("failed to create storage clickhouse client: %w", err) } @@ -49,7 +49,7 @@ func NewManager(ctx context.Context, log logrus.FieldLogger, config *Config) (*M limiterConfig := config.Limiter.Config limiterConfig.Processor = "state-limiter" - limiterClient, err := clickhouse.New(ctx, &limiterConfig) + limiterClient, err := clickhouse.New(&limiterConfig) if err != nil { return nil, fmt.Errorf("failed to create limiter clickhouse client: %w", err) } @@ -74,12 +74,14 @@ func (s *Manager) SetNetwork(network string) { } func (s *Manager) Start(ctx context.Context) error { - if err := s.storageClient.Start(); err != nil { + // Start storage client with infinite retry + if err := s.startClientWithRetry(ctx, s.storageClient, "storage"); err != nil { return fmt.Errorf("failed to start storage client: %w", err) } + // Start limiter client with infinite retry if enabled if s.limiterEnabled && s.limiterClient != nil { - if err := s.limiterClient.Start(); err != nil { + if err := s.startClientWithRetry(ctx, s.limiterClient, "limiter"); err != nil { return fmt.Errorf("failed to start limiter client: %w", err) } } @@ -87,6 +89,49 @@ func (s *Manager) Start(ctx context.Context) error { return nil } +// startClientWithRetry starts a ClickHouse client with infinite retry and capped exponential backoff. +// This ensures the state manager can wait for ClickHouse to become available at startup. +func (s *Manager) startClientWithRetry(ctx context.Context, client clickhouse.ClientInterface, name string) error { + const ( + baseDelay = 100 * time.Millisecond + maxDelay = 10 * time.Second + ) + + attempt := 0 + + for { + err := client.Start() + if err == nil { + if attempt > 0 { + s.log.WithField("client", name).Info("Successfully connected after retries") + } + + return nil + } + + // Calculate delay with exponential backoff, capped at maxDelay + delay := baseDelay * time.Duration(1< maxDelay { + delay = maxDelay + } + + s.log.WithFields(logrus.Fields{ + "client": name, + "attempt": attempt + 1, + "delay": delay, + "error": err, + }).Warn("Failed to start client, retrying...") + + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(delay): + } + + attempt++ + } +} + func (s *Manager) Stop(ctx context.Context) error { var err error From cd9a645a38bce821e955ff79fcc81beb2d4c1535 Mon Sep 17 00:00:00 2001 From: Andrew Davis <1709934+Savid@users.noreply.github.com> Date: Fri, 23 Jan 2026 13:52:13 +1000 Subject: [PATCH 5/5] fix(clickhouse): update integration tests for native protocol - Remove context parameter from New() calls (signature changed) - Add explicit toUInt64() casts in test queries for type strictness The ch-go native protocol is strict about types, so literal values like SELECT 42 return UInt8 instead of UInt64. Tests need explicit casts while production code queries actual UInt64 columns. Co-Authored-By: Claude Opus 4.5 --- pkg/clickhouse/client_integration_test.go | 17 ++++++++--------- pkg/clickhouse/client_test.go | 2 +- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/pkg/clickhouse/client_integration_test.go b/pkg/clickhouse/client_integration_test.go index 491b06b..6497c62 100644 --- a/pkg/clickhouse/client_integration_test.go +++ b/pkg/clickhouse/client_integration_test.go @@ -3,7 +3,6 @@ package clickhouse import ( - "context" "testing" "github.com/ethpandaops/execution-processor/internal/testutil" @@ -24,7 +23,7 @@ func TestClient_Integration_Container_New(t *testing.T) { Compression: "lz4", } - client, err := New(context.Background(), cfg) + client, err := New(cfg) require.NoError(t, err) require.NotNil(t, client) @@ -44,7 +43,7 @@ func TestClient_Integration_Container_StartStop(t *testing.T) { Network: "test", } - client, err := New(context.Background(), cfg) + client, err := New(cfg) require.NoError(t, err) // Start should ping successfully @@ -68,7 +67,7 @@ func TestClient_Integration_Container_Execute(t *testing.T) { Network: "test", } - client, err := New(context.Background(), cfg) + client, err := New(cfg) require.NoError(t, err) defer func() { _ = client.Stop() }() @@ -100,7 +99,7 @@ func TestClient_Integration_Container_QueryUInt64(t *testing.T) { Network: "test", } - client, err := New(context.Background(), cfg) + client, err := New(cfg) require.NoError(t, err) defer func() { _ = client.Stop() }() @@ -109,7 +108,7 @@ func TestClient_Integration_Container_QueryUInt64(t *testing.T) { require.NoError(t, err) // QueryUInt64 returns a single UInt64 value - result, err := client.QueryUInt64(t.Context(), "SELECT 42 as value", "value") + result, err := client.QueryUInt64(t.Context(), "SELECT toUInt64(42) as value", "value") require.NoError(t, err) require.NotNil(t, result) assert.Equal(t, uint64(42), *result) @@ -127,7 +126,7 @@ func TestClient_Integration_Container_QueryMinMaxUInt64(t *testing.T) { Network: "test", } - client, err := New(context.Background(), cfg) + client, err := New(cfg) require.NoError(t, err) defer func() { _ = client.Stop() }() @@ -136,7 +135,7 @@ func TestClient_Integration_Container_QueryMinMaxUInt64(t *testing.T) { require.NoError(t, err) // QueryMinMaxUInt64 returns min and max values - minVal, maxVal, err := client.QueryMinMaxUInt64(t.Context(), "SELECT 10 as min, 100 as max") + minVal, maxVal, err := client.QueryMinMaxUInt64(t.Context(), "SELECT toUInt64(10) as min, toUInt64(100) as max") require.NoError(t, err) require.NotNil(t, minVal) require.NotNil(t, maxVal) @@ -156,7 +155,7 @@ func TestClient_Integration_Container_IsStorageEmpty(t *testing.T) { Network: "test", } - client, err := New(context.Background(), cfg) + client, err := New(cfg) require.NoError(t, err) defer func() { _ = client.Stop() }() diff --git a/pkg/clickhouse/client_test.go b/pkg/clickhouse/client_test.go index 8338d5f..20473e4 100644 --- a/pkg/clickhouse/client_test.go +++ b/pkg/clickhouse/client_test.go @@ -270,7 +270,7 @@ func TestClient_Integration_QueryUInt64(t *testing.T) { err = client.Start() require.NoError(t, err) - result, err := client.QueryUInt64(t.Context(), "SELECT 42 as value", "value") + result, err := client.QueryUInt64(t.Context(), "SELECT toUInt64(42) as value", "value") require.NoError(t, err) require.NotNil(t, result) assert.Equal(t, uint64(42), *result)