diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml index bd32b43..4547fb2 100644 --- a/.github/workflows/go-test.yml +++ b/.github/workflows/go-test.yml @@ -11,14 +11,15 @@ permissions: contents: read jobs: - test: + unit-tests: + name: Unit Tests runs-on: ubuntu-latest steps: - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - uses: ./.github/workflows/go-setup - - name: run tests + - name: Run unit tests run: go test -race -json ./... > test.json - name: Annotate tests @@ -26,3 +27,20 @@ jobs: uses: guyarb/golang-test-annotations@9ab2ea84a399d03ffd114bf49dd23ffadc794541 # v0.6.0 with: test-results: test.json + + integration-tests: + name: Integration Tests + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + - uses: ./.github/workflows/go-setup + + - name: Run integration tests + run: go test -race -tags=integration -json ./... > test.json + + - name: Annotate tests + if: always() + uses: guyarb/golang-test-annotations@9ab2ea84a399d03ffd114bf49dd23ffadc794541 # v0.6.0 + with: + test-results: test.json diff --git a/example_config.yaml b/example_config.yaml index 7643099..51b21ff 100644 --- a/example_config.yaml +++ b/example_config.yaml @@ -17,13 +17,15 @@ redis: stateManager: storage: - url: "http://localhost:8123?database=admin" + addr: "localhost:9000" + database: "admin" table: "execution_block" limiter: # enable limiter to stop processing blocks up to the latest canonical beacon block table # if disabled, the state manager will process blocks up to head of the execution node enabled: false - url: "http://localhost:8123?database=default" + addr: "localhost:9000" + database: "default" table: "canonical_beacon_block" # Processor configuration (block discovery with leader election) @@ -45,7 +47,8 @@ processors: # Processor configuration transactionStructlog: enabled: true - url: "http://localhost:8123?database=default" + addr: "localhost:9000" + database: "default" table: "canonical_execution_transaction_structlog" # debug: false # Enable debug logging for ClickHouse queries # Channel-based batching configuration for memory-efficient processing @@ -62,7 +65,8 @@ processors: # Simple transaction processor (lightweight - no debug traces) transactionSimple: enabled: false - url: "http://localhost:8123?database=default" + addr: "localhost:9000" + database: "default" table: "execution_transaction" # debug: false # Enable debug logging for ClickHouse queries diff --git a/go.mod b/go.mod index bf07bc0..dc38a0d 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,8 @@ module github.com/ethpandaops/execution-processor go 1.25.4 require ( + github.com/ClickHouse/ch-go v0.69.0 + github.com/alicebob/miniredis/v2 v2.36.0 github.com/cenkalti/backoff/v4 v4.3.0 github.com/creasty/defaults v1.8.0 github.com/ethereum/go-ethereum v1.16.7 @@ -14,50 +16,105 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.10.1 github.com/stretchr/testify v1.11.1 - golang.org/x/sync v0.17.0 + github.com/testcontainers/testcontainers-go v0.40.0 + github.com/testcontainers/testcontainers-go/modules/clickhouse v0.40.0 + github.com/testcontainers/testcontainers-go/modules/redis v0.40.0 + golang.org/x/sync v0.18.0 gopkg.in/yaml.v3 v3.0.1 ) require ( + dario.cat/mergo v1.0.2 // indirect + github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/ProjectZKM/Ziren/crates/go-runtime/zkvm_runtime v0.0.0-20251001021608-1fe7b43fc4d6 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.24.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/consensys/gnark-crypto v0.19.0 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/platforms v0.2.1 // indirect + github.com/cpuguy83/dockercfg v0.3.2 // indirect github.com/crate-crypto/go-eth-kzg v1.4.0 // indirect github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/deckarep/golang-set/v2 v2.8.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/dmarkham/enumer v1.6.1 // indirect + github.com/docker/docker v28.5.1+incompatible // indirect + github.com/docker/go-connections v0.6.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/ebitengine/purego v0.8.4 // indirect github.com/ethereum/c-kzg-4844/v2 v2.1.5 // indirect github.com/ethereum/go-verkle v0.2.2 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-faster/city v1.0.1 // indirect + github.com/go-faster/errors v0.7.1 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.5.3 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.5 // indirect + github.com/hashicorp/go-version v1.7.0 // indirect github.com/holiman/uint256 v1.3.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/klauspost/compress v1.17.11 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/klauspost/compress v1.18.0 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 // indirect + github.com/magiconair/properties v1.8.10 // indirect + github.com/mdelapenya/tlscert v0.2.0 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/go-archive v0.1.0 // indirect + github.com/moby/patternmatcher v0.6.0 // indirect + github.com/moby/sys/sequential v0.6.0 // indirect + github.com/moby/sys/user v0.4.0 // indirect + github.com/moby/sys/userns v0.1.0 // indirect + github.com/moby/term v0.5.2 // indirect + github.com/morikuni/aec v1.0.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect + github.com/pascaldekloe/name v1.0.1 // indirect + github.com/pierrec/lz4/v4 v4.1.22 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.64.0 // indirect github.com/prometheus/procfs v0.16.1 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect + github.com/segmentio/asm v1.2.1 // indirect github.com/shirou/gopsutil v3.21.11+incompatible // indirect + github.com/shirou/gopsutil/v4 v4.25.6 // indirect github.com/spf13/cast v1.7.0 // indirect github.com/spf13/pflag v1.0.10 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/supranational/blst v0.3.16 // indirect - github.com/tklauser/go-sysconf v0.3.12 // indirect - github.com/tklauser/numcpus v0.6.1 // indirect - github.com/yusufpapurcu/wmi v1.2.3 // indirect + github.com/tklauser/go-sysconf v0.3.15 // indirect + github.com/tklauser/numcpus v0.10.0 // indirect + github.com/yuin/gopher-lua v1.1.1 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect + go.opentelemetry.io/otel v1.38.0 // indirect + go.opentelemetry.io/otel/metric v1.38.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.38.0 // indirect + go.opentelemetry.io/otel/trace v1.38.0 // indirect go.uber.org/atomic v1.11.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect golang.org/x/crypto v0.43.0 // indirect golang.org/x/exp v0.0.0-20251009144603-d2f985daa21b // indirect - golang.org/x/sys v0.37.0 // indirect + golang.org/x/mod v0.29.0 // indirect + golang.org/x/sys v0.38.0 // indirect golang.org/x/time v0.9.0 // indirect - google.golang.org/protobuf v1.36.6 // indirect + golang.org/x/tools v0.38.0 // indirect + google.golang.org/grpc v1.78.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect ) diff --git a/go.sum b/go.sum index 101195b..c9e71e0 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,13 @@ +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/ClickHouse/ch-go v0.69.0 h1:nO0OJkpxOlN/eaXFj0KzjTz5p7vwP1/y3GN4qc5z/iM= +github.com/ClickHouse/ch-go v0.69.0/go.mod h1:9XeZpSAT4S0kVjOpaJ5186b7PY/NH/hhF8R6u0WIjwg= +github.com/ClickHouse/clickhouse-go/v2 v2.34.0 h1:Y4rqkdrRHgExvC4o/NTbLdY5LFQ3LHS77/RNFxFX3Co= +github.com/ClickHouse/clickhouse-go/v2 v2.34.0/go.mod h1:yioSINoRLVZkLyDzdMXPLRIqhDvel8iLBlwh6Iefso8= github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= @@ -6,6 +16,10 @@ github.com/ProjectZKM/Ziren/crates/go-runtime/zkvm_runtime v0.0.0-20251001021608 github.com/ProjectZKM/Ziren/crates/go-runtime/zkvm_runtime v0.0.0-20251001021608-1fe7b43fc4d6/go.mod h1:ioLG6R+5bUSO1oeGSDxOV3FADARuMoytZCSX6MEMQkI= github.com/VictoriaMetrics/fastcache v1.13.0 h1:AW4mheMR5Vd9FkAPUv+NH6Nhw+fmbTMGMsNAoA/+4G0= github.com/VictoriaMetrics/fastcache v1.13.0/go.mod h1:hHXhl4DA2fTL2HTZDJFXWgW0LNjo6B+4aj2Wmng3TjU= +github.com/alicebob/miniredis/v2 v2.36.0 h1:yKczg+ez0bQYsG/PrgqtMMmCfl820RPu27kVGjP53eY= +github.com/alicebob/miniredis/v2 v2.36.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bits-and-blooms/bitset v1.24.0 h1:H4x4TuulnokZKvHLfzVRTHJfFfnHEeSYJizujEZvmAM= @@ -32,6 +46,16 @@ github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAK github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= github.com/consensys/gnark-crypto v0.19.0 h1:zXCqeY2txSaMl6G5wFpZzMWJU9HPNh8qxPnYJ1BL9vA= github.com/consensys/gnark-crypto v0.19.0/go.mod h1:rT23F0XSZqE0mUA0+pRtnL56IbPxs6gp4CeRsBk4XS0= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= +github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/crate-crypto/go-eth-kzg v1.4.0 h1:WzDGjHk4gFg6YzV0rJOAsTK4z3Qkz5jd4RE3DAvPFkg= @@ -39,6 +63,8 @@ github.com/crate-crypto/go-eth-kzg v1.4.0/go.mod h1:J9/u5sWfznSObptgfa92Jq8rTswn github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a h1:W8mUrRp6NOVl3J+MYp5kPMoUZPp7aOYHtaua31lwRHg= github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a/go.mod h1:sTwzHBvIzm2RfVCGNEBZgRyjwK40bVoun3ZnGOCafNM= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= +github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/creasty/defaults v1.8.0 h1:z27FJxCAa0JKt3utc0sCImAEb+spPucmKoOdLHvHYKk= github.com/creasty/defaults v1.8.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -54,6 +80,18 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 h1:NMZiJj8QnKe1LgsbDayM4UoHwbvw github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/dmarkham/enumer v1.6.1 h1:aSc9awYtZL07TUueWs40QcHtxTvHTAwG0EqrNsK45w4= +github.com/dmarkham/enumer v1.6.1/go.mod h1:yixql+kDDQRYqcuBM2n9Vlt7NoT9ixgXhaXry8vmRg8= +github.com/docker/docker v28.5.1+incompatible h1:Bm8DchhSD2J6PsFzxC35TZo4TLGR2PdW/E69rU45NhM= +github.com/docker/docker v28.5.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= +github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/ebitengine/purego v0.8.4 h1:CF7LEKg5FFOsASUj0+QwaXf8Ht6TlFxg09+S9wz0omw= +github.com/ebitengine/purego v0.8.4/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/emicklei/dot v1.6.2 h1:08GN+DD79cy/tzN6uLCT84+2Wk9u+wvqP+Hkx/dIR8A= github.com/emicklei/dot v1.6.2/go.mod h1:DeV7GvQtIw4h2u73RKBkkFdvVAz0D9fzeJrgPW6gy/s= github.com/ethereum/c-kzg-4844/v2 v2.1.5 h1:aVtoLK5xwJ6c5RiqO8g8ptJ5KU+2Hdquf6G3aXiHh5s= @@ -64,6 +102,8 @@ github.com/ethereum/go-ethereum v1.16.7 h1:qeM4TvbrWK0UC0tgkZ7NiRsmBGwsjqc64BHo2 github.com/ethereum/go-ethereum v1.16.7/go.mod h1:Fs6QebQbavneQTYcA39PEKv2+zIjX7rPUZ14DER46wk= github.com/ethereum/go-verkle v0.2.2 h1:I2W0WjnrFUIzzVPwm8ykY+7pL2d4VhlsePn4j7cnFk8= github.com/ethereum/go-verkle v0.2.2/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/ferranbt/fastssz v0.1.4 h1:OCDB+dYDEQDvAgtAGnTSidK1Pe2tW3nFV40XyMkTeDY= github.com/ferranbt/fastssz v0.1.4/go.mod h1:Ea3+oeoRGGLGm5shYAeDgu6PGUlcvQhE2fILyD9+tGg= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= @@ -72,6 +112,15 @@ github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/go-co-op/gocron v1.37.0 h1:ZYDJGtQ4OMhTLKOKMIch+/CY70Brbb1dGdooLEhh7b0= github.com/go-co-op/gocron v1.37.0/go.mod h1:3L/n6BkO7ABj+TrfSVXLRzsP26zmikL4ISkLQ0O8iNY= +github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= +github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw= +github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg= +github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= @@ -90,8 +139,12 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.5 h1:jP1RStw811EvUDzsUQ9oESqw2e4RqCjSAD9qIL8eMns= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.5/go.mod h1:WXNBZ64q3+ZUemCMXD9kYnr56H7CgZxDBHCVwstfl3s= github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE= github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0= +github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= +github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hibiken/asynq v0.25.1 h1:phj028N0nm15n8O2ims+IvJ2gz4k2auvermngh9JhTw= github.com/hibiken/asynq v0.25.1/go.mod h1:pazWNOLBu0FEynQRBvHA26qdIKRSmfdIfUm4HdsLmXg= github.com/holiman/billy v0.0.0-20250707135307-f2f9b9aae7db h1:IZUYC/xb3giYwBLMnr8d0TGTzPKFGNTCGgGLoyeX330= @@ -104,10 +157,12 @@ github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= -github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= -github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -125,22 +180,56 @@ github.com/leanovate/gopter v0.2.11 h1:vRjThO1EKPb/1NsDXuDrzldR28RLkBflWYcU9CvzW github.com/leanovate/gopter v0.2.11/go.mod h1:aK3tzZP/C+p1m3SPRE4SYZFGP7jjkuSI4f7Xvpt0S9c= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 h1:PpXWgLPs+Fqr325bN2FD2ISlRRztXibcX6e8f5FR5Dc= +github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= +github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= +github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5LJHI= +github.com/mdelapenya/tlscert v0.2.0/go.mod h1:O4njj3ELLnJjGdkN7M/vIVCpZ+Cf0L6muqOG4tLSl8o= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/pointerstructure v1.2.0 h1:O+i9nHnXS3l/9Wu7r4NrEdwA2VFTicjUEN1uBnDo34A= github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/go-archive v0.1.0 h1:Kk/5rdW/g+H8NHdJW2gsXyZ7UnzvJNOy6VKJqueWdcQ= +github.com/moby/go-archive v0.1.0/go.mod h1:G9B+YoujNohJmrIYFBpSd54GTUB4lt9S+xVQvsJyFuo= +github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= +github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= +github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= +github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= +github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= +github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= +github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= +github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= +github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/pascaldekloe/name v1.0.1 h1:9lnXOHeqeHHnWLbKfH6X98+4+ETVqFqxN09UXSjcMb0= +github.com/pascaldekloe/name v1.0.1/go.mod h1:Z//MfYJnH4jVpQ9wkclwu2I2MkHmXTlT9wR5UZScttM= +github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU= +github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= +github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= +github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pion/dtls/v2 v2.2.7 h1:cSUBsETxepsCSFSxC3mc/aDo14qQLMSL+O6IjG28yV8= github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= @@ -156,6 +245,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= @@ -172,14 +263,21 @@ github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/russross/blackfriday v1.6.0 h1:KqfZb0pUVN2lYqZUYRddxF4OR8ZMURnJIG5Y3VRLtww= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/segmentio/asm v1.2.1 h1:DTNbBqs57ioxAD4PrArqftgypG4/qNpXoJx8TVXxPR0= +github.com/segmentio/asm v1.2.1/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/gopsutil/v4 v4.25.6 h1:kLysI2JsKorfaFPcYmcJqbzROzsBWEOAtw6A7dIfqXs= +github.com/shirou/gopsutil/v4 v4.25.6/go.mod h1:PfybzyydfZcN+JMMjkF6Zb8Mq1A/VcogFFg7hj50W9c= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= @@ -205,40 +303,86 @@ github.com/supranational/blst v0.3.16 h1:bTDadT+3fK497EvLdWRQEjiGnUtzJ7jjIUMF0jq github.com/supranational/blst v0.3.16/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= -github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= -github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= -github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= -github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +github.com/testcontainers/testcontainers-go v0.40.0 h1:pSdJYLOVgLE8YdUY2FHQ1Fxu+aMnb6JfVz1mxk7OeMU= +github.com/testcontainers/testcontainers-go v0.40.0/go.mod h1:FSXV5KQtX2HAMlm7U3APNyLkkap35zNLxukw9oBi/MY= +github.com/testcontainers/testcontainers-go/modules/clickhouse v0.40.0 h1:JhYAFtoTCEpzB5jF+wcEP5mL01+JChUUpaaX8sWuEzo= +github.com/testcontainers/testcontainers-go/modules/clickhouse v0.40.0/go.mod h1:UoMHEYTzGmwKyeCQaKfcQHSVs/kQwimfzX+y1gVSRIk= +github.com/testcontainers/testcontainers-go/modules/redis v0.40.0 h1:OG4qwcxp2O0re7V7M9lY9w0v6wWgWf7j7rtkpAnGMd0= +github.com/testcontainers/testcontainers-go/modules/redis v0.40.0/go.mod h1:Bc+EDhKMo5zI5V5zdBkHiMVzeAXbtI4n5isS/nzf6zw= +github.com/tklauser/go-sysconf v0.3.15 h1:VE89k0criAymJ/Os65CSn1IXaol+1wrsFHEB8Ol49K4= +github.com/tklauser/go-sysconf v0.3.15/go.mod h1:Dmjwr6tYFIseJw7a3dRLJfsHAMXZ3nEnL/aZY+0IuI4= +github.com/tklauser/numcpus v0.10.0 h1:18njr6LDBk1zuna922MgdjQuJFjrdppsZG60sHGfjso= +github.com/tklauser/numcpus v0.10.0/go.mod h1:BiTKazU708GQTYF4mB+cmlpT2Is1gLk7XVuEeem8LsQ= github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w= github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= -github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= -github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= +github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= golang.org/x/exp v0.0.0-20251009144603-d2f985daa21b h1:18qgiDvlvH7kk8Ioa8Ov+K6xCi0GMvmGfGW0sgd/SYA= golang.org/x/exp v0.0.0-20251009144603-d2f985daa21b/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70= -golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= -golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= +golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= -golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= -golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q= +golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss= +golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= +golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= -google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= -google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= +golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= +google.golang.org/genproto/googleapis/api v0.0.0-20260120221211-b8f7ae30c516 h1:vmC/ws+pLzWjj/gzApyoZuSVrDtF1aod4u/+bbj8hgM= +google.golang.org/genproto/googleapis/api v0.0.0-20260120221211-b8f7ae30c516/go.mod h1:p3MLuOwURrGBRoEyFHBT3GjUwaCQVKeNqqWxlcISGdw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260120221211-b8f7ae30c516 h1:sNrWoksmOyF5bvJUcnmbeAmQi8baNhqg5IWaI3llQqU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260120221211-b8f7ae30c516/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= +google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= +google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= @@ -251,3 +395,5 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= diff --git a/internal/testutil/testutil.go b/internal/testutil/testutil.go new file mode 100644 index 0000000..696c9ea --- /dev/null +++ b/internal/testutil/testutil.go @@ -0,0 +1,125 @@ +// Package testutil provides test helper utilities for unit and integration tests. +package testutil + +import ( + "context" + "fmt" + "testing" + + "github.com/alicebob/miniredis/v2" + "github.com/redis/go-redis/v9" + "github.com/testcontainers/testcontainers-go" + tcclickhouse "github.com/testcontainers/testcontainers-go/modules/clickhouse" + tcredis "github.com/testcontainers/testcontainers-go/modules/redis" +) + +// NewMiniredis creates an in-memory Redis server for unit tests. +// The server is automatically cleaned up when the test completes. +func NewMiniredis(t *testing.T) *miniredis.Miniredis { + t.Helper() + + s := miniredis.RunT(t) + + return s +} + +// NewMiniredisClient creates a Redis client connected to an in-memory miniredis server. +// Both the server and client are automatically cleaned up when the test completes. +func NewMiniredisClient(t *testing.T) (*redis.Client, *miniredis.Miniredis) { + t.Helper() + + s := miniredis.RunT(t) + client := redis.NewClient(&redis.Options{Addr: s.Addr()}) + + t.Cleanup(func() { + _ = client.Close() + }) + + return client, s +} + +// NewRedisContainer creates a real Redis container for integration tests. +// The container is automatically cleaned up when the test completes. +func NewRedisContainer(t *testing.T) (*redis.Client, string) { + t.Helper() + + ctx := context.Background() + + c, err := tcredis.Run(ctx, "redis:7-alpine") + if err != nil { + t.Fatalf("failed to start redis container: %v", err) + } + + testcontainers.CleanupContainer(t, c) + + connStr, err := c.ConnectionString(ctx) + if err != nil { + t.Fatalf("failed to get redis connection string: %v", err) + } + + // Parse the connection string to get the address + // The connection string is in the format "redis://host:port" + opts, err := redis.ParseURL(connStr) + if err != nil { + t.Fatalf("failed to parse redis connection string: %v", err) + } + + client := redis.NewClient(opts) + + t.Cleanup(func() { + _ = client.Close() + }) + + return client, connStr +} + +// ClickHouseConnection holds ClickHouse connection details. +type ClickHouseConnection struct { + Host string + Port int + Database string + Username string + Password string +} + +// Addr returns the ClickHouse address in host:port format. +func (c ClickHouseConnection) Addr() string { + return fmt.Sprintf("%s:%d", c.Host, c.Port) +} + +// NewClickHouseContainer creates a real ClickHouse container for integration tests. +// The container is automatically cleaned up when the test completes. +func NewClickHouseContainer(t *testing.T) ClickHouseConnection { + t.Helper() + + ctx := context.Background() + + c, err := tcclickhouse.Run(ctx, "clickhouse/clickhouse-server:latest", + tcclickhouse.WithUsername("default"), + tcclickhouse.WithPassword("test"), + tcclickhouse.WithDatabase("default"), + ) + if err != nil { + t.Fatalf("failed to start clickhouse container: %v", err) + } + + testcontainers.CleanupContainer(t, c) + + host, err := c.Host(ctx) + if err != nil { + t.Fatalf("failed to get clickhouse host: %v", err) + } + + port, err := c.MappedPort(ctx, "9000/tcp") + if err != nil { + t.Fatalf("failed to get clickhouse port: %v", err) + } + + return ClickHouseConnection{ + Host: host, + Port: port.Int(), + Database: "default", + Username: "default", + Password: "test", + } +} diff --git a/pkg/clickhouse/client.go b/pkg/clickhouse/client.go index 24a2fbf..d36e78c 100644 --- a/pkg/clickhouse/client.go +++ b/pkg/clickhouse/client.go @@ -1,17 +1,21 @@ package clickhouse import ( - "bytes" "context" - "encoding/json" + "errors" "fmt" "io" - "net/http" - "reflect" + "net" "strings" "sync" + "sync/atomic" + "syscall" "time" + "github.com/ClickHouse/ch-go" + "github.com/ClickHouse/ch-go/chpool" + "github.com/ClickHouse/ch-go/compress" + "github.com/ClickHouse/ch-go/proto" "github.com/sirupsen/logrus" "github.com/ethpandaops/execution-processor/pkg/common" @@ -22,367 +26,443 @@ const ( statusFailed = "failed" ) -// client implements the ClientInterface using HTTP. -type client struct { - log logrus.FieldLogger - httpClient *http.Client - baseURL string - network string - processor string - debug bool - lock sync.RWMutex +// Client implements the ClientInterface using ch-go native protocol. +type Client struct { + pool *chpool.Pool + config *Config + compression ch.Compression + network string + processor string + log logrus.FieldLogger + lock sync.RWMutex + started atomic.Bool + + // Metrics collection + metricsDone chan struct{} + metricsWg sync.WaitGroup } -// New creates a new HTTP-based ClickHouse client. -func New(cfg *Config) (ClientInterface, error) { - if err := cfg.Validate(); err != nil { - return nil, fmt.Errorf("invalid config: %w", err) +// isRetryableError checks if an error is transient and can be retried. +func isRetryableError(err error) bool { + if err == nil { + return false } - // Set defaults - cfg.SetDefaults() - - // Create HTTP client with keep-alive settings - transport := &http.Transport{ - MaxIdleConns: 100, - MaxIdleConnsPerHost: 10, - IdleConnTimeout: cfg.KeepAlive, - DisableKeepAlives: false, + // Check for context errors - these should not be retried + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return false } - httpClient := &http.Client{ - Transport: transport, - Timeout: 0, // We'll set per-request timeouts + // Check for ch-go sentinel errors - client permanently closed + if errors.Is(err, ch.ErrClosed) { + return false } - c := &client{ - log: logrus.WithField("component", "clickhouse-http"), - httpClient: httpClient, - baseURL: strings.TrimRight(cfg.URL, "/"), - network: cfg.Network, - processor: cfg.Processor, - debug: cfg.Debug, + // Check for ch-go server exceptions + if exc, ok := ch.AsException(err); ok { + // Retryable server-side errors + if exc.IsCode( + proto.ErrTimeoutExceeded, // 159 - server timeout + proto.ErrNoFreeConnection, // 203 - server pool exhausted + proto.ErrTooManySimultaneousQueries, // 202 - rate limited + proto.ErrSocketTimeout, // 209 - network timeout + proto.ErrNetworkError, // 210 - generic network + ) { + return true + } + // All other server exceptions are non-retryable (syntax, data errors, etc.) + return false } - return c, nil -} - -func (c *client) Start() error { - // Skip connectivity test if network is not set yet (e.g., during state manager initialization) - // The network will be set later when it's determined from the chain ID - if c.network == "" { - c.log.Debug("Skipping ClickHouse connectivity test - network not yet determined") - - return nil + // Check for data corruption - never retry + var corruptedErr *compress.CorruptedDataErr + if errors.As(err, &corruptedErr) { + return false } - // Test connectivity - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - if err := c.Execute(ctx, "SELECT 1"); err != nil { - return fmt.Errorf("failed to connect to ClickHouse: %w", err) + // Check for connection reset/refused errors (before net.Error since syscall.Errno implements net.Error) + if errors.Is(err, syscall.ECONNRESET) || + errors.Is(err, syscall.ECONNREFUSED) || + errors.Is(err, syscall.EPIPE) || + errors.Is(err, io.EOF) || + errors.Is(err, io.ErrUnexpectedEOF) { + return true } - c.log.Info("Connected to ClickHouse HTTP interface") + // Check for network timeout errors + var netErr net.Error + if errors.As(err, &netErr) && netErr.Timeout() { + return true + } - return nil -} + // Check for common transient error messages (fallback for edge cases) + errStr := err.Error() + transientPatterns := []string{ + "connection reset", + "connection refused", + "broken pipe", + "EOF", + "timeout", + "temporary failure", + "server is overloaded", + "too many connections", + } -func (c *client) SetNetwork(network string) { - c.lock.Lock() - defer c.lock.Unlock() + for _, pattern := range transientPatterns { + if strings.Contains(strings.ToLower(errStr), strings.ToLower(pattern)) { + return true + } + } - c.network = network + return false } -func (c *client) Stop() error { - c.lock.Lock() - defer c.lock.Unlock() - - if c.httpClient != nil { - c.httpClient.CloseIdleConnections() +// withQueryTimeout returns a context with the configured query timeout applied. +// If the context already has a deadline, the original context is returned unchanged. +func (c *Client) withQueryTimeout(ctx context.Context) (context.Context, context.CancelFunc) { + if c.config.QueryTimeout == 0 { + return ctx, func() {} } - c.log.Info("Closed ClickHouse HTTP client") + if _, hasDeadline := ctx.Deadline(); hasDeadline { + return ctx, func() {} + } - return nil + return context.WithTimeout(ctx, c.config.QueryTimeout) } -func (c *client) QueryOne(ctx context.Context, query string, dest interface{}) error { - start := time.Now() - operation := "query_one" - status := statusSuccess +// doWithRetry executes a function with exponential backoff retry logic. +// The function receives a context with the configured query timeout applied per attempt. +func (c *Client) doWithRetry(ctx context.Context, operation string, fn func(ctx context.Context) error) error { + var lastErr error + + for attempt := 0; attempt <= c.config.MaxRetries; attempt++ { + if attempt > 0 { + // Calculate backoff delay with exponential increase, capped at RetryMaxDelay + delay := c.config.RetryBaseDelay * time.Duration(1<<(attempt-1)) + if c.config.RetryMaxDelay > 0 && delay > c.config.RetryMaxDelay { + delay = c.config.RetryMaxDelay + } - defer func() { - c.recordMetrics(operation, status, time.Since(start), query) - }() + c.log.WithFields(logrus.Fields{ + "attempt": attempt, + "max": c.config.MaxRetries, + "delay": delay, + "operation": operation, + "error": lastErr, + }).Debug("Retrying after transient error") + + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(delay): + } + } - // Add FORMAT JSON to query - formattedQuery := query + " FORMAT JSON" + attemptCtx, cancel := c.withQueryTimeout(ctx) + err := fn(attemptCtx) - resp, err := c.executeHTTPRequest(ctx, formattedQuery, c.getTimeout(ctx, "query")) - if err != nil { - status = statusFailed + cancel() - return fmt.Errorf("query execution failed: %w", err) - } + if err == nil { + return nil + } - // Parse response - ClickHouse returns snake_case fields - //nolint:tagliatelle // ClickHouse JSON response format uses snake_case - var result struct { - Data []json.RawMessage `json:"data"` - Meta []struct { - Name string `json:"name"` - Type string `json:"type"` - } `json:"meta"` - Rows int `json:"rows"` - RowsRead int `json:"rows_read"` + lastErr = err + + if !isRetryableError(err) { + return err + } } - if err := json.Unmarshal(resp, &result); err != nil { - status = statusFailed + return fmt.Errorf("max retries (%d) exceeded: %w", c.config.MaxRetries, lastErr) +} - return fmt.Errorf("failed to parse response: %w", err) +// New creates a new ch-go native ClickHouse client. +// The client is not connected until Start() is called. +func New(cfg *Config) (ClientInterface, error) { + if err := cfg.Validate(); err != nil { + return nil, fmt.Errorf("invalid config: %w", err) } - if len(result.Data) == 0 { - // No rows found, return without error but don't unmarshal - return nil - } + cfg.SetDefaults() - // Unmarshal the first row into dest - if err := json.Unmarshal(result.Data[0], dest); err != nil { - status = statusFailed + compression := ch.CompressionLZ4 - return fmt.Errorf("failed to unmarshal result: %w", err) + switch cfg.Compression { + case "zstd": + compression = ch.CompressionZSTD + case "none": + compression = ch.CompressionDisabled } - return nil + log := logrus.WithField("component", "clickhouse-native") + + return &Client{ + config: cfg, + compression: compression, + network: cfg.Network, + processor: cfg.Processor, + log: log, + }, nil } -func (c *client) QueryMany(ctx context.Context, query string, dest interface{}) error { - start := time.Now() - operation := "query_many" - status := statusSuccess +// dialWithRetry wraps the dial operation with exponential backoff retry logic. +func dialWithRetry(ctx context.Context, log logrus.FieldLogger, cfg *Config, fn func() error) error { + var lastErr error - defer func() { - c.recordMetrics(operation, status, time.Since(start), query) - }() + for attempt := 0; attempt <= cfg.MaxRetries; attempt++ { + if attempt > 0 { + // Calculate backoff delay with exponential increase, capped at RetryMaxDelay + delay := cfg.RetryBaseDelay * time.Duration(1<<(attempt-1)) + if cfg.RetryMaxDelay > 0 && delay > cfg.RetryMaxDelay { + delay = cfg.RetryMaxDelay + } - // Validate that dest is a pointer to a slice - destValue := reflect.ValueOf(dest) - if destValue.Kind() != reflect.Ptr || destValue.Elem().Kind() != reflect.Slice { - status = statusFailed + log.WithFields(logrus.Fields{ + "attempt": attempt, + "max": cfg.MaxRetries, + "delay": delay, + "operation": "dial", + "error": lastErr, + }).Debug("Retrying dial after transient error") + + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(delay): + } + } - return fmt.Errorf("dest must be a pointer to a slice") + err := fn() + if err == nil { + return nil + } + + lastErr = err + + if !isRetryableError(err) { + return err + } } - // Add FORMAT JSON to query - formattedQuery := query + " FORMAT JSON" + return fmt.Errorf("max retries (%d) exceeded: %w", cfg.MaxRetries, lastErr) +} - resp, err := c.executeHTTPRequest(ctx, formattedQuery, c.getTimeout(ctx, "query")) - if err != nil { - status = statusFailed +// Start initializes the client by dialing ClickHouse with retry logic. +func (c *Client) Start() error { + // Idempotency guard - prevent multiple successful Start() calls from leaking goroutines. + // We check if pool is already set to allow retries after failures. + c.lock.Lock() - return fmt.Errorf("query execution failed: %w", err) - } + if c.pool != nil { + c.lock.Unlock() + c.log.Debug("Start() already completed successfully, skipping") - // Parse response - ClickHouse returns snake_case fields - //nolint:tagliatelle // ClickHouse JSON response format uses snake_case - var result struct { - Data []json.RawMessage `json:"data"` - Meta []struct { - Name string `json:"name"` - Type string `json:"type"` - } `json:"meta"` - Rows int `json:"rows"` - RowsRead int `json:"rows_read"` + return nil } - if err := json.Unmarshal(resp, &result); err != nil { - status = statusFailed + c.lock.Unlock() - return fmt.Errorf("failed to parse response: %w", err) + // Dial with startup retry logic for transient connection failures + ctx, cancel := context.WithTimeout(context.Background(), c.config.DialTimeout*time.Duration(c.config.MaxRetries+1)) + defer cancel() + + var pool *chpool.Pool + + err := dialWithRetry(ctx, c.log, c.config, func() error { + var dialErr error + + pool, dialErr = chpool.Dial(ctx, chpool.Options{ + ClientOptions: ch.Options{ + Address: c.config.Addr, + Database: c.config.Database, + User: c.config.Username, + Password: c.config.Password, + Compression: c.compression, + DialTimeout: c.config.DialTimeout, + }, + MaxConns: c.config.MaxConns, + MinConns: c.config.MinConns, + MaxConnLifetime: c.config.ConnMaxLifetime, + MaxConnIdleTime: c.config.ConnMaxIdleTime, + HealthCheckPeriod: c.config.HealthCheckPeriod, + }) + + return dialErr + }) + if err != nil { + return fmt.Errorf("failed to dial clickhouse: %w", err) } - // Create a slice of the appropriate type - sliceType := destValue.Elem().Type() - elemType := sliceType.Elem() - newSlice := reflect.MakeSlice(sliceType, len(result.Data), len(result.Data)) + c.lock.Lock() + c.pool = pool + c.lock.Unlock() - // Unmarshal each row - for i, data := range result.Data { - elem := reflect.New(elemType) - if err := json.Unmarshal(data, elem.Interface()); err != nil { - status = statusFailed + c.log.Info("Connected to ClickHouse native interface") - return fmt.Errorf("failed to unmarshal row %d: %w", i, err) - } + // Start pool metrics collection goroutine - use started flag to prevent duplicate goroutines + if !c.started.Swap(true) { + c.metricsDone = make(chan struct{}) + c.metricsWg.Add(1) - newSlice.Index(i).Set(elem.Elem()) + go c.collectPoolMetrics() } - // Set the result - destValue.Elem().Set(newSlice) - return nil } -func (c *client) Execute(ctx context.Context, query string) error { - start := time.Now() - operation := "execute" - status := statusSuccess - - defer func() { - c.recordMetrics(operation, status, time.Since(start), query) - }() - - _, err := c.executeHTTPRequest(ctx, query, c.getTimeout(ctx, "query")) - if err != nil { - status = statusFailed +// Stop closes the connection pool. +func (c *Client) Stop() error { + // Stop metrics collection goroutine + if c.metricsDone != nil { + close(c.metricsDone) + c.metricsWg.Wait() + } - return fmt.Errorf("execution failed: %w", err) + if c.pool != nil { + c.pool.Close() + c.log.Info("Closed ClickHouse connection pool") } return nil } -func (c *client) BulkInsert(ctx context.Context, table string, data interface{}) error { +// SetNetwork updates the network name for metrics labeling. +func (c *Client) SetNetwork(network string) { + c.lock.Lock() + defer c.lock.Unlock() + + c.network = network +} + +// Do executes a query using the pool. +func (c *Client) Do(ctx context.Context, query ch.Query) error { + return c.pool.Do(ctx, query) +} + +// QueryUInt64 executes a query and returns a single UInt64 value from the specified column. +// Returns nil if no rows are found. +func (c *Client) QueryUInt64(ctx context.Context, query string, columnName string) (*uint64, error) { start := time.Now() - operation := "bulk_insert" + operation := "query_uint64" status := statusSuccess defer func() { - c.recordMetrics(operation, status, time.Since(start), table) + c.recordMetrics(operation, status, time.Since(start), query) }() - // Convert data to slice via reflection - dataValue := reflect.ValueOf(data) - if dataValue.Kind() != reflect.Slice { - status = statusFailed - - return fmt.Errorf("data must be a slice") - } + var result *uint64 - if dataValue.Len() == 0 { - return nil // Nothing to insert - } + col := new(proto.ColUInt64) - // Build INSERT query with JSONEachRow format - var buf bytes.Buffer + err := c.doWithRetry(ctx, operation, func(attemptCtx context.Context) error { + col.Reset() - buf.WriteString(fmt.Sprintf("INSERT INTO %s FORMAT JSONEachRow\n", table)) + result = nil - // Marshal each item as JSON - for i := 0; i < dataValue.Len(); i++ { - item := dataValue.Index(i).Interface() - - jsonData, err := json.Marshal(item) - if err != nil { - status = statusFailed - - return fmt.Errorf("failed to marshal row %d: %w", i, err) - } - - buf.Write(jsonData) - buf.WriteByte('\n') - } + return c.pool.Do(attemptCtx, ch.Query{ + Body: query, + Result: proto.Results{ + {Name: columnName, Data: col}, + }, + OnResult: func(ctx context.Context, block proto.Block) error { + if col.Rows() > 0 { + val := col.Row(0) + result = &val + } - // Execute the insert - _, err := c.executeHTTPRequest(ctx, buf.String(), c.getTimeout(ctx, "insert")) + return nil + }, + }) + }) if err != nil { status = statusFailed - return fmt.Errorf("bulk insert failed: %w", err) + return nil, fmt.Errorf("query failed: %w", err) } - return nil + return result, nil } -func (c *client) executeHTTPRequest(ctx context.Context, query string, timeout time.Duration) ([]byte, error) { - // Create request with timeout - reqCtx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - - req, err := http.NewRequestWithContext(reqCtx, "POST", c.baseURL, strings.NewReader(query)) - if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) - } - - // Set headers - req.Header.Set("Content-Type", "text/plain") - req.Header.Set("X-ClickHouse-Format", "JSON") - - // Debug logging - if c.debug { - // For large inserts, truncate the query - logQuery := query - if len(query) > 1000 && strings.Contains(query, "INSERT") { - logQuery = query[:1000] + "... (truncated)" - } +// QueryMinMaxUInt64 executes a query that returns min and max UInt64 values. +// The query must return columns named "min" and "max". +// Returns nil for both values if no rows are found. +func (c *Client) QueryMinMaxUInt64(ctx context.Context, query string) (minVal, maxVal *uint64, err error) { + start := time.Now() + operation := "query_min_max" + status := statusSuccess - c.log.WithField("query", logQuery).Debug("Executing ClickHouse query") - } + defer func() { + c.recordMetrics(operation, status, time.Since(start), query) + }() - // Execute request - resp, err := c.httpClient.Do(req) + colMin := new(proto.ColUInt64) + colMax := new(proto.ColUInt64) + + err = c.doWithRetry(ctx, operation, func(attemptCtx context.Context) error { + colMin.Reset() + colMax.Reset() + + minVal = nil + maxVal = nil + + return c.pool.Do(attemptCtx, ch.Query{ + Body: query, + Result: proto.Results{ + {Name: "min", Data: colMin}, + {Name: "max", Data: colMax}, + }, + OnResult: func(ctx context.Context, block proto.Block) error { + if colMin.Rows() > 0 { + v := colMin.Row(0) + minVal = &v + } + + if colMax.Rows() > 0 { + v := colMax.Row(0) + maxVal = &v + } + + return nil + }, + }) + }) if err != nil { - return nil, fmt.Errorf("request failed: %w", err) - } - defer resp.Body.Close() + status = statusFailed - // Read response body - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read response: %w", err) + return nil, nil, fmt.Errorf("query failed: %w", err) } - // Check status code - if resp.StatusCode != http.StatusOK { - bodyStr := string(body) - if bodyStr == "" { - bodyStr = "(empty response)" - } - - // Try to parse error message from JSON - var errorResp struct { - Exception string `json:"exception"` - } - - if jsonErr := json.Unmarshal(body, &errorResp); jsonErr == nil && errorResp.Exception != "" { - return nil, fmt.Errorf("ClickHouse error (status %d): %s", resp.StatusCode, errorResp.Exception) - } + return minVal, maxVal, nil +} - return nil, fmt.Errorf("ClickHouse error (status %d): %s", resp.StatusCode, bodyStr) - } +// Execute runs a query without expecting results. +func (c *Client) Execute(ctx context.Context, query string) error { + start := time.Now() + operation := "execute" + status := statusSuccess - // Debug logging - if c.debug && len(body) < 1000 { - c.log.WithField("response", string(body)).Debug("ClickHouse response") - } + defer func() { + c.recordMetrics(operation, status, time.Since(start), query) + }() - return body, nil -} + err := c.doWithRetry(ctx, operation, func(attemptCtx context.Context) error { + return c.pool.Do(attemptCtx, ch.Query{ + Body: query, + }) + }) + if err != nil { + status = statusFailed -func (c *client) getTimeout(ctx context.Context, operation string) time.Duration { - // Check if context already has a deadline - if deadline, ok := ctx.Deadline(); ok { - return time.Until(deadline) + return fmt.Errorf("execution failed: %w", err) } - // Use default timeouts based on operation type - switch operation { - case "insert": - return 5 * time.Minute - case "query": - return 30 * time.Second - default: - return 30 * time.Second - } + return nil } -func (c *client) IsStorageEmpty(ctx context.Context, table string, conditions map[string]interface{}) (bool, error) { +// IsStorageEmpty checks if a table has any records matching the given conditions. +func (c *Client) IsStorageEmpty(ctx context.Context, table string, conditions map[string]any) (bool, error) { start := time.Now() operation := "is_storage_empty" status := statusSuccess @@ -392,15 +472,14 @@ func (c *client) IsStorageEmpty(ctx context.Context, table string, conditions ma }() // Build the query - query := fmt.Sprintf("SELECT COUNT(*) as count FROM %s FINAL", table) + query := fmt.Sprintf("SELECT count() as count FROM %s FINAL", table) if len(conditions) > 0 { query += " WHERE " - var conditionParts []string + conditionParts := make([]string, 0, len(conditions)) for key, value := range conditions { - // Handle different value types switch v := value.(type) { case string: conditionParts = append(conditionParts, fmt.Sprintf("%s = '%s'", key, v)) @@ -414,21 +493,31 @@ func (c *client) IsStorageEmpty(ctx context.Context, table string, conditions ma query += strings.Join(conditionParts, " AND ") } - var result struct { - Count json.Number `json:"count"` - } + // Execute and get count + var count uint64 - if err := c.QueryOne(ctx, query, &result); err != nil { - status = statusFailed + err := c.doWithRetry(ctx, operation, func(attemptCtx context.Context) error { + colCount := new(proto.ColUInt64) - return false, fmt.Errorf("failed to check if table is empty: %w", err) - } + if err := c.pool.Do(attemptCtx, ch.Query{ + Body: query, + Result: proto.Results{ + {Name: "count", Data: colCount}, + }, + }); err != nil { + return err + } - count, err := result.Count.Int64() + if colCount.Rows() > 0 { + count = colCount.Row(0) + } + + return nil + }) if err != nil { status = statusFailed - return false, fmt.Errorf("failed to parse count: %w", err) + return false, fmt.Errorf("failed to check if table is empty: %w", err) } return count == 0, nil @@ -436,7 +525,6 @@ func (c *client) IsStorageEmpty(ctx context.Context, table string, conditions ma // extractTableName attempts to extract the table name from various SQL query patterns. func extractTableName(query string) string { - // Work with original query to preserve case trimmedQuery := strings.TrimSpace(query) upperQuery := strings.ToUpper(trimmedQuery) @@ -444,23 +532,17 @@ func extractTableName(query string) string { if strings.HasPrefix(upperQuery, "INSERT INTO") { parts := strings.Fields(trimmedQuery) if len(parts) >= 3 { - // Return the third part which should be the table name - // Handle cases where table name might have backticks or quotes return strings.Trim(parts[2], "`'\"") } } // Handle SELECT ... FROM queries if idx := strings.Index(upperQuery, "FROM"); idx != -1 { - // Get the substring after FROM from the original query afterFrom := strings.TrimSpace(trimmedQuery[idx+4:]) parts := strings.Fields(afterFrom) if len(parts) > 0 { - // Return the first part which should be the table name - // Remove any trailing keywords like WHERE, ORDER BY, etc. tableName := parts[0] - // Handle FINAL keyword (e.g., "FROM table FINAL") if !strings.EqualFold(tableName, "FINAL") { return strings.Trim(tableName, "`'\"") } @@ -486,18 +568,82 @@ func extractTableName(query string) string { return "" } -func (c *client) recordMetrics(operation, status string, duration time.Duration, tableOrQuery string) { - var table string +func (c *Client) recordMetrics(operation, status string, duration time.Duration, tableOrQuery string) { + table := extractTableName(tableOrQuery) - // If it's a bulk_insert operation, tableOrQuery is already the table name - if operation == "bulk_insert" { - table = tableOrQuery - } else { - // Otherwise, try to extract table name from the query - table = extractTableName(tableOrQuery) - } + c.lock.RLock() + network := c.network + c.lock.RUnlock() + + common.ClickHouseOperationDuration.WithLabelValues(network, c.processor, operation, table, status, "").Observe(duration.Seconds()) + common.ClickHouseOperationTotal.WithLabelValues(network, c.processor, operation, table, status, "").Inc() +} + +// collectPoolMetrics periodically collects pool statistics and updates Prometheus metrics. +func (c *Client) collectPoolMetrics() { + defer c.metricsWg.Done() + + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + + // Track previous counter values for delta calculation + var prevAcquireCount, prevEmptyAcquireCount, prevCanceledAcquireCount int64 + + for { + select { + case <-c.metricsDone: + return + case <-ticker.C: + c.lock.RLock() + network := c.network + c.lock.RUnlock() - // Use existing metrics from common package - common.ClickHouseOperationDuration.WithLabelValues(c.network, c.processor, operation, table, status, "").Observe(duration.Seconds()) - common.ClickHouseOperationTotal.WithLabelValues(c.network, c.processor, operation, table, status, "").Inc() + if network == "" { + continue + } + + stat := c.pool.Stat() + + // Set gauge values (current state) + common.ClickHousePoolAcquiredResources.WithLabelValues(network, c.processor).Set(float64(stat.AcquiredResources())) + common.ClickHousePoolIdleResources.WithLabelValues(network, c.processor).Set(float64(stat.IdleResources())) + common.ClickHousePoolConstructingResources.WithLabelValues(network, c.processor).Set(float64(stat.ConstructingResources())) + common.ClickHousePoolTotalResources.WithLabelValues(network, c.processor).Set(float64(stat.TotalResources())) + common.ClickHousePoolMaxResources.WithLabelValues(network, c.processor).Set(float64(stat.MaxResources())) + + // Set cumulative duration gauges + common.ClickHousePoolAcquireDuration.WithLabelValues(network, c.processor).Set(stat.AcquireDuration().Seconds()) + common.ClickHousePoolEmptyAcquireWaitDuration.WithLabelValues(network, c.processor).Set(stat.EmptyAcquireWaitTime().Seconds()) + + // Calculate deltas and add to counters + currentAcquireCount := stat.AcquireCount() + currentEmptyAcquireCount := stat.EmptyAcquireCount() + currentCanceledAcquireCount := stat.CanceledAcquireCount() + + if prevAcquireCount > 0 { + delta := currentAcquireCount - prevAcquireCount + if delta > 0 { + common.ClickHousePoolAcquireTotal.WithLabelValues(network, c.processor).Add(float64(delta)) + } + } + + if prevEmptyAcquireCount > 0 { + delta := currentEmptyAcquireCount - prevEmptyAcquireCount + if delta > 0 { + common.ClickHousePoolEmptyAcquireTotal.WithLabelValues(network, c.processor).Add(float64(delta)) + } + } + + if prevCanceledAcquireCount > 0 { + delta := currentCanceledAcquireCount - prevCanceledAcquireCount + if delta > 0 { + common.ClickHousePoolCanceledAcquireTotal.WithLabelValues(network, c.processor).Add(float64(delta)) + } + } + + prevAcquireCount = currentAcquireCount + prevEmptyAcquireCount = currentEmptyAcquireCount + prevCanceledAcquireCount = currentCanceledAcquireCount + } + } } diff --git a/pkg/clickhouse/client_integration_test.go b/pkg/clickhouse/client_integration_test.go new file mode 100644 index 0000000..6497c62 --- /dev/null +++ b/pkg/clickhouse/client_integration_test.go @@ -0,0 +1,184 @@ +//go:build integration + +package clickhouse + +import ( + "testing" + + "github.com/ethpandaops/execution-processor/internal/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Integration tests using testcontainers - run with: go test -tags=integration ./... + +func TestClient_Integration_Container_New(t *testing.T) { + conn := testutil.NewClickHouseContainer(t) + + cfg := &Config{ + Addr: conn.Addr(), + Database: conn.Database, + Username: conn.Username, + Password: conn.Password, + Compression: "lz4", + } + + client, err := New(cfg) + require.NoError(t, err) + require.NotNil(t, client) + + err = client.Stop() + require.NoError(t, err) +} + +func TestClient_Integration_Container_StartStop(t *testing.T) { + conn := testutil.NewClickHouseContainer(t) + + cfg := &Config{ + Addr: conn.Addr(), + Database: conn.Database, + Username: conn.Username, + Password: conn.Password, + Compression: "lz4", + Network: "test", + } + + client, err := New(cfg) + require.NoError(t, err) + + // Start should ping successfully + err = client.Start() + require.NoError(t, err) + + // Stop should close pool + err = client.Stop() + require.NoError(t, err) +} + +func TestClient_Integration_Container_Execute(t *testing.T) { + conn := testutil.NewClickHouseContainer(t) + + cfg := &Config{ + Addr: conn.Addr(), + Database: conn.Database, + Username: conn.Username, + Password: conn.Password, + Compression: "lz4", + Network: "test", + } + + client, err := New(cfg) + require.NoError(t, err) + + defer func() { _ = client.Stop() }() + + err = client.Start() + require.NoError(t, err) + + // Execute DDL - Create and drop a table + err = client.Execute(t.Context(), ` + CREATE TABLE IF NOT EXISTS test_execute ( + id UInt64 + ) ENGINE = Memory + `) + require.NoError(t, err) + + err = client.Execute(t.Context(), "DROP TABLE IF EXISTS test_execute") + require.NoError(t, err) +} + +func TestClient_Integration_Container_QueryUInt64(t *testing.T) { + conn := testutil.NewClickHouseContainer(t) + + cfg := &Config{ + Addr: conn.Addr(), + Database: conn.Database, + Username: conn.Username, + Password: conn.Password, + Compression: "lz4", + Network: "test", + } + + client, err := New(cfg) + require.NoError(t, err) + + defer func() { _ = client.Stop() }() + + err = client.Start() + require.NoError(t, err) + + // QueryUInt64 returns a single UInt64 value + result, err := client.QueryUInt64(t.Context(), "SELECT toUInt64(42) as value", "value") + require.NoError(t, err) + require.NotNil(t, result) + assert.Equal(t, uint64(42), *result) +} + +func TestClient_Integration_Container_QueryMinMaxUInt64(t *testing.T) { + conn := testutil.NewClickHouseContainer(t) + + cfg := &Config{ + Addr: conn.Addr(), + Database: conn.Database, + Username: conn.Username, + Password: conn.Password, + Compression: "lz4", + Network: "test", + } + + client, err := New(cfg) + require.NoError(t, err) + + defer func() { _ = client.Stop() }() + + err = client.Start() + require.NoError(t, err) + + // QueryMinMaxUInt64 returns min and max values + minVal, maxVal, err := client.QueryMinMaxUInt64(t.Context(), "SELECT toUInt64(10) as min, toUInt64(100) as max") + require.NoError(t, err) + require.NotNil(t, minVal) + require.NotNil(t, maxVal) + assert.Equal(t, uint64(10), *minVal) + assert.Equal(t, uint64(100), *maxVal) +} + +func TestClient_Integration_Container_IsStorageEmpty(t *testing.T) { + conn := testutil.NewClickHouseContainer(t) + + cfg := &Config{ + Addr: conn.Addr(), + Database: conn.Database, + Username: conn.Username, + Password: conn.Password, + Compression: "lz4", + Network: "test", + } + + client, err := New(cfg) + require.NoError(t, err) + + defer func() { _ = client.Stop() }() + + err = client.Start() + require.NoError(t, err) + + // Create a table with ReplacingMergeTree engine (supports FINAL) + err = client.Execute(t.Context(), ` + CREATE TABLE IF NOT EXISTS test_empty_check ( + id UInt64, + name String + ) ENGINE = ReplacingMergeTree() + ORDER BY id + `) + require.NoError(t, err) + + // Should be empty + isEmpty, err := client.IsStorageEmpty(t.Context(), "test_empty_check", nil) + require.NoError(t, err) + assert.True(t, isEmpty) + + // Drop the table + err = client.Execute(t.Context(), "DROP TABLE IF EXISTS test_empty_check") + require.NoError(t, err) +} diff --git a/pkg/clickhouse/client_test.go b/pkg/clickhouse/client_test.go index 7b175c3..20473e4 100644 --- a/pkg/clickhouse/client_test.go +++ b/pkg/clickhouse/client_test.go @@ -2,328 +2,475 @@ package clickhouse import ( "context" - "net/http" - "net/http/httptest" + "errors" + "io" + "net" + "os" + "syscall" "testing" "time" + "github.com/ClickHouse/ch-go" + "github.com/ClickHouse/ch-go/compress" + "github.com/ClickHouse/ch-go/proto" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestHTTPClient_QueryOne(t *testing.T) { - type testResult struct { - Count int `json:"count"` +func TestConfig_Validate(t *testing.T) { + tests := []struct { + name string + config Config + expectError bool + }{ + { + name: "valid config with addr", + config: Config{ + Addr: "localhost:9000", + }, + expectError: false, + }, + { + name: "missing addr", + config: Config{}, + expectError: true, + }, + { + name: "valid config with all fields", + config: Config{ + Addr: "localhost:9000", + Database: "test_db", + Username: "default", + Password: "secret", + MaxConns: 20, + MinConns: 5, + ConnMaxLifetime: 2 * time.Hour, + ConnMaxIdleTime: 1 * time.Hour, + HealthCheckPeriod: 30 * time.Second, + Compression: "zstd", + }, + expectError: false, + }, } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.config.Validate() + if tt.expectError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestConfig_SetDefaults(t *testing.T) { + config := Config{ + Addr: "localhost:9000", + } + + config.SetDefaults() + + assert.Equal(t, "default", config.Database) + assert.Equal(t, int32(10), config.MaxConns) + assert.Equal(t, int32(2), config.MinConns) + assert.Equal(t, time.Hour, config.ConnMaxLifetime) + assert.Equal(t, 30*time.Minute, config.ConnMaxIdleTime) + assert.Equal(t, time.Minute, config.HealthCheckPeriod) + assert.Equal(t, 10*time.Second, config.DialTimeout) + assert.Equal(t, "lz4", config.Compression) + assert.Equal(t, 60*time.Second, config.QueryTimeout) + assert.Equal(t, 10*time.Second, config.RetryMaxDelay) +} + +func TestConfig_SetDefaults_PreservesValues(t *testing.T) { + config := Config{ + Addr: "localhost:9000", + Database: "custom_db", + MaxConns: 50, + MinConns: 10, + ConnMaxLifetime: 3 * time.Hour, + ConnMaxIdleTime: 2 * time.Hour, + HealthCheckPeriod: 5 * time.Minute, + DialTimeout: 30 * time.Second, + Compression: "zstd", + QueryTimeout: 120 * time.Second, + RetryMaxDelay: 30 * time.Second, + } + + config.SetDefaults() + + // Should preserve custom values + assert.Equal(t, "custom_db", config.Database) + assert.Equal(t, int32(50), config.MaxConns) + assert.Equal(t, int32(10), config.MinConns) + assert.Equal(t, 3*time.Hour, config.ConnMaxLifetime) + assert.Equal(t, 2*time.Hour, config.ConnMaxIdleTime) + assert.Equal(t, 5*time.Minute, config.HealthCheckPeriod) + assert.Equal(t, 30*time.Second, config.DialTimeout) + assert.Equal(t, "zstd", config.Compression) + assert.Equal(t, 120*time.Second, config.QueryTimeout) + assert.Equal(t, 30*time.Second, config.RetryMaxDelay) +} + +func TestClient_withQueryTimeout(t *testing.T) { tests := []struct { - name string - query string - serverResponse string - serverStatus int - expectedResult *testResult - expectedError bool + name string + queryTimeout time.Duration + ctxHasDeadline bool + expectNewContext bool }{ { - name: "successful query with result", - query: "SELECT count FROM test_table", - serverResponse: `{ - "data": [{"count": 42}], - "meta": [{"name": "count", "type": "UInt64"}], - "rows": 1, - "rows_read": 1 - }`, - serverStatus: http.StatusOK, - expectedResult: &testResult{Count: 42}, - expectedError: false, + name: "applies timeout when no deadline exists", + queryTimeout: 5 * time.Second, + ctxHasDeadline: false, + expectNewContext: true, }, { - name: "query with no results", - query: "SELECT count FROM test_table WHERE id = 999", - serverResponse: `{ - "data": [], - "meta": [{"name": "count", "type": "UInt64"}], - "rows": 0, - "rows_read": 0 - }`, - serverStatus: http.StatusOK, - expectedResult: nil, - expectedError: false, + name: "preserves existing deadline", + queryTimeout: 5 * time.Second, + ctxHasDeadline: true, + expectNewContext: false, }, { - name: "server error", - query: "SELECT invalid", - serverResponse: `{"exception": "DB::Exception: Unknown identifier: invalid"}`, - serverStatus: http.StatusBadRequest, - expectedResult: nil, - expectedError: true, + name: "no-op when timeout is zero", + queryTimeout: 0, + ctxHasDeadline: false, + expectNewContext: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - // Create test server - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - assert.Equal(t, "POST", r.Method) - assert.Contains(t, r.Header.Get("Content-Type"), "text/plain") + client := &Client{ + config: &Config{ + QueryTimeout: tt.queryTimeout, + }, + } - w.WriteHeader(tt.serverStatus) - _, _ = w.Write([]byte(tt.serverResponse)) - })) - defer server.Close() + var ctx context.Context - // Create client - client, err := New(&Config{ - URL: server.URL, - }) - require.NoError(t, err) + var originalCancel context.CancelFunc - // Execute query - var result testResult + if tt.ctxHasDeadline { + ctx, originalCancel = context.WithTimeout(context.Background(), 10*time.Second) + defer originalCancel() + } else { + ctx = context.Background() + } - err = client.QueryOne(context.Background(), tt.query, &result) + newCtx, cancel := client.withQueryTimeout(ctx) + defer cancel() - if tt.expectedError { - assert.Error(t, err) - } else { - assert.NoError(t, err) + _, hasDeadline := newCtx.Deadline() - if tt.expectedResult != nil { - assert.Equal(t, *tt.expectedResult, result) - } + if tt.expectNewContext { + require.True(t, hasDeadline, "expected context to have deadline") + } else if tt.ctxHasDeadline { + require.True(t, hasDeadline, "expected original deadline to be preserved") + } else { + require.False(t, hasDeadline, "expected no deadline") } }) } } -func TestHTTPClient_QueryMany(t *testing.T) { - // Create test server - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - response := `{ - "data": [ - {"id": 1, "name": "first"}, - {"id": 2, "name": "second"}, - {"id": 3, "name": "third"} - ], - "meta": [ - {"name": "id", "type": "UInt64"}, - {"name": "name", "type": "String"} - ], - "rows": 3, - "rows_read": 3 - }` - - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte(response)) - })) - defer server.Close() - - // Create client - client, err := New(&Config{ - URL: server.URL, - }) - require.NoError(t, err) +// Integration tests - require CLICKHOUSE_ADDR environment variable - // Execute query - type testRow struct { - ID int `json:"id"` - Name string `json:"name"` +func TestClient_Integration_New(t *testing.T) { + addr := os.Getenv("CLICKHOUSE_ADDR") + if addr == "" { + t.Skip("CLICKHOUSE_ADDR not set, skipping integration test") } - var results []testRow + cfg := &Config{ + Addr: addr, + Database: "default", + Compression: "lz4", + } - err = client.QueryMany(context.Background(), "SELECT id, name FROM test_table", &results) + client, err := New(cfg) require.NoError(t, err) + require.NotNil(t, client) - // Verify results - assert.Len(t, results, 3) - assert.Equal(t, 1, results[0].ID) - assert.Equal(t, "first", results[0].Name) - assert.Equal(t, 2, results[1].ID) - assert.Equal(t, "second", results[1].Name) + err = client.Stop() + require.NoError(t, err) } -func TestHTTPClient_BulkInsert(t *testing.T) { - var receivedBody string +func TestClient_Integration_StartStop(t *testing.T) { + addr := os.Getenv("CLICKHOUSE_ADDR") + if addr == "" { + t.Skip("CLICKHOUSE_ADDR not set, skipping integration test") + } - // Create test server - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - assert.Equal(t, "POST", r.Method) + cfg := &Config{ + Addr: addr, + Database: "default", + Compression: "lz4", + Network: "test", + } - // Read body - body := make([]byte, r.ContentLength) - _, _ = r.Body.Read(body) - receivedBody = string(body) + client, err := New(cfg) + require.NoError(t, err) - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte("Ok.")) - })) - defer server.Close() + // Start should dial and connect successfully + err = client.Start() + require.NoError(t, err) - // Create client - client, err := New(&Config{ - URL: server.URL, - }) + // Stop should close pool + err = client.Stop() require.NoError(t, err) +} - // Test data - type testData struct { - ID int `json:"id"` - Name string `json:"name"` - Timestamp int64 `json:"timestamp"` +func TestClient_Integration_Execute(t *testing.T) { + addr := os.Getenv("CLICKHOUSE_ADDR") + if addr == "" { + t.Skip("CLICKHOUSE_ADDR not set, skipping integration test") } - data := []testData{ - {ID: 1, Name: "first", Timestamp: 1000}, - {ID: 2, Name: "second", Timestamp: 2000}, - {ID: 3, Name: "third", Timestamp: 3000}, + cfg := &Config{ + Addr: addr, + Database: "default", + Compression: "lz4", + Network: "test", } - // Execute bulk insert - err = client.BulkInsert(context.Background(), "test_table", data) + client, err := New(cfg) require.NoError(t, err) - // Verify the request body - assert.Contains(t, receivedBody, "INSERT INTO test_table FORMAT JSONEachRow") - assert.Contains(t, receivedBody, `{"id":1,"name":"first","timestamp":1000}`) - assert.Contains(t, receivedBody, `{"id":2,"name":"second","timestamp":2000}`) - assert.Contains(t, receivedBody, `{"id":3,"name":"third","timestamp":3000}`) -} + defer func() { _ = client.Stop() }() -func TestHTTPClient_Execute(t *testing.T) { - // Create test server - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte("Ok.")) - })) - defer server.Close() - - // Create client - client, err := New(&Config{ - URL: server.URL, - }) + err = client.Start() require.NoError(t, err) - // Execute command - err = client.Execute(context.Background(), "CREATE TABLE test_table (id UInt64) ENGINE = Memory") + // Execute a simple query + err = client.Execute(t.Context(), "SELECT 1") require.NoError(t, err) } -func TestHTTPClient_Timeouts(t *testing.T) { - // Create a slow server - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - time.Sleep(2 * time.Second) - w.WriteHeader(http.StatusOK) - })) - defer server.Close() - - // Create client with short timeout - client, err := New(&Config{ - URL: server.URL, - QueryTimeout: 100 * time.Millisecond, - }) +func TestClient_Integration_QueryUInt64(t *testing.T) { + addr := os.Getenv("CLICKHOUSE_ADDR") + if addr == "" { + t.Skip("CLICKHOUSE_ADDR not set, skipping integration test") + } + + cfg := &Config{ + Addr: addr, + Database: "default", + Compression: "lz4", + Network: "test", + } + + client, err := New(cfg) require.NoError(t, err) - // Execute query - should timeout - ctx := context.Background() + defer func() { _ = client.Stop() }() - var result struct{} + err = client.Start() + require.NoError(t, err) - err = client.QueryOne(ctx, "SELECT 1", &result) - assert.Error(t, err) - // The timeout causes the request to fail - we just need to ensure an error occurred + result, err := client.QueryUInt64(t.Context(), "SELECT toUInt64(42) as value", "value") + require.NoError(t, err) + require.NotNil(t, result) + assert.Equal(t, uint64(42), *result) } -func TestHTTPClient_StartStop(t *testing.T) { - // Create test server - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte("Ok.")) - })) - defer server.Close() - - // Create client - client, err := New(&Config{ - URL: server.URL, - }) +func TestClient_Integration_IsStorageEmpty(t *testing.T) { + addr := os.Getenv("CLICKHOUSE_ADDR") + if addr == "" { + t.Skip("CLICKHOUSE_ADDR not set, skipping integration test") + } + + cfg := &Config{ + Addr: addr, + Database: "default", + Compression: "lz4", + Network: "test", + } + + client, err := New(cfg) require.NoError(t, err) - // Start should succeed + defer func() { _ = client.Stop() }() + err = client.Start() - assert.NoError(t, err) + require.NoError(t, err) - // Stop should succeed - err = client.Stop() - assert.NoError(t, err) + // Create a temporary table + err = client.Execute(t.Context(), ` + CREATE TABLE IF NOT EXISTS test_empty_check ( + id UInt64, + name String + ) ENGINE = Memory + `) + require.NoError(t, err) + + // Should be empty + isEmpty, err := client.IsStorageEmpty(t.Context(), "test_empty_check", nil) + require.NoError(t, err) + assert.True(t, isEmpty) + + // Drop the table + err = client.Execute(t.Context(), "DROP TABLE IF EXISTS test_empty_check") + require.NoError(t, err) } -func TestConfig_Validate(t *testing.T) { +// mockNetError implements net.Error for testing. +type mockNetError struct { + timeout bool + temporary bool +} + +func (e *mockNetError) Error() string { return "mock network error" } +func (e *mockNetError) Timeout() bool { return e.timeout } +func (e *mockNetError) Temporary() bool { return e.temporary } + +func TestIsRetryableError(t *testing.T) { tests := []struct { - name string - config Config - expectError bool + name string + err error + expected bool }{ + // Nil error { - name: "valid config", - config: Config{ - URL: "http://localhost:8123", - }, - expectError: false, + name: "nil error", + err: nil, + expected: false, }, + // Context errors - non-retryable { - name: "missing URL", - config: Config{}, - expectError: true, + name: "context canceled", + err: context.Canceled, + expected: false, + }, + { + name: "context deadline exceeded", + err: context.DeadlineExceeded, + expected: false, + }, + // ch-go sentinel errors - non-retryable + { + name: "ch.ErrClosed", + err: ch.ErrClosed, + expected: false, + }, + { + name: "wrapped ch.ErrClosed", + err: errors.Join(errors.New("operation failed"), ch.ErrClosed), + expected: false, + }, + // ch-go server exceptions - retryable codes (using ch.Exception which implements error) + { + name: "proto.ErrTimeoutExceeded", + err: &ch.Exception{Code: proto.ErrTimeoutExceeded, Message: "timeout"}, + expected: true, + }, + { + name: "proto.ErrNoFreeConnection", + err: &ch.Exception{Code: proto.ErrNoFreeConnection, Message: "no free connection"}, + expected: true, + }, + { + name: "proto.ErrTooManySimultaneousQueries", + err: &ch.Exception{Code: proto.ErrTooManySimultaneousQueries, Message: "rate limited"}, + expected: true, + }, + { + name: "proto.ErrSocketTimeout", + err: &ch.Exception{Code: proto.ErrSocketTimeout, Message: "socket timeout"}, + expected: true, + }, + { + name: "proto.ErrNetworkError", + err: &ch.Exception{Code: proto.ErrNetworkError, Message: "network error"}, + expected: true, + }, + // ch-go server exceptions - non-retryable codes + { + name: "proto.ErrBadArguments", + err: &ch.Exception{Code: proto.ErrBadArguments, Message: "bad arguments"}, + expected: false, + }, + { + name: "proto.ErrUnknownTable", + err: &ch.Exception{Code: proto.ErrUnknownTable, Message: "unknown table"}, + expected: false, + }, + // Data corruption - non-retryable + { + name: "compress.CorruptedDataErr", + err: &compress.CorruptedDataErr{}, + expected: false, + }, + // Network errors + { + name: "network timeout error", + err: &mockNetError{timeout: true}, + expected: true, + }, + { + name: "network non-timeout error", + err: &mockNetError{timeout: false}, + expected: false, + }, + // Syscall errors - retryable + { + name: "syscall.ECONNRESET", + err: syscall.ECONNRESET, + expected: true, + }, + { + name: "syscall.ECONNREFUSED", + err: syscall.ECONNREFUSED, + expected: true, + }, + { + name: "syscall.EPIPE", + err: syscall.EPIPE, + expected: true, + }, + { + name: "io.EOF", + err: io.EOF, + expected: true, + }, + { + name: "io.ErrUnexpectedEOF", + err: io.ErrUnexpectedEOF, + expected: true, + }, + // String pattern fallback - retryable + { + name: "connection reset string", + err: errors.New("connection reset by peer"), + expected: true, + }, + { + name: "server overloaded string", + err: errors.New("server is overloaded"), + expected: true, + }, + { + name: "too many connections string", + err: errors.New("too many connections"), + expected: true, + }, + // Unknown errors - non-retryable + { + name: "unknown error", + err: errors.New("some unknown error"), + expected: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := tt.config.Validate() - if tt.expectError { - assert.Error(t, err) - } else { - assert.NoError(t, err) - } + result := isRetryableError(tt.err) + assert.Equal(t, tt.expected, result) }) } } -func TestConfig_SetDefaults(t *testing.T) { - config := Config{ - URL: "http://localhost:8123", - } - - config.SetDefaults() - - assert.Equal(t, 30*time.Second, config.QueryTimeout) - assert.Equal(t, 5*time.Minute, config.InsertTimeout) - assert.Equal(t, 30*time.Second, config.KeepAlive) -} - -func TestHTTPClient_DebugLogging(t *testing.T) { - // Create test server - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - response := `{"data": [{"value": 1}], "meta": [], "rows": 1}` - - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte(response)) - })) - defer server.Close() - - // Create client with debug enabled - client, err := New(&Config{ - URL: server.URL, - Debug: true, - }) - require.NoError(t, err) - - // Execute query - debug logging should not cause errors - var result struct { - Value int `json:"value"` - } - - err = client.QueryOne(context.Background(), "SELECT 1 as value", &result) - assert.NoError(t, err) - assert.Equal(t, 1, result.Value) -} +// Ensure mockNetError implements net.Error. +var _ net.Error = (*mockNetError)(nil) diff --git a/pkg/clickhouse/config.go b/pkg/clickhouse/config.go index fe218b4..62410e0 100644 --- a/pkg/clickhouse/config.go +++ b/pkg/clickhouse/config.go @@ -5,35 +5,97 @@ import ( "time" ) +// Config holds configuration for ClickHouse ch-go native client. +// //nolint:tagliatelle // YAML config uses snake_case by convention type Config struct { - URL string `yaml:"url"` - QueryTimeout time.Duration `yaml:"query_timeout"` - InsertTimeout time.Duration `yaml:"insert_timeout"` - Network string `yaml:"network"` - Processor string `yaml:"processor"` - Debug bool `yaml:"debug"` - KeepAlive time.Duration `yaml:"keep_alive"` + // Connection + Addr string `yaml:"addr"` // Native protocol address, e.g., "localhost:9000" + Database string `yaml:"database"` // Database name, default: "default" + Username string `yaml:"username"` + Password string `yaml:"password"` + + // Pool settings + MaxConns int32 `yaml:"max_conns"` // Maximum connections in pool, default: 10 + MinConns int32 `yaml:"min_conns"` // Minimum connections in pool, default: 2 + ConnMaxLifetime time.Duration `yaml:"conn_max_lifetime"` // Maximum connection lifetime, default: 1h + ConnMaxIdleTime time.Duration `yaml:"conn_max_idle_time"` // Maximum idle time, default: 30m + HealthCheckPeriod time.Duration `yaml:"health_check_period"` // Health check period, default: 1m + DialTimeout time.Duration `yaml:"dial_timeout"` // Dial timeout, default: 10s + + // Performance + Compression string `yaml:"compression"` // Compression: lz4, zstd, none (default: lz4) + + // Retry settings + MaxRetries int `yaml:"max_retries"` // Maximum retry attempts, default: 3 + RetryBaseDelay time.Duration `yaml:"retry_base_delay"` // Base delay for exponential backoff, default: 100ms + RetryMaxDelay time.Duration `yaml:"retry_max_delay"` // Max delay between retries, default: 10s + + // Timeout settings + QueryTimeout time.Duration `yaml:"query_timeout"` // Query timeout per attempt, default: 60s + + // Metrics labels + Network string `yaml:"network"` + Processor string `yaml:"processor"` + Debug bool `yaml:"debug"` } +// Validate checks if the config is valid. func (c *Config) Validate() error { - if c.URL == "" { - return fmt.Errorf("URL is required") + if c.Addr == "" { + return fmt.Errorf("addr is required") } return nil } +// SetDefaults sets default values for unset fields. func (c *Config) SetDefaults() { - if c.QueryTimeout == 0 { - c.QueryTimeout = 30 * time.Second + if c.Database == "" { + c.Database = "default" } - if c.InsertTimeout == 0 { - c.InsertTimeout = 5 * time.Minute + if c.MaxConns == 0 { + c.MaxConns = 10 } - if c.KeepAlive == 0 { - c.KeepAlive = 30 * time.Second + if c.DialTimeout == 0 { + c.DialTimeout = 10 * time.Second + } + + if c.MinConns == 0 { + c.MinConns = 2 + } + + if c.ConnMaxLifetime == 0 { + c.ConnMaxLifetime = time.Hour + } + + if c.ConnMaxIdleTime == 0 { + c.ConnMaxIdleTime = 30 * time.Minute + } + + if c.HealthCheckPeriod == 0 { + c.HealthCheckPeriod = time.Minute + } + + if c.Compression == "" { + c.Compression = "lz4" + } + + if c.MaxRetries == 0 { + c.MaxRetries = 3 + } + + if c.RetryBaseDelay == 0 { + c.RetryBaseDelay = 100 * time.Millisecond + } + + if c.RetryMaxDelay == 0 { + c.RetryMaxDelay = 10 * time.Second + } + + if c.QueryTimeout == 0 { + c.QueryTimeout = 60 * time.Second } } diff --git a/pkg/clickhouse/interface.go b/pkg/clickhouse/interface.go index 2b56635..ff506d6 100644 --- a/pkg/clickhouse/interface.go +++ b/pkg/clickhouse/interface.go @@ -1,23 +1,31 @@ package clickhouse -import "context" +import ( + "context" + + "github.com/ClickHouse/ch-go" +) // ClientInterface defines the methods for interacting with ClickHouse. type ClientInterface interface { - // QueryOne executes a query and returns a single result - QueryOne(ctx context.Context, query string, dest interface{}) error - // QueryMany executes a query and returns multiple results - QueryMany(ctx context.Context, query string, dest interface{}) error - // Execute runs a query without expecting results - Execute(ctx context.Context, query string) error - // BulkInsert performs a bulk insert operation - BulkInsert(ctx context.Context, table string, data interface{}) error - // IsStorageEmpty checks if a table has any records matching the given conditions - IsStorageEmpty(ctx context.Context, table string, conditions map[string]interface{}) (bool, error) - // SetNetwork updates the network name for metrics labeling - SetNetwork(network string) // Start initializes the client Start() error // Stop closes the client Stop() error + // SetNetwork updates the network name for metrics labeling + SetNetwork(network string) + // Do executes a ch-go query directly for streaming operations + Do(ctx context.Context, query ch.Query) error + // Execute runs a query without expecting results + Execute(ctx context.Context, query string) error + // IsStorageEmpty checks if a table has any records matching the given conditions + IsStorageEmpty(ctx context.Context, table string, conditions map[string]any) (bool, error) + + // QueryUInt64 executes a query and returns a single UInt64 value from the specified column. + // Returns nil if no rows are found. + QueryUInt64(ctx context.Context, query string, columnName string) (*uint64, error) + // QueryMinMaxUInt64 executes a query that returns min and max UInt64 values. + // The query must return columns named "min" and "max". + // Returns nil for both values if no rows are found. + QueryMinMaxUInt64(ctx context.Context, query string) (minVal, maxVal *uint64, err error) } diff --git a/pkg/clickhouse/mock.go b/pkg/clickhouse/mock.go index 9473136..ff1bc93 100644 --- a/pkg/clickhouse/mock.go +++ b/pkg/clickhouse/mock.go @@ -4,21 +4,21 @@ package clickhouse import ( "context" - "encoding/json" - "fmt" - "reflect" + + "github.com/ClickHouse/ch-go" ) // MockClient is a mock implementation of ClientInterface for testing. // It should only be used in test files, not in production code. type MockClient struct { // Function fields that can be set by tests - QueryOneFunc func(ctx context.Context, query string, dest interface{}) error - QueryManyFunc func(ctx context.Context, query string, dest interface{}) error - ExecuteFunc func(ctx context.Context, query string) error - BulkInsertFunc func(ctx context.Context, table string, data interface{}) error - StartFunc func() error - StopFunc func() error + ExecuteFunc func(ctx context.Context, query string) error + IsStorageEmptyFunc func(ctx context.Context, table string, conditions map[string]any) (bool, error) + StartFunc func() error + StopFunc func() error + DoFunc func(ctx context.Context, query ch.Query) error + QueryUInt64Func func(ctx context.Context, query string, columnName string) (*uint64, error) + QueryMinMaxUInt64Func func(ctx context.Context, query string) (minVal, maxVal *uint64, err error) // Track calls for assertions Calls []MockCall @@ -27,23 +27,17 @@ type MockClient struct { // MockCall represents a method call made to the mock. type MockCall struct { Method string - Args []interface{} + Args []any } // NewMockClient creates a new mock client with default implementations. func NewMockClient() *MockClient { return &MockClient{ - QueryOneFunc: func(ctx context.Context, query string, dest interface{}) error { - return nil - }, - QueryManyFunc: func(ctx context.Context, query string, dest interface{}) error { - return nil - }, ExecuteFunc: func(ctx context.Context, query string) error { return nil }, - BulkInsertFunc: func(ctx context.Context, table string, data interface{}) error { - return nil + IsStorageEmptyFunc: func(ctx context.Context, table string, conditions map[string]any) (bool, error) { + return true, nil }, StartFunc: func() error { return nil @@ -51,43 +45,52 @@ func NewMockClient() *MockClient { StopFunc: func() error { return nil }, + DoFunc: func(ctx context.Context, query ch.Query) error { + return nil + }, + QueryUInt64Func: func(ctx context.Context, query string, columnName string) (*uint64, error) { + return nil, nil + }, + QueryMinMaxUInt64Func: func(ctx context.Context, query string) (minVal, maxVal *uint64, err error) { + return nil, nil, nil + }, Calls: make([]MockCall, 0), } } -// QueryOne implements ClientInterface. -func (m *MockClient) QueryOne(ctx context.Context, query string, dest interface{}) error { +// QueryUInt64 implements ClientInterface. +func (m *MockClient) QueryUInt64(ctx context.Context, query string, columnName string) (*uint64, error) { m.Calls = append(m.Calls, MockCall{ - Method: "QueryOne", - Args: []interface{}{ctx, query, dest}, + Method: "QueryUInt64", + Args: []any{ctx, query, columnName}, }) - if m.QueryOneFunc != nil { - return m.QueryOneFunc(ctx, query, dest) + if m.QueryUInt64Func != nil { + return m.QueryUInt64Func(ctx, query, columnName) } - return nil + return nil, nil //nolint:nilnil // valid for mock: nil means no value found } -// QueryMany implements ClientInterface. -func (m *MockClient) QueryMany(ctx context.Context, query string, dest interface{}) error { +// QueryMinMaxUInt64 implements ClientInterface. +func (m *MockClient) QueryMinMaxUInt64(ctx context.Context, query string) (minVal, maxVal *uint64, err error) { m.Calls = append(m.Calls, MockCall{ - Method: "QueryMany", - Args: []interface{}{ctx, query, dest}, + Method: "QueryMinMaxUInt64", + Args: []any{ctx, query}, }) - if m.QueryManyFunc != nil { - return m.QueryManyFunc(ctx, query, dest) + if m.QueryMinMaxUInt64Func != nil { + return m.QueryMinMaxUInt64Func(ctx, query) } - return nil + return nil, nil, nil } // Execute implements ClientInterface. func (m *MockClient) Execute(ctx context.Context, query string) error { m.Calls = append(m.Calls, MockCall{ Method: "Execute", - Args: []interface{}{ctx, query}, + Args: []any{ctx, query}, }) if m.ExecuteFunc != nil { @@ -97,25 +100,33 @@ func (m *MockClient) Execute(ctx context.Context, query string) error { return nil } -// BulkInsert implements ClientInterface. -func (m *MockClient) BulkInsert(ctx context.Context, table string, data interface{}) error { +// IsStorageEmpty implements ClientInterface. +func (m *MockClient) IsStorageEmpty(ctx context.Context, table string, conditions map[string]any) (bool, error) { m.Calls = append(m.Calls, MockCall{ - Method: "BulkInsert", - Args: []interface{}{ctx, table, data}, + Method: "IsStorageEmpty", + Args: []any{ctx, table, conditions}, }) - if m.BulkInsertFunc != nil { - return m.BulkInsertFunc(ctx, table, data) + if m.IsStorageEmptyFunc != nil { + return m.IsStorageEmptyFunc(ctx, table, conditions) } - return nil + return true, nil +} + +// SetNetwork implements ClientInterface. +func (m *MockClient) SetNetwork(network string) { + m.Calls = append(m.Calls, MockCall{ + Method: "SetNetwork", + Args: []any{network}, + }) } // Start implements ClientInterface. func (m *MockClient) Start() error { m.Calls = append(m.Calls, MockCall{ Method: "Start", - Args: []interface{}{}, + Args: []any{}, }) if m.StartFunc != nil { @@ -129,7 +140,7 @@ func (m *MockClient) Start() error { func (m *MockClient) Stop() error { m.Calls = append(m.Calls, MockCall{ Method: "Stop", - Args: []interface{}{}, + Args: []any{}, }) if m.StopFunc != nil { @@ -139,6 +150,20 @@ func (m *MockClient) Stop() error { return nil } +// Do implements ClientInterface. +func (m *MockClient) Do(ctx context.Context, query ch.Query) error { + m.Calls = append(m.Calls, MockCall{ + Method: "Do", + Args: []any{ctx, query}, + }) + + if m.DoFunc != nil { + return m.DoFunc(ctx, query) + } + + return nil +} + // GetCallCount returns the number of times a method was called. func (m *MockClient) GetCallCount(method string) int { count := 0 @@ -164,49 +189,27 @@ func (m *MockClient) Reset() { // Helper functions for common test scenarios -// SetQueryOneResponse sets up the mock to return specific data for QueryOne. -func (m *MockClient) SetQueryOneResponse(data interface{}) { - m.QueryOneFunc = func(ctx context.Context, query string, dest interface{}) error { - // Marshal the data to JSON and then unmarshal into dest - jsonData, err := json.Marshal(data) - if err != nil { - return err - } - - return json.Unmarshal(jsonData, dest) +// SetQueryUInt64Response sets up the mock to return a specific value for QueryUInt64. +func (m *MockClient) SetQueryUInt64Response(value *uint64) { + m.QueryUInt64Func = func(ctx context.Context, query string, columnName string) (*uint64, error) { + return value, nil } } -// SetQueryManyResponse sets up the mock to return specific data for QueryMany. -func (m *MockClient) SetQueryManyResponse(data interface{}) { - m.QueryManyFunc = func(ctx context.Context, query string, dest interface{}) error { - // Use reflection to set the slice - destValue := reflect.ValueOf(dest).Elem() - srcValue := reflect.ValueOf(data) - - if destValue.Kind() != reflect.Slice || srcValue.Kind() != reflect.Slice { - return fmt.Errorf("both dest and data must be slices") - } - - destValue.Set(srcValue) - - return nil +// SetQueryMinMaxUInt64Response sets up the mock to return specific values for QueryMinMaxUInt64. +func (m *MockClient) SetQueryMinMaxUInt64Response(minVal, maxVal *uint64) { + m.QueryMinMaxUInt64Func = func(ctx context.Context, query string) (*uint64, *uint64, error) { + return minVal, maxVal, nil } } // SetError sets all functions to return the specified error. func (m *MockClient) SetError(err error) { - m.QueryOneFunc = func(ctx context.Context, query string, dest interface{}) error { - return err - } - m.QueryManyFunc = func(ctx context.Context, query string, dest interface{}) error { - return err - } m.ExecuteFunc = func(ctx context.Context, query string) error { return err } - m.BulkInsertFunc = func(ctx context.Context, table string, data interface{}) error { - return err + m.IsStorageEmptyFunc = func(ctx context.Context, table string, conditions map[string]any) (bool, error) { + return false, err } m.StartFunc = func() error { return err @@ -214,4 +217,13 @@ func (m *MockClient) SetError(err error) { m.StopFunc = func() error { return err } + m.DoFunc = func(ctx context.Context, query ch.Query) error { + return err + } + m.QueryUInt64Func = func(ctx context.Context, query string, columnName string) (*uint64, error) { + return nil, err + } + m.QueryMinMaxUInt64Func = func(ctx context.Context, query string) (*uint64, *uint64, error) { + return nil, nil, err + } } diff --git a/pkg/common/metrics.go b/pkg/common/metrics.go index 2552d6f..f452116 100644 --- a/pkg/common/metrics.go +++ b/pkg/common/metrics.go @@ -152,4 +152,57 @@ var ( Name: "execution_processor_retry_count_total", Help: "Total number of retry attempts", }, []string{"network", "processor", "reason"}) + + // ClickHouse pool metrics - gauges for current state. + ClickHousePoolAcquiredResources = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "execution_processor_clickhouse_pool_acquired_resources", + Help: "Number of currently acquired resources in the ClickHouse connection pool", + }, []string{"network", "processor"}) + + ClickHousePoolIdleResources = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "execution_processor_clickhouse_pool_idle_resources", + Help: "Number of currently idle resources in the ClickHouse connection pool", + }, []string{"network", "processor"}) + + ClickHousePoolConstructingResources = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "execution_processor_clickhouse_pool_constructing_resources", + Help: "Number of resources currently being constructed in the ClickHouse connection pool", + }, []string{"network", "processor"}) + + ClickHousePoolTotalResources = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "execution_processor_clickhouse_pool_total_resources", + Help: "Total number of resources in the ClickHouse connection pool", + }, []string{"network", "processor"}) + + ClickHousePoolMaxResources = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "execution_processor_clickhouse_pool_max_resources", + Help: "Maximum number of resources allowed in the ClickHouse connection pool", + }, []string{"network", "processor"}) + + // ClickHouse pool metrics - counters for cumulative values. + ClickHousePoolAcquireTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "execution_processor_clickhouse_pool_acquire_total", + Help: "Total number of successful resource acquisitions from the ClickHouse connection pool", + }, []string{"network", "processor"}) + + ClickHousePoolEmptyAcquireTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "execution_processor_clickhouse_pool_empty_acquire_total", + Help: "Total number of acquires that waited for a resource because the pool was empty", + }, []string{"network", "processor"}) + + ClickHousePoolCanceledAcquireTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "execution_processor_clickhouse_pool_canceled_acquire_total", + Help: "Total number of acquires that were canceled due to context cancellation", + }, []string{"network", "processor"}) + + // ClickHouse pool timing metrics - cumulative durations. + ClickHousePoolAcquireDuration = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "execution_processor_clickhouse_pool_acquire_duration_seconds", + Help: "Cumulative time spent acquiring resources from the ClickHouse connection pool", + }, []string{"network", "processor"}) + + ClickHousePoolEmptyAcquireWaitDuration = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "execution_processor_clickhouse_pool_empty_acquire_wait_duration_seconds", + Help: "Cumulative time spent waiting for a resource when pool was empty", + }, []string{"network", "processor"}) ) diff --git a/pkg/leaderelection/redis_election_test.go b/pkg/leaderelection/redis_election_test.go index 5c78e42..b772413 100644 --- a/pkg/leaderelection/redis_election_test.go +++ b/pkg/leaderelection/redis_election_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/alicebob/miniredis/v2" "github.com/ethpandaops/execution-processor/pkg/leaderelection" "github.com/redis/go-redis/v9" "github.com/sirupsen/logrus" @@ -14,24 +15,17 @@ import ( "github.com/stretchr/testify/require" ) -// skipIfRedisUnavailable checks if Redis is available and skips the test if not. -func skipIfRedisUnavailable(t *testing.T) *redis.Client { +// newTestRedis creates an in-memory Redis server for testing. +// The server and client are automatically cleaned up when the test completes. +func newTestRedis(t *testing.T) *redis.Client { t.Helper() - client := redis.NewClient(&redis.Options{ - Addr: "localhost:6379", - DB: 15, // Use high DB number for tests - }) - - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() + s := miniredis.RunT(t) + client := redis.NewClient(&redis.Options{Addr: s.Addr()}) - if err := client.Ping(ctx).Err(); err != nil { - client.Close() - t.Skipf("Redis unavailable at localhost:6379: %v", err) - - return nil - } + t.Cleanup(func() { + _ = client.Close() + }) return client } @@ -41,11 +35,7 @@ func skipIfRedisUnavailable(t *testing.T) *redis.Client { // ===================================== func TestNewRedisElector(t *testing.T) { - client := skipIfRedisUnavailable(t) - if client == nil { - return - } - defer client.Close() + client := newTestRedis(t) log := logrus.New() log.SetLevel(logrus.ErrorLevel) @@ -127,11 +117,7 @@ func TestNewRedisElector(t *testing.T) { } func TestRedisElector_GetLeaderID_NoLeader(t *testing.T) { - client := skipIfRedisUnavailable(t) - if client == nil { - return - } - defer client.Close() + client := newTestRedis(t) log := logrus.New() log.SetLevel(logrus.ErrorLevel) @@ -162,11 +148,7 @@ func TestRedisElector_GetLeaderID_NoLeader(t *testing.T) { } func TestRedisElector_StartStop(t *testing.T) { - client := skipIfRedisUnavailable(t) - if client == nil { - return - } - defer client.Close() + client := newTestRedis(t) log := logrus.New() log.SetLevel(logrus.ErrorLevel) @@ -212,11 +194,7 @@ func TestRedisElector_StartStop(t *testing.T) { } func TestRedisElector_LeadershipAcquisition(t *testing.T) { - client := skipIfRedisUnavailable(t) - if client == nil { - return - } - defer client.Close() + client := newTestRedis(t) log := logrus.New() log.SetLevel(logrus.ErrorLevel) @@ -285,11 +263,7 @@ func TestRedisElector_LeadershipAcquisition(t *testing.T) { } func TestRedisElector_MultipleNodes(t *testing.T) { - client := skipIfRedisUnavailable(t) - if client == nil { - return - } - defer client.Close() + client := newTestRedis(t) log := logrus.New() log.SetLevel(logrus.ErrorLevel) @@ -368,11 +342,7 @@ func TestRedisElector_MultipleNodes(t *testing.T) { } func TestRedisElector_LeadershipTransition(t *testing.T) { - client := skipIfRedisUnavailable(t) - if client == nil { - return - } - defer client.Close() + client := newTestRedis(t) log := logrus.New() log.SetLevel(logrus.ErrorLevel) @@ -479,11 +449,7 @@ func TestRedisElector_LeadershipTransition(t *testing.T) { // ===================================== func TestRedisElector_RenewalFailure(t *testing.T) { - client := skipIfRedisUnavailable(t) - if client == nil { - return - } - defer client.Close() + client := newTestRedis(t) log := logrus.New() log.SetLevel(logrus.ErrorLevel) @@ -545,11 +511,7 @@ func TestRedisElector_RenewalFailure(t *testing.T) { } func TestRedisElector_ConcurrentElectors(t *testing.T) { - client := skipIfRedisUnavailable(t) - if client == nil { - return - } - defer client.Close() + client := newTestRedis(t) log := logrus.New() log.SetLevel(logrus.ErrorLevel) @@ -638,11 +600,7 @@ func TestRedisElector_ConcurrentElectors(t *testing.T) { } func TestRedisElector_StopWithoutStart(t *testing.T) { - client := skipIfRedisUnavailable(t) - if client == nil { - return - } - defer client.Close() + client := newTestRedis(t) log := logrus.New() log.SetLevel(logrus.ErrorLevel) @@ -665,11 +623,7 @@ func TestRedisElector_StopWithoutStart(t *testing.T) { } func TestRedisElector_MultipleStops(t *testing.T) { - client := skipIfRedisUnavailable(t) - if client == nil { - return - } - defer client.Close() + client := newTestRedis(t) log := logrus.New() log.SetLevel(logrus.ErrorLevel) @@ -709,11 +663,7 @@ func TestRedisElector_MultipleStops(t *testing.T) { } func TestRedisElector_ContextCancellation(t *testing.T) { - client := skipIfRedisUnavailable(t) - if client == nil { - return - } - defer client.Close() + client := newTestRedis(t) log := logrus.New() log.SetLevel(logrus.ErrorLevel) diff --git a/pkg/processor/config.go b/pkg/processor/config.go index 71a9c77..d64fe57 100644 --- a/pkg/processor/config.go +++ b/pkg/processor/config.go @@ -99,8 +99,8 @@ func (c *Config) Validate() error { } if c.TransactionStructlog.Enabled { - if c.TransactionStructlog.URL == "" { - return fmt.Errorf("transaction structlog URL is required when enabled") + if c.TransactionStructlog.Addr == "" { + return fmt.Errorf("transaction structlog addr is required when enabled") } if c.TransactionStructlog.Table == "" { diff --git a/pkg/processor/manager.go b/pkg/processor/manager.go index 5ce39b6..cf5e2ac 100644 --- a/pkg/processor/manager.go +++ b/pkg/processor/manager.go @@ -312,7 +312,7 @@ func (m *Manager) initializeProcessors(ctx context.Context) error { if m.config.TransactionStructlog.Enabled { m.log.Debug("Transaction structlog processor is enabled, initializing...") - processor, err := transaction_structlog.New(ctx, &transaction_structlog.Dependencies{ + processor, err := transaction_structlog.New(&transaction_structlog.Dependencies{ Log: m.log.WithField("processor", "transaction_structlog"), Pool: m.pool, State: m.state, @@ -331,7 +331,7 @@ func (m *Manager) initializeProcessors(ctx context.Context) error { m.log.WithField("processor", "transaction_structlog").Info("Initialized processor") - if err := processor.Start(ctx); err != nil { + if err := m.startProcessorWithRetry(ctx, processor, "transaction_structlog"); err != nil { return fmt.Errorf("failed to start transaction_structlog processor: %w", err) } } else { @@ -342,7 +342,7 @@ func (m *Manager) initializeProcessors(ctx context.Context) error { if m.config.TransactionSimple.Enabled { m.log.Debug("Transaction simple processor is enabled, initializing...") - processor, err := transaction_simple.New(ctx, &transaction_simple.Dependencies{ + processor, err := transaction_simple.New(&transaction_simple.Dependencies{ Log: m.log.WithField("processor", "transaction_simple"), Pool: m.pool, State: m.state, @@ -361,7 +361,7 @@ func (m *Manager) initializeProcessors(ctx context.Context) error { m.log.WithField("processor", "transaction_simple").Info("Initialized processor") - if err := processor.Start(ctx); err != nil { + if err := m.startProcessorWithRetry(ctx, processor, "transaction_simple"); err != nil { return fmt.Errorf("failed to start transaction_simple processor: %w", err) } } else { @@ -373,6 +373,49 @@ func (m *Manager) initializeProcessors(ctx context.Context) error { return nil } +// startProcessorWithRetry starts a processor with infinite retry and capped exponential backoff. +// This ensures processors can wait for their dependencies (like ClickHouse) to become available. +func (m *Manager) startProcessorWithRetry(ctx context.Context, processor c.BlockProcessor, name string) error { + const ( + baseDelay = 100 * time.Millisecond + maxDelay = 10 * time.Second + ) + + attempt := 0 + + for { + err := processor.Start(ctx) + if err == nil { + if attempt > 0 { + m.log.WithField("processor", name).Info("Processor started successfully after retries") + } + + return nil + } + + // Calculate delay with exponential backoff, capped at maxDelay + delay := baseDelay * time.Duration(1< maxDelay { + delay = maxDelay + } + + m.log.WithFields(logrus.Fields{ + "processor": name, + "attempt": attempt + 1, + "delay": delay, + "error": err, + }).Warn("Failed to start processor, retrying...") + + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(delay): + } + + attempt++ + } +} + func (m *Manager) processBlocks(ctx context.Context) { m.log.WithField("processor_count", len(m.processors)).Debug("Starting to process blocks") diff --git a/pkg/processor/manager_test.go b/pkg/processor/manager_test.go index ed34068..c21c97d 100644 --- a/pkg/processor/manager_test.go +++ b/pkg/processor/manager_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "github.com/alicebob/miniredis/v2" "github.com/ethpandaops/execution-processor/pkg/clickhouse" "github.com/ethpandaops/execution-processor/pkg/ethereum" "github.com/ethpandaops/execution-processor/pkg/ethereum/execution" @@ -18,6 +19,20 @@ import ( "github.com/stretchr/testify/require" ) +// newTestRedis creates an in-memory Redis server for testing. +func newTestRedis(t *testing.T) *redis.Client { + t.Helper() + + s := miniredis.RunT(t) + client := redis.NewClient(&redis.Options{Addr: s.Addr()}) + + t.Cleanup(func() { + _ = client.Close() + }) + + return client +} + // Basic functionality tests func TestManager_Creation(t *testing.T) { @@ -39,11 +54,7 @@ func TestManager_Creation(t *testing.T) { }, } - redisClient := redis.NewClient(&redis.Options{ - Addr: "localhost:6379", - DB: 15, // Use high DB number for tests - }) - defer redisClient.Close() + redisClient := newTestRedis(t) // Create proper pool and state mocks poolConfig := ðereum.Config{ @@ -59,7 +70,7 @@ func TestManager_Creation(t *testing.T) { stateConfig := &state.Config{ Storage: state.StorageConfig{ Config: clickhouse.Config{ - URL: "http://localhost:8123", + Addr: "localhost:9000", }, Table: "test_manager_blocks", }, @@ -67,7 +78,7 @@ func TestManager_Creation(t *testing.T) { Enabled: false, }, } - stateManager, err := state.NewManager(context.Background(), log.WithField("component", "state"), stateConfig) + stateManager, err := state.NewManager(log.WithField("component", "state"), stateConfig) require.NoError(t, err) manager, err := processor.NewManager(log, config, pool, stateManager, redisClient, "test-prefix") @@ -94,11 +105,7 @@ func TestManager_StartStop(t *testing.T) { }, } - redisClient := redis.NewClient(&redis.Options{ - Addr: "localhost:6379", - DB: 15, - }) - defer redisClient.Close() + redisClient := newTestRedis(t) // Create proper components poolConfig := ðereum.Config{ @@ -114,7 +121,7 @@ func TestManager_StartStop(t *testing.T) { stateConfig := &state.Config{ Storage: state.StorageConfig{ Config: clickhouse.Config{ - URL: "http://localhost:8123", + Addr: "localhost:9000", }, Table: "test_manager_start_stop_blocks", }, @@ -122,7 +129,7 @@ func TestManager_StartStop(t *testing.T) { Enabled: false, }, } - stateManager, err := state.NewManager(context.Background(), log.WithField("component", "state"), stateConfig) + stateManager, err := state.NewManager(log.WithField("component", "state"), stateConfig) require.NoError(t, err) manager, err := processor.NewManager(log, config, pool, stateManager, redisClient, "test-prefix") @@ -176,11 +183,7 @@ func TestManager_MultipleStops(t *testing.T) { }, } - redisClient := redis.NewClient(&redis.Options{ - Addr: "localhost:6379", - DB: 14, - }) - defer redisClient.Close() + redisClient := newTestRedis(t) // Create proper components poolConfig := ðereum.Config{ @@ -196,7 +199,7 @@ func TestManager_MultipleStops(t *testing.T) { stateConfig := &state.Config{ Storage: state.StorageConfig{ Config: clickhouse.Config{ - URL: "http://localhost:8123", + Addr: "localhost:9000", }, Table: "test_manager_multi_stop_blocks", }, @@ -204,7 +207,7 @@ func TestManager_MultipleStops(t *testing.T) { Enabled: false, }, } - stateManager, err := state.NewManager(context.Background(), log.WithField("component", "state"), stateConfig) + stateManager, err := state.NewManager(log.WithField("component", "state"), stateConfig) require.NoError(t, err) manager, err := processor.NewManager(log, config, pool, stateManager, redisClient, "test-prefix") @@ -263,12 +266,8 @@ func TestManager_ModeSpecificLeaderElection(t *testing.T) { Enabled: false, // Disable to avoid ClickHouse requirements Table: "test_structlog", Config: clickhouse.Config{ - URL: "http://localhost:8123", + Addr: "localhost:9000", }, - BigTransactionThreshold: 500000, - BatchInsertThreshold: 50000, - BatchFlushInterval: 5 * time.Second, - BatchMaxSize: 100000, }, } @@ -286,7 +285,7 @@ func TestManager_ModeSpecificLeaderElection(t *testing.T) { stateConfig := &state.Config{ Storage: state.StorageConfig{ Config: clickhouse.Config{ - URL: "http://localhost:8123", + Addr: "localhost:9000", }, Table: "test_mode_blocks", }, @@ -294,14 +293,10 @@ func TestManager_ModeSpecificLeaderElection(t *testing.T) { Enabled: false, }, } - stateManager, err := state.NewManager(context.Background(), log.WithField("component", "state"), stateConfig) + stateManager, err := state.NewManager(log.WithField("component", "state"), stateConfig) require.NoError(t, err) - redisClient := redis.NewClient(&redis.Options{ - Addr: "localhost:6379", - DB: 15, // Use test DB - }) - defer redisClient.Close() + redisClient := newTestRedis(t) // Create manager (leader election will be initialized) manager, err := processor.NewManager(log, config, pool, stateManager, redisClient, "test-prefix") @@ -363,7 +358,7 @@ func TestManager_ConcurrentModes(t *testing.T) { stateConfig := &state.Config{ Storage: state.StorageConfig{ Config: clickhouse.Config{ - URL: "http://localhost:8123", + Addr: "localhost:9000", }, Table: "test_concurrent_blocks", }, @@ -371,21 +366,12 @@ func TestManager_ConcurrentModes(t *testing.T) { Enabled: false, }, } - stateManager, err := state.NewManager(context.Background(), log.WithField("component", "state"), stateConfig) + stateManager, err := state.NewManager(log.WithField("component", "state"), stateConfig) require.NoError(t, err) - // Use different Redis DBs to simulate separate Redis instances - forwardsRedis := redis.NewClient(&redis.Options{ - Addr: "localhost:6379", - DB: 10, // Forwards Redis DB - }) - defer forwardsRedis.Close() - - backwardsRedis := redis.NewClient(&redis.Options{ - Addr: "localhost:6379", - DB: 11, // Backwards Redis DB - }) - defer backwardsRedis.Close() + // Each miniredis instance is isolated, simulating separate Redis servers + forwardsRedis := newTestRedis(t) + backwardsRedis := newTestRedis(t) // Create both managers - they should not conflict with mode-specific leader keys forwardsManager, err := processor.NewManager(log.WithField("mode", "forwards"), forwardsConfig, pool, stateManager, forwardsRedis, "test-prefix") @@ -427,7 +413,7 @@ func TestManager_LeaderElectionDisabled(t *testing.T) { stateConfig := &state.Config{ Storage: state.StorageConfig{ Config: clickhouse.Config{ - URL: "http://localhost:8123", + Addr: "localhost:9000", }, Table: "test_no_leader_blocks", }, @@ -435,14 +421,10 @@ func TestManager_LeaderElectionDisabled(t *testing.T) { Enabled: false, }, } - stateManager, err := state.NewManager(context.Background(), log.WithField("component", "state"), stateConfig) + stateManager, err := state.NewManager(log.WithField("component", "state"), stateConfig) require.NoError(t, err) - redisClient := redis.NewClient(&redis.Options{ - Addr: "localhost:6379", - DB: 15, - }) - defer redisClient.Close() + redisClient := newTestRedis(t) // Should work fine even with leader election disabled manager, err := processor.NewManager(log, config, pool, stateManager, redisClient, "test-prefix") @@ -487,7 +469,7 @@ func TestManager_RaceConditions(t *testing.T) { stateConfig := &state.Config{ Storage: state.StorageConfig{ Config: clickhouse.Config{ - URL: "http://localhost:8123", + Addr: "localhost:9000", }, Table: "test_race_blocks", }, @@ -495,14 +477,10 @@ func TestManager_RaceConditions(t *testing.T) { Enabled: false, }, } - stateManager, err := state.NewManager(context.Background(), log.WithField("component", "state"), stateConfig) + stateManager, err := state.NewManager(log.WithField("component", "state"), stateConfig) require.NoError(t, err) - redisClient := redis.NewClient(&redis.Options{ - Addr: "localhost:6379", - DB: 15, - }) - defer redisClient.Close() + redisClient := newTestRedis(t) manager, err := processor.NewManager(log, config, pool, stateManager, redisClient, "test-prefix") require.NoError(t, err) @@ -572,7 +550,7 @@ func TestManager_ConcurrentConfiguration(t *testing.T) { stateConfig := &state.Config{ Storage: state.StorageConfig{ Config: clickhouse.Config{ - URL: "http://localhost:8123", + Addr: "localhost:9000", }, Table: "test_config_blocks", }, @@ -580,14 +558,10 @@ func TestManager_ConcurrentConfiguration(t *testing.T) { Enabled: false, }, } - stateManager, err := state.NewManager(context.Background(), log.WithField("component", "state"), stateConfig) + stateManager, err := state.NewManager(log.WithField("component", "state"), stateConfig) require.NoError(t, err) - redisClient := redis.NewClient(&redis.Options{ - Addr: "localhost:6379", - DB: 14, - }) - defer redisClient.Close() + redisClient := newTestRedis(t) _, err = processor.NewManager(log, config, pool, stateManager, redisClient, "test-prefix") require.NoError(t, err) diff --git a/pkg/processor/transaction/simple/columns.go b/pkg/processor/transaction/simple/columns.go new file mode 100644 index 0000000..7a87ed1 --- /dev/null +++ b/pkg/processor/transaction/simple/columns.go @@ -0,0 +1,196 @@ +package simple + +import ( + "math/big" + "time" + + "github.com/ClickHouse/ch-go/proto" +) + +// Columns holds all columns for transaction batch insert using ch-go columnar protocol. +type Columns struct { + UpdatedDateTime proto.ColDateTime + BlockNumber proto.ColUInt64 + BlockHash proto.ColStr + ParentHash proto.ColStr + Position proto.ColUInt32 + Hash proto.ColStr + From proto.ColStr + To *proto.ColNullable[string] + Nonce proto.ColUInt64 + GasPrice proto.ColUInt128 + Gas proto.ColUInt64 + GasTipCap *proto.ColNullable[proto.UInt128] + GasFeeCap *proto.ColNullable[proto.UInt128] + Value proto.ColUInt128 + Type proto.ColUInt8 + Size proto.ColUInt32 + CallDataSize proto.ColUInt32 + BlobGas *proto.ColNullable[uint64] + BlobGasFeeCap *proto.ColNullable[proto.UInt128] + BlobHashes *proto.ColArr[string] + Success proto.ColBool + NInputBytes proto.ColUInt32 + NInputZeroBytes proto.ColUInt32 + NInputNonzeroBytes proto.ColUInt32 + MetaNetworkName proto.ColStr +} + +// NewColumns creates a new Columns instance with all nullable and array columns initialized. +func NewColumns() *Columns { + return &Columns{ + To: new(proto.ColStr).Nullable(), + GasTipCap: new(proto.ColUInt128).Nullable(), + GasFeeCap: new(proto.ColUInt128).Nullable(), + BlobGas: new(proto.ColUInt64).Nullable(), + BlobGasFeeCap: new(proto.ColUInt128).Nullable(), + BlobHashes: new(proto.ColStr).Array(), + } +} + +// Append adds a Transaction row to all columns. +func (c *Columns) Append(tx Transaction) { + c.UpdatedDateTime.Append(time.Time(tx.UpdatedDateTime)) + c.BlockNumber.Append(tx.BlockNumber) + c.BlockHash.Append(tx.BlockHash) + c.ParentHash.Append(tx.ParentHash) + c.Position.Append(tx.Position) + c.Hash.Append(tx.Hash) + c.From.Append(tx.From) + c.To.Append(nullableStr(tx.To)) + c.Nonce.Append(tx.Nonce) + c.GasPrice.Append(parseUInt128(tx.GasPrice)) + c.Gas.Append(tx.Gas) + c.GasTipCap.Append(nullableUInt128(tx.GasTipCap)) + c.GasFeeCap.Append(nullableUInt128(tx.GasFeeCap)) + c.Value.Append(parseUInt128(tx.Value)) + c.Type.Append(tx.Type) + c.Size.Append(tx.Size) + c.CallDataSize.Append(tx.CallDataSize) + c.BlobGas.Append(nullableUint64(tx.BlobGas)) + c.BlobGasFeeCap.Append(nullableUInt128(tx.BlobGasFeeCap)) + c.BlobHashes.Append(tx.BlobHashes) + c.Success.Append(tx.Success) + c.NInputBytes.Append(tx.NInputBytes) + c.NInputZeroBytes.Append(tx.NInputZeroBytes) + c.NInputNonzeroBytes.Append(tx.NInputNonzeroBytes) + c.MetaNetworkName.Append(tx.MetaNetworkName) +} + +// Reset clears all columns for reuse. +func (c *Columns) Reset() { + c.UpdatedDateTime.Reset() + c.BlockNumber.Reset() + c.BlockHash.Reset() + c.ParentHash.Reset() + c.Position.Reset() + c.Hash.Reset() + c.From.Reset() + c.To.Reset() + c.Nonce.Reset() + c.GasPrice.Reset() + c.Gas.Reset() + c.GasTipCap.Reset() + c.GasFeeCap.Reset() + c.Value.Reset() + c.Type.Reset() + c.Size.Reset() + c.CallDataSize.Reset() + c.BlobGas.Reset() + c.BlobGasFeeCap.Reset() + c.BlobHashes.Reset() + c.Success.Reset() + c.NInputBytes.Reset() + c.NInputZeroBytes.Reset() + c.NInputNonzeroBytes.Reset() + c.MetaNetworkName.Reset() +} + +// Input returns the proto.Input for inserting data. +func (c *Columns) Input() proto.Input { + return proto.Input{ + {Name: "updated_date_time", Data: &c.UpdatedDateTime}, + {Name: "block_number", Data: &c.BlockNumber}, + {Name: "block_hash", Data: &c.BlockHash}, + {Name: "parent_hash", Data: &c.ParentHash}, + {Name: "position", Data: &c.Position}, + {Name: "hash", Data: &c.Hash}, + {Name: "from", Data: &c.From}, + {Name: "to", Data: c.To}, + {Name: "nonce", Data: &c.Nonce}, + {Name: "gas_price", Data: &c.GasPrice}, + {Name: "gas", Data: &c.Gas}, + {Name: "gas_tip_cap", Data: c.GasTipCap}, + {Name: "gas_fee_cap", Data: c.GasFeeCap}, + {Name: "value", Data: &c.Value}, + {Name: "type", Data: &c.Type}, + {Name: "size", Data: &c.Size}, + {Name: "call_data_size", Data: &c.CallDataSize}, + {Name: "blob_gas", Data: c.BlobGas}, + {Name: "blob_gas_fee_cap", Data: c.BlobGasFeeCap}, + {Name: "blob_hashes", Data: c.BlobHashes}, + {Name: "success", Data: &c.Success}, + {Name: "n_input_bytes", Data: &c.NInputBytes}, + {Name: "n_input_zero_bytes", Data: &c.NInputZeroBytes}, + {Name: "n_input_nonzero_bytes", Data: &c.NInputNonzeroBytes}, + {Name: "meta_network_name", Data: &c.MetaNetworkName}, + } +} + +// Rows returns the number of rows in the columns. +func (c *Columns) Rows() int { + return c.BlockNumber.Rows() +} + +// parseUInt128 parses a decimal string to proto.UInt128. +// Returns zero if the string is empty or invalid. +func parseUInt128(s string) proto.UInt128 { + if s == "" { + return proto.UInt128{} + } + + bi := new(big.Int) + if _, ok := bi.SetString(s, 10); !ok { + return proto.UInt128{} + } + + // Extract low and high 64-bit words + // UInt128 = High * 2^64 + Low + maxUint64 := new(big.Int).SetUint64(^uint64(0)) + maxUint64.Add(maxUint64, big.NewInt(1)) // 2^64 + + low := new(big.Int).And(bi, new(big.Int).Sub(maxUint64, big.NewInt(1))) + high := new(big.Int).Rsh(bi, 64) + + return proto.UInt128{ + Low: low.Uint64(), + High: high.Uint64(), + } +} + +// nullableStr converts a *string to proto.Nullable[string]. +func nullableStr(s *string) proto.Nullable[string] { + if s == nil { + return proto.Null[string]() + } + + return proto.NewNullable(*s) +} + +// nullableUint64 converts a *uint64 to proto.Nullable[uint64]. +func nullableUint64(v *uint64) proto.Nullable[uint64] { + if v == nil { + return proto.Null[uint64]() + } + + return proto.NewNullable(*v) +} + +// nullableUInt128 converts a *string (decimal) to proto.Nullable[proto.UInt128]. +func nullableUInt128(s *string) proto.Nullable[proto.UInt128] { + if s == nil { + return proto.Null[proto.UInt128]() + } + + return proto.NewNullable(parseUInt128(*s)) +} diff --git a/pkg/processor/transaction/simple/handlers.go b/pkg/processor/transaction/simple/handlers.go index 2bbf6be..18d4ef3 100644 --- a/pkg/processor/transaction/simple/handlers.go +++ b/pkg/processor/transaction/simple/handlers.go @@ -6,44 +6,52 @@ import ( "math/big" "time" + "github.com/ClickHouse/ch-go" "github.com/ethereum/go-ethereum/core/types" "github.com/hibiken/asynq" "github.com/sirupsen/logrus" "github.com/ethpandaops/execution-processor/pkg/common" c "github.com/ethpandaops/execution-processor/pkg/processor/common" - "github.com/ethpandaops/execution-processor/pkg/processor/transaction/structlog" ) +// ClickHouseDateTime is a time.Time wrapper that formats correctly for ClickHouse JSON. +type ClickHouseDateTime time.Time + +// MarshalJSON formats time for ClickHouse DateTime column. +func (t ClickHouseDateTime) MarshalJSON() ([]byte, error) { + return []byte(fmt.Sprintf(`"%s"`, time.Time(t).UTC().Format("2006-01-02 15:04:05"))), nil +} + // Transaction represents a row in the execution_transaction table. // //nolint:tagliatelle // ClickHouse uses snake_case column names type Transaction struct { - UpdatedDateTime structlog.ClickHouseTime `json:"updated_date_time"` - BlockNumber uint64 `json:"block_number"` - BlockHash string `json:"block_hash"` - ParentHash string `json:"parent_hash"` - Position uint32 `json:"position"` - Hash string `json:"hash"` - From string `json:"from"` - To *string `json:"to"` - Nonce uint64 `json:"nonce"` - GasPrice string `json:"gas_price"` // Effective gas price as UInt128 string - Gas uint64 `json:"gas"` // Gas limit - GasTipCap *string `json:"gas_tip_cap"` // Nullable UInt128 string - GasFeeCap *string `json:"gas_fee_cap"` // Nullable UInt128 string - Value string `json:"value"` // UInt128 string - Type uint8 `json:"type"` // Transaction type - Size uint32 `json:"size"` // Transaction size in bytes - CallDataSize uint32 `json:"call_data_size"` // Size of call data - BlobGas *uint64 `json:"blob_gas"` // Nullable - for type 3 txs - BlobGasFeeCap *string `json:"blob_gas_fee_cap"` // Nullable UInt128 string - BlobHashes []string `json:"blob_hashes"` // Array of versioned hashes - Success bool `json:"success"` - NInputBytes uint32 `json:"n_input_bytes"` - NInputZeroBytes uint32 `json:"n_input_zero_bytes"` - NInputNonzeroBytes uint32 `json:"n_input_nonzero_bytes"` - MetaNetworkName string `json:"meta_network_name"` + UpdatedDateTime ClickHouseDateTime `json:"updated_date_time"` + BlockNumber uint64 `json:"block_number"` + BlockHash string `json:"block_hash"` + ParentHash string `json:"parent_hash"` + Position uint32 `json:"position"` + Hash string `json:"hash"` + From string `json:"from"` + To *string `json:"to"` + Nonce uint64 `json:"nonce"` + GasPrice string `json:"gas_price"` // Effective gas price as UInt128 string + Gas uint64 `json:"gas"` // Gas limit + GasTipCap *string `json:"gas_tip_cap"` // Nullable UInt128 string + GasFeeCap *string `json:"gas_fee_cap"` // Nullable UInt128 string + Value string `json:"value"` // UInt128 string + Type uint8 `json:"type"` // Transaction type + Size uint32 `json:"size"` // Transaction size in bytes + CallDataSize uint32 `json:"call_data_size"` // Size of call data + BlobGas *uint64 `json:"blob_gas"` // Nullable - for type 3 txs + BlobGasFeeCap *string `json:"blob_gas_fee_cap"` // Nullable UInt128 string + BlobHashes []string `json:"blob_hashes"` // Array of versioned hashes + Success bool `json:"success"` + NInputBytes uint32 `json:"n_input_bytes"` + NInputZeroBytes uint32 `json:"n_input_zero_bytes"` + NInputNonzeroBytes uint32 `json:"n_input_nonzero_bytes"` + MetaNetworkName string `json:"meta_network_name"` } // GetHandlers returns the task handlers for this processor. @@ -240,7 +248,7 @@ func (p *Processor) buildTransactionRow( // Build transaction row txRow := Transaction{ - UpdatedDateTime: structlog.NewClickHouseTime(time.Now()), + UpdatedDateTime: ClickHouseDateTime(time.Now()), BlockNumber: block.Number().Uint64(), BlockHash: block.Hash().String(), ParentHash: block.ParentHash().String(), @@ -322,13 +330,23 @@ func calculateEffectiveGasPrice(block *types.Block, tx *types.Transaction) *big. return effectiveGasPrice } -// insertTransactions inserts transactions into ClickHouse. +// insertTransactions inserts transactions into ClickHouse using columnar protocol. func (p *Processor) insertTransactions(ctx context.Context, transactions []Transaction) error { if len(transactions) == 0 { return nil } - if err := p.clickhouse.BulkInsert(ctx, p.config.Table, transactions); err != nil { + cols := NewColumns() + for _, tx := range transactions { + cols.Append(tx) + } + + input := cols.Input() + + if err := p.clickhouse.Do(ctx, ch.Query{ + Body: input.Into(p.config.Table), + Input: input, + }); err != nil { common.ClickHouseInsertsRows.WithLabelValues( p.network.Name, ProcessorName, diff --git a/pkg/processor/transaction/simple/processor.go b/pkg/processor/transaction/simple/processor.go index b417185..aa6d9bd 100644 --- a/pkg/processor/transaction/simple/processor.go +++ b/pkg/processor/transaction/simple/processor.go @@ -40,7 +40,7 @@ type Processor struct { } // New creates a new simple transaction processor. -func New(ctx context.Context, deps *Dependencies, config *Config) (*Processor, error) { +func New(deps *Dependencies, config *Config) (*Processor, error) { if err := config.Validate(); err != nil { return nil, fmt.Errorf("invalid config: %w", err) } diff --git a/pkg/processor/transaction/structlog/batch_manager.go b/pkg/processor/transaction/structlog/batch_manager.go deleted file mode 100644 index 8c4d083..0000000 --- a/pkg/processor/transaction/structlog/batch_manager.go +++ /dev/null @@ -1,225 +0,0 @@ -package structlog - -import ( - "context" - "sync" - "time" - - "github.com/ethpandaops/execution-processor/pkg/common" - "github.com/hibiken/asynq" - "github.com/sirupsen/logrus" -) - -// BatchItem represents a single transaction's data within a batch. -type BatchItem struct { - structlogs []Structlog - task *asynq.Task - payload *ProcessPayload -} - -// BatchManager accumulates small transactions and flushes them in batches. -type BatchManager struct { - mu sync.Mutex - items []BatchItem - currentSize int64 - flushThreshold int64 - maxSize int64 - flushInterval time.Duration - flushTimer *time.Timer - flushCh chan struct{} - processor *Processor - cancel context.CancelFunc - wg sync.WaitGroup - stopCh chan struct{} -} - -// NewBatchManager creates a new batch manager. -func NewBatchManager(processor *Processor, config *Config) *BatchManager { - _, cancel := context.WithCancel(context.Background()) - - return &BatchManager{ - items: make([]BatchItem, 0, 100), - flushThreshold: config.BatchInsertThreshold, - maxSize: config.BatchMaxSize, - flushInterval: config.BatchFlushInterval, - flushCh: make(chan struct{}, 1), - processor: processor, - cancel: cancel, - stopCh: make(chan struct{}), - } -} - -// Start begins the batch manager's flush goroutine. -func (bm *BatchManager) Start() error { - bm.wg.Add(1) - - go bm.flushLoop() - - bm.processor.log.WithFields(logrus.Fields{ - "flush_threshold": bm.flushThreshold, - "max_size": bm.maxSize, - "flush_interval": bm.flushInterval, - }).Info("batch manager started") - - return nil -} - -// Stop gracefully shuts down the batch manager. -func (bm *BatchManager) Stop() { - bm.cancel() - close(bm.stopCh) - - // Flush any remaining items - bm.Flush() - - // Wait for flush loop to exit - bm.wg.Wait() - - bm.processor.log.Info("batch manager stopped") -} - -// Add adds a transaction's structlogs to the batch. -func (bm *BatchManager) Add(structlogs []Structlog, task *asynq.Task, payload *ProcessPayload) error { - bm.mu.Lock() - defer bm.mu.Unlock() - - // Check if adding this would exceed max size - newSize := bm.currentSize + int64(len(structlogs)) - if newSize > bm.maxSize { - // Flush current batch first - bm.flushLocked() - } - - // Add to batch - bm.items = append(bm.items, BatchItem{ - structlogs: structlogs, - task: task, - payload: payload, - }) - bm.currentSize += int64(len(structlogs)) - - // Reset flush timer - if bm.flushTimer != nil { - bm.flushTimer.Stop() - } - - bm.flushTimer = time.AfterFunc(bm.flushInterval, func() { - select { - case bm.flushCh <- struct{}{}: - default: - } - }) - - // Check if we should flush - bm.FlushIfNeeded() - - return nil -} - -// FlushIfNeeded checks if the batch should be flushed based on size threshold. -func (bm *BatchManager) FlushIfNeeded() { - if bm.currentSize >= bm.flushThreshold { - bm.flushLocked() - } -} - -// Flush performs a batch flush (thread-safe). -func (bm *BatchManager) Flush() { - bm.mu.Lock() - defer bm.mu.Unlock() - - bm.flushLocked() -} - -// flushLocked performs the actual flush (must be called with lock held). -func (bm *BatchManager) flushLocked() { - if len(bm.items) == 0 { - return - } - - // Stop the flush timer - if bm.flushTimer != nil { - bm.flushTimer.Stop() - bm.flushTimer = nil - } - - // Collect all structlogs - var allStructlogs []Structlog - for _, item := range bm.items { - allStructlogs = append(allStructlogs, item.structlogs...) - } - - // Log batch details - bm.processor.log.WithFields(logrus.Fields{ - "batch_items": len(bm.items), - "batch_structlogs": len(allStructlogs), - }).Debug("flushing batch") - - // Perform bulk insert - startTime := time.Now() - - err := bm.processor.insertStructlogs(context.Background(), allStructlogs) - if err != nil { - // Failed - fail all tasks in batch - bm.processor.log.WithError(err).Error("batch insert failed") - - for _, item := range bm.items { - // Increment error metrics - common.TasksErrored.WithLabelValues( - bm.processor.network.Name, - ProcessorName, - item.payload.ProcessingMode, - ProcessForwardsTaskType, - "batch_insert_error", - ).Inc() - } - } else { - // Success - complete all tasks - for _, item := range bm.items { - // Increment success metrics - common.TasksProcessed.WithLabelValues( - bm.processor.network.Name, - ProcessorName, - item.payload.ProcessingMode, - ProcessForwardsTaskType, - "success", - ).Inc() - } - - // Record batch metrics using ClickHouse metrics - common.ClickHouseInsertsRows.WithLabelValues( - bm.processor.network.Name, - ProcessorName, - "structlogs_batch", - "success", - "", - ).Add(float64(len(allStructlogs))) - - common.ClickHouseOperationDuration.WithLabelValues( - bm.processor.network.Name, - ProcessorName, - "batch_insert", - "structlogs_batch", - "success", - "", - ).Observe(time.Since(startTime).Seconds()) - } - - // Clear the batch - bm.items = bm.items[:0] - bm.currentSize = 0 -} - -// flushLoop runs the time-based flush goroutine. -func (bm *BatchManager) flushLoop() { - defer bm.wg.Done() - - for { - select { - case <-bm.stopCh: - return - case <-bm.flushCh: - bm.Flush() - } - } -} diff --git a/pkg/processor/transaction/structlog/batch_manager_test.go b/pkg/processor/transaction/structlog/batch_manager_test.go deleted file mode 100644 index 26a94b4..0000000 --- a/pkg/processor/transaction/structlog/batch_manager_test.go +++ /dev/null @@ -1,377 +0,0 @@ -package structlog_test - -import ( - "context" - "sync" - "testing" - "time" - - "github.com/ethpandaops/execution-processor/pkg/clickhouse" - "github.com/ethpandaops/execution-processor/pkg/processor/transaction/structlog" - "github.com/hibiken/asynq" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" -) - -// MockClickHouseClient is a mock implementation of clickhouse.ClientInterface. -type MockClickHouseClient struct { - mock.Mock -} - -func (m *MockClickHouseClient) Start() error { - args := m.Called() - - return args.Error(0) -} - -func (m *MockClickHouseClient) Stop() error { - args := m.Called() - - return args.Error(0) -} - -func (m *MockClickHouseClient) BulkInsert(ctx context.Context, table string, data interface{}) error { - args := m.Called(ctx, table, data) - - return args.Error(0) -} - -func (m *MockClickHouseClient) QueryOne(ctx context.Context, query string, dest interface{}) error { - args := m.Called(ctx, query, dest) - - return args.Error(0) -} - -func (m *MockClickHouseClient) QueryMany(ctx context.Context, query string, dest interface{}) error { - args := m.Called(ctx, query, dest) - - return args.Error(0) -} - -func (m *MockClickHouseClient) Execute(ctx context.Context, query string) error { - args := m.Called(ctx, query) - - return args.Error(0) -} - -func (m *MockClickHouseClient) IsStorageEmpty(ctx context.Context, table string, conditions map[string]interface{}) (bool, error) { - args := m.Called(ctx, table, conditions) - - return args.Bool(0), args.Error(1) -} - -func (m *MockClickHouseClient) SetNetwork(network string) { - m.Called(network) -} - -// TestBatchManager_Accumulation tests that the batch manager accumulates transactions up to threshold. -func TestBatchManager_Accumulation(t *testing.T) { - t.Skip("Skipping due to inability to properly mock processor internals") -} - -// TestBatchManager_TimeBasedFlush tests that batches are flushed based on time. -func TestBatchManager_TimeBasedFlush(t *testing.T) { - t.Skip("Skipping due to inability to properly mock processor internals") -} - -// TestBatchManager_BigTransactionFlush tests that batches are flushed before big transactions. -func TestBatchManager_BigTransactionFlush(t *testing.T) { - t.Skip("Skipping due to inability to properly mock processor internals") -} - -// TestProcessor_RoutingLogic tests that transactions are correctly routed based on size. -func TestProcessor_RoutingLogic(t *testing.T) { - config := &structlog.Config{ - Enabled: true, - Table: "test_structlog", - BatchInsertThreshold: 100, - BatchFlushInterval: 5 * time.Second, - BatchMaxSize: 200, - BigTransactionThreshold: 500000, - Config: clickhouse.Config{ - URL: "http://localhost:8123", - }, - } - - // Test the ShouldBatch logic directly - // The logic is: count < BatchInsertThreshold - - // Test small transaction - should be batched - assert.True(t, 50 < int(config.BatchInsertThreshold)) - assert.True(t, 99 < int(config.BatchInsertThreshold)) - - // Test at threshold - should NOT be batched - assert.False(t, 100 < int(config.BatchInsertThreshold)) - - // Test large transaction - should NOT be batched - assert.False(t, 1000 < int(config.BatchInsertThreshold)) - assert.False(t, 500000 < int(config.BatchInsertThreshold)) - - // Test edge cases - assert.True(t, 0 < int(config.BatchInsertThreshold)) // Empty transaction should be less than threshold - assert.True(t, -1 < int(config.BatchInsertThreshold)) // Invalid count is still technically less than threshold -} - -// TestBatchManager_ErrorHandling tests batch failure handling. -func TestBatchManager_ErrorHandling(t *testing.T) { - t.Skip("Skipping due to inability to properly mock processor internals") -} - -// TestBatchManager_Shutdown tests graceful shutdown with pending batches. -func TestBatchManager_Shutdown(t *testing.T) { - t.Skip("Skipping due to inability to properly mock processor internals") -} - -// TestBatchManager_ConcurrentAdd tests thread-safe concurrent additions. -func TestBatchManager_ConcurrentAdd(t *testing.T) { - t.Skip("Skipping due to inability to properly mock processor internals") -} - -// TestBatchManager_ThresholdBoundaries tests behavior at exact threshold values. -func TestBatchManager_ThresholdBoundaries(t *testing.T) { - t.Skip("Skipping due to inability to properly mock processor internals") -} - -// TestBatchManager_PerformanceImprovement benchmarks batching benefits. -func TestBatchManager_PerformanceImprovement(t *testing.T) { - if testing.Short() { - t.Skip("Skipping performance test in short mode") - } - - // Test configuration - numTransactions := 1000 - - // Non-batched timing simulation - nonBatchedStart := time.Now() - - for i := 0; i < numTransactions; i++ { - // Simulate individual insert delay - time.Sleep(100 * time.Microsecond) - } - - nonBatchedDuration := time.Since(nonBatchedStart) - - // Batched timing simulation (batch size 100) - batchedStart := time.Now() - batchSize := 100 - numBatches := numTransactions / batchSize - - for i := 0; i < numBatches; i++ { - // Simulate batch insert delay (slightly longer per batch, but fewer operations) - time.Sleep(500 * time.Microsecond) - } - - batchedDuration := time.Since(batchedStart) - - // Calculate improvement - improvement := float64(nonBatchedDuration) / float64(batchedDuration) - - t.Logf("Non-batched duration: %v", nonBatchedDuration) - t.Logf("Batched duration: %v", batchedDuration) - t.Logf("Performance improvement: %.2fx", improvement) - - // Batching should be faster - assert.Less(t, batchedDuration, nonBatchedDuration) - assert.Greater(t, improvement, 1.0) -} - -// TestConfigValidation_BatchFields tests validation of batch configuration fields. -func TestConfigValidation_BatchFields(t *testing.T) { - testCases := []struct { - name string - config structlog.Config - expectError bool - errorMsg string - }{ - { - name: "valid batch config", - config: structlog.Config{ - Enabled: true, - Table: "test", - BatchInsertThreshold: 100, - BatchFlushInterval: 5 * time.Second, - BatchMaxSize: 200, - BigTransactionThreshold: 500000, - Config: clickhouse.Config{ - URL: "http://localhost:8123", - }, - }, - expectError: false, - }, - { - name: "batch threshold zero", - config: structlog.Config{ - Enabled: true, - Table: "test", - BatchInsertThreshold: 0, - BatchFlushInterval: 5 * time.Second, - BatchMaxSize: 200, - BigTransactionThreshold: 500000, - Config: clickhouse.Config{ - URL: "http://localhost:8123", - }, - }, - expectError: true, - errorMsg: "batchInsertThreshold must be greater than 0", - }, - { - name: "batch threshold exceeds big transaction threshold", - config: structlog.Config{ - Enabled: true, - Table: "test", - BatchInsertThreshold: 600000, - BatchFlushInterval: 5 * time.Second, - BatchMaxSize: 700000, - BigTransactionThreshold: 500000, - Config: clickhouse.Config{ - URL: "http://localhost:8123", - }, - }, - expectError: true, - errorMsg: "batchInsertThreshold must be less than bigTransactionThreshold", - }, - { - name: "batch flush interval zero", - config: structlog.Config{ - Enabled: true, - Table: "test", - BatchInsertThreshold: 100, - BatchFlushInterval: 0, - BatchMaxSize: 200, - BigTransactionThreshold: 500000, - Config: clickhouse.Config{ - URL: "http://localhost:8123", - }, - }, - expectError: true, - errorMsg: "batchFlushInterval must be greater than 0", - }, - { - name: "batch max size less than threshold", - config: structlog.Config{ - Enabled: true, - Table: "test", - BatchInsertThreshold: 200, - BatchFlushInterval: 5 * time.Second, - BatchMaxSize: 100, - BigTransactionThreshold: 500000, - Config: clickhouse.Config{ - URL: "http://localhost:8123", - }, - }, - expectError: true, - errorMsg: "batchMaxSize must be greater than or equal to batchInsertThreshold", - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - err := tc.config.Validate() - if tc.expectError { - assert.Error(t, err) - - if tc.errorMsg != "" { - assert.Contains(t, err.Error(), tc.errorMsg) - } - } else { - assert.NoError(t, err) - } - }) - } -} - -// TestBatchManagerConcepts tests the concepts behind batch management. -func TestBatchManagerConcepts(t *testing.T) { - // Test batch size calculations - t.Run("batch size tracking", func(t *testing.T) { - var currentSize int64 - - threshold := int64(100) - - // Simulate adding items - items := []int64{10, 20, 30, 40, 50} - - for _, size := range items { - currentSize += size - - if currentSize >= threshold { - // Would trigger flush - assert.GreaterOrEqual(t, currentSize, threshold) - - break - } - } - }) - - // Test timer-based flushing concept - t.Run("timer based flush", func(t *testing.T) { - flushInterval := 100 * time.Millisecond - timer := time.NewTimer(flushInterval) - - select { - case <-timer.C: - // Timer expired, would trigger flush - assert.True(t, true, "Timer expired as expected") - case <-time.After(200 * time.Millisecond): - t.Fatal("Timer did not expire in expected time") - } - }) - - // Test concurrent access patterns - t.Run("concurrent access", func(t *testing.T) { - var mu sync.Mutex - - var items []int - - var wg sync.WaitGroup - - for i := 0; i < 10; i++ { - wg.Add(1) - - go func(val int) { - defer wg.Done() - - mu.Lock() - defer mu.Unlock() - - items = append(items, val) - }(i) - } - - wg.Wait() - assert.Len(t, items, 10) - }) -} - -// TestBatchItemStructure tests the BatchItem structure. -func TestBatchItemStructure(t *testing.T) { - // Test creating batch items - structlogs := []structlog.Structlog{ - { - BlockNumber: 12345, - TransactionHash: "0xtest", - Index: 0, - }, - } - - task := asynq.NewTask("test_type", []byte("test_payload")) - payload := &structlog.ProcessPayload{ - TransactionHash: "0xtest", - } - - // Test that we can create a batch item with these components - batchItem := struct { - structlogs []structlog.Structlog - task *asynq.Task - payload *structlog.ProcessPayload - }{ - structlogs: structlogs, - task: task, - payload: payload, - } - - assert.NotNil(t, batchItem.structlogs) - assert.NotNil(t, batchItem.task) - assert.NotNil(t, batchItem.payload) - assert.Equal(t, "0xtest", batchItem.payload.TransactionHash) -} diff --git a/pkg/processor/transaction/structlog/big_transaction_manager.go b/pkg/processor/transaction/structlog/big_transaction_manager.go deleted file mode 100644 index 0945773..0000000 --- a/pkg/processor/transaction/structlog/big_transaction_manager.go +++ /dev/null @@ -1,69 +0,0 @@ -package structlog - -import ( - "sync/atomic" - - "github.com/sirupsen/logrus" -) - -// BigTransactionManager coordinates processing of large transactions to prevent OOM. -type BigTransactionManager struct { - // Active big transaction count - currentBigCount atomic.Int32 - pauseNewWork atomic.Bool - - // Configuration - threshold int // Default: 500k structlogs - log logrus.FieldLogger -} - -// NewBigTransactionManager creates a new manager for big transaction coordination. -func NewBigTransactionManager(threshold int, log logrus.FieldLogger) *BigTransactionManager { - return &BigTransactionManager{ - threshold: threshold, - log: log, - } -} - -// RegisterBigTransaction marks a big transaction as actively processing. -func (btm *BigTransactionManager) RegisterBigTransaction(txHash string, processor *Processor) { - // Flush any pending batches before blocking - if processor.batchManager != nil { - processor.batchManager.Flush() - } - - count := btm.currentBigCount.Add(1) - - // Set pause flag when ANY big transaction is active - btm.pauseNewWork.Store(true) - btm.log.WithFields(logrus.Fields{ - "tx_hash": txHash, - "total_big_txs": count, - }).Debug("Registered big transaction") -} - -// UnregisterBigTransaction marks a big transaction as complete. -func (btm *BigTransactionManager) UnregisterBigTransaction(txHash string) { - newCount := btm.currentBigCount.Add(-1) - - // Only clear pause when NO big transactions remain - if newCount == 0 { - btm.pauseNewWork.Store(false) - btm.log.Info("All big transactions complete, resuming normal processing") - } else { - btm.log.WithFields(logrus.Fields{ - "tx_hash": txHash, - "remaining_big_txs": newCount, - }).Debug("Big transaction complete") - } -} - -// ShouldPauseNewWork returns true if new work should be paused. -func (btm *BigTransactionManager) ShouldPauseNewWork() bool { - return btm.pauseNewWork.Load() -} - -// GetThreshold returns the threshold for big transactions. -func (btm *BigTransactionManager) GetThreshold() int { - return btm.threshold -} diff --git a/pkg/processor/transaction/structlog/clickhouse_time.go b/pkg/processor/transaction/structlog/clickhouse_time.go deleted file mode 100644 index 5e7fcd7..0000000 --- a/pkg/processor/transaction/structlog/clickhouse_time.go +++ /dev/null @@ -1,43 +0,0 @@ -package structlog - -import ( - "fmt" - "time" -) - -// ClickHouseTime wraps time.Time to provide custom JSON marshaling for ClickHouse. -type ClickHouseTime struct { - time.Time -} - -// MarshalJSON formats the time for ClickHouse DateTime format. -func (t ClickHouseTime) MarshalJSON() ([]byte, error) { - // ClickHouse expects DateTime in the format: "2006-01-02 15:04:05" - formatted := fmt.Sprintf("%q", t.Format("2006-01-02 15:04:05")) - - return []byte(formatted), nil -} - -// UnmarshalJSON parses the time from ClickHouse DateTime format. -func (t *ClickHouseTime) UnmarshalJSON(data []byte) error { - // Remove quotes - if len(data) < 2 || data[0] != '"' || data[len(data)-1] != '"' { - return fmt.Errorf("invalid time format") - } - - timeStr := string(data[1 : len(data)-1]) - - parsed, err := time.Parse("2006-01-02 15:04:05", timeStr) - if err != nil { - return err - } - - t.Time = parsed - - return nil -} - -// NewClickHouseTime creates a new ClickHouseTime from a time.Time. -func NewClickHouseTime(t time.Time) ClickHouseTime { - return ClickHouseTime{Time: t} -} diff --git a/pkg/processor/transaction/structlog/columns.go b/pkg/processor/transaction/structlog/columns.go new file mode 100644 index 0000000..a08f2af --- /dev/null +++ b/pkg/processor/transaction/structlog/columns.go @@ -0,0 +1,155 @@ +package structlog + +import ( + "time" + + "github.com/ClickHouse/ch-go/proto" +) + +// Columns holds all columns for structlog batch insert using ch-go columnar protocol. +type Columns struct { + UpdatedDateTime proto.ColDateTime + BlockNumber proto.ColUInt64 + TransactionHash proto.ColStr + TransactionIndex proto.ColUInt32 + TransactionGas proto.ColUInt64 + TransactionFailed proto.ColBool + TransactionReturnValue *proto.ColNullable[string] + Index proto.ColUInt32 + ProgramCounter proto.ColUInt32 + Operation proto.ColStr + Gas proto.ColUInt64 + GasCost proto.ColUInt64 + GasUsed proto.ColUInt64 + Depth proto.ColUInt64 + ReturnData *proto.ColNullable[string] + Refund *proto.ColNullable[uint64] + Error *proto.ColNullable[string] + CallToAddress *proto.ColNullable[string] + MetaNetworkName proto.ColStr +} + +// NewColumns creates a new Columns instance with all columns initialized. +func NewColumns() *Columns { + return &Columns{ + TransactionReturnValue: new(proto.ColStr).Nullable(), + ReturnData: new(proto.ColStr).Nullable(), + Refund: new(proto.ColUInt64).Nullable(), + Error: new(proto.ColStr).Nullable(), + CallToAddress: new(proto.ColStr).Nullable(), + } +} + +// Append adds a row to all columns. +func (c *Columns) Append( + updatedDateTime time.Time, + blockNumber uint64, + txHash string, + txIndex uint32, + txGas uint64, + txFailed bool, + txReturnValue *string, + index uint32, + pc uint32, + op string, + gas uint64, + gasCost uint64, + gasUsed uint64, + depth uint64, + returnData *string, + refund *uint64, + errStr *string, + callTo *string, + network string, +) { + c.UpdatedDateTime.Append(updatedDateTime) + c.BlockNumber.Append(blockNumber) + c.TransactionHash.Append(txHash) + c.TransactionIndex.Append(txIndex) + c.TransactionGas.Append(txGas) + c.TransactionFailed.Append(txFailed) + c.TransactionReturnValue.Append(nullableStr(txReturnValue)) + c.Index.Append(index) + c.ProgramCounter.Append(pc) + c.Operation.Append(op) + c.Gas.Append(gas) + c.GasCost.Append(gasCost) + c.GasUsed.Append(gasUsed) + c.Depth.Append(depth) + c.ReturnData.Append(nullableStr(returnData)) + c.Refund.Append(nullableUint64(refund)) + c.Error.Append(nullableStr(errStr)) + c.CallToAddress.Append(nullableStr(callTo)) + c.MetaNetworkName.Append(network) +} + +// Reset clears all columns for reuse. +func (c *Columns) Reset() { + c.UpdatedDateTime.Reset() + c.BlockNumber.Reset() + c.TransactionHash.Reset() + c.TransactionIndex.Reset() + c.TransactionGas.Reset() + c.TransactionFailed.Reset() + c.TransactionReturnValue.Reset() + c.Index.Reset() + c.ProgramCounter.Reset() + c.Operation.Reset() + c.Gas.Reset() + c.GasCost.Reset() + c.GasUsed.Reset() + c.Depth.Reset() + c.ReturnData.Reset() + c.Refund.Reset() + c.Error.Reset() + c.CallToAddress.Reset() + c.MetaNetworkName.Reset() +} + +// Input returns the proto.Input for inserting data. +func (c *Columns) Input() proto.Input { + return proto.Input{ + {Name: "updated_date_time", Data: &c.UpdatedDateTime}, + {Name: "block_number", Data: &c.BlockNumber}, + {Name: "transaction_hash", Data: &c.TransactionHash}, + {Name: "transaction_index", Data: &c.TransactionIndex}, + {Name: "transaction_gas", Data: &c.TransactionGas}, + {Name: "transaction_failed", Data: &c.TransactionFailed}, + {Name: "transaction_return_value", Data: c.TransactionReturnValue}, + {Name: "index", Data: &c.Index}, + {Name: "program_counter", Data: &c.ProgramCounter}, + {Name: "operation", Data: &c.Operation}, + {Name: "gas", Data: &c.Gas}, + {Name: "gas_cost", Data: &c.GasCost}, + {Name: "gas_used", Data: &c.GasUsed}, + {Name: "depth", Data: &c.Depth}, + {Name: "return_data", Data: c.ReturnData}, + {Name: "refund", Data: c.Refund}, + {Name: "error", Data: c.Error}, + {Name: "call_to_address", Data: c.CallToAddress}, + {Name: "meta_network_name", Data: &c.MetaNetworkName}, + } +} + +// Rows returns the number of rows in the columns. +func (c *Columns) Rows() int { + return c.BlockNumber.Rows() +} + +// nullableStr converts a *string to proto.Nullable[string]. +func nullableStr(s *string) proto.Nullable[string] { + if s == nil { + return proto.Null[string]() + } + + return proto.NewNullable(*s) +} + +// nullableUint64 converts a *uint64 to proto.Nullable[uint64]. +func nullableUint64(v *uint64) proto.Nullable[uint64] { + if v == nil { + return proto.Null[uint64]() + } + + return proto.NewNullable(*v) +} diff --git a/pkg/processor/transaction/structlog/config.go b/pkg/processor/transaction/structlog/config.go index 909f4a2..57d809f 100644 --- a/pkg/processor/transaction/structlog/config.go +++ b/pkg/processor/transaction/structlog/config.go @@ -2,7 +2,6 @@ package structlog import ( "fmt" - "time" "github.com/ethpandaops/execution-processor/pkg/clickhouse" ) @@ -13,24 +12,12 @@ type Config struct { Enabled bool `yaml:"enabled"` Table string `yaml:"table"` - // Big transaction handling - BigTransactionThreshold int `yaml:"bigTransactionThreshold"` // Default: 500,000 - ChunkSize int `yaml:"chunkSize"` // Default: 10,000 - ChannelBufferSize int `yaml:"channelBufferSize"` // Default: 2 - ProgressLogThreshold int `yaml:"progressLogThreshold"` // Default: 100,000 - - // Batch processing configuration - // BatchInsertThreshold is the minimum number of structlogs to accumulate before batch insert - // Transactions with more structlogs than this will bypass batching - BatchInsertThreshold int64 `yaml:"batchInsertThreshold" default:"50000"` - - // BatchFlushInterval is the maximum time to wait before flushing a batch - BatchFlushInterval time.Duration `yaml:"batchFlushInterval" default:"5s"` - - // BatchMaxSize is the maximum number of structlogs to accumulate in a batch - BatchMaxSize int64 `yaml:"batchMaxSize" default:"100000"` + // Streaming settings + ChunkSize int `yaml:"chunkSize"` // Default: 10,000 rows per OnInput iteration + ProgressLogThreshold int `yaml:"progressLogThreshold"` // Default: 100,000 - log progress for large txs } +// Validate validates the configuration. func (c *Config) Validate() error { if !c.Enabled { return nil @@ -45,22 +32,5 @@ func (c *Config) Validate() error { return fmt.Errorf("transaction structlog table is required when enabled") } - // Validate batch configuration - if c.BatchInsertThreshold <= 0 { - return fmt.Errorf("batchInsertThreshold must be greater than 0") - } - - if c.BatchInsertThreshold >= int64(c.BigTransactionThreshold) { - return fmt.Errorf("batchInsertThreshold must be less than bigTransactionThreshold to prevent routing conflicts") - } - - if c.BatchFlushInterval <= 0 { - return fmt.Errorf("batchFlushInterval must be greater than 0") - } - - if c.BatchMaxSize < c.BatchInsertThreshold { - return fmt.Errorf("batchMaxSize must be greater than or equal to batchInsertThreshold") - } - return nil } diff --git a/pkg/processor/transaction/structlog/handlers.go b/pkg/processor/transaction/structlog/handlers.go index d425038..e4cecbf 100644 --- a/pkg/processor/transaction/structlog/handlers.go +++ b/pkg/processor/transaction/structlog/handlers.go @@ -28,9 +28,6 @@ func (p *Processor) handleProcessForwardsTask(ctx context.Context, task *asynq.T return fmt.Errorf("failed to unmarshal process payload: %w", err) } - // Wait for any active big transactions before starting - p.waitForBigTransactions("process_forwards") - // Get healthy execution node node := p.pool.GetHealthyExecutionNode() if node == nil { @@ -55,30 +52,8 @@ func (p *Processor) handleProcessForwardsTask(ctx context.Context, task *asynq.T return fmt.Errorf("transaction hash mismatch: expected %s, got %s", payload.TransactionHash, tx.Hash().String()) } - // Extract structlogs from the transaction - structlogs, err := p.ExtractStructlogs(ctx, block, int(payload.TransactionIndex), tx) - if err != nil { - common.TasksErrored.WithLabelValues(p.network.Name, ProcessorName, c.ProcessForwardsQueue(ProcessorName), ProcessForwardsTaskType, "extraction_error").Inc() - - return fmt.Errorf("failed to extract structlogs: %w", err) - } - - structlogCount := int64(len(structlogs)) - - // Check if transaction should be batched - if p.ShouldBatch(structlogCount) { - // Route to batch manager - if addErr := p.batchManager.Add(structlogs, task, &payload); addErr != nil { - common.TasksErrored.WithLabelValues(p.network.Name, ProcessorName, c.ProcessForwardsQueue(ProcessorName), ProcessForwardsTaskType, "batch_add_error").Inc() - - return fmt.Errorf("failed to add to batch: %w", addErr) - } - // Note: Task completion handled by batch manager - return nil - } - - // Process large transaction using existing logic - _, err = p.ProcessTransaction(ctx, block, int(payload.TransactionIndex), tx) + // Process transaction using ch-go streaming + structlogCount, err := p.ProcessTransaction(ctx, block, int(payload.TransactionIndex), tx) if err != nil { common.TasksErrored.WithLabelValues(p.network.Name, ProcessorName, c.ProcessForwardsQueue(ProcessorName), ProcessForwardsTaskType, "processing_error").Inc() @@ -112,9 +87,6 @@ func (p *Processor) handleProcessBackwardsTask(ctx context.Context, task *asynq. return fmt.Errorf("failed to unmarshal process payload: %w", err) } - // Wait for any active big transactions before starting - p.waitForBigTransactions("process_backwards") - // Get healthy execution node node := p.pool.GetHealthyExecutionNode() if node == nil { @@ -139,34 +111,12 @@ func (p *Processor) handleProcessBackwardsTask(ctx context.Context, task *asynq. return fmt.Errorf("transaction hash mismatch: expected %s, got %s", payload.TransactionHash, tx.Hash().String()) } - // Extract structlogs from the transaction - structlogs, err := p.ExtractStructlogs(ctx, block, int(payload.TransactionIndex), tx) + // Process transaction using ch-go streaming + structlogCount, err := p.ProcessTransaction(ctx, block, int(payload.TransactionIndex), tx) if err != nil { - common.TasksErrored.WithLabelValues(p.network.Name, ProcessorName, c.ProcessBackwardsQueue(ProcessorName), ProcessBackwardsTaskType, "extraction_error").Inc() - - return fmt.Errorf("failed to extract structlogs: %w", err) - } - - structlogCount := int64(len(structlogs)) - - // Check if transaction should be batched - if p.ShouldBatch(structlogCount) { - // Route to batch manager - if addErr := p.batchManager.Add(structlogs, task, &payload); addErr != nil { - common.TasksErrored.WithLabelValues(p.network.Name, ProcessorName, c.ProcessBackwardsQueue(ProcessorName), ProcessBackwardsTaskType, "batch_add_error").Inc() - - return fmt.Errorf("failed to add to batch: %w", addErr) - } - // Note: Task completion handled by batch manager - return nil - } - - // Process large transaction using existing logic - _, processErr := p.ProcessTransaction(ctx, block, int(payload.TransactionIndex), tx) - if processErr != nil { common.TasksErrored.WithLabelValues(p.network.Name, ProcessorName, c.ProcessBackwardsQueue(ProcessorName), ProcessBackwardsTaskType, "processing_error").Inc() - return fmt.Errorf("failed to process transaction: %w", processErr) + return fmt.Errorf("failed to process transaction: %w", err) } // Record successful processing diff --git a/pkg/processor/transaction/structlog/processor.go b/pkg/processor/transaction/structlog/processor.go index 0b82d4b..65ba613 100644 --- a/pkg/processor/transaction/structlog/processor.go +++ b/pkg/processor/transaction/structlog/processor.go @@ -3,11 +3,8 @@ package structlog import ( "context" "fmt" - "strings" - "time" "github.com/ethpandaops/execution-processor/pkg/clickhouse" - "github.com/ethpandaops/execution-processor/pkg/common" "github.com/ethpandaops/execution-processor/pkg/ethereum" c "github.com/ethpandaops/execution-processor/pkg/processor/common" "github.com/ethpandaops/execution-processor/pkg/state" @@ -36,12 +33,10 @@ type Processor struct { asynqClient *asynq.Client processingMode string redisPrefix string - bigTxManager *BigTransactionManager - batchManager *BatchManager } // New creates a new transaction structlog processor. -func New(ctx context.Context, deps *Dependencies, config *Config) (*Processor, error) { +func New(deps *Dependencies, config *Config) (*Processor, error) { // Create a copy of the embedded config and set processor-specific values clickhouseConfig := config.Config clickhouseConfig.Network = deps.Network.Name @@ -74,24 +69,6 @@ func New(ctx context.Context, deps *Dependencies, config *Config) (*Processor, e // Start starts the processor. func (p *Processor) Start(ctx context.Context) error { - // Use configured value or default - threshold := p.config.BigTransactionThreshold - if threshold == 0 { - threshold = 500_000 // Default - } - - p.bigTxManager = NewBigTransactionManager(threshold, p.log) - - p.log.WithFields(logrus.Fields{ - "big_transaction_threshold": threshold, - }).Info("Initialized big transaction manager") - - // Create and start batch manager - p.batchManager = NewBatchManager(p, p.config) - if err := p.batchManager.Start(); err != nil { - return fmt.Errorf("failed to start batch manager: %w", err) - } - // Start the ClickHouse client if err := p.clickhouse.Start(); err != nil { return fmt.Errorf("failed to start ClickHouse client: %w", err) @@ -106,11 +83,6 @@ func (p *Processor) Start(ctx context.Context) error { func (p *Processor) Stop(ctx context.Context) error { p.log.Info("Stopping transaction structlog processor") - // Stop the batch manager first to flush any pending batches - if p.batchManager != nil { - p.batchManager.Stop() - } - // Stop the ClickHouse client return p.clickhouse.Stop() } @@ -164,90 +136,3 @@ func (p *Processor) getProcessForwardsQueue() string { func (p *Processor) getProcessBackwardsQueue() string { return c.PrefixedProcessBackwardsQueue(ProcessorName, p.redisPrefix) } - -// insertStructlogs inserts structlog rows directly into ClickHouse. -func (p *Processor) insertStructlogs(ctx context.Context, structlogs []Structlog) error { - // Short-circuit for empty structlog arrays - if len(structlogs) == 0 { - return nil - } - - // Create timeout context for insert - timeout := 5 * time.Minute - insertCtx, cancel := context.WithTimeout(ctx, timeout) - - defer cancel() - - // Perform the insert with metrics tracking - start := time.Now() - err := p.clickhouse.BulkInsert(insertCtx, p.config.Table, structlogs) - duration := time.Since(start) - - if err != nil { - code := parseErrorCode(err) - common.ClickHouseOperationDuration.WithLabelValues(p.network.Name, ProcessorName, "bulk_insert", p.config.Table, "failed", code).Observe(duration.Seconds()) - common.ClickHouseOperationTotal.WithLabelValues(p.network.Name, ProcessorName, "bulk_insert", p.config.Table, "failed", code).Inc() - common.ClickHouseInsertsRows.WithLabelValues(p.network.Name, ProcessorName, p.config.Table, "failed", code).Add(float64(len(structlogs))) - - p.log.WithFields(logrus.Fields{ - "table": p.config.Table, - "error": err.Error(), - "structlog_count": len(structlogs), - }).Error("Failed to insert structlogs") - - return fmt.Errorf("failed to insert %d structlogs: %w", len(structlogs), err) - } - - common.ClickHouseOperationDuration.WithLabelValues(p.network.Name, ProcessorName, "bulk_insert", p.config.Table, "success", "").Observe(duration.Seconds()) - common.ClickHouseOperationTotal.WithLabelValues(p.network.Name, ProcessorName, "bulk_insert", p.config.Table, "success", "").Inc() - common.ClickHouseInsertsRows.WithLabelValues(p.network.Name, ProcessorName, p.config.Table, "success", "").Add(float64(len(structlogs))) - - return nil -} - -// waitForBigTransactions waits for big transactions to complete before proceeding. -func (p *Processor) waitForBigTransactions(taskType string) { - if p.bigTxManager.ShouldPauseNewWork() { - p.log.WithFields(logrus.Fields{ - "task_type": taskType, - "active_big_txs": p.bigTxManager.currentBigCount.Load(), - }).Debug("Task waiting for big transactions to complete") - - startWait := time.Now() - - for p.bigTxManager.ShouldPauseNewWork() { - time.Sleep(100 * time.Millisecond) - } - - waitDuration := time.Since(startWait) - p.log.WithFields(logrus.Fields{ - "task_type": taskType, - "wait_duration": waitDuration.String(), - }).Debug("Task resumed after big transactions completed") - } -} - -// parseErrorCode extracts error code from error message. -func parseErrorCode(err error) string { - if err == nil { - return "" - } - // For HTTP client errors, we don't have specific error codes - // Return a generic code based on error type - errStr := err.Error() - - if strings.Contains(errStr, "timeout") { - return "TIMEOUT" - } - - if strings.Contains(errStr, "connection") { - return "CONNECTION" - } - - return "UNKNOWN" -} - -// ShouldBatch determines if a transaction should be batched based on structlog count. -func (p *Processor) ShouldBatch(structlogCount int64) bool { - return structlogCount > 0 && structlogCount < p.config.BatchInsertThreshold -} diff --git a/pkg/processor/transaction/structlog/processor_test.go b/pkg/processor/transaction/structlog/processor_test.go index 033a642..add904d 100644 --- a/pkg/processor/transaction/structlog/processor_test.go +++ b/pkg/processor/transaction/structlog/processor_test.go @@ -19,12 +19,9 @@ func TestProcessor_Creation(t *testing.T) { Enabled: true, Table: "test_structlog", Config: clickhouse.Config{ - URL: "http://localhost:8123", + Addr: "localhost:9000", }, - BigTransactionThreshold: 500000, - BatchInsertThreshold: 50000, - BatchFlushInterval: 5 * time.Second, - BatchMaxSize: 100000, + ChunkSize: 10000, } // Test config validation @@ -44,12 +41,9 @@ func TestProcessor_ConfigValidation(t *testing.T) { Enabled: true, Table: "test_table", Config: clickhouse.Config{ - URL: "http://localhost:8123", + Addr: "localhost:9000", }, - BigTransactionThreshold: 500000, - BatchInsertThreshold: 50000, - BatchFlushInterval: 5 * time.Second, - BatchMaxSize: 100000, + ChunkSize: 10000, }, expectError: false, }, @@ -61,14 +55,11 @@ func TestProcessor_ConfigValidation(t *testing.T) { expectError: false, }, { - name: "missing URL", + name: "missing addr", config: transaction_structlog.Config{ - Enabled: true, - Table: "test_table", - BigTransactionThreshold: 500000, - BatchInsertThreshold: 50000, - BatchFlushInterval: 5 * time.Second, - BatchMaxSize: 100000, + Enabled: true, + Table: "test_table", + ChunkSize: 10000, }, expectError: true, }, @@ -77,12 +68,9 @@ func TestProcessor_ConfigValidation(t *testing.T) { config: transaction_structlog.Config{ Enabled: true, Config: clickhouse.Config{ - URL: "http://localhost:8123", + Addr: "localhost:9000", }, - BigTransactionThreshold: 500000, - BatchInsertThreshold: 50000, - BatchFlushInterval: 5 * time.Second, - BatchMaxSize: 100000, + ChunkSize: 10000, }, expectError: true, }, @@ -113,12 +101,9 @@ func TestProcessor_ConcurrentConfigValidation(t *testing.T) { Enabled: true, Table: "test_concurrent", Config: clickhouse.Config{ - URL: "http://localhost:8123", + Addr: "localhost:9000", }, - BigTransactionThreshold: 500000, - BatchInsertThreshold: 50000, - BatchFlushInterval: 5 * time.Second, - BatchMaxSize: 100000, + ChunkSize: 10000, } results <- cfg.Validate() }() @@ -146,60 +131,55 @@ func TestStructlogCountReturn(t *testing.T) { }, } - // Note: cannot directly create processor with private fields in _test package - // This test validates the counting logic conceptually - // Test the count calculation and return expectedCount := len(mockTrace.Structlogs) - // We can not easily test the full processTransaction method without a real execution node, - // but we can test the logic of creating structlogs and counting them - structlogs := make([]transaction_structlog.Structlog, 0, expectedCount) - + // Test using Columns type for counting + cols := transaction_structlog.NewColumns() now := time.Now() - // Simulate the structlog creation logic + // Simulate the structlog appending logic for i, structLog := range mockTrace.Structlogs { - row := transaction_structlog.Structlog{ - UpdatedDateTime: transaction_structlog.NewClickHouseTime(now), - BlockNumber: 12345, - TransactionHash: "0x1234567890abcdef", - TransactionIndex: 0, - TransactionGas: mockTrace.Gas, - TransactionFailed: mockTrace.Failed, - TransactionReturnValue: mockTrace.ReturnValue, - Index: uint32(i), - ProgramCounter: structLog.PC, - Operation: structLog.Op, - Gas: structLog.Gas, - GasCost: structLog.GasCost, - Depth: structLog.Depth, - ReturnData: structLog.ReturnData, - Refund: structLog.Refund, - Error: structLog.Error, - MetaNetworkName: "test", - } - - structlogs = append(structlogs, row) - } - - // Test the key fix: save count before clearing slice - structlogCount := len(structlogs) - - // Clear slice like the real code does - structlogs = nil + cols.Append( + now, + 12345, // blockNumber + "0x1234567890abcdef", // txHash + 0, // txIndex + mockTrace.Gas, // txGas + mockTrace.Failed, // txFailed + mockTrace.ReturnValue, // txReturnValue + uint32(i), // index + structLog.PC, // pc + structLog.Op, // op + structLog.Gas, // gas + structLog.GasCost, // gasCost + structLog.GasCost, // gasUsed (simplified) + structLog.Depth, // depth + structLog.ReturnData, // returnData + structLog.Refund, // refund + structLog.Error, // error + nil, // callTo + "test", // network + ) + } + + // Test the key fix: count is tracked by Columns + structlogCount := cols.Rows() // Verify the count was saved correctly if structlogCount != expectedCount { t.Errorf("Expected count %d, but got %d", expectedCount, structlogCount) } - // Verify that len(structlogs) is now 0 (which would be the bug) - if len(structlogs) != 0 { - t.Errorf("Expected cleared slice to have length 0, but got %d", len(structlogs)) + // Reset columns (like the real code does between chunks) + cols.Reset() + + // Verify that Rows() is now 0 (which would be the bug if we returned this after reset) + if cols.Rows() != 0 { + t.Errorf("Expected reset columns to have 0 rows, but got %d", cols.Rows()) } - // The fix ensures we return structlogCount, not len(structlogs) + // The fix ensures we track count before reset if structlogCount == 0 { t.Error("structlogCount should not be 0 after processing valid structlogs") } @@ -235,45 +215,52 @@ func TestMemoryManagement(t *testing.T) { runtime.ReadMemStats(&initialMemStats) - // Create a large slice of structlogs to simulate memory usage - largeStructlogs := make([]transaction_structlog.Structlog, 10000) + // Create columns and simulate large batch of structlogs + cols := transaction_structlog.NewColumns() now := time.Now() - for i := range largeStructlogs { - largeStructlogs[i] = transaction_structlog.Structlog{ - UpdatedDateTime: transaction_structlog.NewClickHouseTime(now), - BlockNumber: uint64(i), - TransactionHash: "0x1234567890abcdef1234567890abcdef12345678", - TransactionIndex: uint32(i % 100), - TransactionGas: 21000, - TransactionFailed: false, - TransactionReturnValue: nil, - Index: uint32(i), - ProgramCounter: uint32(i * 2), - Operation: "SSTORE", - Gas: uint64(21000 - i), - GasCost: 5000, - Depth: 1, - ReturnData: nil, - Refund: nil, - Error: nil, - MetaNetworkName: "mainnet", - } - } + const rowCount = 10000 + + for i := 0; i < rowCount; i++ { + cols.Append( + now, + uint64(i), // blockNumber + "0x1234567890abcdef1234567890abcdef12345678", // txHash + uint32(i%100), // txIndex + 21000, // txGas + false, // txFailed + nil, // txReturnValue + uint32(i), // index + uint32(i*2), // pc + "SSTORE", // op + uint64(21000-i), // gas + 5000, // gasCost + 5000, // gasUsed + 1, // depth + nil, // returnData + nil, // refund + nil, // error + nil, // callTo + "mainnet", // network + ) + } + + assert.Equal(t, rowCount, cols.Rows(), "Should have correct row count") // Test that chunking calculations work properly const chunkSize = 100 - expectedChunks := (len(largeStructlogs) + chunkSize - 1) / chunkSize + expectedChunks := (rowCount + chunkSize - 1) / chunkSize // Verify chunking logic actualChunks := 0 - for i := 0; i < len(largeStructlogs); i += chunkSize { + + for i := 0; i < rowCount; i += chunkSize { actualChunks++ end := i + chunkSize - if end > len(largeStructlogs) { - end = len(largeStructlogs) + if end > rowCount { + end = rowCount } // Verify chunk size constraints @@ -287,6 +274,10 @@ func TestMemoryManagement(t *testing.T) { t.Errorf("Expected %d chunks, got %d", expectedChunks, actualChunks) } + // Reset columns to free memory + cols.Reset() + assert.Equal(t, 0, cols.Rows(), "Reset should clear all rows") + runtime.GC() runtime.ReadMemStats(&finalMemStats) @@ -341,8 +332,20 @@ func TestChunkProcessing(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - // Create input data - structlogs := make([]transaction_structlog.Structlog, tt.inputSize) + // Test chunking logic using Columns + cols := transaction_structlog.NewColumns() + now := time.Now() + + // Fill columns with test data + for i := 0; i < tt.inputSize; i++ { + cols.Append( + now, uint64(i), "0xtest", 0, 21000, false, nil, + uint32(i), uint32(i), "PUSH1", 20000, 3, 3, 1, + nil, nil, nil, nil, "test", + ) + } + + assert.Equal(t, tt.inputSize, cols.Rows(), "Should have correct row count") // Calculate expected chunks expectedChunks := (tt.inputSize + tt.chunkSize - 1) / tt.chunkSize @@ -353,12 +356,13 @@ func TestChunkProcessing(t *testing.T) { // Test that the chunking logic would work correctly chunkCount := 0 - for i := 0; i < len(structlogs); i += tt.chunkSize { + + for i := 0; i < tt.inputSize; i += tt.chunkSize { chunkCount++ end := i + tt.chunkSize - if end > len(structlogs) { - end = len(structlogs) + if end > tt.inputSize { + end = tt.inputSize } // Verify chunk boundaries if end <= i { @@ -373,6 +377,51 @@ func TestChunkProcessing(t *testing.T) { } } +func TestColumnsAppendAndReset(t *testing.T) { + cols := transaction_structlog.NewColumns() + now := time.Now() + + // Initially empty + assert.Equal(t, 0, cols.Rows()) + + // Append a row + str := "test" + num := uint64(42) + + cols.Append( + now, 100, "0xabc", 0, 21000, false, &str, + 0, 100, "PUSH1", 20000, 3, 3, 1, + nil, &num, nil, nil, "mainnet", + ) + + assert.Equal(t, 1, cols.Rows()) + + // Append more rows + for i := 0; i < 99; i++ { + cols.Append( + now, 100, "0xabc", 0, 21000, false, nil, + uint32(i+1), 100, "PUSH1", 20000, 3, 3, 1, + nil, nil, nil, nil, "mainnet", + ) + } + + assert.Equal(t, 100, cols.Rows()) + + // Reset + cols.Reset() + assert.Equal(t, 0, cols.Rows()) +} + +func TestColumnsInput(t *testing.T) { + cols := transaction_structlog.NewColumns() + input := cols.Input() + + // Verify all 19 columns are present + assert.Len(t, input, 19) + assert.Equal(t, "updated_date_time", input[0].Name) + assert.Equal(t, "meta_network_name", input[18].Name) +} + // Tests from tasks_test.go func TestProcessPayload(t *testing.T) { diff --git a/pkg/processor/transaction/structlog/transaction_processing.go b/pkg/processor/transaction/structlog/transaction_processing.go index 1031ab8..ce0f54c 100644 --- a/pkg/processor/transaction/structlog/transaction_processing.go +++ b/pkg/processor/transaction/structlog/transaction_processing.go @@ -3,8 +3,10 @@ package structlog import ( "context" "fmt" + "io" "time" + "github.com/ClickHouse/ch-go" "github.com/ethereum/go-ethereum/core/types" "github.com/sirupsen/logrus" @@ -12,194 +14,106 @@ import ( "github.com/ethpandaops/execution-processor/pkg/ethereum/execution" ) -//nolint:tagliatelle // ClickHouse uses snake_case column names -type Structlog struct { - UpdatedDateTime ClickHouseTime `json:"updated_date_time"` - BlockNumber uint64 `json:"block_number"` - TransactionHash string `json:"transaction_hash"` - TransactionIndex uint32 `json:"transaction_index"` - TransactionGas uint64 `json:"transaction_gas"` - TransactionFailed bool `json:"transaction_failed"` - TransactionReturnValue *string `json:"transaction_return_value"` - Index uint32 `json:"index"` - ProgramCounter uint32 `json:"program_counter"` - Operation string `json:"operation"` - Gas uint64 `json:"gas"` - GasCost uint64 `json:"gas_cost"` - GasUsed uint64 `json:"gas_used"` - Depth uint64 `json:"depth"` - ReturnData *string `json:"return_data"` - Refund *uint64 `json:"refund"` - Error *string `json:"error"` - CallToAddress *string `json:"call_to_address"` - MetaNetworkName string `json:"meta_network_name"` -} - -// ProcessSingleTransaction processes a single transaction and inserts its structlogs directly to ClickHouse. -func (p *Processor) ProcessSingleTransaction(ctx context.Context, block *types.Block, index int, tx *types.Transaction) (int, error) { - // Extract structlog data - structlogs, err := p.ExtractStructlogs(ctx, block, index, tx) - if err != nil { - return 0, err - } - - // Store count before processing - structlogCount := len(structlogs) - - // Ensure we clear the slice on exit to allow GC - defer func() { - // Clear the slice to release memory - structlogs = nil - }() - - // Send for direct insertion - if err := p.insertStructlogs(ctx, structlogs); err != nil { - common.TransactionsProcessed.WithLabelValues(p.network.Name, "structlog", "failed").Inc() - - return 0, fmt.Errorf("failed to insert structlogs: %w", err) - } - - // Record success metrics - common.TransactionsProcessed.WithLabelValues(p.network.Name, "structlog", "success").Inc() - - return structlogCount, nil -} - -// ProcessTransaction processes a transaction using memory-efficient channel-based batching. +// ProcessTransaction processes a transaction using ch-go columnar streaming. func (p *Processor) ProcessTransaction(ctx context.Context, block *types.Block, index int, tx *types.Transaction) (int, error) { - // Get trace from execution node trace, err := p.getTransactionTrace(ctx, tx, block) if err != nil { return 0, fmt.Errorf("failed to get trace: %w", err) } totalCount := len(trace.Structlogs) + if totalCount == 0 { + return 0, nil + } // Compute actual gas used for each structlog gasUsed := ComputeGasUsed(trace.Structlogs) - // Check if this is a big transaction and register if needed - if totalCount >= p.bigTxManager.GetThreshold() { - p.bigTxManager.RegisterBigTransaction(tx.Hash().String(), p) - defer p.bigTxManager.UnregisterBigTransaction(tx.Hash().String()) - - p.log.WithFields(logrus.Fields{ - "tx_hash": tx.Hash().String(), - "structlog_count": totalCount, - "current_big_count": p.bigTxManager.currentBigCount.Load(), - }).Info("Processing big transaction") - } - chunkSize := p.config.ChunkSize if chunkSize == 0 { - chunkSize = 10_000 // Default - } - - // Buffered channel holds configured number of chunks - bufferSize := p.config.ChannelBufferSize - if bufferSize == 0 { - bufferSize = 2 // Default + chunkSize = 10_000 } - batchChan := make(chan []Structlog, bufferSize) - errChan := make(chan error, 1) - - // Consumer goroutine - inserts to ClickHouse - go func() { - inserted := 0 - - for batch := range batchChan { - if err := p.insertStructlogs(ctx, batch); err != nil { - errChan <- fmt.Errorf("failed to insert at %d: %w", inserted, err) + cols := NewColumns() + input := cols.Input() + + currentIdx := 0 + blockNum := block.Number().Uint64() + txHash := tx.Hash().String() + txIndex := uint32(index) //nolint:gosec // index is bounded by block.Transactions() length + now := time.Now() + + // Use Do with OnInput for streaming insert + err = p.clickhouse.Do(ctx, ch.Query{ + Body: input.Into(p.config.Table), + Input: input, + OnInput: func(ctx context.Context) error { + // Reset columns for next chunk + cols.Reset() + + if currentIdx >= totalCount { + return io.EOF + } - return + // Fill columns with next chunk + end := currentIdx + chunkSize + if end > totalCount { + end = totalCount } - inserted += len(batch) + for i := currentIdx; i < end; i++ { + sl := &trace.Structlogs[i] + cols.Append( + now, + blockNum, + txHash, + txIndex, + trace.Gas, + trace.Failed, + trace.ReturnValue, + uint32(i), //nolint:gosec // index is bounded by structlogs length + sl.PC, + sl.Op, + sl.Gas, + sl.GasCost, + gasUsed[i], + sl.Depth, + sl.ReturnData, + sl.Refund, + sl.Error, + p.extractCallAddress(sl), + p.network.Name, + ) + + // Free original trace data immediately to help GC + trace.Structlogs[i] = execution.StructLog{} + } // Log progress for large transactions progressThreshold := p.config.ProgressLogThreshold if progressThreshold == 0 { - progressThreshold = 100_000 // Default + progressThreshold = 100_000 } - if totalCount > progressThreshold && inserted%progressThreshold < chunkSize { + if totalCount > progressThreshold && end%progressThreshold < chunkSize { p.log.WithFields(logrus.Fields{ - "tx_hash": tx.Hash(), - "progress": fmt.Sprintf("%d/%d", inserted, totalCount), + "tx_hash": txHash, + "progress": fmt.Sprintf("%d/%d", end, totalCount), }).Debug("Processing large transaction") } - } - - errChan <- nil - }() - - // Producer - convert and send batches - batch := make([]Structlog, 0, chunkSize) - for i := 0; i < totalCount; i++ { - // Convert structlog - batch = append(batch, Structlog{ - UpdatedDateTime: NewClickHouseTime(time.Now()), - BlockNumber: block.Number().Uint64(), - TransactionHash: tx.Hash().String(), - TransactionIndex: uint32(index), //nolint:gosec // index is bounded by block.Transactions() length - TransactionGas: trace.Gas, - TransactionFailed: trace.Failed, - TransactionReturnValue: trace.ReturnValue, - Index: uint32(i), //nolint:gosec // index is bounded by structlogs length - ProgramCounter: trace.Structlogs[i].PC, - Operation: trace.Structlogs[i].Op, - Gas: trace.Structlogs[i].Gas, - GasCost: trace.Structlogs[i].GasCost, - GasUsed: gasUsed[i], - Depth: trace.Structlogs[i].Depth, - ReturnData: trace.Structlogs[i].ReturnData, - Refund: trace.Structlogs[i].Refund, - Error: trace.Structlogs[i].Error, - CallToAddress: p.extractCallAddress(&trace.Structlogs[i]), - MetaNetworkName: p.network.Name, - }) - // CRITICAL: Free original trace data immediately - trace.Structlogs[i] = execution.StructLog{} - - // Send full batch - if len(batch) == chunkSize { - select { - case batchChan <- batch: - batch = make([]Structlog, 0, chunkSize) - case <-ctx.Done(): - close(batchChan) - - return 0, ctx.Err() - } - } - } - - // Clear trace reference to help GC - trace = nil - - // Send final batch if any - if len(batch) > 0 { - select { - case batchChan <- batch: - case <-ctx.Done(): - close(batchChan) - - return 0, ctx.Err() - } - } + currentIdx = end - // Signal completion and wait - close(batchChan) - - // Wait for consumer to finish - if err := <-errChan; err != nil { - return 0, err + return nil + }, + }) + if err != nil { + return 0, fmt.Errorf("insert failed: %w", err) } // Record success metrics common.TransactionsProcessed.WithLabelValues(p.network.Name, "structlog", "success").Inc() + common.ClickHouseInsertsRows.WithLabelValues(p.network.Name, ProcessorName, p.config.Table, "success", "").Add(float64(totalCount)) return totalCount, nil } @@ -235,80 +149,3 @@ func (p *Processor) extractCallAddress(structLog *execution.StructLog) *string { return nil } - -// ExtractStructlogs extracts structlog data from a transaction without inserting to database. -func (p *Processor) ExtractStructlogs(ctx context.Context, block *types.Block, index int, tx *types.Transaction) ([]Structlog, error) { - start := time.Now() - - defer func() { - duration := time.Since(start) - common.TransactionProcessingDuration.WithLabelValues(p.network.Name, "structlog").Observe(duration.Seconds()) - }() - - // Get execution node - node := p.pool.GetHealthyExecutionNode() - if node == nil { - return nil, fmt.Errorf("no healthy execution node available") - } - - // Process transaction with timeout - processCtx, cancel := context.WithTimeout(ctx, 30*time.Second) - defer cancel() - - // Get transaction trace - trace, err := node.DebugTraceTransaction(processCtx, tx.Hash().String(), block.Number(), execution.StackTraceOptions()) - if err != nil { - return nil, fmt.Errorf("failed to trace transaction: %w", err) - } - - // Convert trace to structlog rows - var structlogs []Structlog - - uIndex := uint32(index) //nolint:gosec // index is bounded by block.Transactions() length - - if trace != nil { - // Compute actual gas used for each structlog - gasUsed := ComputeGasUsed(trace.Structlogs) - - // Pre-allocate slice for better memory efficiency - structlogs = make([]Structlog, 0, len(trace.Structlogs)) - - for i, structLog := range trace.Structlogs { - var callToAddress *string - - if structLog.Op == "CALL" && structLog.Stack != nil && len(*structLog.Stack) > 1 { - stackValue := (*structLog.Stack)[len(*structLog.Stack)-2] - callToAddress = &stackValue - } - - row := Structlog{ - UpdatedDateTime: NewClickHouseTime(time.Now()), - BlockNumber: block.Number().Uint64(), - TransactionHash: tx.Hash().String(), - TransactionIndex: uIndex, - TransactionGas: trace.Gas, - TransactionFailed: trace.Failed, - TransactionReturnValue: trace.ReturnValue, - Index: uint32(i), //nolint:gosec // index is bounded by structlogs length - ProgramCounter: structLog.PC, - Operation: structLog.Op, - Gas: structLog.Gas, - GasCost: structLog.GasCost, - GasUsed: gasUsed[i], - Depth: structLog.Depth, - ReturnData: structLog.ReturnData, - Refund: structLog.Refund, - Error: structLog.Error, - CallToAddress: callToAddress, - MetaNetworkName: p.network.Name, - } - - structlogs = append(structlogs, row) - } - - // Clear the original trace data to free memory - trace.Structlogs = nil - } - - return structlogs, nil -} diff --git a/pkg/server/server.go b/pkg/server/server.go index fa8485e..25a21d6 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -49,7 +49,7 @@ func NewServer(ctx context.Context, log logrus.FieldLogger, namespace string, co pool := ethereum.NewPool(log.WithField("component", "ethereum"), namespace, &config.Ethereum) - stateManager, err := state.NewManager(ctx, log.WithField("component", "state"), &config.StateManager) + stateManager, err := state.NewManager(log.WithField("component", "state"), &config.StateManager) if err != nil { return nil, fmt.Errorf("failed to create state manager: %w", err) } @@ -145,20 +145,32 @@ func (s *Server) Start(ctx context.Context) error { return nil }) + // Channel to signal when state manager is ready + stateReady := make(chan struct{}) + g.Go(func() error { if err := s.state.Start(ctx); err != nil { s.log.WithError(err).Error("State manager start failed") + close(stateReady) // Signal even on failure so processor doesn't hang return err } + close(stateReady) // Signal that state manager is ready <-ctx.Done() return nil }) - // Start processor + // Start processor - must wait for state manager to be ready g.Go(func() error { + // Wait for state manager to be ready before starting processor + select { + case <-stateReady: + case <-ctx.Done(): + return ctx.Err() + } + return s.processor.Start(ctx) }) diff --git a/pkg/state/json_number.go b/pkg/state/json_number.go deleted file mode 100644 index 6c5cf17..0000000 --- a/pkg/state/json_number.go +++ /dev/null @@ -1,126 +0,0 @@ -package state - -import ( - "encoding/json" - "strconv" -) - -// JSONInt64 handles JSON numbers that might be strings or numbers. -type JSONInt64 int64 - -// UnmarshalJSON implements json.Unmarshaler. -func (j *JSONInt64) UnmarshalJSON(data []byte) error { - // Try to unmarshal as number first - var num int64 - if err := json.Unmarshal(data, &num); err == nil { - *j = JSONInt64(num) - - return nil - } - - // Try to unmarshal as string - var str string - if err := json.Unmarshal(data, &str); err != nil { - return err - } - - // Parse string to int64 - num, err := strconv.ParseInt(str, 10, 64) - if err != nil { - return err - } - - *j = JSONInt64(num) - - return nil -} - -// MarshalJSON implements json.Marshaler. -func (j JSONInt64) MarshalJSON() ([]byte, error) { - return json.Marshal(int64(j)) -} - -// Int64 returns the value as int64. -func (j JSONInt64) Int64() int64 { - return int64(j) -} - -// JSONInt handles JSON numbers that might be strings or numbers. -type JSONInt int - -// UnmarshalJSON implements json.Unmarshaler. -func (j *JSONInt) UnmarshalJSON(data []byte) error { - // Try to unmarshal as number first - var num int - if err := json.Unmarshal(data, &num); err == nil { - *j = JSONInt(num) - - return nil - } - - // Try to unmarshal as string - var str string - if err := json.Unmarshal(data, &str); err != nil { - return err - } - - // Parse string to int - num, err := strconv.Atoi(str) - if err != nil { - return err - } - - *j = JSONInt(num) - - return nil -} - -// MarshalJSON implements json.Marshaler. -func (j JSONInt) MarshalJSON() ([]byte, error) { - return json.Marshal(int(j)) -} - -// Int returns the value as int. -func (j JSONInt) Int() int { - return int(j) -} - -// JSONUint64 handles JSON numbers that might be strings or numbers (unsigned). -type JSONUint64 uint64 - -// UnmarshalJSON implements json.Unmarshaler. -func (j *JSONUint64) UnmarshalJSON(data []byte) error { - // Try to unmarshal as number first - var num uint64 - if err := json.Unmarshal(data, &num); err == nil { - *j = JSONUint64(num) - - return nil - } - - // Try to unmarshal as string - var str string - if err := json.Unmarshal(data, &str); err != nil { - return err - } - - // Parse string to uint64 - num, err := strconv.ParseUint(str, 10, 64) - if err != nil { - return err - } - - *j = JSONUint64(num) - - return nil -} - -// MarshalJSON implements json.Marshaler. -func (j JSONUint64) MarshalJSON() ([]byte, error) { - return json.Marshal(uint64(j)) -} - -// Uint64 returns the value as uint64. -func (j JSONUint64) Uint64() uint64 { - return uint64(j) -} diff --git a/pkg/state/manager.go b/pkg/state/manager.go index 3e5c0d5..dcaa4d9 100644 --- a/pkg/state/manager.go +++ b/pkg/state/manager.go @@ -17,19 +17,6 @@ var ( ErrNoMoreBlocks = errors.New("no more blocks to process") ) -// blockNumberResult is used for queries that return a single block number. -// -//nolint:tagliatelle // ClickHouse uses snake_case column names -type blockNumberResult struct { - BlockNumber *JSONInt64 `json:"block_number"` -} - -// minMaxResult is used for queries that return min and max values. -type minMaxResult struct { - Min *JSONInt64 `json:"min"` - Max *JSONInt64 `json:"max"` -} - type Manager struct { log logrus.FieldLogger storageClient clickhouse.ClientInterface @@ -40,7 +27,7 @@ type Manager struct { network string } -func NewManager(ctx context.Context, log logrus.FieldLogger, config *Config) (*Manager, error) { +func NewManager(log logrus.FieldLogger, config *Config) (*Manager, error) { // Create storage client storageConfig := config.Storage.Config storageConfig.Processor = "state" @@ -87,12 +74,14 @@ func (s *Manager) SetNetwork(network string) { } func (s *Manager) Start(ctx context.Context) error { - if err := s.storageClient.Start(); err != nil { + // Start storage client with infinite retry + if err := s.startClientWithRetry(ctx, s.storageClient, "storage"); err != nil { return fmt.Errorf("failed to start storage client: %w", err) } + // Start limiter client with infinite retry if enabled if s.limiterEnabled && s.limiterClient != nil { - if err := s.limiterClient.Start(); err != nil { + if err := s.startClientWithRetry(ctx, s.limiterClient, "limiter"); err != nil { return fmt.Errorf("failed to start limiter client: %w", err) } } @@ -100,6 +89,49 @@ func (s *Manager) Start(ctx context.Context) error { return nil } +// startClientWithRetry starts a ClickHouse client with infinite retry and capped exponential backoff. +// This ensures the state manager can wait for ClickHouse to become available at startup. +func (s *Manager) startClientWithRetry(ctx context.Context, client clickhouse.ClientInterface, name string) error { + const ( + baseDelay = 100 * time.Millisecond + maxDelay = 10 * time.Second + ) + + attempt := 0 + + for { + err := client.Start() + if err == nil { + if attempt > 0 { + s.log.WithField("client", name).Info("Successfully connected after retries") + } + + return nil + } + + // Calculate delay with exponential backoff, capped at maxDelay + delay := baseDelay * time.Duration(1< maxDelay { + delay = maxDelay + } + + s.log.WithFields(logrus.Fields{ + "client": name, + "attempt": attempt + 1, + "delay": delay, + "error": err, + }).Warn("Failed to start client, retrying...") + + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(delay): + } + + attempt++ + } +} + func (s *Manager) Stop(ctx context.Context) error { var err error @@ -237,15 +269,13 @@ func (s *Manager) getProgressiveNextBlock(ctx context.Context, processor, networ "table": s.storageTable, }).Debug("Querying for last processed block") - var result blockNumberResult - - err := s.storageClient.QueryOne(ctx, query, &result) + blockNumber, err := s.storageClient.QueryUInt64(ctx, query, "block_number") if err != nil { return nil, false, fmt.Errorf("failed to get next block from %s: %w", s.storageTable, err) } // Check if we got a result - if result.BlockNumber == nil { + if blockNumber == nil { // Double-check if this is actually empty or no data isEmpty, err := s.storageClient.IsStorageEmpty(ctx, s.storageTable, map[string]interface{}{ "processor": processor, @@ -277,20 +307,20 @@ func (s *Manager) getProgressiveNextBlock(ctx context.Context, processor, networ } } - // Block 0 was actually processed but BlockNumber is nil (shouldn't happen but be defensive) + // Block 0 was actually processed but blockNumber is nil (shouldn't happen but be defensive) s.log.WithFields(logrus.Fields{ "processor": processor, "network": network, - }).Warn("Unexpected state: BlockNumber is nil but storage is not empty") + }).Warn("Unexpected state: block_number is nil but storage is not empty") return big.NewInt(0), false, nil } - nextBlock := big.NewInt(result.BlockNumber.Int64() + 1) + nextBlock := new(big.Int).SetUint64(*blockNumber + 1) s.log.WithFields(logrus.Fields{ "processor": processor, "network": network, - "last_processed": result.BlockNumber.Int64(), + "last_processed": *blockNumber, "progressive_next": nextBlock.String(), }).Debug("Found last processed block, calculated progressive next block") @@ -313,15 +343,13 @@ func (s *Manager) getProgressiveNextBlockBackwards(ctx context.Context, processo "table": s.storageTable, }).Debug("Querying for earliest processed block (backwards mode)") - var result blockNumberResult - - err := s.storageClient.QueryOne(ctx, query, &result) + blockNumber, err := s.storageClient.QueryUInt64(ctx, query, "block_number") if err != nil { return nil, fmt.Errorf("failed to get earliest block from %s: %w", s.storageTable, err) } // Check if we got a result - if result.BlockNumber == nil { + if blockNumber == nil { // No entry in table, need to start from chain tip for backwards processing if chainHead != nil && chainHead.Int64() > 0 { s.log.WithFields(logrus.Fields{ @@ -343,7 +371,7 @@ func (s *Manager) getProgressiveNextBlockBackwards(ctx context.Context, processo } // Calculate previous block (go backwards) - if result.BlockNumber.Int64() <= 0 { + if *blockNumber == 0 { // Already at genesis, no more blocks to process backwards s.log.WithFields(logrus.Fields{ "processor": processor, @@ -353,11 +381,11 @@ func (s *Manager) getProgressiveNextBlockBackwards(ctx context.Context, processo return nil, ErrNoMoreBlocks } - prevBlock := big.NewInt(result.BlockNumber.Int64() - 1) + prevBlock := new(big.Int).SetUint64(*blockNumber - 1) s.log.WithFields(logrus.Fields{ "processor": processor, "network": network, - "earliest_processed": result.BlockNumber.Int64(), + "earliest_processed": *blockNumber, "progressive_prev": prevBlock.String(), }).Debug("Found earliest processed block, calculated previous block for backwards processing") @@ -376,15 +404,13 @@ func (s *Manager) getLimiterMaxBlock(ctx context.Context, network string) (*big. "table": s.limiterTable, }).Debug("Querying for maximum execution payload block number") - var result blockNumberResult - - err := s.limiterClient.QueryOne(ctx, query, &result) + blockNumber, err := s.limiterClient.QueryUInt64(ctx, query, "block_number") if err != nil { return nil, fmt.Errorf("failed to get max execution payload block from %s: %w", s.limiterTable, err) } // Check if we got a result - if result.BlockNumber == nil { + if blockNumber == nil { // No blocks in limiter table, return genesis s.log.WithFields(logrus.Fields{ "network": network, @@ -393,7 +419,7 @@ func (s *Manager) getLimiterMaxBlock(ctx context.Context, network string) (*big. return big.NewInt(0), nil } - maxBlock := big.NewInt(result.BlockNumber.Int64()) + maxBlock := new(big.Int).SetUint64(*blockNumber) s.log.WithFields(logrus.Fields{ "network": network, "limiter_max": maxBlock.String(), @@ -434,15 +460,13 @@ func (s *Manager) GetMinMaxStoredBlocks(ctx context.Context, network, processor "table": s.storageTable, }).Debug("Querying for min/max stored blocks") - var result minMaxResult - - err = s.storageClient.QueryOne(ctx, query, &result) + minResult, maxResult, err := s.storageClient.QueryMinMaxUInt64(ctx, query) if err != nil { return nil, nil, fmt.Errorf("failed to get min/max blocks: %w", err) } // Handle case where no blocks are stored - if result.Min == nil || result.Max == nil { + if minResult == nil || maxResult == nil { s.log.WithFields(logrus.Fields{ "network": network, "processor": processor, @@ -456,11 +480,11 @@ func (s *Manager) GetMinMaxStoredBlocks(ctx context.Context, network, processor "network": network, "processor": processor, "table": s.storageTable, - "min": result.Min.Int64(), - "max": result.Max.Int64(), + "min": *minResult, + "max": *maxResult, }).Debug("Found min/max blocks") - return big.NewInt(result.Min.Int64()), big.NewInt(result.Max.Int64()), nil + return new(big.Int).SetUint64(*minResult), new(big.Int).SetUint64(*maxResult), nil } // IsBlockRecentlyProcessed checks if a block was processed within the specified number of seconds. @@ -482,16 +506,12 @@ func (s *Manager) IsBlockRecentlyProcessed(ctx context.Context, blockNumber uint "table": s.storageTable, }).Debug("Checking if block was recently processed") - var result struct { - Count JSONInt64 `json:"count"` - } - - err := s.storageClient.QueryOne(ctx, query, &result) + count, err := s.storageClient.QueryUInt64(ctx, query, "count") if err != nil { return false, fmt.Errorf("failed to check recent block processing: %w", err) } - return result.Count.Int64() > 0, nil + return count != nil && *count > 0, nil } // GetHeadDistance calculates the distance between current processing block and the relevant head. diff --git a/pkg/state/manager_test.go b/pkg/state/manager_test.go index 6e0b829..da33f5b 100644 --- a/pkg/state/manager_test.go +++ b/pkg/state/manager_test.go @@ -5,6 +5,7 @@ import ( "math/big" "testing" + "github.com/ClickHouse/ch-go" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -15,26 +16,36 @@ type MockClickHouseClient struct { mock.Mock } -func (m *MockClickHouseClient) QueryOne(ctx context.Context, query string, dest interface{}) error { - args := m.Called(ctx, query, dest) +func (m *MockClickHouseClient) QueryUInt64(ctx context.Context, query string, columnName string) (*uint64, error) { + args := m.Called(ctx, query, columnName) - return args.Error(0) -} + if args.Get(0) == nil { + return nil, args.Error(1) + } -func (m *MockClickHouseClient) QueryMany(ctx context.Context, query string, dest interface{}) error { - args := m.Called(ctx, query, dest) + val, _ := args.Get(0).(*uint64) - return args.Error(0) + return val, args.Error(1) } -func (m *MockClickHouseClient) Execute(ctx context.Context, query string) error { +func (m *MockClickHouseClient) QueryMinMaxUInt64(ctx context.Context, query string) (*uint64, *uint64, error) { args := m.Called(ctx, query) - return args.Error(0) + var minResult, maxResult *uint64 + + if args.Get(0) != nil { + minResult = args.Get(0).(*uint64) //nolint:errcheck // test mock + } + + if args.Get(1) != nil { + maxResult = args.Get(1).(*uint64) //nolint:errcheck // test mock + } + + return minResult, maxResult, args.Error(2) } -func (m *MockClickHouseClient) BulkInsert(ctx context.Context, table string, data interface{}) error { - args := m.Called(ctx, table, data) +func (m *MockClickHouseClient) Execute(ctx context.Context, query string) error { + args := m.Called(ctx, query) return args.Error(0) } @@ -61,6 +72,17 @@ func (m *MockClickHouseClient) SetNetwork(network string) { m.Called(network) } +func (m *MockClickHouseClient) Do(ctx context.Context, query ch.Query) error { + args := m.Called(ctx, query) + + return args.Error(0) +} + +// Helper to create uint64 pointer. +func uint64Ptr(v uint64) *uint64 { + return &v +} + func TestNextBlock_NoResultsVsBlock0(t *testing.T) { ctx := context.Background() log := logrus.New() @@ -75,12 +97,9 @@ func TestNextBlock_NoResultsVsBlock0(t *testing.T) { { name: "No results - should return chain head", setupMock: func(m *MockClickHouseClient) { - // QueryOne returns nil error but doesn't modify the dest struct - // This simulates the behavior when no rows are found - m.On("QueryOne", ctx, mock.AnythingOfType("string"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { - // Don't modify dest - this is what happens when no rows are found - }) - // IsStorageEmpty is called inside getProgressiveNextBlock when result is nil + // QueryUInt64 returns nil (no rows found) + m.On("QueryUInt64", ctx, mock.AnythingOfType("string"), "block_number").Return(nil, nil) + // IsStorageEmpty is called when result is nil m.On("IsStorageEmpty", ctx, "test_table", map[string]interface{}{ "processor": "test_processor", "meta_network_name": "mainnet", @@ -92,12 +111,7 @@ func TestNextBlock_NoResultsVsBlock0(t *testing.T) { { name: "Block 0 found - should return 1", setupMock: func(m *MockClickHouseClient) { - m.On("QueryOne", ctx, mock.AnythingOfType("string"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { - // Simulate finding block 0 - result := args.Get(2).(*blockNumberResult) //nolint:errcheck // type assertion in test - blockNum := JSONInt64(0) - result.BlockNumber = &blockNum - }) + m.On("QueryUInt64", ctx, mock.AnythingOfType("string"), "block_number").Return(uint64Ptr(0), nil) }, expectedNextBlock: big.NewInt(1), // Next block after 0 expectError: false, @@ -105,12 +119,7 @@ func TestNextBlock_NoResultsVsBlock0(t *testing.T) { { name: "Block 100 found - should return 101", setupMock: func(m *MockClickHouseClient) { - m.On("QueryOne", ctx, mock.AnythingOfType("string"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { - // Simulate finding block 100 - result := args.Get(2).(*blockNumberResult) //nolint:errcheck // type assertion in test - blockNum := JSONInt64(100) - result.BlockNumber = &blockNum - }) + m.On("QueryUInt64", ctx, mock.AnythingOfType("string"), "block_number").Return(uint64Ptr(100), nil) }, expectedNextBlock: big.NewInt(101), // Next block after 100 expectError: false, @@ -151,28 +160,28 @@ func TestNextBlock_EmptyStorageWithLimiter(t *testing.T) { tests := []struct { name string chainHead *big.Int - limiterMax *big.Int + limiterMax *uint64 limiterEnabled bool expectedNext *big.Int }{ { name: "empty storage with limiter and high chain head", chainHead: big.NewInt(5000), - limiterMax: big.NewInt(1000), + limiterMax: uint64Ptr(1000), limiterEnabled: true, expectedNext: big.NewInt(999), // limiterMax - 1 }, { name: "empty storage with limiter and nil chain head", chainHead: nil, - limiterMax: big.NewInt(1000), + limiterMax: uint64Ptr(1000), limiterEnabled: true, expectedNext: big.NewInt(999), // limiterMax - 1 }, { name: "empty storage with limiter and zero chain head", chainHead: big.NewInt(0), - limiterMax: big.NewInt(1000), + limiterMax: uint64Ptr(1000), limiterEnabled: true, expectedNext: big.NewInt(999), // limiterMax - 1 }, @@ -189,9 +198,7 @@ func TestNextBlock_EmptyStorageWithLimiter(t *testing.T) { t.Run(tt.name, func(t *testing.T) { // Create mock storage that reports empty mockStorage := new(MockClickHouseClient) - mockStorage.On("QueryOne", ctx, mock.AnythingOfType("string"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { - // Don't modify dest - storage is empty - }) + mockStorage.On("QueryUInt64", ctx, mock.AnythingOfType("string"), "block_number").Return(nil, nil) mockStorage.On("IsStorageEmpty", ctx, "test_table", map[string]interface{}{ "processor": "test-processor", "meta_network_name": "mainnet", @@ -201,15 +208,7 @@ func TestNextBlock_EmptyStorageWithLimiter(t *testing.T) { var mockLimiter *MockClickHouseClient if tt.limiterEnabled { mockLimiter = new(MockClickHouseClient) - mockLimiter.On("QueryOne", ctx, mock.AnythingOfType("string"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { - // Set the limiter max block - result := args.Get(2).(*blockNumberResult) //nolint:errcheck // type assertion in test - - if tt.limiterMax != nil { - blockNum := JSONInt64(tt.limiterMax.Int64()) - result.BlockNumber = &blockNum - } - }) + mockLimiter.On("QueryUInt64", ctx, mock.AnythingOfType("string"), "block_number").Return(tt.limiterMax, nil) } // Create manager @@ -290,20 +289,14 @@ func TestGetProgressiveNextBlock_EmptyStorage(t *testing.T) { if tt.storageEmpty { // Empty storage case - mockStorage.On("QueryOne", ctx, mock.AnythingOfType("string"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { - // Don't modify dest - storage is empty - }) + mockStorage.On("QueryUInt64", ctx, mock.AnythingOfType("string"), "block_number").Return(nil, nil) mockStorage.On("IsStorageEmpty", ctx, "test_table", map[string]interface{}{ "processor": "test-processor", "meta_network_name": "mainnet", }).Return(true, nil) } else { // Non-empty storage case - mockStorage.On("QueryOne", ctx, mock.AnythingOfType("string"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { - result := args.Get(2).(*blockNumberResult) //nolint:errcheck // type assertion in test - blockNum := JSONInt64(tt.storedBlock) - result.BlockNumber = &blockNum - }) + mockStorage.On("QueryUInt64", ctx, mock.AnythingOfType("string"), "block_number").Return(uint64Ptr(tt.storedBlock), nil) } manager := &Manager{ @@ -338,9 +331,7 @@ func TestGetProgressiveNextBlock_NoResultsVsBlock0(t *testing.T) { { name: "No results with chain head - should return chain head", setupMock: func(m *MockClickHouseClient) { - m.On("QueryOne", ctx, mock.AnythingOfType("string"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { - // Don't modify dest - this is what happens when no rows are found - }) + m.On("QueryUInt64", ctx, mock.AnythingOfType("string"), "block_number").Return(nil, nil) // IsStorageEmpty is called when result is nil m.On("IsStorageEmpty", ctx, "test_table", map[string]interface{}{ "processor": "test_processor", @@ -354,9 +345,7 @@ func TestGetProgressiveNextBlock_NoResultsVsBlock0(t *testing.T) { { name: "No results without chain head - should return 0", setupMock: func(m *MockClickHouseClient) { - m.On("QueryOne", ctx, mock.AnythingOfType("string"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { - // Don't modify dest - this is what happens when no rows are found - }) + m.On("QueryUInt64", ctx, mock.AnythingOfType("string"), "block_number").Return(nil, nil) // IsStorageEmpty is called when result is nil m.On("IsStorageEmpty", ctx, "test_table", map[string]interface{}{ "processor": "test_processor", @@ -370,11 +359,7 @@ func TestGetProgressiveNextBlock_NoResultsVsBlock0(t *testing.T) { { name: "Block 0 found - should return 1", setupMock: func(m *MockClickHouseClient) { - m.On("QueryOne", ctx, mock.AnythingOfType("string"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { - result := args.Get(2).(*blockNumberResult) //nolint:errcheck // type assertion in test - blockNum := JSONInt64(0) - result.BlockNumber = &blockNum - }) + m.On("QueryUInt64", ctx, mock.AnythingOfType("string"), "block_number").Return(uint64Ptr(0), nil) }, chainHead: big.NewInt(1000), expectedNextBlock: big.NewInt(1),