diff --git a/cmd/ingest.go b/cmd/ingest.go index d1ce7a5cb..cb96f6525 100644 --- a/cmd/ingest.go +++ b/cmd/ingest.go @@ -59,29 +59,53 @@ func (c *ingestCmd) Command() *cobra.Command { Required: true, }, { - Name: "backfill-workers", - Usage: "Maximum concurrent workers for backfill processing. Defaults to number of CPUs. Lower values reduce RAM usage at cost of throughput.", + Name: "backfill-process-workers", + Usage: "Number of Stage 2 process workers in the backfill pipeline. Defaults to number of CPUs.", OptType: types.Int, - ConfigKey: &cfg.BackfillWorkers, - FlagDefault: 0, + ConfigKey: &cfg.BackfillProcessWorkers, + FlagDefault: 2, Required: false, }, { - Name: "backfill-batch-size", - Usage: "Number of ledgers per batch during backfill. Defaults to 250. Lower values reduce RAM usage at cost of more DB transactions.", + Name: "backfill-flush-workers", + Usage: "Number of Stage 3 flush workers in the backfill pipeline. Each uses 5 parallel DB connections.", OptType: types.Int, - ConfigKey: &cfg.BackfillBatchSize, - FlagDefault: 250, + ConfigKey: &cfg.BackfillFlushWorkers, + FlagDefault: 2, Required: false, }, { Name: "backfill-db-insert-batch-size", - Usage: "Number of ledgers to process before flushing buffer to DB during backfill. Defaults to 100. Lower values reduce RAM usage at cost of more DB transactions.", + Usage: "Number of ledgers to process before flushing buffer to DB during backfill. Lower values reduce RAM usage at cost of more DB transactions.", OptType: types.Int, ConfigKey: &cfg.BackfillDBInsertBatchSize, FlagDefault: 100, Required: false, }, + { + Name: "backfill-ledger-chan-size", + Usage: "Bounded channel size between the dispatcher and process workers in the backfill pipeline.", + OptType: types.Int, + ConfigKey: &cfg.BackfillLedgerChanSize, + FlagDefault: 200, + Required: false, + }, + { + Name: "backfill-flush-chan-size", + Usage: "Bounded channel size between process workers and flush workers in the backfill pipeline.", + OptType: types.Int, + ConfigKey: &cfg.BackfillFlushChanSize, + FlagDefault: 200, + Required: false, + }, + { + Name: "backfill-fetch-workers", + Usage: "Number of parallel S3 download workers in the backfill fetcher. Each worker downloads and decodes one file at a time.", + OptType: types.Int, + ConfigKey: &cfg.BackfillFetchWorkers, + FlagDefault: 40, + Required: false, + }, { Name: "archive-url", Usage: "Archive URL for history archives", diff --git a/config/datastore-pubnet.toml b/config/datastore-pubnet.toml index 51381c209..278ff47a8 100644 --- a/config/datastore-pubnet.toml +++ b/config/datastore-pubnet.toml @@ -13,9 +13,9 @@ region = "us-east-2" [buffered_storage_backend_config] # Buffer size for reading ledgers -buffer_size = 100 +buffer_size = 200 # Number of concurrent workers for reading -num_workers = 10 +num_workers = 15 # Number of retries for failed operations retry_limit = 3 # Wait time between retries diff --git a/docker-compose.yaml b/docker-compose.yaml index acbae6bee..d487c0468 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -8,7 +8,24 @@ services: db: container_name: db image: timescale/timescaledb:2.25.0-pg17 - command: ["postgres", "-c", "timescaledb.enable_chunk_skipping=on", "-c", "timescaledb.enable_sparse_index_bloom=on"] + command: [ + "postgres", + "-c", "timescaledb.enable_chunk_skipping=on", + "-c", "timescaledb.enable_sparse_index_bloom=on", + "-c", "shared_buffers=8GB", + "-c", "work_mem=256MB", + "-c", "maintenance_work_mem=2GB", + "-c", "wal_buffers=64MB", + "-c", "max_wal_size=32GB", + "-c", "wal_compression=lz4", + "-c", "checkpoint_completion_target=0.9", + "-c", "checkpoint_timeout=30min", + "-c", "max_connections=100", + "-c", "effective_io_concurrency=200", + "-c", "synchronous_commit=off", + "-c", "max_parallel_maintenance_workers=4", + "-c", "autovacuum=off" + ] healthcheck: test: ["CMD-SHELL", "pg_isready -U postgres -d wallet-backend"] interval: 10s diff --git a/docs/plans/2026-04-02-backfill-observability-design.md b/docs/plans/2026-04-02-backfill-observability-design.md new file mode 100644 index 000000000..6ed15d014 --- /dev/null +++ b/docs/plans/2026-04-02-backfill-observability-design.md @@ -0,0 +1,168 @@ +# Backfill Pipeline Observability Design + +**Date:** 2026-04-02 +**Status:** Approved +**Goal:** Add per-stage timing, bottleneck identification, and throughput metrics to the 3-stage backfill pipeline (dispatcher -> process -> flush). + +## Problem + +The backfill pipeline in `internal/services/ingest_backfill.go` has minimal observability. Only `RetriesTotal` and `RetryExhaustionsTotal` with `"batch_flush"` labels exist. There is no visibility into: + +- How much time each pipeline stage takes +- Whether workers are blocked waiting on channels (backpressure) +- What the throughput rate is +- Whether batches are full or partial + +Without this, optimization is guesswork. + +## Approach + +**Approach B: Prometheus metrics + structured gap summary logs.** Uses the existing `IngestionMetrics` struct and custom registry. No new dependencies. + +## New Prometheus Metrics (8 total, 14 series) + +### 1. Per-Stage Durations — Reuse existing `PhaseDuration` HistogramVec + +Add 3 new phase labels to the existing `wallet_ingestion_phase_duration_seconds`: + +| Phase Label | Stage | Measures | +|---|---|---| +| `backfill_fetch` | Stage 1 | Time for a single successful ledger fetch (excludes retry overhead) | +| `backfill_process` | Stage 2 | Time to process one ledger into an IndexerBuffer | +| `backfill_flush` | Stage 3 | Time for one `flushBufferWithRetry` call | + +### 2. Channel Wait Time Histogram (1 new metric, 4 series) + +``` +wallet_ingestion_backfill_channel_wait_seconds {channel, direction} +``` + +- **Type:** Histogram +- **Labels:** `channel` (`"ledger"`, `"flush"`) x `direction` (`"send"`, `"receive"`) +- **Buckets:** `[0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10, 30]` +- **Diagnostic:** High send wait = downstream bottleneck. High receive wait = upstream bottleneck. + +### 3. Channel Utilization Gauges (1 new metric, 2 series) + +``` +wallet_ingestion_backfill_channel_utilization_ratio {channel} +``` + +- **Type:** GaugeVec +- **Labels:** `channel` (`"ledger"`, `"flush"`) +- **Sampled:** Every 1 second by a goroutine scoped to the gap's context +- **Range:** 0.0 (empty) to 1.0 (full) +- **Diagnostic:** Sustained 1.0 = downstream can't keep up. Sustained 0.0 = upstream is slow. + +### 4. Backfill Throughput Counter (1 new metric, 1 series) + +``` +wallet_ingestion_backfill_ledgers_flushed_total +``` + +- **Type:** Counter +- **Incremented:** By batch size when flush worker completes +- **Query:** `rate(...[5m])` for ledgers/sec + +### 5. Backfill Batch Size Histogram (1 new metric, 1 series) + +``` +wallet_ingestion_backfill_batch_size +``` + +- **Type:** Histogram +- **Buckets:** `LinearBuckets(10, 10, 10)` -> 10, 20, ..., 100 +- **Diagnostic:** Most batches at 100 = healthy. Many partial batches = process workers starved. + +### 6. Gap Progress Gauge (1 new metric, 1 series) + +``` +wallet_ingestion_backfill_gap_progress_ratio +``` + +- **Type:** Gauge +- **Updated:** When watermark advances, set to `(cursor - start) / (end - start)` +- **Diagnostic:** Steadily climbing = healthy. Plateau = stalled pipeline. + +### 7. Gap Boundary Gauges (2 new metrics, 2 series) + +``` +wallet_ingestion_backfill_gap_start_ledger +wallet_ingestion_backfill_gap_end_ledger +``` + +- **Type:** Gauge +- **Set:** When gap processing begins; reset to 0 when done +- **Diagnostic:** Shows which ledger range is active for dashboard correlation. + +### Cardinality Summary + +| Metric | Series | +|---|---| +| PhaseDuration (3 new labels) | 3 | +| channel_wait_seconds | 4 | +| channel_utilization_ratio | 2 | +| backfill_ledgers_flushed_total | 1 | +| backfill_batch_size | 1 | +| backfill_gap_progress_ratio | 1 | +| gap_start/end_ledger | 2 | +| **Total** | **14** | + +## Structured Logging + +### Gap Summary Log + +Emitted at gap completion by `processGap`. Each pipeline stage accumulates local stats (no contention), reported at shutdown: + +``` +Gap [1000-50000] complete in 2m34s: + fetch: 1m12s total (avg 1.4ms/ledger) + process: 48s total (avg 0.98ms/ledger) + flush: 34s total (avg 340ms/batch, 145 batches) + channel_wait: ledger_send=2.1s ledger_recv=45s flush_send=12s flush_recv=0.3s + throughput: 318 ledgers/sec +``` + +### Implementation Pattern + +A `backfillStats` struct with per-worker accumulators: + +- Each worker goroutine maintains a local `backfillWorkerStats` (no mutex needed) +- Workers report stats through a channel or at function return +- `processGap` aggregates all worker stats into the summary log + +## Channel Utilization Sampler + +A lightweight goroutine started per gap: + +- Ticks every 1 second +- Reads `len(ledgerCh)/cap(ledgerCh)` and `len(flushCh)/cap(flushCh)` +- Sets the gauge values +- Exits when the gap context is cancelled + +Since Go channels expose `len()` and `cap()` as non-blocking reads, this has negligible overhead. + +## Bottleneck Identification Cheat Sheet + +| Symptom | Bottleneck | Action | +|---|---|---| +| `ledger/send` wait high, `ledger/receive` wait low | Stage 2 (process) | Increase `backfillProcessWorkers` | +| `flush/send` wait high, `flush/receive` wait low | Stage 3 (flush/DB) | Increase `backfillFlushWorkers` or tune DB | +| `ledger/receive` high, `flush/receive` high | Stage 1 (fetch/RPC) | RPC is slow; nothing to tune internally | +| `ledger` utilization ~1.0 | Stage 2 can't keep up | More process workers or increase `backfillLedgerChanSize` | +| `flush` utilization ~1.0 | Stage 3 can't keep up | More flush workers or increase `backfillFlushChanSize` | +| Batch sizes mostly partial | Process workers starved | Stage 1 fetch is the bottleneck | + +## Files Modified + +| File | Changes | +|---|---| +| `internal/metrics/ingestion.go` | Add 8 new metric fields + registration | +| `internal/services/ingest_backfill.go` | Instrument all 3 stages, add channel sampler, add stats aggregation, add gap summary log | +| `internal/services/backfill_stats.go` | New file: `backfillStats` and `backfillWorkerStats` types | + +## Non-Goals + +- OpenTelemetry tracing +- Per-ledger log lines (too noisy) +- Alerting rules (separate concern, can be added to Grafana later) diff --git a/docs/plans/2026-04-02-backfill-observability.md b/docs/plans/2026-04-02-backfill-observability.md new file mode 100644 index 000000000..7ae187f47 --- /dev/null +++ b/docs/plans/2026-04-02-backfill-observability.md @@ -0,0 +1,843 @@ +# Backfill Pipeline Observability Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Add per-stage timing, channel wait/utilization metrics, throughput counters, and structured gap summary logs to the 3-stage backfill pipeline. + +**Architecture:** Extend the existing `IngestionMetrics` struct with 8 new Prometheus metrics (14 series). Each pipeline stage instruments its own timing and channel waits. A per-gap stats accumulator collects worker-local data for a summary log at gap completion. A lightweight channel utilization sampler goroutine runs per gap. + +**Tech Stack:** `prometheus/client_golang`, existing `metrics.Metrics` DI pattern, Go channels, `sync/atomic` for stats. + +**Design doc:** `docs/plans/2026-04-02-backfill-observability-design.md` + +--- + +### Task 1: Add new backfill metrics to IngestionMetrics + +**Files:** +- Modify: `internal/metrics/ingestion.go` +- Modify: `internal/metrics/ingestion_test.go` + +**Step 1: Write the failing tests for new metric fields** + +Add to `internal/metrics/ingestion_test.go`: + +```go +func TestIngestionMetrics_BackfillRegistration(t *testing.T) { + reg := prometheus.NewRegistry() + m := newIngestionMetrics(reg) + + require.NotNil(t, m.BackfillChannelWait) + require.NotNil(t, m.BackfillChannelUtilization) + require.NotNil(t, m.BackfillLedgersFlushed) + require.NotNil(t, m.BackfillBatchSize) + require.NotNil(t, m.BackfillGapProgress) + require.NotNil(t, m.BackfillGapStartLedger) + require.NotNil(t, m.BackfillGapEndLedger) +} + +func TestIngestionMetrics_BackfillChannelWait_Buckets(t *testing.T) { + reg := prometheus.NewRegistry() + m := newIngestionMetrics(reg) + + m.BackfillChannelWait.WithLabelValues("ledger", "send").Observe(0.1) + + families, err := reg.Gather() + require.NoError(t, err) + for _, f := range families { + if f.GetName() == "wallet_ingestion_backfill_channel_wait_seconds" { + h := f.GetMetric()[0].GetHistogram() + assert.Len(t, h.GetBucket(), 10) + } + } +} + +func TestIngestionMetrics_BackfillBatchSize_Buckets(t *testing.T) { + reg := prometheus.NewRegistry() + m := newIngestionMetrics(reg) + + m.BackfillBatchSize.Observe(50) + + families, err := reg.Gather() + require.NoError(t, err) + for _, f := range families { + if f.GetName() == "wallet_ingestion_backfill_batch_size" { + h := f.GetMetric()[0].GetHistogram() + assert.Len(t, h.GetBucket(), 10) // LinearBuckets(10, 10, 10) + } + } +} + +func TestIngestionMetrics_BackfillChannelWait_GoldenExposition(t *testing.T) { + reg := prometheus.NewRegistry() + m := newIngestionMetrics(reg) + + m.BackfillChannelWait.WithLabelValues("ledger", "send").Observe(0.05) + m.BackfillChannelWait.WithLabelValues("flush", "receive").Observe(1.0) + + // Verify labels exist and are correct + assert.Equal(t, 1.0, testutil.ToFloat64( + m.BackfillChannelWait.WithLabelValues("ledger", "send").(*prometheus.HistogramVec... // skip: just assert non-panic + )) + // Simpler: just ensure no panic on valid label combos + assert.NotPanics(t, func() { + m.BackfillChannelWait.WithLabelValues("ledger", "send").Observe(0.1) + m.BackfillChannelWait.WithLabelValues("ledger", "receive").Observe(0.1) + m.BackfillChannelWait.WithLabelValues("flush", "send").Observe(0.1) + m.BackfillChannelWait.WithLabelValues("flush", "receive").Observe(0.1) + }) +} +``` + +Also update the lint test to include new collectors: + +```go +// In TestIngestionMetrics_Lint, add to the collector slice: +m.BackfillChannelWait, m.BackfillChannelUtilization, +m.BackfillLedgersFlushed, m.BackfillBatchSize, +m.BackfillGapProgress, m.BackfillGapStartLedger, m.BackfillGapEndLedger, +``` + +**Step 2: Run tests to verify they fail** + +Run: `go test -v ./internal/metrics/ -run TestIngestionMetrics_Backfill -timeout 30s` +Expected: FAIL — fields don't exist on `IngestionMetrics` + +**Step 3: Add new metric fields and registration to ingestion.go** + +Add these fields to the `IngestionMetrics` struct in `internal/metrics/ingestion.go`: + +```go +// --- Backfill Pipeline Metrics --- + +// BackfillChannelWait observes time goroutines spend blocked on channel operations. +// Labels: channel ("ledger", "flush"), direction ("send", "receive"). +// PromQL: histogram_quantile(0.99, rate(wallet_ingestion_backfill_channel_wait_seconds_bucket{channel="ledger",direction="send"}[5m])) +BackfillChannelWait *prometheus.HistogramVec +// BackfillChannelUtilization reports channel fill ratio (0.0-1.0), sampled every second. +// Labels: channel ("ledger", "flush"). +// PromQL: wallet_ingestion_backfill_channel_utilization_ratio{channel="ledger"} +BackfillChannelUtilization *prometheus.GaugeVec +// BackfillLedgersFlushed counts ledgers successfully flushed to DB during backfill. +// PromQL: rate(wallet_ingestion_backfill_ledgers_flushed_total[5m]) +BackfillLedgersFlushed prometheus.Counter +// BackfillBatchSize observes the number of ledgers per flush batch. +// PromQL: histogram_quantile(0.5, rate(wallet_ingestion_backfill_batch_size_bucket[5m])) +BackfillBatchSize prometheus.Histogram +// BackfillGapProgress reports completion ratio (0.0-1.0) of the current gap. +// PromQL: wallet_ingestion_backfill_gap_progress_ratio +BackfillGapProgress prometheus.Gauge +// BackfillGapStartLedger is the start ledger of the gap currently being processed. +// PromQL: wallet_ingestion_backfill_gap_start_ledger +BackfillGapStartLedger prometheus.Gauge +// BackfillGapEndLedger is the end ledger of the gap currently being processed. +// PromQL: wallet_ingestion_backfill_gap_end_ledger +BackfillGapEndLedger prometheus.Gauge +``` + +Add initialization in `newIngestionMetrics()`: + +```go +BackfillChannelWait: prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "wallet_ingestion_backfill_channel_wait_seconds", + Help: "Time goroutines spend blocked on backfill pipeline channel operations.", + Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10, 30}, +}, []string{"channel", "direction"}), +BackfillChannelUtilization: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "wallet_ingestion_backfill_channel_utilization_ratio", + Help: "Fill ratio (0.0-1.0) of backfill pipeline channels, sampled every second.", +}, []string{"channel"}), +BackfillLedgersFlushed: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "wallet_ingestion_backfill_ledgers_flushed_total", + Help: "Total ledgers successfully flushed to database during backfill.", +}), +BackfillBatchSize: prometheus.NewHistogram(prometheus.HistogramOpts{ + Name: "wallet_ingestion_backfill_batch_size", + Help: "Number of ledgers per flush batch during backfill.", + Buckets: prometheus.LinearBuckets(10, 10, 10), +}), +BackfillGapProgress: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "wallet_ingestion_backfill_gap_progress_ratio", + Help: "Completion ratio (0.0-1.0) of the backfill gap currently being processed.", +}), +BackfillGapStartLedger: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "wallet_ingestion_backfill_gap_start_ledger", + Help: "Start ledger of the backfill gap currently being processed (0 when idle).", +}), +BackfillGapEndLedger: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "wallet_ingestion_backfill_gap_end_ledger", + Help: "End ledger of the backfill gap currently being processed (0 when idle).", +}), +``` + +Add to `reg.MustRegister(...)`: + +```go +m.BackfillChannelWait, +m.BackfillChannelUtilization, +m.BackfillLedgersFlushed, +m.BackfillBatchSize, +m.BackfillGapProgress, +m.BackfillGapStartLedger, +m.BackfillGapEndLedger, +``` + +**Step 4: Run tests to verify they pass** + +Run: `go test -v ./internal/metrics/ -timeout 30s` +Expected: ALL PASS (including lint test with new collectors) + +**Step 5: Commit** + +```bash +git add internal/metrics/ingestion.go internal/metrics/ingestion_test.go +git commit -m "Add backfill pipeline Prometheus metrics to IngestionMetrics" +``` + +--- + +### Task 2: Create backfill stats types + +**Files:** +- Create: `internal/services/backfill_stats.go` +- Create: `internal/services/backfill_stats_test.go` + +**Step 1: Write the failing tests** + +Add to `internal/services/backfill_stats_test.go`: + +```go +package services + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestBackfillWorkerStats_Add(t *testing.T) { + s := &backfillWorkerStats{} + + s.addFetch(100 * time.Millisecond) + s.addFetch(200 * time.Millisecond) + s.addProcess(50 * time.Millisecond) + s.addChannelWait("ledger", "send", 10*time.Millisecond) + + assert.Equal(t, 2, s.fetchCount) + assert.Equal(t, 300*time.Millisecond, s.fetchTotal) + assert.Equal(t, 1, s.processCount) + assert.Equal(t, 50*time.Millisecond, s.processTotal) + assert.Equal(t, 10*time.Millisecond, s.channelWait["ledger:send"]) +} + +func TestBackfillGapStats_Merge(t *testing.T) { + gs := newBackfillGapStats() + + w1 := &backfillWorkerStats{} + w1.addFetch(100 * time.Millisecond) + w1.addProcess(50 * time.Millisecond) + w1.addChannelWait("ledger", "send", 10*time.Millisecond) + + w2 := &backfillWorkerStats{} + w2.addFetch(200 * time.Millisecond) + w2.addProcess(75 * time.Millisecond) + w2.addChannelWait("ledger", "send", 20*time.Millisecond) + + gs.mergeWorker(w1) + gs.mergeWorker(w2) + + assert.Equal(t, 3, gs.fetchCount) // NOT merged — wait, this should be sum: 1+1=2? No: let me re-check... + // Actually w1 has fetchCount=1, w2 has fetchCount=1 => merged = 2 + // Let me fix: the test above sets w1.fetchCount=2 via two addFetch calls + // So merged = 2 + 1 = 3 + assert.Equal(t, 300*time.Millisecond, gs.fetchTotal) + assert.Equal(t, 2, gs.processCount) + assert.Equal(t, 125*time.Millisecond, gs.processTotal) + assert.Equal(t, 30*time.Millisecond, gs.channelWait["ledger:send"]) +} + +func TestBackfillGapStats_MergeFlush(t *testing.T) { + gs := newBackfillGapStats() + + w := &backfillFlushWorkerStats{} + w.addFlush(500 * time.Millisecond) + w.addFlush(300 * time.Millisecond) + w.addChannelWait("flush", "receive", 100*time.Millisecond) + + gs.mergeFlushWorker(w) + + assert.Equal(t, 2, gs.flushCount) + assert.Equal(t, 800*time.Millisecond, gs.flushTotal) + assert.Equal(t, 100*time.Millisecond, gs.channelWait["flush:receive"]) +} +``` + +**Step 2: Run tests to verify they fail** + +Run: `go test -v ./internal/services/ -run TestBackfill.*Stats -timeout 30s` +Expected: FAIL — types don't exist + +**Step 3: Implement backfill stats types** + +Create `internal/services/backfill_stats.go`: + +```go +package services + +import ( + "fmt" + "time" +) + +// backfillWorkerStats accumulates timing for a single process worker. +// Each worker owns its instance — no mutex needed. +type backfillWorkerStats struct { + fetchCount int + fetchTotal time.Duration + processCount int + processTotal time.Duration + channelWait map[string]time.Duration // key: "channel:direction" +} + +func (s *backfillWorkerStats) addFetch(d time.Duration) { + s.fetchCount++ + s.fetchTotal += d +} + +func (s *backfillWorkerStats) addProcess(d time.Duration) { + s.processCount++ + s.processTotal += d +} + +func (s *backfillWorkerStats) addChannelWait(channel, direction string, d time.Duration) { + if s.channelWait == nil { + s.channelWait = make(map[string]time.Duration) + } + s.channelWait[fmt.Sprintf("%s:%s", channel, direction)] += d +} + +// backfillFlushWorkerStats accumulates timing for a single flush worker. +type backfillFlushWorkerStats struct { + flushCount int + flushTotal time.Duration + channelWait map[string]time.Duration +} + +func (s *backfillFlushWorkerStats) addFlush(d time.Duration) { + s.flushCount++ + s.flushTotal += d +} + +func (s *backfillFlushWorkerStats) addChannelWait(channel, direction string, d time.Duration) { + if s.channelWait == nil { + s.channelWait = make(map[string]time.Duration) + } + s.channelWait[fmt.Sprintf("%s:%s", channel, direction)] += d +} + +// backfillGapStats aggregates stats from all workers for a single gap. +// Only accessed from processGap after workers complete — no mutex needed. +type backfillGapStats struct { + fetchCount int + fetchTotal time.Duration + processCount int + processTotal time.Duration + flushCount int + flushTotal time.Duration + channelWait map[string]time.Duration +} + +func newBackfillGapStats() *backfillGapStats { + return &backfillGapStats{ + channelWait: make(map[string]time.Duration), + } +} + +func (gs *backfillGapStats) mergeWorker(w *backfillWorkerStats) { + gs.fetchCount += w.fetchCount + gs.fetchTotal += w.fetchTotal + gs.processCount += w.processCount + gs.processTotal += w.processTotal + for k, v := range w.channelWait { + gs.channelWait[k] += v + } +} + +func (gs *backfillGapStats) mergeFlushWorker(w *backfillFlushWorkerStats) { + gs.flushCount += w.flushCount + gs.flushTotal += w.flushTotal + for k, v := range w.channelWait { + gs.channelWait[k] += v + } +} + +// avgOrZero returns average duration, or zero if count is 0. +func avgOrZero(total time.Duration, count int) time.Duration { + if count == 0 { + return 0 + } + return total / time.Duration(count) +} +``` + +**Step 4: Run tests to verify they pass** + +Run: `go test -v ./internal/services/ -run TestBackfill.*Stats -timeout 30s` +Expected: ALL PASS + +**Step 5: Commit** + +```bash +git add internal/services/backfill_stats.go internal/services/backfill_stats_test.go +git commit -m "Add backfill stats accumulator types for gap summary logging" +``` + +--- + +### Task 3: Instrument Stage 1 (Dispatcher) with fetch timing and channel wait + +**Files:** +- Modify: `internal/services/ingest_backfill.go` — `runDispatcher` function + +**Step 1: Add fetch timing and ledger-send channel wait instrumentation** + +Modify `runDispatcher` in `internal/services/ingest_backfill.go`. The function currently: +1. Loops `gap.GapStart` to `gap.GapEnd` +2. Calls `getLedgerWithRetry` (already has `LedgerFetchDuration` metric for total including retries) +3. Sends to `ledgerCh` + +Change the signature to accept `*backfillWorkerStats` and return it filled. Add timing around the fetch call (for the `backfill_fetch` phase — just the successful call time, different from `LedgerFetchDuration` which includes retries), and timing around the channel send: + +```go +func (m *ingestService) runDispatcher( + ctx context.Context, + cancel context.CancelCauseFunc, + backend ledgerbackend.LedgerBackend, + gap data.LedgerRange, + ledgerCh chan<- xdr.LedgerCloseMeta, +) *backfillWorkerStats { + defer close(ledgerCh) + stats := &backfillWorkerStats{} + + for seq := gap.GapStart; seq <= gap.GapEnd; seq++ { + if ctx.Err() != nil { + return stats + } + + fetchStart := time.Now() + lcm, err := m.getLedgerWithRetry(ctx, backend, seq) + fetchDur := time.Since(fetchStart) + if err != nil { + cancel(fmt.Errorf("fetching ledger %d: %w", seq, err)) + return stats + } + stats.addFetch(fetchDur) + m.appMetrics.Ingestion.PhaseDuration.WithLabelValues("backfill_fetch").Observe(fetchDur.Seconds()) + + sendStart := time.Now() + select { + case ledgerCh <- lcm: + sendDur := time.Since(sendStart) + stats.addChannelWait("ledger", "send", sendDur) + m.appMetrics.Ingestion.BackfillChannelWait.WithLabelValues("ledger", "send").Observe(sendDur.Seconds()) + case <-ctx.Done(): + return stats + } + } + return stats +} +``` + +**Step 2: Update the call site in `processGap`** + +In `processGap`, change from: +```go +m.runDispatcher(gapCtx, gapCancel, backend, gap, ledgerCh) +``` +to: +```go +dispatcherStats := m.runDispatcher(gapCtx, gapCancel, backend, gap, ledgerCh) +``` + +Store `dispatcherStats` for later use in the gap summary (Task 6). + +**Step 3: Run existing tests + vet** + +Run: `go vet ./internal/services/` +Expected: PASS (no compilation errors) + +**Step 4: Commit** + +```bash +git add internal/services/ingest_backfill.go +git commit -m "Instrument backfill dispatcher with fetch timing and channel wait metrics" +``` + +--- + +### Task 4: Instrument Stage 2 (Process Workers) with process timing and channel waits + +**Files:** +- Modify: `internal/services/ingest_backfill.go` — `runProcessWorkers` function + +**Step 1: Add process timing and channel wait instrumentation** + +Modify `runProcessWorkers` to: +1. Accept a `chan<- *backfillWorkerStats` to report per-worker stats +2. Time each `processLedger` call +3. Time the `ledgerCh` receive wait (time blocked in `range ledgerCh`) +4. Time the `flushCh` send wait + +Change the signature: +```go +func (m *ingestService) runProcessWorkers( + ctx context.Context, + cancel context.CancelCauseFunc, + ledgerCh <-chan xdr.LedgerCloseMeta, + flushCh chan<- flushItem, + statsCh chan<- *backfillWorkerStats, +) +``` + +Each worker goroutine creates its own `backfillWorkerStats` and sends it to `statsCh` on exit. The channel receive wait is measured by timing before/after pulling from `ledgerCh`. Since we use `range`, we need to switch to an explicit loop with `select` to time the receive: + +```go +for range m.backfillProcessWorkers { + wg.Add(1) + go func() { + defer wg.Done() + stats := &backfillWorkerStats{} + defer func() { statsCh <- stats }() + + buffer := indexer.NewIndexerBuffer() + var ledgers []uint32 + + flush := func() { + if len(ledgers) == 0 { + return + } + sendStart := time.Now() + select { + case flushCh <- flushItem{Buffer: buffer, Ledgers: ledgers}: + sendDur := time.Since(sendStart) + stats.addChannelWait("flush", "send", sendDur) + m.appMetrics.Ingestion.BackfillChannelWait.WithLabelValues("flush", "send").Observe(sendDur.Seconds()) + case <-ctx.Done(): + return + } + buffer = indexer.NewIndexerBuffer() + ledgers = nil + } + + for { + recvStart := time.Now() + lcm, ok := <-ledgerCh + if !ok { + break + } + recvDur := time.Since(recvStart) + stats.addChannelWait("ledger", "receive", recvDur) + m.appMetrics.Ingestion.BackfillChannelWait.WithLabelValues("ledger", "receive").Observe(recvDur.Seconds()) + + if ctx.Err() != nil { + return + } + + processStart := time.Now() + if err := m.processLedger(ctx, lcm, buffer); err != nil { + cancel(fmt.Errorf("processing ledger %d: %w", lcm.LedgerSequence(), err)) + return + } + processDur := time.Since(processStart) + stats.addProcess(processDur) + m.appMetrics.Ingestion.PhaseDuration.WithLabelValues("backfill_process").Observe(processDur.Seconds()) + + ledgers = append(ledgers, lcm.LedgerSequence()) + + if uint32(len(ledgers)) >= m.backfillFlushBatchSize { + flush() + } + } + + flush() + }() +} +``` + +**Step 2: Update the call site in `processGap`** + +Add a `statsCh` channel and collect worker stats: + +```go +processStatsCh := make(chan *backfillWorkerStats, m.backfillProcessWorkers) +``` + +Pass `processStatsCh` to `runProcessWorkers`. After `pipelineWg.Wait()`, drain the channel: + +```go +close(processStatsCh) +gapStats := newBackfillGapStats() +gapStats.mergeWorker(dispatcherStats) +for ws := range processStatsCh { + gapStats.mergeWorker(ws) +} +``` + +**Step 3: Run vet** + +Run: `go vet ./internal/services/` +Expected: PASS + +**Step 4: Commit** + +```bash +git add internal/services/ingest_backfill.go +git commit -m "Instrument backfill process workers with timing and channel wait metrics" +``` + +--- + +### Task 5: Instrument Stage 3 (Flush Workers) with flush timing, throughput, batch size, and gap progress + +**Files:** +- Modify: `internal/services/ingest_backfill.go` — `runFlushWorkers` function + +**Step 1: Add flush timing, throughput counter, batch size, and gap progress** + +Modify `runFlushWorkers` signature to accept `chan<- *backfillFlushWorkerStats`: + +```go +func (m *ingestService) runFlushWorkers( + ctx context.Context, + flushCh <-chan flushItem, + watermark *backfillWatermark, + numWorkers int, + gap data.LedgerRange, + statsCh chan<- *backfillFlushWorkerStats, +) +``` + +Each worker: +1. Times the `flushCh` receive +2. Times each `flushBufferWithRetry` call → observe `PhaseDuration` with `"backfill_flush"` +3. Observes `BackfillBatchSize` with `len(item.Ledgers)` +4. Increments `BackfillLedgersFlushed` by `len(item.Ledgers)` on success +5. Updates `BackfillGapProgress` when watermark advances + +```go +for i := range numWorkers { + wg.Add(1) + go func(workerID int) { + defer wg.Done() + stats := &backfillFlushWorkerStats{} + defer func() { statsCh <- stats }() + + for { + recvStart := time.Now() + item, ok := <-flushCh + if !ok { + return + } + recvDur := time.Since(recvStart) + stats.addChannelWait("flush", "receive", recvDur) + m.appMetrics.Ingestion.BackfillChannelWait.WithLabelValues("flush", "receive").Observe(recvDur.Seconds()) + + if ctx.Err() != nil { + return + } + + m.appMetrics.Ingestion.BackfillBatchSize.Observe(float64(len(item.Ledgers))) + + flushStart := time.Now() + if err := m.flushBufferWithRetry(ctx, item.Buffer); err != nil { + log.Ctx(ctx).Errorf("Flush worker %d: %d ledgers failed: %v", + workerID, len(item.Ledgers), err) + continue + } + flushDur := time.Since(flushStart) + stats.addFlush(flushDur) + m.appMetrics.Ingestion.PhaseDuration.WithLabelValues("backfill_flush").Observe(flushDur.Seconds()) + m.appMetrics.Ingestion.BackfillLedgersFlushed.Add(float64(len(item.Ledgers))) + + if advanced := watermark.MarkFlushed(item.Ledgers); advanced { + gapSize := float64(gap.GapEnd - gap.GapStart + 1) + progress := float64(watermark.Cursor()-gap.GapStart+1) / gapSize + m.appMetrics.Ingestion.BackfillGapProgress.Set(progress) + + if err := m.updateOldestCursor(ctx, watermark.Cursor()); err != nil { + log.Ctx(ctx).Warnf("Flush worker %d: cursor update failed: %v", + workerID, err) + } + } + } + }(i) +} +``` + +**Step 2: Update the call site in `processGap`** + +Add flush stats channel, pass `gap` to `runFlushWorkers`: + +```go +flushStatsCh := make(chan *backfillFlushWorkerStats, m.backfillFlushWorkers) +``` + +Pass to `runFlushWorkers`. After pipeline completes, drain: + +```go +close(flushStatsCh) +for fs := range flushStatsCh { + gapStats.mergeFlushWorker(fs) +} +``` + +**Step 3: Run vet** + +Run: `go vet ./internal/services/` +Expected: PASS + +**Step 4: Commit** + +```bash +git add internal/services/ingest_backfill.go +git commit -m "Instrument backfill flush workers with timing, throughput, and gap progress metrics" +``` + +--- + +### Task 6: Add channel utilization sampler and gap boundary gauges + +**Files:** +- Modify: `internal/services/ingest_backfill.go` — `processGap` function + +**Step 1: Add channel utilization sampler goroutine** + +In `processGap`, after creating `ledgerCh` and `flushCh`, start a sampler goroutine: + +```go +// Set gap boundary gauges +m.appMetrics.Ingestion.BackfillGapStartLedger.Set(float64(gap.GapStart)) +m.appMetrics.Ingestion.BackfillGapEndLedger.Set(float64(gap.GapEnd)) +m.appMetrics.Ingestion.BackfillGapProgress.Set(0) +defer func() { + m.appMetrics.Ingestion.BackfillGapStartLedger.Set(0) + m.appMetrics.Ingestion.BackfillGapEndLedger.Set(0) +}() + +// Channel utilization sampler — snapshots fill ratios every second +pipelineWg.Add(1) +go func() { + defer pipelineWg.Done() + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + for { + select { + case <-gapCtx.Done(): + return + case <-ticker.C: + if cap(ledgerCh) > 0 { + m.appMetrics.Ingestion.BackfillChannelUtilization.WithLabelValues("ledger").Set( + float64(len(ledgerCh)) / float64(cap(ledgerCh)), + ) + } + if cap(flushCh) > 0 { + m.appMetrics.Ingestion.BackfillChannelUtilization.WithLabelValues("flush").Set( + float64(len(flushCh)) / float64(cap(flushCh)), + ) + } + } + } +}() +``` + +**Step 2: Run vet** + +Run: `go vet ./internal/services/` +Expected: PASS + +**Step 3: Commit** + +```bash +git add internal/services/ingest_backfill.go +git commit -m "Add channel utilization sampler and gap boundary gauges" +``` + +--- + +### Task 7: Add gap summary log + +**Files:** +- Modify: `internal/services/ingest_backfill.go` — `processGap` function + +**Step 1: Add gap summary logging after pipeline completion** + +After all stats are collected (post `pipelineWg.Wait()` and stats channel draining), add the summary log: + +```go +// Log gap summary +total := gap.GapEnd - gap.GapStart + 1 +elapsed := time.Since(overallStart) +ledgersPerSec := float64(0) +if elapsed > 0 { + ledgersPerSec = float64(total) / elapsed.Seconds() +} + +log.Ctx(ctx).Infof("Gap [%d-%d] summary (%v, %.0f ledgers/sec):\n"+ + " fetch: %v total, %v avg (%d calls)\n"+ + " process: %v total, %v avg (%d calls)\n"+ + " flush: %v total, %v avg (%d batches)\n"+ + " channel_wait: ledger_send=%v ledger_recv=%v flush_send=%v flush_recv=%v", + gap.GapStart, gap.GapEnd, elapsed, ledgersPerSec, + gapStats.fetchTotal, avgOrZero(gapStats.fetchTotal, gapStats.fetchCount), gapStats.fetchCount, + gapStats.processTotal, avgOrZero(gapStats.processTotal, gapStats.processCount), gapStats.processCount, + gapStats.flushTotal, avgOrZero(gapStats.flushTotal, gapStats.flushCount), gapStats.flushCount, + gapStats.channelWait["ledger:send"], + gapStats.channelWait["ledger:receive"], + gapStats.channelWait["flush:send"], + gapStats.channelWait["flush:receive"], +) +``` + +This replaces or augments the existing completion log in `processGap`. + +**Step 2: Run vet** + +Run: `go vet ./internal/services/` +Expected: PASS + +**Step 3: Commit** + +```bash +git add internal/services/ingest_backfill.go +git commit -m "Add structured gap summary log with per-stage timing breakdown" +``` + +--- + +### Task 8: Integration verification + +**Files:** +- None new — verification only + +**Step 1: Run unit tests** + +Run: `go test -v -race ./internal/services/ -timeout 3m` +Expected: ALL PASS + +**Step 2: Run metrics tests** + +Run: `go test -v ./internal/metrics/ -timeout 30s` +Expected: ALL PASS + +**Step 3: Run linter** + +Run: `make tidy && go vet ./...` +Expected: PASS + +**Step 4: Final commit (if any tidy changes)** + +```bash +git add -A +git commit -m "tidy: fix formatting after backfill observability changes" +``` diff --git a/docs/plans/2026-04-02-backfill-pipeline-refactor-design.md b/docs/plans/2026-04-02-backfill-pipeline-refactor-design.md new file mode 100644 index 000000000..5299b7c1b --- /dev/null +++ b/docs/plans/2026-04-02-backfill-pipeline-refactor-design.md @@ -0,0 +1,153 @@ +# Backfill Pipeline Refactor: High-Throughput 3-Stage Design + +**Date:** 2026-04-02 +**Branch:** optimize-backfill-code +**Scope:** `internal/services/ingest_backfill.go` — complete refactor + +## Problem + +The current backfill implementation creates one `optimizedStorageBackend` per batch, processes ledgers sequentially within each batch, and performs stop-the-world DB flushes. These design choices leave significant throughput on the table: + +- Redundant S3 backends for contiguous ranges within the same gap +- No pipelining between ledger processing and DB writes +- Single-writer flushes underutilize TimescaleDB's chunk-parallel write capability + +## Design Decisions + +| Decision | Choice | Rationale | +|----------|--------|-----------| +| Backend per gap | 1 backend per gap | Avoids redundant S3 connections; `optimizedStorageBackend` already handles sequential prefetch internally | +| Gap concurrency | Sequential | Typically 1 large gap in practice; adding a semaphore later is trivial | +| Processing parallelism | Fan-out via bounded channel | Idiomatic Go pipeline; backpressure automatic via bounded channels | +| Indexer pool | Shared pool, per-worker groups | `pond.NewGroupContext` allows concurrent groups on the same pool; natural concurrency bound | +| Flush parallelism | M concurrent flush workers, 5 parallel COPYs each | TimescaleDB docs recommend parallel COPY; different time ranges hit different chunks, minimizing lock contention | +| Transaction wrapping | None (fire-and-forget COPYs) | Backfill is idempotent via UniqueViolation handling; atomicity buys nothing | +| Cursor management | Watermark tracker (bitmap) | Flush order is non-deterministic; watermark advances cursor only when contiguous range is safe | +| Error: flush failure | Retry with exponential backoff | Transient DB issues (load, connections) are common and self-resolving | +| Error: download/process | Fail-fast, cancel gap context | Indicates bad data or config; won't self-resolve | +| Progressive recompressor | Removed | Will be re-added in a separate PR | + +## Architecture + +### Configuration Parameters + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `BackfillProcessWorkers` | `runtime.NumCPU()` | Stage 2 processing workers per gap | +| `BackfillFlushWorkers` | 4 | Stage 3 concurrent flush workers | +| `BackfillFlushBatchSize` | 100 | Ledgers accumulated per buffer before flush | +| `BackfillLedgerChannelSize` | 256 | Bounded channel: dispatcher to process workers | +| `BackfillFlushChannelSize` | 8 | Bounded channel: process workers to flush workers | + +### Startup Validation + +``` +requiredConns = BackfillFlushWorkers * 5 + 5 (headroom) +if pgxpool.MaxConns < requiredConns → error +``` + +### Pipeline Flow (per gap, gaps processed sequentially) + +``` + 1 Backend (optimizedStorageBackend for full gap range) + | + v + 1 Dispatcher goroutine + calls GetLedger(seq++) sequentially + | + | ledgerCh (bounded, size 256) + v + N Process Workers (default: NumCPU) + +-------------------------------------+ + | Each worker: | + | - Owns its own IndexerBuffer | + | - Pulls ledgers from ledgerCh | + | - Calls processLedger() using | + | shared indexer pool for tx | + | parallelism within each ledger | + | - After FlushBatchSize ledgers, | + | sends buffer + ledger set to | + | flushCh | + | - Creates fresh buffer, continues | + +-------------------------------------+ + | + | flushCh (bounded, size 8) + v + M Flush Workers (default: 4) + +-------------------------------------+ + | Each flush: | + | - 5 parallel COPYs via errgroup | + | (transactions, tx_accounts, | + | operations, op_accounts, | + | state_changes) | + | - Each COPY: own pool conn, | + | synchronous_commit=off | + | - UniqueViolation = success | + | - Retry with exponential backoff | + | - On success: report ledger set | + | to watermark tracker | + +-------------------------------------+ + | + v + Watermark Tracker + +-------------------------------------+ + | - Bitmap of flushed ledger seqs | + | - Advances cursor to highest | + | contiguous flushed ledger | + | - Updates cursor in DB | + +-------------------------------------+ +``` + +### Backpressure + +All automatic via bounded channels — no manual flow control: + +- DB slow -> flushCh fills -> workers block on send -> ledgerCh fills -> dispatcher blocks -> backend pauses S3 downloads +- Processing slow -> ledgerCh stays full -> dispatcher blocks -> backend paces itself + +### Error Handling + +- **Flush failure:** retry with exponential backoff (max 10 attempts, capped at 30s). After exhaustion, cancel gap context. +- **Download/processing failure:** fail-fast, cancel gap context immediately. All in-flight work for the gap drains. +- **Gap isolation:** each gap has its own context. A failed gap does not prevent subsequent gaps from running. +- **Partial progress:** watermark tracker records which ledgers were successfully flushed. On restart, `calculateBackfillGaps` detects remaining gaps and only re-processes what's missing. + +### Watermark Tracker Detail + +The watermark tracker replaces per-batch cursor updates. Since M flush workers process buffers concurrently, and each buffer contains non-contiguous ledgers (workers pull from a shared channel), flush completion order is non-deterministic. + +The tracker maintains a bitmap indexed by ledger sequence. When a flush worker reports success, it marks those ledger sequences as done. The tracker scans forward from the current cursor position to find the highest contiguous flushed ledger and updates the DB cursor to that value. + +Cursor updates are batched (not per-flush) to avoid excessive DB writes. + +### Connection Budget + +At peak throughput with default config: + +| Consumer | Connections | +|----------|-------------| +| Flush workers (4 x 5 COPYs) | 20 | +| Watermark cursor updates | 1 | +| Backend (S3, not DB) | 0 | +| Headroom | 4 | +| **Total** | **25** | + +Pool max connections should be >= 25 for default config. + +## What's NOT In Scope + +- **Progressive recompressor** — removed entirely, will be re-added in a separate PR +- **Upsert operations** — skipped in backfill (balance tables represent current state, not historical) +- **Gap concurrency** — gaps run sequentially; concurrency can be added later via semaphore if needed + +## Key Changes from Current Implementation + +| Current | New | +|---------|-----| +| 1 backend per batch | 1 backend per gap | +| Independent batch workers (pond pool) | 3-stage pipeline with channels | +| Stop-the-world flush within batch | Decoupled flush workers, processing continues | +| 1 flush at a time per batch | M concurrent flush workers | +| Wrapping transaction around COPYs | Fire-and-forget parallel COPYs | +| Per-batch cursor update | Watermark-based contiguous cursor | +| Progressive recompressor integrated | Removed (separate PR) | diff --git a/docs/plans/2026-04-02-backfill-pipeline-refactor.md b/docs/plans/2026-04-02-backfill-pipeline-refactor.md new file mode 100644 index 000000000..476f69d54 --- /dev/null +++ b/docs/plans/2026-04-02-backfill-pipeline-refactor.md @@ -0,0 +1,837 @@ +# Backfill Pipeline Refactor Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Replace the batch-per-backend backfill with a 3-stage pipelined architecture (dispatcher → process workers → flush workers) for maximum throughput. + +**Architecture:** Single `optimizedStorageBackend` per gap, fan-out to N process workers via bounded channel, M flush workers with 5 parallel COPYs each, watermark-based cursor tracking. Gaps processed sequentially. + +**Tech Stack:** Go channels, `sync.WaitGroup`, `errgroup`, `pond.Pool` (shared indexer pool), `pgxpool` + +**Design Doc:** `docs/plans/2026-04-02-backfill-pipeline-refactor-design.md` + +--- + +## Task 1: Add New Config Fields to `ingestService` + +**Files:** +- Modify: `internal/services/ingest.go:46-79` (IngestServiceConfig) +- Modify: `internal/services/ingest.go:99-121` (ingestService struct) +- Modify: `internal/services/ingest.go:123-161` (NewIngestService) + +**Step 1: Add config fields** + +Add new fields to `IngestServiceConfig`: + +```go +// === Backfill Tuning === +BackfillWorkers int +BackfillBatchSize int +BackfillDBInsertBatchSize int +BackfillProcessWorkers int // NEW: Stage 2 workers (default: NumCPU) +BackfillFlushWorkers int // NEW: Stage 3 flush workers (default: 4) +BackfillLedgerChanSize int // NEW: ledgerCh buffer size (default: 256) +BackfillFlushChanSize int // NEW: flushCh buffer size (default: 8) +``` + +**Step 2: Add fields to `ingestService` struct** + +Replace `backfillPool`, `backfillBatchSize`, `backfillDBInsertBatchSize` with: + +```go +// Backfill pipeline config +backfillProcessWorkers int +backfillFlushWorkers int +backfillFlushBatchSize uint32 +backfillLedgerChanSize int +backfillFlushChanSize int +``` + +Note: `backfillBatchSize` (ledgers per backend range) is no longer needed since we use 1 backend per gap. `backfillDBInsertBatchSize` is renamed to `backfillFlushBatchSize` for clarity. + +**Step 3: Update `NewIngestService`** + +Remove the `backfillPool` creation. Apply defaults for new fields: + +```go +processWorkers := cfg.BackfillProcessWorkers +if processWorkers <= 0 { + processWorkers = runtime.NumCPU() +} +flushWorkers := cfg.BackfillFlushWorkers +if flushWorkers <= 0 { + flushWorkers = 4 +} +flushBatchSize := cfg.BackfillDBInsertBatchSize +if flushBatchSize <= 0 { + flushBatchSize = 100 +} +ledgerChanSize := cfg.BackfillLedgerChanSize +if ledgerChanSize <= 0 { + ledgerChanSize = 256 +} +flushChanSize := cfg.BackfillFlushChanSize +if flushChanSize <= 0 { + flushChanSize = 8 +} +``` + +Remove `backfillPool` from struct initialization. Remove pool metrics registration for "backfill" pool. Add the new fields to the struct literal. + +**Step 4: Fix all compilation errors** + +Any code referencing `m.backfillPool`, `m.backfillBatchSize`, or `m.backfillDBInsertBatchSize` will break. The backfill file will be rewritten in later tasks, but fix any references in `ingest.go` and test files to compile. + +**Step 5: Run tests** + +Run: `go build ./internal/services/...` +Expected: Compiles successfully (tests may fail — that's OK, we're rewriting the backfill file next) + +**Step 6: Commit** + +```bash +git add internal/services/ingest.go +git commit -m "Refactor ingestService config for pipeline-based backfill" +``` + +--- + +## Task 2: Implement the Watermark Tracker + +**Files:** +- Create: `internal/services/backfill_watermark.go` +- Create: `internal/services/backfill_watermark_test.go` + +This is a standalone component with no dependencies on the pipeline, so we build and test it first. + +**Step 1: Write the failing tests** + +Create `internal/services/backfill_watermark_test.go`: + +```go +package services + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Test_backfillWatermark_MarkFlushed_sequential(t *testing.T) { + w := newBackfillWatermark(100, 109) // 10 ledgers: 100-109 + + // Flush ledgers 100-104 + advanced := w.MarkFlushed([]uint32{100, 101, 102, 103, 104}) + assert.True(t, advanced) + assert.Equal(t, uint32(104), w.Cursor()) + + // Flush ledgers 105-109 + advanced = w.MarkFlushed([]uint32{105, 106, 107, 108, 109}) + assert.True(t, advanced) + assert.Equal(t, uint32(109), w.Cursor()) + assert.True(t, w.Complete()) +} + +func Test_backfillWatermark_MarkFlushed_outOfOrder(t *testing.T) { + w := newBackfillWatermark(100, 109) + + // Flush ledgers 105-109 first — cursor can't advance + advanced := w.MarkFlushed([]uint32{105, 106, 107, 108, 109}) + assert.False(t, advanced) + assert.Equal(t, uint32(0), w.Cursor()) // no contiguous range from start + + // Flush ledgers 100-104 — cursor jumps to 109 + advanced = w.MarkFlushed([]uint32{100, 101, 102, 103, 104}) + assert.True(t, advanced) + assert.Equal(t, uint32(109), w.Cursor()) + assert.True(t, w.Complete()) +} + +func Test_backfillWatermark_MarkFlushed_withGap(t *testing.T) { + w := newBackfillWatermark(100, 109) + + // Flush 100-102 + w.MarkFlushed([]uint32{100, 101, 102}) + assert.Equal(t, uint32(102), w.Cursor()) + + // Flush 105-107 (skip 103-104) + advanced := w.MarkFlushed([]uint32{105, 106, 107}) + assert.False(t, advanced) // cursor stuck at 102 + + // Flush 103-104 — cursor jumps to 107 + advanced = w.MarkFlushed([]uint32{103, 104}) + assert.True(t, advanced) + assert.Equal(t, uint32(107), w.Cursor()) + assert.False(t, w.Complete()) // 108, 109 still missing +} + +func Test_backfillWatermark_singleLedger(t *testing.T) { + w := newBackfillWatermark(500, 500) + + advanced := w.MarkFlushed([]uint32{500}) + assert.True(t, advanced) + assert.Equal(t, uint32(500), w.Cursor()) + assert.True(t, w.Complete()) +} + +func Test_backfillWatermark_emptyFlush(t *testing.T) { + w := newBackfillWatermark(100, 109) + + advanced := w.MarkFlushed([]uint32{}) + assert.False(t, advanced) + assert.Equal(t, uint32(0), w.Cursor()) +} +``` + +**Step 2: Run tests to verify they fail** + +Run: `go test -v ./internal/services -run Test_backfillWatermark -timeout 30s` +Expected: FAIL — `newBackfillWatermark` not defined + +**Step 3: Implement the watermark tracker** + +Create `internal/services/backfill_watermark.go`: + +```go +package services + +import "sync" + +// backfillWatermark tracks which ledgers in a gap have been flushed to DB, +// and reports the highest contiguous flushed ledger for cursor updates. +// Thread-safe: multiple flush workers call MarkFlushed concurrently. +type backfillWatermark struct { + mu sync.Mutex + flushed []bool // indexed by (ledgerSeq - startLedger) + start uint32 // first ledger in gap + end uint32 // last ledger in gap + cursor uint32 // highest contiguous flushed ledger (0 = none) + scanFrom uint32 // optimization: resume scan from last cursor position +} + +// newBackfillWatermark creates a watermark tracker for the gap [start, end]. +func newBackfillWatermark(start, end uint32) *backfillWatermark { + return &backfillWatermark{ + flushed: make([]bool, end-start+1), + start: start, + end: end, + scanFrom: start, + } +} + +// MarkFlushed records the given ledger sequences as flushed and advances the +// cursor if possible. Returns true if the cursor advanced. +func (w *backfillWatermark) MarkFlushed(ledgers []uint32) bool { + w.mu.Lock() + defer w.mu.Unlock() + + for _, seq := range ledgers { + if seq >= w.start && seq <= w.end { + w.flushed[seq-w.start] = true + } + } + + // Advance cursor from last known position + oldCursor := w.cursor + for w.scanFrom <= w.end && w.flushed[w.scanFrom-w.start] { + w.cursor = w.scanFrom + w.scanFrom++ + } + + return w.cursor != oldCursor +} + +// Cursor returns the highest contiguous flushed ledger, or 0 if none. +func (w *backfillWatermark) Cursor() uint32 { + w.mu.Lock() + defer w.mu.Unlock() + return w.cursor +} + +// Complete returns true if all ledgers in the gap have been flushed. +func (w *backfillWatermark) Complete() bool { + w.mu.Lock() + defer w.mu.Unlock() + return w.cursor == w.end +} +``` + +**Step 4: Run tests to verify they pass** + +Run: `go test -v ./internal/services -run Test_backfillWatermark -timeout 30s` +Expected: PASS (all 5 tests) + +**Step 5: Commit** + +```bash +git add internal/services/backfill_watermark.go internal/services/backfill_watermark_test.go +git commit -m "Add backfill watermark tracker for contiguous cursor advancement" +``` + +--- + +## Task 3: Implement the `flushItem` Type and Flush Worker + +**Files:** +- Create: `internal/services/backfill_flush.go` +- Create: `internal/services/backfill_flush_test.go` + +The flush worker receives filled buffers and writes them to DB with parallel COPYs. Tested with mocks. + +**Step 1: Write the flush item type and flush function** + +Create `internal/services/backfill_flush.go`: + +```go +package services + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/stellar/go-stellar-sdk/support/log" + "golang.org/x/sync/errgroup" + + "github.com/stellar/wallet-backend/internal/indexer" + "github.com/stellar/wallet-backend/internal/metrics" +) + +// flushItem is the unit of work sent from process workers to flush workers. +// It contains a filled IndexerBuffer and the ledger sequences it covers, +// so the watermark tracker knows which ledgers were persisted. +type flushItem struct { + Buffer *indexer.IndexerBuffer + Ledgers []uint32 +} + +// runFlushWorkers starts M flush worker goroutines that read from flushCh, +// write data to DB via parallel COPYs, and report success to the watermark. +// Returns when flushCh is closed and all in-flight flushes complete. +func (m *ingestService) runFlushWorkers( + ctx context.Context, + flushCh <-chan flushItem, + watermark *backfillWatermark, + numWorkers int, +) { + var wg sync.WaitGroup + for i := 0; i < numWorkers; i++ { + wg.Add(1) + go func(workerID int) { + defer wg.Done() + for item := range flushCh { + if ctx.Err() != nil { + return + } + if err := m.flushBufferWithRetry(ctx, item.Buffer); err != nil { + log.Ctx(ctx).Errorf("Flush worker %d: failed to flush %d ledgers: %v", + workerID, len(item.Ledgers), err) + continue + } + if advanced := watermark.MarkFlushed(item.Ledgers); advanced { + cursor := watermark.Cursor() + if err := m.updateOldestCursor(ctx, cursor); err != nil { + log.Ctx(ctx).Warnf("Flush worker %d: failed to update cursor to %d: %v", + workerID, cursor, err) + } + } + } + }(i) + } + wg.Wait() +} + +// flushBufferWithRetry persists a buffer's data to DB via 5 parallel COPYs +// with exponential backoff retry. +func (m *ingestService) flushBufferWithRetry(ctx context.Context, buffer *indexer.IndexerBuffer) error { + var lastErr error + for attempt := 0; attempt < maxIngestProcessedDataRetries; attempt++ { + select { + case <-ctx.Done(): + return fmt.Errorf("context cancelled: %w", ctx.Err()) + default: + } + + txs := buffer.GetTransactions() + if _, _, err := m.insertAndUpsertParallel(ctx, txs, buffer); err != nil { + lastErr = err + m.appMetrics.Ingestion.RetriesTotal.WithLabelValues("batch_flush").Inc() + + backoff := time.Duration(1< maxIngestProcessedDataRetryBackoff { + backoff = maxIngestProcessedDataRetryBackoff + } + log.Ctx(ctx).Warnf("Error flushing buffer (attempt %d/%d): %v, retrying in %v...", + attempt+1, maxIngestProcessedDataRetries, lastErr, backoff) + + select { + case <-ctx.Done(): + return fmt.Errorf("context cancelled during backoff: %w", ctx.Err()) + case <-time.After(backoff): + } + continue + } + return nil + } + m.appMetrics.Ingestion.RetryExhaustionsTotal.WithLabelValues("batch_flush").Inc() + return lastErr +} +``` + +**Step 2: Run compilation check** + +Run: `go build ./internal/services/...` +Expected: Compiles (depends on Task 1 being done first) + +**Step 3: Commit** + +```bash +git add internal/services/backfill_flush.go +git commit -m "Add flush workers with parallel COPY and retry for backfill pipeline" +``` + +--- + +## Task 4: Implement the 3-Stage Pipeline Orchestrator + +**Files:** +- Rewrite: `internal/services/ingest_backfill.go` + +This is the core refactor. Replace all existing backfill code (except `calculateBackfillGaps` which is kept as-is) with the 3-stage pipeline. + +**Step 1: Rewrite `ingest_backfill.go`** + +The file should contain: + +1. **`startBackfilling`** — orchestrator: calculate gaps, process each gap sequentially +2. **`calculateBackfillGaps`** — KEEP AS-IS (lines 100-157 of current file) +3. **`processGap`** — sets up backend, launches 3 stages, waits for completion +4. **`runDispatcher`** — Stage 1: sequential GetLedger, sends to ledgerCh +5. **`runProcessWorkers`** — Stage 2: N workers pull from ledgerCh, process, send to flushCh + +```go +package services + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/stellar/go-stellar-sdk/ingest/ledgerbackend" + "github.com/stellar/go-stellar-sdk/support/log" + "github.com/stellar/go-stellar-sdk/xdr" + + "github.com/stellar/wallet-backend/internal/data" + "github.com/stellar/wallet-backend/internal/indexer" +) + +// startBackfilling processes ledgers in the specified range, identifying gaps +// and processing them sequentially via a 3-stage pipeline. +func (m *ingestService) startBackfilling(ctx context.Context, startLedger, endLedger uint32) error { + if startLedger > endLedger { + return fmt.Errorf("start ledger cannot be greater than end ledger") + } + + latestIngestedLedger, err := m.models.IngestStore.Get(ctx, m.latestLedgerCursorName) + if err != nil { + return fmt.Errorf("getting latest ledger cursor: %w", err) + } + + if endLedger > latestIngestedLedger { + return fmt.Errorf("end ledger %d cannot be greater than latest ingested ledger %d for backfilling", endLedger, latestIngestedLedger) + } + + gaps, err := m.calculateBackfillGaps(ctx, startLedger, endLedger) + if err != nil { + return fmt.Errorf("calculating backfill gaps: %w", err) + } + if len(gaps) == 0 { + log.Ctx(ctx).Infof("No gaps to backfill in range [%d - %d]", startLedger, endLedger) + return nil + } + + overallStart := time.Now() + for i, gap := range gaps { + log.Ctx(ctx).Infof("Processing gap %d/%d [%d - %d]", i+1, len(gaps), gap.GapStart, gap.GapEnd) + if err := m.processGap(ctx, gap); err != nil { + log.Ctx(ctx).Errorf("Gap %d/%d [%d - %d] failed: %v", i+1, len(gaps), gap.GapStart, gap.GapEnd, err) + // Continue to next gap — partial progress is preserved via watermark + continue + } + } + + log.Ctx(ctx).Infof("Backfilling completed in %v: %d gaps", time.Since(overallStart), len(gaps)) + return nil +} + +// calculateBackfillGaps determines which ledger ranges need to be backfilled. +// KEEP EXISTING IMPLEMENTATION (lines 100-157 of current file) — copy as-is. + +// processGap runs the 3-stage pipeline for a single contiguous gap. +func (m *ingestService) processGap(ctx context.Context, gap data.LedgerRange) error { + gapCtx, gapCancel := context.WithCancelCause(ctx) + defer gapCancel(nil) + + // Stage 0: Create backend for the full gap range + backend, err := m.ledgerBackendFactory(gapCtx) + if err != nil { + return fmt.Errorf("creating ledger backend: %w", err) + } + defer func() { + if closeErr := backend.Close(); closeErr != nil { + log.Ctx(ctx).Warnf("Error closing ledger backend for gap [%d-%d]: %v", + gap.GapStart, gap.GapEnd, closeErr) + } + }() + + ledgerRange := ledgerbackend.BoundedRange(gap.GapStart, gap.GapEnd) + if err := backend.PrepareRange(gapCtx, ledgerRange); err != nil { + return fmt.Errorf("preparing backend range [%d-%d]: %w", gap.GapStart, gap.GapEnd, err) + } + + // Create channels + ledgerCh := make(chan xdr.LedgerCloseMeta, m.backfillLedgerChanSize) + flushCh := make(chan flushItem, m.backfillFlushChanSize) + + // Create watermark tracker + watermark := newBackfillWatermark(gap.GapStart, gap.GapEnd) + + var pipelineWg sync.WaitGroup + + // Stage 3: Flush workers (start first so they're ready when data arrives) + pipelineWg.Add(1) + go func() { + defer pipelineWg.Done() + m.runFlushWorkers(gapCtx, flushCh, watermark, m.backfillFlushWorkers) + }() + + // Stage 2: Process workers + pipelineWg.Add(1) + go func() { + defer pipelineWg.Done() + defer close(flushCh) // signal flush workers when all processing is done + m.runProcessWorkers(gapCtx, gapCancel, ledgerCh, flushCh) + }() + + // Stage 1: Dispatcher (runs on this goroutine) + m.runDispatcher(gapCtx, gapCancel, backend, gap, ledgerCh) + + // Wait for pipeline to drain + pipelineWg.Wait() + + // Check if pipeline was cancelled due to error + if cause := context.Cause(gapCtx); cause != nil && cause != context.Canceled { + return fmt.Errorf("pipeline failed: %w", cause) + } + + // Log final state + cursor := watermark.Cursor() + total := gap.GapEnd - gap.GapStart + 1 + if watermark.Complete() { + log.Ctx(ctx).Infof("Gap [%d-%d] complete: %d ledgers", gap.GapStart, gap.GapEnd, total) + } else { + log.Ctx(ctx).Warnf("Gap [%d-%d] partial: cursor at %d of %d", gap.GapStart, gap.GapEnd, cursor, gap.GapEnd) + } + + return nil +} + +// runDispatcher is Stage 1: sequentially fetches ledgers from the backend +// and sends them to ledgerCh. Closes ledgerCh when done or on error. +func (m *ingestService) runDispatcher( + ctx context.Context, + cancel context.CancelCauseFunc, + backend ledgerbackend.LedgerBackend, + gap data.LedgerRange, + ledgerCh chan<- xdr.LedgerCloseMeta, +) { + defer close(ledgerCh) + + for seq := gap.GapStart; seq <= gap.GapEnd; seq++ { + if ctx.Err() != nil { + return + } + + lcm, err := m.getLedgerWithRetry(ctx, backend, seq) + if err != nil { + cancel(fmt.Errorf("fetching ledger %d: %w", seq, err)) + return + } + + select { + case ledgerCh <- lcm: + case <-ctx.Done(): + return + } + } +} + +// runProcessWorkers is Stage 2: N workers pull ledgers from ledgerCh, +// process them into IndexerBuffers, and send filled buffers to flushCh. +func (m *ingestService) runProcessWorkers( + ctx context.Context, + cancel context.CancelCauseFunc, + ledgerCh <-chan xdr.LedgerCloseMeta, + flushCh chan<- flushItem, +) { + var wg sync.WaitGroup + for i := 0; i < m.backfillProcessWorkers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + m.processWorkerLoop(ctx, cancel, ledgerCh, flushCh) + }() + } + wg.Wait() +} + +// processWorkerLoop is the main loop for a single process worker. +func (m *ingestService) processWorkerLoop( + ctx context.Context, + cancel context.CancelCauseFunc, + ledgerCh <-chan xdr.LedgerCloseMeta, + flushCh chan<- flushItem, +) { + buffer := indexer.NewIndexerBuffer() + var ledgers []uint32 + + flush := func() { + if len(ledgers) == 0 { + return + } + item := flushItem{Buffer: buffer, Ledgers: ledgers} + select { + case flushCh <- item: + case <-ctx.Done(): + return + } + buffer = indexer.NewIndexerBuffer() + ledgers = nil + } + + for lcm := range ledgerCh { + if ctx.Err() != nil { + return + } + + if err := m.processLedger(ctx, lcm, buffer); err != nil { + cancel(fmt.Errorf("processing ledger %d: %w", lcm.LedgerSequence(), err)) + return + } + ledgers = append(ledgers, lcm.LedgerSequence()) + + if uint32(len(ledgers)) >= m.backfillFlushBatchSize { + flush() + } + } + + // Flush remaining + flush() +} +``` + +**Step 2: Remove old types and functions** + +Delete from the file: +- `BackfillBatch` type +- `BackfillResult` type +- `analyzeBatchResults` function +- `splitGapsIntoBatches` method +- `processBackfillBatchesParallel` method +- `processSingleBatch` method +- `setupBatchBackend` method +- `flushBatchBufferWithRetry` method +- `processLedgersInBatch` method +- `updateOldestCursor` method — KEEP this, it's used by flush workers +- `progressiveRecompressor` type and all its methods + +**Step 3: Verify compilation** + +Run: `go build ./internal/services/...` +Expected: Compiles. Some tests will fail (addressed in Task 5). + +**Step 4: Commit** + +```bash +git add internal/services/ingest_backfill.go +git commit -m "Rewrite backfill as 3-stage pipeline: dispatcher, process workers, flush workers" +``` + +--- + +## Task 5: Update Tests + +**Files:** +- Rewrite: `internal/services/ingest_backfill_test.go` +- Modify: `internal/services/ingest_test.go` (remove/update tests for deleted functions) + +**Step 1: Identify tests to remove from `ingest_test.go`** + +The following tests reference deleted functions and must be removed or updated: +- `Test_ingestService_splitGapsIntoBatches` — DELETE (function removed) +- `Test_ingestService_setupBatchBackend` — DELETE (function removed) +- `Test_ingestService_processSingleBatch` — DELETE (function removed) +- `Test_ingestService_startBackfilling_HistoricalMode_MixedResults` — REWRITE for new pipeline +- `Test_ingestService_startBackfilling_HistoricalMode_AllBatchesFail_CursorUnchanged` — REWRITE for new pipeline +- `Test_ingestProcessedDataWithRetry` — UPDATE to test `flushBufferWithRetry` + +Keep unchanged: +- `Test_ingestService_calculateBackfillGaps` — KEEP (function unchanged) +- `Test_ingestService_updateOldestCursor` — KEEP (function unchanged) + +**Step 2: Replace `ingest_backfill_test.go`** + +Delete all progressive recompressor tests. The watermark tests are already in `backfill_watermark_test.go` (Task 2). + +The file can either be deleted entirely or repurposed for pipeline integration tests if needed. + +**Step 3: Update test service construction** + +All tests that construct `ingestService` need to use the new config fields instead of the old ones. Replace: +```go +BackfillBatchSize: 10, +BackfillDBInsertBatchSize: 50, +``` +With: +```go +BackfillProcessWorkers: 2, +BackfillFlushWorkers: 1, +BackfillDBInsertBatchSize: 50, +BackfillLedgerChanSize: 10, +BackfillFlushChanSize: 2, +``` + +**Step 4: Run all tests** + +Run: `go test -v ./internal/services/... -timeout 3m` +Expected: PASS — all remaining tests pass, deleted tests are gone + +**Step 5: Commit** + +```bash +git add internal/services/ingest_backfill_test.go internal/services/ingest_test.go +git commit -m "Update tests for pipeline-based backfill refactor" +``` + +--- + +## Task 6: Clean Up Unused References + +**Files:** +- Modify: `internal/services/ingest.go` (remove unused imports, fields) +- Possibly modify: any other files that reference removed types (`BackfillBatch`, `BackfillResult`, `progressiveRecompressor`) + +**Step 1: Search for references to removed types** + +Search for: `BackfillBatch`, `BackfillResult`, `progressiveRecompressor`, `analyzeBatchResults`, `splitGapsIntoBatches`, `backfillPool` + +Remove all dead references. + +**Step 2: Remove `backfillPool` from metrics** + +The "backfill" pool metrics registration in `NewIngestService` should be removed since the pond pool is gone. + +**Step 3: Remove unused imports** + +After removing all dead code, clean up imports across modified files. + +**Step 4: Run linter and tests** + +Run: `make tidy && go test -v -race ./internal/services/... -timeout 3m` +Expected: PASS, no lint errors + +**Step 5: Commit** + +```bash +git add -u +git commit -m "Remove unused backfill types and pool references" +``` + +--- + +## Task 7: Connection Pool Validation + +**Files:** +- Modify: `internal/services/ingest.go` (add validation in `NewIngestService`) + +**Step 1: Add pool size validation** + +In `NewIngestService`, after computing `flushWorkers`, validate: + +```go +// Each flush worker runs 5 parallel COPYs, each needing its own connection. +requiredConns := int32(flushWorkers*5 + 5) // 5 headroom for cursor updates etc. +maxConns := cfg.Models.DB.Config().MaxConns +if maxConns > 0 && maxConns < requiredConns { + return nil, fmt.Errorf( + "pgxpool max connections (%d) too low for %d flush workers (need at least %d: %d workers * 5 COPYs + 5 headroom)", + maxConns, flushWorkers, requiredConns, flushWorkers) +} +``` + +**Step 2: Write test for validation** + +Add to `ingest_test.go`: + +```go +func Test_NewIngestService_poolValidation(t *testing.T) { + // Test that NewIngestService returns error when pool is too small + // for the configured flush workers. + // Use a real dbtest pool with known MaxConns. +} +``` + +**Step 3: Run tests** + +Run: `go test -v ./internal/services -run Test_NewIngestService -timeout 30s` +Expected: PASS + +**Step 4: Commit** + +```bash +git add internal/services/ingest.go internal/services/ingest_test.go +git commit -m "Validate connection pool size against flush worker count at startup" +``` + +--- + +## Task 8: Final Verification + +**Step 1: Run full lint and test suite** + +Run: `make tidy && make check && make unit-test` +Expected: All pass + +**Step 2: Verify no references to removed code** + +Run: `grep -r "progressiveRecompressor\|BackfillBatch\|BackfillResult\|splitGapsIntoBatches\|backfillPool\|analyzeBatchResults" internal/services/` +Expected: No matches (except possibly in comments or the design doc) + +**Step 3: Commit any fixes** + +If any issues found, fix and commit. + +--- + +## Summary of Files Changed + +| File | Action | +|------|--------| +| `internal/services/ingest.go` | Modify: config fields, remove pool, add validation | +| `internal/services/ingest_backfill.go` | Rewrite: 3-stage pipeline | +| `internal/services/backfill_watermark.go` | Create: watermark tracker | +| `internal/services/backfill_watermark_test.go` | Create: watermark tests | +| `internal/services/backfill_flush.go` | Create: flush workers | +| `internal/services/ingest_backfill_test.go` | Rewrite: remove recompressor tests | +| `internal/services/ingest_test.go` | Modify: update/remove tests for deleted functions | + +## Execution Order + +Tasks 1-3 can be done somewhat independently but should be committed in order. Task 4 depends on Tasks 1-3. Tasks 5-7 depend on Task 4. Task 8 is the final check. + +``` +Task 1 (config) → Task 2 (watermark) → Task 3 (flush) → Task 4 (pipeline) → Task 5 (tests) → Task 6 (cleanup) → Task 7 (validation) → Task 8 (verify) +``` diff --git a/docs/plans/2026-04-03-backfill-parallel-fetcher-design.md b/docs/plans/2026-04-03-backfill-parallel-fetcher-design.md new file mode 100644 index 000000000..c80e9ffb4 --- /dev/null +++ b/docs/plans/2026-04-03-backfill-parallel-fetcher-design.md @@ -0,0 +1,198 @@ +# Backfill Parallel Fetcher Design + +**Date**: 2026-04-03 +**Status**: Approved +**Problem**: The backfill pipeline achieves 60 ledgers/sec, bottlenecked by a single dispatcher goroutine calling `GetLedger` sequentially at 16.38ms/ledger. + +## Context + +The current backfill pipeline uses `optimizedStorageBackend` (forked from the SDK's `BufferedStorageBackend`) which was designed for sequential, single-consumer access. It internally uses: +- 15 S3 download workers that fetch files out-of-order +- A **priority queue** to re-order decoded batches +- A `batchQueue` that delivers batches in strict sequence +- A `GetLedger(seq)` API that enforces sequential consumption + +The dispatcher goroutine sits on top, calling `GetLedger` in a loop and forwarding individual `LedgerCloseMeta` entries to `ledgerCh`. This sequential bottleneck limits throughput to ~61 ledgers/sec regardless of how many process/flush workers are configured. + +**Key insight**: For backfill, ordering is unnecessary. Process workers handle ledgers independently. The watermark tracker handles out-of-order flushes. The priority queue and sequential `GetLedger` contract are pure overhead. + +## Design: Backfill S3 Fetcher + +Replace both the `optimizedStorageBackend` and the dispatcher with a new `backfillFetcher` component that downloads S3 files and pushes individual `LedgerCloseMeta` entries directly to `ledgerCh`. + +### Architecture + +``` +┌─────────────────────────────────────────────────────┐ +│ backfillFetcher │ +│ │ +│ taskCh (file-start sequences for the gap range) │ +│ ↓ │ +│ N S3 Workers ──download+decode──→ unpack batch │ +│ ↓ │ +│ push each LCM to │ +│ external ledgerCh │ +└──────────────────────────┬──────────────────────────┘ + │ + ledgerCh (shared) + │ + process workers (Stage 2) +``` + +### What's Removed (vs current pipeline) + +| Component | Purpose | Why Unnecessary | +|-----------|---------|-----------------| +| Priority queue + lock | Re-order out-of-order S3 downloads | Process workers don't need order | +| `batchQueue` channel | Buffer ordered batches for `GetLedger` | No intermediate buffering needed | +| `GetLedger` sequential contract | Deliver one ledger at a time in order | Workers push directly to `ledgerCh` | +| `nextLedger` tracking | Enforce sequential consumption | No sequential contract | +| Dispatcher goroutine (`runDispatcher`) | Bridge backend → `ledgerCh` | Workers push directly | + +### `backfillFetcher` Struct + +```go +// backfillFetcher downloads S3 ledger files in parallel and fans out +// individual LedgerCloseMeta entries directly to an external channel. +// No ordering guarantees — designed for backfill where consumers +// handle ledgers independently. +type backfillFetcher struct { + dataStore datastore.DataStore + schema datastore.DataStoreSchema + config BackfillFetcherConfig + + taskCh chan uint32 // file-start sequences for workers + ledgerCh chan<- xdr.LedgerCloseMeta // external channel owned by caller + + ctx context.Context + cancel context.CancelCauseFunc + wg sync.WaitGroup +} + +type BackfillFetcherConfig struct { + NumWorkers uint32 + RetryLimit uint32 + RetryWait time.Duration + GapStart uint32 + GapEnd uint32 +} +``` + +### File Location + +`internal/ingest/backfill_fetcher.go` — co-located with `storage_backend.go`. + +### Worker Logic + +Each worker: +1. Reads a file-start sequence from `taskCh` +2. Downloads and stream-decodes the S3 file (same `downloadAndDecode` logic as today) +3. Iterates over `batch.LedgerCloseMetas` +4. For each `LedgerCloseMeta` in the gap range, pushes to `ledgerCh` +5. Repeats until `taskCh` is closed or context cancelled + +Retry logic: Same exponential backoff as `storageBuffer.downloadAndStore` — retries transient S3 errors, hard-fails on missing files in bounded ranges. + +### Task Generation + +A goroutine (or the `Run` method itself) computes file-start boundaries: +```go +for seq := schema.GetSequenceNumberStartBoundary(gap.GapStart); seq <= gap.GapEnd; seq += schema.LedgersPerFile { + taskCh <- seq +} +close(taskCh) +``` + +This is simpler than the current `pushTaskQueue` replenishment pattern because the range is bounded and known up-front. + +### Ledger Filtering + +Each S3 file may contain ledgers outside the gap range (e.g., first/last file). Workers filter: +```go +for _, lcm := range batch.LedgerCloseMetas { + seq := lcm.LedgerSequence() + if seq < gap.GapStart || seq > gap.GapEnd { + continue + } + ledgerCh <- lcm +} +``` + +### Stats Collection + +Workers track fetch duration and channel wait times (same as current `backfillWorkerStats`). Stats are returned via a channel for aggregation into the gap summary log. + +## Pipeline Integration + +### Changes to `processGap` + +**Before** (current): +```go +backend, err := m.ledgerBackendFactory(gapCtx) +backend.PrepareRange(gapCtx, BoundedRange(gap.GapStart, gap.GapEnd)) +// ... +dispatcherStats := m.runDispatcher(gapCtx, gapCancel, backend, gap, ledgerCh) +``` + +**After**: +```go +fetcher := ingest.NewBackfillFetcher(ingest.BackfillFetcherConfig{...}, dataStore, schema, ledgerCh) +fetcherStats := fetcher.Run(gapCtx, gapCancel) +// fetcher.Run closes ledgerCh when done (same contract as runDispatcher) +``` + +### New Dependencies on `ingestService` + +The service needs access to `dataStore` and `schema` (currently encapsulated inside the backend). Options: +1. Store them on `ingestService` during construction (passed from `ingest.Configs`) +2. Create a `BackfillFetcherFactory` function (similar to `ledgerBackendFactory`) + +Option 2 is cleaner — avoids exposing datastore internals to the service layer. + +### New Config Field + +`BackfillFetchWorkers` (default: 15) — number of S3 download goroutines. Exposed as a CLI flag. + +### What Stays the Same + +- `optimizedStorageBackend` — unchanged, still used for live ingestion +- `runProcessWorkers` — unchanged, reads from same `ledgerCh` +- `runFlushWorkers` — unchanged +- `backfillWatermark` — unchanged, handles out-of-order flushes +- `getLedgerWithRetry` — kept for live ingestion path +- `ledgerBackendFactory` — kept for `fetchLedgerCloseTime` (chunk pre-creation) +- `fetchLedgerCloseTime` — unchanged (only fetches 2 boundary ledgers) + +### What's Removed + +- `runDispatcher` — replaced by `backfillFetcher` +- Backend creation in `processGap` — replaced by fetcher creation + +## Expected Performance + +### Current Bottleneck +Single dispatcher: 61 ledgers/sec fetch → 60 ledgers/sec pipeline. + +### After Change +S3 workers push directly. With 15 workers, fetch capacity is hundreds of ledgers/sec. **Bottleneck shifts to process workers** (CPU-bound XDR parsing at ~119ms/ledger). + +### Scaling Targets + +| Fetch Workers | Process Workers | Flush Workers | DB Conns Needed | Expected Throughput | +|--------------|----------------|---------------|-----------------|-------------------| +| 15 | 8 (current) | 2 (current) | 15 | ~67/sec (process-limited) | +| 15 | 16 | 4 | 25 | ~130/sec | +| 15 | 24 | 5 | 30 | ~200/sec | +| 15 | 32 | 8 | 45 | ~260/sec | + +**2-4× improvement** depending on process/flush worker scaling. + +## Verification + +1. **Unit tests**: Test `backfillFetcher` with mock `DataStore` — verify all ledgers in gap range are pushed to `ledgerCh`, out-of-range ledgers are filtered, retries work, context cancellation stops workers. +2. **Integration**: Run backfill on a small gap (1000 ledgers) and verify: + - All ledgers ingested (no gaps in DB) + - Watermark cursor reaches gap end + - Gap summary log shows fetch stats +3. **Performance**: Run on the same gap range (61602559-61637119) and compare ledgers/sec. +4. **Live ingestion unaffected**: Run `make unit-test` — no changes to live path. diff --git a/docs/plans/2026-04-03-backfill-parallel-fetcher.md b/docs/plans/2026-04-03-backfill-parallel-fetcher.md new file mode 100644 index 000000000..1d2f70b70 --- /dev/null +++ b/docs/plans/2026-04-03-backfill-parallel-fetcher.md @@ -0,0 +1,854 @@ +# Backfill Parallel Fetcher Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Replace the single-threaded dispatcher + `optimizedStorageBackend` bottleneck with a `backfillFetcher` that downloads S3 files in parallel and pushes `LedgerCloseMeta` directly to `ledgerCh`, eliminating the priority queue and sequential `GetLedger` overhead. + +**Architecture:** New `backfillFetcher` struct in `internal/ingest/` with N worker goroutines that download+decode S3 files and fan out individual ledgers to an external channel. A `BackfillFetcherFactory` closure (created in `setupDeps`) is passed through `IngestServiceConfig` to `ingestService`, replacing the backend+dispatcher in `processGap`. The existing `optimizedStorageBackend` stays untouched for live ingestion. + +**Tech Stack:** Go concurrency (goroutines, channels, sync.WaitGroup), `datastore.DataStore` (S3 client), `compressxdr.NewXDRDecoder` (zstd+XDR stream decode), `xdr.LedgerCloseMetaBatch` + +**Design doc:** `docs/plans/2026-04-03-backfill-parallel-fetcher-design.md` + +--- + +## Task 1: Create `backfillFetcher` with tests + +**Files:** +- Create: `internal/ingest/backfill_fetcher.go` +- Create: `internal/ingest/backfill_fetcher_test.go` + +### Step 1: Write the failing test + +Create `internal/ingest/backfill_fetcher_test.go`. Test that a fetcher delivers all ledgers in a gap range to `ledgerCh`, unordered. + +```go +package ingest + +import ( + "context" + "testing" + "time" + + "github.com/stellar/go-stellar-sdk/support/datastore" + "github.com/stellar/go-stellar-sdk/xdr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBackfillFetcher_DeliversAllLedgers(t *testing.T) { + // Setup: mock datastore returns batches of 10 ledgers per file. + // Gap: ledgers 100-129 = 3 files (100-109, 110-119, 120-129). + ledgersPerFile := uint32(10) + schema := testSchema(ledgersPerFile) + ds := &mockDataStore{} + + // Register 3 files covering ledgers 100-129 + for fileStart := uint32(100); fileStart < 130; fileStart += ledgersPerFile { + batch := makeBatch(fileStart, fileStart+ledgersPerFile-1) + objectKey := schema.GetObjectKeyFromSequenceNumber(fileStart) + ds.On("GetFile", mock.Anything, objectKey).Return( + newBatchReader(batch), nil, + ).Once() + } + + ledgerCh := make(chan xdr.LedgerCloseMeta, 100) + + fetcher := NewBackfillFetcher(BackfillFetcherConfig{ + NumWorkers: 3, + RetryLimit: 1, + RetryWait: 10 * time.Millisecond, + GapStart: 100, + GapEnd: 129, + }, ds, schema, ledgerCh) + + ctx := context.Background() + fetchCtx, fetchCancel := context.WithCancelCause(ctx) + defer fetchCancel(nil) + + stats := fetcher.Run(fetchCtx, fetchCancel) + + // ledgerCh should be closed by Run + var received []uint32 + for lcm := range ledgerCh { + received = append(received, lcm.LedgerSequence()) + } + + // All 30 ledgers delivered (order doesn't matter) + assert.Len(t, received, 30) + assert.Equal(t, 30, stats.fetchCount) + + // Verify all sequences present + seqSet := make(map[uint32]bool) + for _, seq := range received { + seqSet[seq] = true + } + for seq := uint32(100); seq <= 129; seq++ { + assert.True(t, seqSet[seq], "missing ledger %d", seq) + } +} +``` + +Note: This test uses `testSchema` and `mockDataStore` from `storage_backend_test.go`. You will need a helper `newBatchReader` that encodes a `LedgerCloseMetaBatch` through `compressxdr` and returns an `io.ReadCloser`. Model it after how the existing tests create mock S3 responses. + +### Step 2: Run test to verify it fails + +```bash +go test -v ./internal/ingest/ -run TestBackfillFetcher_DeliversAllLedgers -timeout 30s +``` + +Expected: FAIL — `NewBackfillFetcher` not defined. + +### Step 3: Write minimal `backfillFetcher` implementation + +Create `internal/ingest/backfill_fetcher.go`: + +```go +// backfillFetcher downloads S3 ledger files in parallel and fans out +// individual LedgerCloseMeta entries directly to an external channel. +// No ordering guarantees — designed for backfill where consumers +// handle ledgers independently and the watermark tracker handles +// out-of-order flushes. +package ingest + +import ( + "context" + "fmt" + "os" + "sync" + "time" + + "github.com/stellar/go-stellar-sdk/support/compressxdr" + "github.com/stellar/go-stellar-sdk/support/datastore" + "github.com/stellar/go-stellar-sdk/support/log" + "github.com/stellar/go-stellar-sdk/xdr" +) + +// BackfillFetcherConfig configures the parallel S3 fetcher for backfill. +type BackfillFetcherConfig struct { + NumWorkers uint32 + RetryLimit uint32 + RetryWait time.Duration + GapStart uint32 + GapEnd uint32 +} + +// BackfillFetchStats accumulates timing from all fetch workers. +// Returned by Run for gap summary log aggregation. +type BackfillFetchStats struct { + FetchCount int + FetchTotal time.Duration + ChannelWait map[string]time.Duration // key: "channel:direction" +} + +// backfillFetcher downloads S3 ledger files in parallel and pushes +// individual LedgerCloseMeta entries to an external channel. +type backfillFetcher struct { + dataStore datastore.DataStore + schema datastore.DataStoreSchema + config BackfillFetcherConfig + + ledgerCh chan<- xdr.LedgerCloseMeta // external, caller-owned +} + +// NewBackfillFetcher creates a fetcher that will push ledgers from the +// configured gap range to ledgerCh. Call Run to start workers. +func NewBackfillFetcher( + config BackfillFetcherConfig, + ds datastore.DataStore, + schema datastore.DataStoreSchema, + ledgerCh chan<- xdr.LedgerCloseMeta, +) *backfillFetcher { + return &backfillFetcher{ + dataStore: ds, + schema: schema, + config: config, + ledgerCh: ledgerCh, + } +} + +// Run starts fetch workers, waits for them to complete, then closes ledgerCh. +// Returns aggregated fetch stats for the gap summary log. +// On error, calls cancel with the cause; the caller checks context.Cause. +func (f *backfillFetcher) Run(ctx context.Context, cancel context.CancelCauseFunc) *BackfillFetchStats { + defer close(f.ledgerCh) + + // Compute all file-start sequences up-front (bounded range). + startBoundary := f.schema.GetSequenceNumberStartBoundary(f.config.GapStart) + endBoundary := f.schema.GetSequenceNumberStartBoundary(f.config.GapEnd) + + taskCh := make(chan uint32, f.config.NumWorkers) + + // Seed tasks in a goroutine to avoid blocking if taskCh fills. + go func() { + defer close(taskCh) + for seq := startBoundary; seq <= endBoundary; seq += f.schema.LedgersPerFile { + select { + case taskCh <- seq: + case <-ctx.Done(): + return + } + } + }() + + // Collect per-worker stats. + statsCh := make(chan *BackfillFetchStats, f.config.NumWorkers) + + var wg sync.WaitGroup + for i := uint32(0); i < f.config.NumWorkers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + stats := f.worker(ctx, cancel, taskCh) + statsCh <- stats + }() + } + wg.Wait() + close(statsCh) + + // Aggregate stats from all workers. + agg := &BackfillFetchStats{ChannelWait: make(map[string]time.Duration)} + for ws := range statsCh { + agg.FetchCount += ws.FetchCount + agg.FetchTotal += ws.FetchTotal + for k, v := range ws.ChannelWait { + agg.ChannelWait[k] += v + } + } + return agg +} + +// worker processes file-start sequences from taskCh until the channel +// closes or the context is cancelled. +func (f *backfillFetcher) worker(ctx context.Context, cancel context.CancelCauseFunc, taskCh <-chan uint32) *BackfillFetchStats { + stats := &BackfillFetchStats{ChannelWait: make(map[string]time.Duration)} + + for sequence := range taskCh { + if ctx.Err() != nil { + return stats + } + + fetchStart := time.Now() + batch, err := f.downloadWithRetry(ctx, cancel, sequence) + fetchDur := time.Since(fetchStart) + if err != nil { + return stats // cancel already called by downloadWithRetry + } + stats.FetchCount++ + stats.FetchTotal += fetchDur + + // Fan out individual ledgers to ledgerCh, filtering to gap range. + for i := range batch.LedgerCloseMetas { + lcm := batch.LedgerCloseMetas[i] + seq := uint32(lcm.LedgerSequence()) + if seq < f.config.GapStart || seq > f.config.GapEnd { + continue + } + + sendStart := time.Now() + select { + case f.ledgerCh <- lcm: + stats.ChannelWait[fmt.Sprintf("%s:%s", "ledger", "send")] += time.Since(sendStart) + case <-ctx.Done(): + return stats + } + } + } + return stats +} + +// downloadWithRetry downloads and decodes a single S3 file with retry. +// On permanent failure, calls cancel and returns the error. +func (f *backfillFetcher) downloadWithRetry(ctx context.Context, cancel context.CancelCauseFunc, sequence uint32) (xdr.LedgerCloseMetaBatch, error) { + for attempt := uint32(0); attempt <= f.config.RetryLimit; attempt++ { + batch, err := f.downloadAndDecode(ctx, sequence) + if err != nil { + if ctx.Err() != nil { + return xdr.LedgerCloseMetaBatch{}, ctx.Err() + } + if os.IsNotExist(err) { + cancel(fmt.Errorf("ledger file for sequence %d not found: %w", sequence, err)) + return xdr.LedgerCloseMetaBatch{}, err + } + if attempt == f.config.RetryLimit { + cancel(fmt.Errorf("downloading ledger file for sequence %d: maximum retries (%d) exceeded: %w", + sequence, f.config.RetryLimit, err)) + return xdr.LedgerCloseMetaBatch{}, err + } + log.WithField("sequence", sequence).WithError(err). + Warnf("Backfill fetch error (attempt %d/%d), retrying...", attempt+1, f.config.RetryLimit) + if !sleepWithContext(ctx, f.config.RetryWait) { + return xdr.LedgerCloseMetaBatch{}, ctx.Err() + } + continue + } + return batch, nil + } + // Unreachable, but Go requires a return. + return xdr.LedgerCloseMetaBatch{}, fmt.Errorf("unreachable: retry loop exited for sequence %d", sequence) +} + +// downloadAndDecode fetches and stream-decodes a single S3 file. +// Same streaming optimization as storageBuffer.downloadAndDecode. +func (f *backfillFetcher) downloadAndDecode(ctx context.Context, sequence uint32) (xdr.LedgerCloseMetaBatch, error) { + objectKey := f.schema.GetObjectKeyFromSequenceNumber(sequence) + reader, err := f.dataStore.GetFile(ctx, objectKey) + if err != nil { + return xdr.LedgerCloseMetaBatch{}, fmt.Errorf("fetching ledger file %s: %w", objectKey, err) + } + defer reader.Close() //nolint:errcheck + + var batch xdr.LedgerCloseMetaBatch + decoder := compressxdr.NewXDRDecoder(compressxdr.DefaultCompressor, &batch) + if _, err = decoder.ReadFrom(reader); err != nil { + return xdr.LedgerCloseMetaBatch{}, fmt.Errorf("decoding ledger file %s: %w", objectKey, err) + } + return batch, nil +} +``` + +### Step 4: Run test to verify it passes + +```bash +go test -v ./internal/ingest/ -run TestBackfillFetcher -timeout 30s +``` + +Expected: PASS. You may need to add the `newBatchReader` test helper that compresses a batch so `mockDataStore.GetFile` can return it. Look at how the existing `storage_backend_test.go` mocks S3 responses — if it doesn't have one, create a helper that uses `compressxdr.NewXDREncoder` to write a batch to a buffer and returns `io.NopCloser(bytes.NewReader(buf))`. + +### Step 5: Add edge-case tests + +Add to `backfill_fetcher_test.go`: + +1. **Partial first/last file filtering** — gap 105-124 with `LedgersPerFile=10` should only deliver ledgers 105-124 (not 100-104 or 125-129). +2. **Context cancellation** — cancel context mid-fetch, verify workers exit without deadlock. +3. **Retry on transient error** — mock `GetFile` to fail once then succeed, verify retry works. +4. **Missing file (NotExist)** — mock `GetFile` to return `os.ErrNotExist`, verify `cancel` is called. + +### Step 6: Run all tests + +```bash +go test -v ./internal/ingest/ -timeout 60s +``` + +Expected: All pass, including existing `storage_backend_test.go` tests (unchanged). + +### Step 7: Commit + +```bash +git add internal/ingest/backfill_fetcher.go internal/ingest/backfill_fetcher_test.go +git commit -m "feat: add backfillFetcher for parallel S3 ledger fetching + +Replaces the sequential dispatcher + optimizedStorageBackend bottleneck +in the backfill pipeline. Downloads S3 files with N parallel workers and +pushes individual LedgerCloseMeta entries directly to ledgerCh, removing +the priority queue and sequential GetLedger overhead." +``` + +--- + +## Task 2: Create `BackfillFetcherFactory` and wire through config + +**Files:** +- Modify: `internal/ingest/ingest.go:142-229` (setupDeps + new factory) +- Modify: `internal/ingest/ledger_backend.go:27-57` (extract dataStore/schema creation) +- Modify: `internal/services/ingest.go:45-81` (IngestServiceConfig) +- Modify: `internal/services/ingest.go:101-125` (ingestService struct) +- Modify: `internal/services/ingest.go:127-177` (NewIngestService) +- Modify: `cmd/ingest.go:61-100` (new CLI flag) + +### Step 1: Extract dataStore and schema creation + +In `internal/ingest/ledger_backend.go`, extract the datastore+schema creation from `newDatastoreLedgerBackend` into a shared function so both the legacy backend and the new fetcher can use it: + +```go +// newDatastoreResources creates the DataStore client and loads the schema +// from the S3 manifest. Shared by both optimizedStorageBackend (live) and +// backfillFetcher (backfill). +func newDatastoreResources(ctx context.Context, configPath string, networkPassphrase string) ( + datastore.DataStore, datastore.DataStoreSchema, ledgerbackend.BufferedStorageBackendConfig, error, +) { + storageBackendConfig, err := loadDatastoreBackendConfig(configPath) + if err != nil { + return nil, datastore.DataStoreSchema{}, ledgerbackend.BufferedStorageBackendConfig{}, + fmt.Errorf("loading datastore config: %w", err) + } + storageBackendConfig.DataStoreConfig.NetworkPassphrase = networkPassphrase + + ds, err := datastore.NewDataStore(ctx, storageBackendConfig.DataStoreConfig) + if err != nil { + return nil, datastore.DataStoreSchema{}, ledgerbackend.BufferedStorageBackendConfig{}, + fmt.Errorf("creating datastore: %w", err) + } + + schema, err := datastore.LoadSchema(ctx, ds, storageBackendConfig.DataStoreConfig) + if err != nil { + return nil, datastore.DataStoreSchema{}, ledgerbackend.BufferedStorageBackendConfig{}, + fmt.Errorf("loading datastore schema: %w", err) + } + + return ds, schema, storageBackendConfig.BufferedStorageBackendConfig, nil +} +``` + +Refactor `newDatastoreLedgerBackend` to use it: + +```go +func newDatastoreLedgerBackend(ctx context.Context, datastoreConfigPath string, networkPassphrase string) (ledgerbackend.LedgerBackend, error) { + ds, schema, bufConfig, err := newDatastoreResources(ctx, datastoreConfigPath, networkPassphrase) + if err != nil { + return nil, err + } + + ledgerBackend, err := newOptimizedStorageBackend(bufConfig, ds, schema) + if err != nil { + return nil, fmt.Errorf("creating optimized storage backend: %w", err) + } + + log.Infof("Using optimized storage backend with buffer size %d, %d workers", + bufConfig.BufferSize, bufConfig.NumWorkers) + return ledgerBackend, nil +} +``` + +### Step 2: Define the factory type and add to config + +In `internal/services/ingest.go`, add the factory type alongside the existing `LedgerBackendFactory`: + +```go +// BackfillFetcherFactory creates a backfillFetcher that pushes ledgers to ledgerCh. +// Each gap gets its own fetcher. The factory encapsulates dataStore/schema creation. +type BackfillFetcherFactory func(ctx context.Context, cancel context.CancelCauseFunc, config ingest.BackfillFetcherConfig, ledgerCh chan<- xdr.LedgerCloseMeta) *ingest.BackfillFetcher +``` + +Wait — we should avoid the import cycle (`services` importing `ingest`). The `BackfillFetcherConfig` and `BackfillFetchStats` types live in `ingest` package. The service layer already imports `ingest` types indirectly via `indexer`. Let me check... + +Actually, `internal/services/` does NOT import `internal/ingest/` currently — the `ledgerBackendFactory` returns `ledgerbackend.LedgerBackend` (from the SDK). We need a factory that returns a runner interface to avoid the import cycle. + +**Better approach**: Define an interface in `services` that the fetcher satisfies: + +```go +// BackfillFetcher runs parallel S3 download workers and pushes ledgers to a channel. +type BackfillFetcher interface { + // Run starts workers, blocks until done, closes ledgerCh, returns stats. + Run(ctx context.Context, cancel context.CancelCauseFunc) *BackfillFetchStats +} + +// BackfillFetchStats holds aggregated timing from fetch workers. +type BackfillFetchStats struct { + FetchCount int + FetchTotal time.Duration + ChannelWait map[string]time.Duration +} + +// BackfillFetcherFactory creates a fetcher for a specific gap. +type BackfillFetcherFactory func(ctx context.Context, config BackfillFetcherConfig, ledgerCh chan<- xdr.LedgerCloseMeta) BackfillFetcher + +// BackfillFetcherConfig configures a single fetcher instance. +type BackfillFetcherConfig struct { + NumWorkers uint32 + RetryLimit uint32 + RetryWait time.Duration + GapStart uint32 + GapEnd uint32 +} +``` + +Then move the config/stats types from `ingest` to `services` (or define them in `services` and have the `ingest` fetcher return the `services` type — but that's also a cycle). + +**Cleanest approach**: Keep `BackfillFetcherConfig` and `BackfillFetchStats` in the `ingest` package. Define the factory in `ingest` package too. The services package imports `ingest` just like it imports `indexer`. Check if this import already exists or is safe: + +Look at imports in `internal/services/ingest.go` — it imports `indexer` already: +```go +"github.com/stellar/wallet-backend/internal/indexer" +``` + +Adding `"github.com/stellar/wallet-backend/internal/ingest"` should be fine as long as `ingest` doesn't import `services`. Let me verify — `ingest/ingest.go` imports `services` at line 207 (`services.NewIngestService`). **There IS a cycle**: `ingest → services` already exists, so `services → ingest` would create a cycle. + +**Resolution**: The factory is a closure created in `setupDeps` (in the `ingest` package). It returns an interface defined in `services`. The stats type needs to live somewhere both can see — either a shared package or just use a generic struct. + +**Simplest solution**: Use the same pattern as `LedgerBackendFactory` — return a function type that returns a concrete runner. The runner is a `func() *backfillWorkerStats` (which already exists in services). The factory closure captures `dataStore` and `schema` from `setupDeps`. + +```go +// In services/ingest.go: +// BackfillFetcherFactory creates a fetcher for a gap and returns a Run function. +// The Run function starts workers, blocks until done, closes ledgerCh, and returns +// fetch stats compatible with the gap summary log. +type BackfillFetcherFactory func( + ctx context.Context, + cancel context.CancelCauseFunc, + gapStart, gapEnd uint32, + ledgerCh chan<- xdr.LedgerCloseMeta, +) *backfillWorkerStats +``` + +This avoids any new types crossing the package boundary — `backfillWorkerStats` already exists in `services`, and `xdr.LedgerCloseMeta` is from the SDK. The factory is a closure that internally creates a `backfillFetcher` (in `ingest` package) and adapts its stats to `*backfillWorkerStats`. + +Add to `IngestServiceConfig` (after line 80): + +```go +// === Backfill Tuning === +BackfillProcessWorkers int +BackfillFlushWorkers int +BackfillDBInsertBatchSize int +BackfillLedgerChanSize int +BackfillFlushChanSize int +BackfillFetcherFactory BackfillFetcherFactory // NEW: parallel S3 fetcher for backfill +BackfillFetchWorkers int // NEW: S3 download goroutines (default: 15) +``` + +Add to `ingestService` struct (after line 123): + +```go +backfillFetcherFactory BackfillFetcherFactory +backfillFetchWorkers int +``` + +Assign in `NewIngestService` (after line 174): + +```go +backfillFetcherFactory: cfg.BackfillFetcherFactory, +backfillFetchWorkers: cfg.BackfillFetchWorkers, +``` + +### Step 3: Create the factory in `setupDeps` + +In `internal/ingest/ingest.go`, after the existing `ledgerBackendFactory` (line 205), add: + +```go +// Create factory for the parallel backfill fetcher. +// Each gap gets its own fetcher with fresh S3 connections. +var backfillFetcherFactory services.BackfillFetcherFactory +if cfg.LedgerBackendType == LedgerBackendTypeDatastore { + backfillFetcherFactory = func( + ctx context.Context, + cancel context.CancelCauseFunc, + gapStart, gapEnd uint32, + ledgerCh chan<- xdr.LedgerCloseMeta, + ) *services.BackfillWorkerStats { + ds, schema, _, err := newDatastoreResources(ctx, cfg.DatastoreConfigPath, cfg.NetworkPassphrase) + if err != nil { + cancel(fmt.Errorf("creating datastore resources: %w", err)) + return &services.BackfillWorkerStats{} + } + + fetcher := NewBackfillFetcher(BackfillFetcherConfig{ + NumWorkers: uint32(cfg.BackfillFetchWorkers), + RetryLimit: 3, + RetryWait: 5 * time.Second, + GapStart: gapStart, + GapEnd: gapEnd, + }, ds, schema, ledgerCh) + + fetchStats := fetcher.Run(ctx, cancel) + + // Adapt ingest.BackfillFetchStats → services.backfillWorkerStats + ws := &services.BackfillWorkerStats{} + // ... map fields + return ws + } +} +``` + +**Wait** — this requires `services.BackfillWorkerStats` to be exported, and creates a `services → ingest` import path issue. Let me reconsider. + +**Final, cleanest approach**: Make the factory return a **function** — no cross-package types needed: + +```go +// In services/ingest.go: +// BackfillFetcherRunner starts fetch workers and returns when done. +// It must close ledgerCh before returning. +// Returns (fetchCount, fetchTotal, channelWait). +type BackfillFetcherRunner func(ctx context.Context, cancel context.CancelCauseFunc) (int, time.Duration, map[string]time.Duration) + +// BackfillFetcherFactory creates a runner for a specific gap. +type BackfillFetcherFactory func(gapStart, gapEnd uint32, ledgerCh chan<- xdr.LedgerCloseMeta) BackfillFetcherRunner +``` + +In `setupDeps`, the factory closure creates the fetcher and returns a runner that adapts the results: + +```go +backfillFetcherFactory := func(gapStart, gapEnd uint32, ledgerCh chan<- xdr.LedgerCloseMeta) services.BackfillFetcherRunner { + return func(ctx context.Context, cancel context.CancelCauseFunc) (int, time.Duration, map[string]time.Duration) { + ds, schema, _, err := newDatastoreResources(ctx, cfg.DatastoreConfigPath, cfg.NetworkPassphrase) + if err != nil { + cancel(fmt.Errorf("creating datastore resources: %w", err)) + return 0, 0, nil + } + fetcher := NewBackfillFetcher(BackfillFetcherConfig{ + NumWorkers: uint32(cfg.BackfillFetchWorkers), + RetryLimit: 3, + RetryWait: 5 * time.Second, + GapStart: gapStart, + GapEnd: gapEnd, + }, ds, schema, ledgerCh) + stats := fetcher.Run(ctx, cancel) + return stats.FetchCount, stats.FetchTotal, stats.ChannelWait + } +} +``` + +No new cross-package types — only primitives and `map[string]time.Duration`. + +### Step 4: Add CLI flag for `BackfillFetchWorkers` + +In `cmd/ingest.go`, add after the `backfill-flush-chan-size` flag (after line 100): + +```go +{ + Name: "backfill-fetch-workers", + Usage: "Number of parallel S3 download workers in the backfill fetcher. Each worker downloads and decodes one file at a time.", + OptType: types.Int, + ConfigKey: &cfg.BackfillFetchWorkers, + FlagDefault: 15, + Required: false, +}, +``` + +Add `BackfillFetchWorkers int` to `ingest.Configs` struct (after line 86). + +### Step 5: Run existing tests + +```bash +go test -v ./internal/ingest/ -timeout 60s +go test -v ./internal/services/ -timeout 3m +``` + +Expected: All existing tests pass. The new factory is nil for non-datastore backends and for tests that don't set it. + +### Step 6: Commit + +```bash +git add internal/ingest/ledger_backend.go internal/ingest/ingest.go internal/services/ingest.go cmd/ingest.go +git commit -m "feat: wire BackfillFetcherFactory through config pipeline + +Extract newDatastoreResources from newDatastoreLedgerBackend for reuse. +Add BackfillFetcherFactory and BackfillFetchWorkers to IngestServiceConfig. +Add --backfill-fetch-workers CLI flag (default: 15)." +``` + +--- + +## Task 3: Replace `runDispatcher` with `backfillFetcher` in `processGap` + +**Files:** +- Modify: `internal/services/ingest_backfill.go:152-291` (processGap) +- Modify: `internal/services/ingest_backfill.go:296-332` (runDispatcher — remove) + +### Step 1: Modify `processGap` to use the fetcher + +In `processGap`, replace the backend creation + dispatcher with the fetcher factory. + +**Remove** (lines 161-174): +```go +backend, err := m.ledgerBackendFactory(gapCtx) +// ... PrepareRange, defer Close +``` + +**Remove** (line 237): +```go +dispatcherStats := m.runDispatcher(gapCtx, gapCancel, backend, gap, ledgerCh) +``` + +**Replace with**: +```go +// Stage 1: parallel S3 fetcher (replaces dispatcher + backend) +var fetchCount int +var fetchTotal time.Duration +var fetchChannelWait map[string]time.Duration + +pipelineWg.Add(1) +go func() { + defer pipelineWg.Done() + runner := m.backfillFetcherFactory(gap.GapStart, gap.GapEnd, ledgerCh) + fetchCount, fetchTotal, fetchChannelWait = runner(gapCtx, gapCancel) +}() +``` + +Note: The fetcher's `Run` closes `ledgerCh`, which is the same contract as `runDispatcher` (which calls `defer close(ledgerCh)` at line 303). The process workers' `for lcm := range ledgerCh` loop exits when `ledgerCh` closes, then they close `flushCh`. + +**Update the stats aggregation** (lines 248-257). Replace: +```go +gapStats.mergeWorker(dispatcherStats) +``` + +With: +```go +// Merge fetcher stats into gap stats +gapStats.fetchCount += fetchCount +gapStats.fetchTotal += fetchTotal +for k, v := range fetchChannelWait { + gapStats.channelWait[k] += v +} +``` + +### Step 2: Remove `runDispatcher` + +Delete the `runDispatcher` method (lines 296-332 in `ingest_backfill.go`). It's no longer called. + +Also remove the `getLedgerWithRetry` import if it's only used by the dispatcher — but check first: it's also used by live ingestion, so likely keep it. + +### Step 3: Run tests + +```bash +go test -v ./internal/services/ -timeout 3m +``` + +Expected: Tests pass. Any test that mocks `ledgerBackendFactory` for backfill scenarios will need updating to mock `backfillFetcherFactory` instead. + +### Step 4: Commit + +```bash +git add internal/services/ingest_backfill.go +git commit -m "feat: replace dispatcher with parallel backfillFetcher in processGap + +Remove runDispatcher and backend creation from processGap. +Use BackfillFetcherFactory to create a parallel S3 fetcher per gap. +Workers push LedgerCloseMeta directly to ledgerCh, eliminating the +priority queue and sequential GetLedger bottleneck." +``` + +--- + +## Task 4: Add Prometheus metrics to the fetcher + +**Files:** +- Modify: `internal/ingest/backfill_fetcher.go` (add metrics observations) +- Modify: `internal/ingest/backfill_fetcher.go` (accept metrics in config or via option) + +### Step 1: Add metrics to fetcher worker + +The existing dispatcher observes two metrics: +- `PhaseDuration.WithLabelValues("backfill_fetch").Observe(fetchDur.Seconds())` +- `BackfillChannelWait.WithLabelValues("ledger", "send").Observe(sendDur.Seconds())` + +Since the fetcher lives in `internal/ingest/` (not `services/`), it can't directly access `appMetrics`. Two options: +1. Pass the specific metric observers as function callbacks +2. Pass the full `metrics.Ingestion` struct + +Use option 1 — add optional observer callbacks to the config: + +```go +type BackfillFetcherConfig struct { + NumWorkers uint32 + RetryLimit uint32 + RetryWait time.Duration + GapStart uint32 + GapEnd uint32 + // Optional Prometheus observers (nil-safe — no metrics if nil). + OnFetchDuration func(seconds float64) + OnChannelWait func(channel, direction string, seconds float64) +} +``` + +In the worker, after recording stats: + +```go +if f.config.OnFetchDuration != nil { + f.config.OnFetchDuration(fetchDur.Seconds()) +} +// ... and for channel wait: +if f.config.OnChannelWait != nil { + f.config.OnChannelWait("ledger", "send", sendDur.Seconds()) +} +``` + +Wire in `setupDeps` when creating the factory: + +```go +OnFetchDuration: func(s float64) { + m.Ingestion.PhaseDuration.WithLabelValues("backfill_fetch").Observe(s) +}, +OnChannelWait: func(ch, dir string, s float64) { + m.Ingestion.BackfillChannelWait.WithLabelValues(ch, dir).Observe(s) +}, +``` + +### Step 2: Run tests + +```bash +go test -v ./internal/ingest/ -run TestBackfillFetcher -timeout 30s +``` + +Expected: Pass (callbacks are nil in tests — no metrics observed). + +### Step 3: Commit + +```bash +git add internal/ingest/backfill_fetcher.go internal/ingest/ingest.go +git commit -m "feat: add Prometheus metric callbacks to backfillFetcher + +Observe backfill_fetch phase duration and ledger channel send wait +via optional callbacks, wired to the existing Prometheus metrics." +``` + +--- + +## Task 5: Update channel utilization sampler + +**Files:** +- Modify: `internal/services/ingest_backfill.go:192-217` (sampler goroutine) + +### Step 1: Review sampler + +The sampler currently samples `ledgerCh` and `flushCh` fill ratios. With the fetcher owning `ledgerCh` production, the sampler still works — it just reads `len(ledgerCh)/cap(ledgerCh)`. No code changes needed for the sampler itself. + +However, the sampler is started **before** the fetcher, and the fetcher closes `ledgerCh`. After close, `len(ledgerCh)` returns 0 and `cap(ledgerCh)` is still valid — so reading fill ratio of a closed channel is safe in Go. + +**No changes needed.** Move to next task. + +### Step 2: Commit (skip — no changes) + +--- + +## Task 6: Clean up and verify + +**Files:** +- All modified files + +### Step 1: Run linting + +```bash +make tidy +make check +``` + +Fix any issues. + +### Step 2: Run full unit tests + +```bash +make unit-test +``` + +Expected: All pass. + +### Step 3: Run backfill on a small gap + +```bash +go run main.go ingest --mode=backfill --start-ledger=61602559 --end-ledger=61603559 --backfill-fetch-workers=15 --backfill-process-workers=8 --backfill-flush-workers=2 --log-level=debug +``` + +Verify: +- Gap completes without errors +- Summary log shows fetch stats (fetch count, fetch total, channel wait) +- No missing ledgers in DB + +### Step 4: Run performance comparison + +Run on the same 34,561-ledger gap and compare: +- Previous: 60 ledgers/sec +- Expected: >60 ledgers/sec (process-worker-limited, ~67 with 8 workers) + +Then scale up process workers: +```bash +--backfill-process-workers=16 --backfill-flush-workers=4 +``` + +Expected: ~130 ledgers/sec. + +### Step 5: Final commit + +```bash +git add -A +git commit -m "chore: clean up after backfill parallel fetcher integration" +``` diff --git a/go.mod b/go.mod index 5550af9b3..37fd339ff 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,9 @@ require ( github.com/alitto/pond/v2 v2.5.0 github.com/avast/retry-go/v4 v4.6.1 github.com/aws/aws-sdk-go v1.55.7 + github.com/aws/aws-sdk-go-v2 v1.36.5 + github.com/aws/aws-sdk-go-v2/config v1.29.17 + github.com/aws/aws-sdk-go-v2/service/s3 v1.83.0 github.com/basemachina/gqlgen-complexity-reporter v0.1.2 github.com/deckarep/golang-set/v2 v2.8.0 github.com/docker/go-connections v0.5.0 @@ -19,6 +22,7 @@ require ( github.com/google/uuid v1.6.0 github.com/guregu/null v4.0.0+incompatible github.com/jackc/pgx/v5 v5.7.6 + github.com/klauspost/compress v1.18.0 github.com/pelletier/go-toml v1.9.5 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.22.0 @@ -35,7 +39,6 @@ require ( github.com/tetratelabs/wazero v1.10.1 github.com/vektah/gqlparser/v2 v2.5.30 github.com/vikstrous/dataloadgen v0.0.9 - golang.org/x/sync v0.16.0 golang.org/x/term v0.33.0 golang.org/x/text v0.27.0 ) @@ -59,9 +62,7 @@ require ( github.com/Microsoft/go-winio v0.6.2 // indirect github.com/agnivade/levenshtein v1.2.1 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect - github.com/aws/aws-sdk-go-v2 v1.36.5 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.11 // indirect - github.com/aws/aws-sdk-go-v2/config v1.29.17 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.17.70 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32 // indirect github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.83 // indirect @@ -73,7 +74,6 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.4 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.17 // indirect - github.com/aws/aws-sdk-go-v2/service/s3 v1.83.0 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.25.5 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.34.0 // indirect @@ -123,7 +123,6 @@ require ( github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jmoiron/sqlx v1.4.0 // indirect - github.com/klauspost/compress v1.18.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect @@ -183,6 +182,7 @@ require ( golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect golang.org/x/net v0.42.0 // indirect golang.org/x/oauth2 v0.30.0 // indirect + golang.org/x/sync v0.16.0 // indirect golang.org/x/sys v0.35.0 // indirect golang.org/x/time v0.8.0 // indirect google.golang.org/api v0.215.0 // indirect diff --git a/internal/data/operations.go b/internal/data/operations.go index c08ab08f9..02e054a4f 100644 --- a/internal/data/operations.go +++ b/internal/data/operations.go @@ -294,8 +294,8 @@ func (m *OperationModel) BatchGetByAccountAddress(ctx context.Context, accountAd func (m *OperationModel) BatchGetByStateChangeIDs(ctx context.Context, scToIDs []int64, scOpIDs []int64, stateChangeIDs []int64, columns string) ([]*types.OperationWithStateChangeID, error) { columns = prepareColumnsWithID(columns, types.Operation{}, "operations", "id") - // Build tuples for the IN clause. Since (to_id, operation_id, state_change_id) is the primary key of state_changes, - // it will be faster to search on this tuple. + // Build tuples for the IN clause. (to_id, operation_id, state_change_id) is indexed on state_changes, + // so searching on this tuple is efficient. tuples := make([]string, len(stateChangeIDs)) for i := range stateChangeIDs { tuples[i] = fmt.Sprintf("(%d, %d, %d)", scToIDs[i], scOpIDs[i], stateChangeIDs[i]) diff --git a/internal/data/statechanges.go b/internal/data/statechanges.go index 398d0e087..c6e9c2ac1 100644 --- a/internal/data/statechanges.go +++ b/internal/data/statechanges.go @@ -209,82 +209,88 @@ func (m *StateChangeModel) BatchCopy( "signer_weight_old", "signer_weight_new", "threshold_old", "threshold_new", "trustline_limit_old", "trustline_limit_new", "flags", "key_value", }, - pgx.CopyFromSlice(len(stateChanges), func(i int) ([]any, error) { - sc := stateChanges[i] - - // Generate a fresh random ID at insertion time so retries on PK collision - // produce new IDs automatically. - stateChangeID, err := generateStateChangeID() - if err != nil { - return nil, err - } - - // Convert account_id to BYTEA (required field) - accountVal, err := sc.AccountID.Value() - if err != nil { - return nil, fmt.Errorf("converting account_id: %w", err) - } - - // Convert nullable account_id fields to BYTEA - signerBytes, err := pgtypeBytesFromNullAddressBytea(sc.SignerAccountID) - if err != nil { - return nil, fmt.Errorf("converting signer_account_id: %w", err) - } - spenderBytes, err := pgtypeBytesFromNullAddressBytea(sc.SpenderAccountID) - if err != nil { - return nil, fmt.Errorf("converting spender_account_id: %w", err) - } - sponsoredBytes, err := pgtypeBytesFromNullAddressBytea(sc.SponsoredAccountID) - if err != nil { - return nil, fmt.Errorf("converting sponsored_account_id: %w", err) - } - sponsorBytes, err := pgtypeBytesFromNullAddressBytea(sc.SponsorAccountID) - if err != nil { - return nil, fmt.Errorf("converting sponsor_account_id: %w", err) - } - deployerBytes, err := pgtypeBytesFromNullAddressBytea(sc.DeployerAccountID) - if err != nil { - return nil, fmt.Errorf("converting deployer_account_id: %w", err) - } - funderBytes, err := pgtypeBytesFromNullAddressBytea(sc.FunderAccountID) - if err != nil { - return nil, fmt.Errorf("converting funder_account_id: %w", err) - } - tokenBytes, err := pgtypeBytesFromNullAddressBytea(sc.TokenID) - if err != nil { - return nil, fmt.Errorf("converting token_id: %w", err) - } - - return []any{ - pgtype.Int8{Int64: sc.ToID, Valid: true}, - pgtype.Int8{Int64: stateChangeID, Valid: true}, - pgtype.Text{String: string(sc.StateChangeCategory), Valid: true}, - pgtypeTextFromReason(sc.StateChangeReason), - pgtype.Timestamptz{Time: sc.LedgerCreatedAt, Valid: true}, - pgtype.Int4{Int32: int32(sc.LedgerNumber), Valid: true}, - accountVal.([]byte), - pgtype.Int8{Int64: sc.OperationID, Valid: true}, - tokenBytes, - pgtypeTextFromNullString(sc.Amount), - signerBytes, - spenderBytes, - sponsoredBytes, - sponsorBytes, - deployerBytes, - funderBytes, - pgtypeTextFromNullString(sc.ClaimableBalanceID), - pgtypeTextFromNullString(sc.LiquidityPoolID), - pgtypeTextFromNullString(sc.SponsoredData), - pgtypeInt2FromNullInt16(sc.SignerWeightOld), - pgtypeInt2FromNullInt16(sc.SignerWeightNew), - pgtypeInt2FromNullInt16(sc.ThresholdOld), - pgtypeInt2FromNullInt16(sc.ThresholdNew), - pgtypeTextFromNullString(sc.TrustlineLimitOld), - pgtypeTextFromNullString(sc.TrustlineLimitNew), - pgtypeInt2FromNullInt16(sc.Flags), - jsonbFromMap(sc.KeyValue), - }, nil - }), + // Use CopyFromFunc with a reusable row buffer to avoid allocating + // a fresh []any (27 elements) per row — eliminates ~61M allocations per gap. + // Safe because pgx encodes each row immediately and never retains the slice. + func() pgx.CopyFromSource { + idx := 0 + row := make([]any, 27) + return pgx.CopyFromFunc(func() ([]any, error) { + if idx >= len(stateChanges) { + return nil, nil + } + sc := stateChanges[idx] + idx++ + + stateChangeID, err := generateStateChangeID() + if err != nil { + return nil, err + } + + accountVal, err := sc.AccountID.Value() + if err != nil { + return nil, fmt.Errorf("converting account_id: %w", err) + } + + signerBytes, err := pgtypeBytesFromNullAddressBytea(sc.SignerAccountID) + if err != nil { + return nil, fmt.Errorf("converting signer_account_id: %w", err) + } + spenderBytes, err := pgtypeBytesFromNullAddressBytea(sc.SpenderAccountID) + if err != nil { + return nil, fmt.Errorf("converting spender_account_id: %w", err) + } + sponsoredBytes, err := pgtypeBytesFromNullAddressBytea(sc.SponsoredAccountID) + if err != nil { + return nil, fmt.Errorf("converting sponsored_account_id: %w", err) + } + sponsorBytes, err := pgtypeBytesFromNullAddressBytea(sc.SponsorAccountID) + if err != nil { + return nil, fmt.Errorf("converting sponsor_account_id: %w", err) + } + deployerBytes, err := pgtypeBytesFromNullAddressBytea(sc.DeployerAccountID) + if err != nil { + return nil, fmt.Errorf("converting deployer_account_id: %w", err) + } + funderBytes, err := pgtypeBytesFromNullAddressBytea(sc.FunderAccountID) + if err != nil { + return nil, fmt.Errorf("converting funder_account_id: %w", err) + } + tokenBytes, err := pgtypeBytesFromNullAddressBytea(sc.TokenID) + if err != nil { + return nil, fmt.Errorf("converting token_id: %w", err) + } + + row[0] = pgtype.Int8{Int64: sc.ToID, Valid: true} + row[1] = pgtype.Int8{Int64: stateChangeID, Valid: true} + row[2] = pgtype.Text{String: string(sc.StateChangeCategory), Valid: true} + row[3] = pgtypeTextFromReason(sc.StateChangeReason) + row[4] = pgtype.Timestamptz{Time: sc.LedgerCreatedAt, Valid: true} + row[5] = pgtype.Int4{Int32: int32(sc.LedgerNumber), Valid: true} + row[6] = accountVal.([]byte) + row[7] = pgtype.Int8{Int64: sc.OperationID, Valid: true} + row[8] = tokenBytes + row[9] = pgtypeTextFromNullString(sc.Amount) + row[10] = signerBytes + row[11] = spenderBytes + row[12] = sponsoredBytes + row[13] = sponsorBytes + row[14] = deployerBytes + row[15] = funderBytes + row[16] = pgtypeTextFromNullString(sc.ClaimableBalanceID) + row[17] = pgtypeTextFromNullString(sc.LiquidityPoolID) + row[18] = pgtypeTextFromNullString(sc.SponsoredData) + row[19] = pgtypeInt2FromNullInt16(sc.SignerWeightOld) + row[20] = pgtypeInt2FromNullInt16(sc.SignerWeightNew) + row[21] = pgtypeInt2FromNullInt16(sc.ThresholdOld) + row[22] = pgtypeInt2FromNullInt16(sc.ThresholdNew) + row[23] = pgtypeTextFromNullString(sc.TrustlineLimitOld) + row[24] = pgtypeTextFromNullString(sc.TrustlineLimitNew) + row[25] = pgtypeInt2FromNullInt16(sc.Flags) + row[26] = jsonbFromMap(sc.KeyValue) + return row, nil + }) + }(), ) if err != nil { m.Metrics.QueryErrors.WithLabelValues("BatchCopy", "state_changes", utils.GetDBErrorType(err)).Inc() diff --git a/internal/data/transactions.go b/internal/data/transactions.go index d9cb7e69d..35c71fc1f 100644 --- a/internal/data/transactions.go +++ b/internal/data/transactions.go @@ -206,8 +206,8 @@ func (m *TransactionModel) BatchGetByOperationIDs(ctx context.Context, operation func (m *TransactionModel) BatchGetByStateChangeIDs(ctx context.Context, scToIDs []int64, scOpIDs []int64, scOrders []int64, columns string) ([]*types.TransactionWithStateChangeID, error) { columns = prepareColumnsWithID(columns, types.Transaction{}, "transactions", "to_id") - // Build tuples for the IN clause. Since (to_id, operation_id, state_change_id) is the primary key of state_changes, - // it will be faster to search on this tuple. + // Build tuples for the IN clause. (to_id, operation_id, state_change_id) is indexed on state_changes, + // so searching on this tuple is efficient. tuples := make([]string, len(scOrders)) for i := range scOrders { tuples[i] = fmt.Sprintf("(%d, %d, %d)", scToIDs[i], scOpIDs[i], scOrders[i]) diff --git a/internal/db/migrations/2025-06-10.2-transactions.sql b/internal/db/migrations/2025-06-10.2-transactions.sql index 6043f34a2..b8eed22ef 100644 --- a/internal/db/migrations/2025-06-10.2-transactions.sql +++ b/internal/db/migrations/2025-06-10.2-transactions.sql @@ -9,8 +9,7 @@ CREATE TABLE transactions ( ledger_number INTEGER NOT NULL, is_fee_bump BOOLEAN NOT NULL DEFAULT false, ingested_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - ledger_created_at TIMESTAMPTZ NOT NULL, - PRIMARY KEY (to_id, ledger_created_at) + ledger_created_at TIMESTAMPTZ NOT NULL ) WITH ( tsdb.hypertable, tsdb.partition_column = 'ledger_created_at', @@ -22,13 +21,13 @@ CREATE TABLE transactions ( SELECT enable_chunk_skipping('transactions', 'to_id'); CREATE INDEX idx_transactions_hash ON transactions(hash); +CREATE INDEX idx_transactions_toid_time ON transactions(to_id, ledger_created_at); -- Table: transactions_accounts (TimescaleDB hypertable for automatic cleanup with retention) CREATE TABLE transactions_accounts ( tx_to_id BIGINT NOT NULL, account_id BYTEA NOT NULL, - ledger_created_at TIMESTAMPTZ NOT NULL, - PRIMARY KEY (account_id, tx_to_id, ledger_created_at) + ledger_created_at TIMESTAMPTZ NOT NULL ) WITH ( tsdb.hypertable, tsdb.partition_column = 'ledger_created_at', diff --git a/internal/db/migrations/2025-06-10.3-operations.sql b/internal/db/migrations/2025-06-10.3-operations.sql index 77929329f..d01ead07e 100644 --- a/internal/db/migrations/2025-06-10.3-operations.sql +++ b/internal/db/migrations/2025-06-10.3-operations.sql @@ -22,8 +22,7 @@ CREATE TABLE operations ( successful BOOLEAN NOT NULL, ledger_number INTEGER NOT NULL, ingested_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - ledger_created_at TIMESTAMPTZ NOT NULL, - PRIMARY KEY (id, ledger_created_at) + ledger_created_at TIMESTAMPTZ NOT NULL ) WITH ( tsdb.hypertable, tsdb.partition_column = 'ledger_created_at', @@ -34,12 +33,13 @@ CREATE TABLE operations ( SELECT enable_chunk_skipping('operations', 'id'); +CREATE INDEX idx_operations_id_time ON operations(id, ledger_created_at); + -- Table: operations_accounts (TimescaleDB hypertable for automatic cleanup with retention) CREATE TABLE operations_accounts ( operation_id BIGINT NOT NULL, account_id BYTEA NOT NULL, - ledger_created_at TIMESTAMPTZ NOT NULL, - PRIMARY KEY (account_id, operation_id, ledger_created_at) + ledger_created_at TIMESTAMPTZ NOT NULL ) WITH ( tsdb.hypertable, tsdb.partition_column = 'ledger_created_at', diff --git a/internal/db/migrations/2025-06-10.4-statechanges.sql b/internal/db/migrations/2025-06-10.4-statechanges.sql index cbc30228d..e3a8bf66d 100644 --- a/internal/db/migrations/2025-06-10.4-statechanges.sql +++ b/internal/db/migrations/2025-06-10.4-statechanges.sql @@ -42,8 +42,7 @@ CREATE TABLE state_changes ( trustline_limit_new TEXT, flags SMALLINT, key_value JSONB, - ledger_created_at TIMESTAMPTZ NOT NULL, - PRIMARY KEY (to_id, operation_id, state_change_id, ledger_created_at) + ledger_created_at TIMESTAMPTZ NOT NULL ) WITH ( tsdb.hypertable, tsdb.partition_column = 'ledger_created_at', @@ -57,6 +56,7 @@ SELECT enable_chunk_skipping('state_changes', 'to_id'); SELECT enable_chunk_skipping('state_changes', 'operation_id'); CREATE INDEX idx_state_changes_operation_id ON state_changes(operation_id); +CREATE INDEX idx_state_changes_toid_opid_scid_time ON state_changes(to_id, operation_id, state_change_id, ledger_created_at); CREATE INDEX idx_state_changes_account_category ON state_changes(account_id, state_change_category, state_change_reason, ledger_created_at DESC, to_id DESC, operation_id DESC, state_change_id DESC); -- +migrate Down diff --git a/internal/db/timescaledb_chunks.go b/internal/db/timescaledb_chunks.go new file mode 100644 index 000000000..14cacbe0d --- /dev/null +++ b/internal/db/timescaledb_chunks.go @@ -0,0 +1,274 @@ +package db + +import ( + "context" + "fmt" + "strings" + "sync/atomic" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/stellar/go-stellar-sdk/support/log" +) + +// droppableIndexes lists the parent index names (from migrations) that should be +// dropped on backfill chunks. TimescaleDB chunk indexes embed the parent name, +// e.g. parent "idx_transactions_hash" becomes "_hyper_1_5_chunk_idx_transactions_hash". +// +// Update this list when adding new indexes to hypertable migrations. +var droppableIndexes = []string{ + // TimescaleDB auto-created time-dimension indexes (safe to drop on backfill chunks; + // only used for chunk exclusion on uncompressed reads, not needed during bulk writes + // and replaced by segmentby/orderby metadata after compression) + "transactions_ledger_created_at_idx", + "transactions_accounts_ledger_created_at_idx", + "operations_ledger_created_at_idx", + "operations_accounts_ledger_created_at_idx", + "state_changes_ledger_created_at_idx", + // transactions (2025-06-10.2-transactions.sql) + "idx_transactions_hash", + "idx_transactions_toid_time", + "idx_transactions_accounts_tx_to_id", + "idx_transactions_accounts_account_id", + // operations (2025-06-10.3-operations.sql) + "idx_operations_id_time", + "idx_operations_accounts_operation_id", + "idx_operations_accounts_account_id", + // state_changes (2025-06-10.4-statechanges.sql) + "idx_state_changes_operation_id", + "idx_state_changes_toid_opid_scid_time", + "idx_state_changes_account_category", +} + +type Chunk struct { + Name string + Start time.Time + End time.Time + NumWriters atomic.Int64 +} + +// chunkRange is a time boundary for a single chunk. +type chunkRange struct { + start time.Time + end time.Time +} + +// PreCreateChunks pre-creates empty TimescaleDB chunks for the given hypertables +// covering the time range [rangeStart, rangeEnd]. Chunk boundaries are aligned to +// match TimescaleDB's internal alignment by reading interval_length from the catalog +// and replicating the same integer division used in calculate_open_range_default() +// (see timescale/timescaledb src/dimension.c): +// +// range_start = (value / interval_length) * interval_length +// +// Existing chunks are skipped (create_chunk is idempotent for exact boundary matches). +// +// This is used before backfill to ensure chunks exist so their indexes can be +// dropped before bulk INSERTs, avoiding the ~40% write overhead from B-tree maintenance. +// +// Returns the aligned start time of the first chunk boundary (for use as a lower bound +// in downstream chunk queries like the progressive recompressor). +func PreCreateChunks(ctx context.Context, pool *pgxpool.Pool, hypertables []string, rangeStart, rangeEnd time.Time) ([]*Chunk, error) { + // Generate aligned chunk boundaries using the same integer division as TimescaleDB's + // C code. We read interval_length (microseconds) from the catalog and use + // generate_series over chunk indices: floor(usec / interval) to ceil(usec / interval). + rows, err := pool.Query(ctx, ` + WITH dim AS ( + SELECT d.interval_length + FROM _timescaledb_catalog.dimension d + JOIN _timescaledb_catalog.hypertable h ON d.hypertable_id = h.id + WHERE h.table_name = $3 AND d.column_name = 'ledger_created_at' + ) + SELECT + to_timestamp((gs * dim.interval_length)::double precision / 1000000) AS chunk_start, + to_timestamp(((gs + 1) * dim.interval_length)::double precision / 1000000) AS chunk_end + FROM dim, + LATERAL generate_series( + (extract(epoch from $1::timestamptz) * 1000000)::bigint / dim.interval_length, + (extract(epoch from $2::timestamptz) * 1000000)::bigint / dim.interval_length + ) AS gs + ORDER BY gs`, + rangeStart, rangeEnd, hypertables[0], + ) + if err != nil { + return []*Chunk{}, fmt.Errorf("generating chunk boundaries: %w", err) + } + var boundaries []chunkRange + for rows.Next() { + var boundary chunkRange + if scanErr := rows.Scan(&boundary.start, &boundary.end); scanErr != nil { + rows.Close() + return []*Chunk{}, fmt.Errorf("scanning chunk boundary: %w", scanErr) + } + boundaries = append(boundaries, boundary) + } + rows.Close() + if err = rows.Err(); err != nil { + return []*Chunk{}, fmt.Errorf("iterating chunk boundaries: %w", err) + } + + var chunks []*Chunk + for _, table := range hypertables { + start := time.Now() + numChunks := 0 + for _, boundary := range boundaries { + chunk, err := prepareNewChunk(ctx, pool, table, boundary) + if err != nil { + return nil, err + } + if chunk != nil { + chunks = append(chunks, chunk) + numChunks++ + } + } + log.Ctx(ctx).Infof("Pre-created %d chunks for %s covering [%s, %s] in %v", + numChunks, table, rangeStart.Format(time.RFC3339), rangeEnd.Format(time.RFC3339), time.Since(start)) + } + return chunks, nil +} + +// prepareNewChunk creates a chunk for the given boundary, drops its indexes, sets it +// UNLOGGED, and disables autovacuum. Returns nil if the chunk already existed. +func prepareNewChunk(ctx context.Context, pool *pgxpool.Pool, table string, boundary chunkRange) (*Chunk, error) { + chunk := Chunk{Start: boundary.start, End: boundary.end} + slices := fmt.Sprintf(`{"ledger_created_at": [%d, %d]}`, + boundary.start.UnixMicro(), boundary.end.UnixMicro()) + var created bool + if err := pool.QueryRow(ctx, + "SELECT schema_name || '.' || table_name, created FROM _timescaledb_functions.create_chunk($1::regclass, $2::jsonb)", + table, slices, + ).Scan(&chunk.Name, &created); err != nil { + return nil, fmt.Errorf("creating chunk for %s at %s: %w", table, boundary.start.Format(time.RFC3339), err) + } + if !created { + return nil, nil + } + + numDropped, err := dropChunkIndexes(ctx, pool, chunk.Name) + if err != nil { + return nil, fmt.Errorf("dropping indexes on %s: %w", chunk.Name, err) + } + log.Ctx(ctx).Infof("Dropped %d indexes on chunk %s", numDropped, chunk.Name) + + // UNLOGGED disables WAL writes (~2-3x faster inserts, ~20x less WAL). + // Safe for backfill data that can be re-ingested. Set back to LOGGED after compression. + if err := executeChunkDDL(ctx, pool, chunk.Name, "SET UNLOGGED"); err != nil { + return nil, err + } + // Disable autovacuum — no useful work during bulk COPY into UNLOGGED tables. + // Re-enabled in SetChunkLogged after compression. + if err := executeChunkDDL(ctx, pool, chunk.Name, "SET (autovacuum_enabled = false)"); err != nil { + return nil, err + } + return &chunk, nil +} + +// dropChunkIndexes drops indexes on a single chunk that match the droppableIndexes list. +// TimescaleDB chunk indexes embed the parent index name as a suffix, so we match using +// strings.HasSuffix. Returns the number of indexes dropped. +func dropChunkIndexes(ctx context.Context, pool *pgxpool.Pool, chunkName string) (int, error) { + parts := strings.SplitN(chunkName, ".", 2) + schemaName, tableName := parts[0], parts[1] + + rows, err := pool.Query(ctx, ` + SELECT indexname + FROM pg_indexes + WHERE schemaname = $1 AND tablename = $2 + `, schemaName, tableName) + if err != nil { + return 0, fmt.Errorf("querying indexes: %w", err) + } + + var chunkIndexes []string + for rows.Next() { + var name string + if scanErr := rows.Scan(&name); scanErr != nil { + rows.Close() + return 0, fmt.Errorf("scanning index name: %w", scanErr) + } + chunkIndexes = append(chunkIndexes, name) + } + rows.Close() + if err = rows.Err(); err != nil { + return 0, fmt.Errorf("iterating index rows: %w", err) + } + + dropped := 0 + for _, idx := range chunkIndexes { + if !shouldDropIndex(idx) { + continue + } + dropSQL := fmt.Sprintf("DROP INDEX IF EXISTS %s.%s", schemaName, idx) + if _, execErr := pool.Exec(ctx, dropSQL); execErr != nil { + return dropped, fmt.Errorf("dropping index %s: %w", idx, execErr) + } + log.Ctx(ctx).Debugf("Dropped index %s.%s on %s", schemaName, idx, chunkName) + dropped++ + } + + return dropped, nil +} + +// shouldDropIndex checks if a chunk index name matches any parent index in droppableIndexes. +// TimescaleDB names chunk indexes as "_", +// so we check if the chunk index name ends with the parent index name. +func shouldDropIndex(chunkIndexName string) bool { + for _, parentIdx := range droppableIndexes { + if strings.HasSuffix(chunkIndexName, parentIdx) { + return true + } + } + return false +} + +// DisableInsertAutovacuum suppresses insert-triggered autovacuum on the given +// hypertables by setting autovacuum_vacuum_insert_threshold = -1. This prevents +// autovacuum from competing for I/O during bulk backfill. Call RestoreInsertAutovacuum +// after backfill to restore default behavior. +func DisableInsertAutovacuum(ctx context.Context, pool *pgxpool.Pool, hypertables []string) error { + for _, table := range hypertables { + sql := fmt.Sprintf("ALTER TABLE %s SET (autovacuum_vacuum_insert_threshold = -1)", table) + if _, err := pool.Exec(ctx, sql); err != nil { + return fmt.Errorf("disabling insert autovacuum on %s: %w", table, err) + } + log.Ctx(ctx).Infof("Disabled insert-triggered autovacuum on %s", table) + } + return nil +} + +// RestoreInsertAutovacuum resets autovacuum_vacuum_insert_threshold to the +// Postgres default on the given hypertables. +func RestoreInsertAutovacuum(ctx context.Context, pool *pgxpool.Pool, hypertables []string) error { + for _, table := range hypertables { + sql := fmt.Sprintf("ALTER TABLE %s RESET (autovacuum_vacuum_insert_threshold)", table) + if _, err := pool.Exec(ctx, sql); err != nil { + return fmt.Errorf("restoring insert autovacuum on %s: %w", table, err) + } + log.Ctx(ctx).Infof("Restored insert-triggered autovacuum on %s", table) + } + return nil +} + +// executeChunkDDL runs a single ALTER TABLE statement on a chunk. +func executeChunkDDL(ctx context.Context, pool *pgxpool.Pool, chunkName, ddl string) error { + sql := fmt.Sprintf("ALTER TABLE %s %s", chunkName, ddl) + if _, err := pool.Exec(ctx, sql); err != nil { + return fmt.Errorf("alter table %s (%s): %w", chunkName, ddl, err) + } + return nil +} + +// SetChunkLogged sets a single chunk back to LOGGED and re-enables autovacuum. +// Call this after compress_chunk() to restore crash safety and normal maintenance +// on the compressed data. TimescaleDB auto-propagates the LOGGED change to the +// internal compressed chunk. +func SetChunkLogged(ctx context.Context, pool *pgxpool.Pool, chunkName string) error { + if err := executeChunkDDL(ctx, pool, chunkName, "SET LOGGED"); err != nil { + return err + } + // Re-enable autovacuum (disabled during backfill by PrepareChunksForBackfill). + if err := executeChunkDDL(ctx, pool, chunkName, "SET (autovacuum_enabled = true)"); err != nil { + return err + } + return nil +} diff --git a/internal/db/timescaledb_chunks_test.go b/internal/db/timescaledb_chunks_test.go new file mode 100644 index 000000000..5003bbf50 --- /dev/null +++ b/internal/db/timescaledb_chunks_test.go @@ -0,0 +1,545 @@ +package db + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stellar/wallet-backend/internal/db/dbtest" +) + +// queryRelpersistence returns 'p' (LOGGED/permanent) or 'u' (UNLOGGED) for the given +// schema-qualified chunk name (e.g. "_timescaledb_internal.chunk_name"). +func queryRelpersistence(t *testing.T, ctx context.Context, pool *pgxpool.Pool, chunkName string) string { + t.Helper() + parts := strings.SplitN(chunkName, ".", 2) + require.Len(t, parts, 2, "chunkName must be schema.table") + var persistence string + err := pool.QueryRow(ctx, ` + SELECT relpersistence::text + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = $1 AND c.relname = $2 + `, parts[0], parts[1]).Scan(&persistence) + require.NoError(t, err, "querying relpersistence for %s", chunkName) + return persistence +} + +// countChunkIndexes returns the number of indexes on the given schema-qualified chunk. +func countChunkIndexes(t *testing.T, ctx context.Context, pool *pgxpool.Pool, chunkName string) int { + t.Helper() + parts := strings.SplitN(chunkName, ".", 2) + require.Len(t, parts, 2) + var count int + err := pool.QueryRow(ctx, ` + SELECT count(*) + FROM pg_indexes + WHERE schemaname = $1 AND tablename = $2 + `, parts[0], parts[1]).Scan(&count) + require.NoError(t, err, "counting indexes for %s", chunkName) + return count +} + +// countDroppableIndexes returns the number of indexes on the chunk that match droppableIndexes. +func countDroppableIndexes(t *testing.T, ctx context.Context, pool *pgxpool.Pool, chunkName string) int { + t.Helper() + parts := strings.SplitN(chunkName, ".", 2) + require.Len(t, parts, 2) + + rows, err := pool.Query(ctx, ` + SELECT indexname + FROM pg_indexes + WHERE schemaname = $1 AND tablename = $2 + `, parts[0], parts[1]) + require.NoError(t, err) + + count := 0 + for rows.Next() { + var name string + require.NoError(t, rows.Scan(&name)) + if shouldDropIndex(name) { + count++ + } + } + rows.Close() + require.NoError(t, rows.Err()) + return count +} + +// hasReloption checks if the given schema-qualified table has the specified reloption. +// The option should be in "key=value" format (e.g. "autovacuum_enabled=false"). +func hasReloption(t *testing.T, ctx context.Context, pool *pgxpool.Pool, tableName, option string) bool { + t.Helper() + parts := strings.SplitN(tableName, ".", 2) + schema, table := "public", tableName + if len(parts) == 2 { + schema, table = parts[0], parts[1] + } + var has bool + // Use fmt.Sprintf for the option value because pgx doesn't support parameterized + // array literals like ARRAY[$3::text] (it counts placeholders incorrectly). + query := fmt.Sprintf(` + SELECT coalesce(c.reloptions @> ARRAY['%s'], false) + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = $1 AND c.relname = $2 + `, option) + err := pool.QueryRow(ctx, query, schema, table).Scan(&has) + require.NoError(t, err, "checking reloption %q on %s", option, tableName) + return has +} + +// createTestChunk creates a single chunk via _timescaledb_functions.create_chunk and returns +// its schema-qualified name (e.g. "_timescaledb_internal._hyper_1_1_chunk"). +func createTestChunk(t *testing.T, ctx context.Context, pool *pgxpool.Pool, table string, start, end time.Time) string { + t.Helper() + slices := fmt.Sprintf(`{"ledger_created_at": [%d, %d]}`, start.UnixMicro(), end.UnixMicro()) + var chunkName string + var created bool + err := pool.QueryRow(ctx, + "SELECT schema_name || '.' || table_name, created FROM _timescaledb_functions.create_chunk($1::regclass, $2::jsonb)", + table, slices, + ).Scan(&chunkName, &created) + require.NoError(t, err, "creating test chunk for %s", table) + return chunkName +} + +// openChunksTestPool opens a migrated test DB and returns a pgx pool suitable for chunk tests. +func openChunksTestPool(t *testing.T) (*pgxpool.Pool, context.Context) { + t.Helper() + dbt := dbtest.Open(t) + t.Cleanup(dbt.Close) + ctx := context.Background() + pool, err := OpenDBConnectionPool(ctx, dbt.DSN) + require.NoError(t, err) + t.Cleanup(pool.Close) + return pool, ctx +} + +// --- Test: shouldDropIndex (pure function, no DB) --- + +func TestShouldDropIndex(t *testing.T) { + tests := []struct { + name string + input string + expected bool + }{ + { + name: "matches chunk-prefixed droppable index (transactions hash)", + input: "_hyper_1_5_chunk_idx_transactions_hash", + expected: true, + }, + { + name: "matches time-dimension index (transactions)", + input: "_hyper_1_5_chunk_transactions_ledger_created_at_idx", + expected: true, + }, + { + name: "matches transactions_accounts tx_to_id", + input: "_hyper_2_10_chunk_idx_transactions_accounts_tx_to_id", + expected: true, + }, + { + name: "matches transactions_accounts account_id", + input: "_hyper_2_10_chunk_idx_transactions_accounts_account_id", + expected: true, + }, + { + name: "matches operations_accounts operation_id", + input: "_hyper_3_15_chunk_idx_operations_accounts_operation_id", + expected: true, + }, + { + name: "matches operations_accounts account_id", + input: "_hyper_3_15_chunk_idx_operations_accounts_account_id", + expected: true, + }, + { + name: "matches state_changes operation_id", + input: "_hyper_5_20_chunk_idx_state_changes_operation_id", + expected: true, + }, + { + name: "matches state_changes account_category", + input: "_hyper_5_20_chunk_idx_state_changes_account_category", + expected: true, + }, + { + name: "matches operations time-dimension index", + input: "_hyper_3_15_chunk_operations_ledger_created_at_idx", + expected: true, + }, + { + name: "matches state_changes time-dimension index", + input: "_hyper_5_20_chunk_state_changes_ledger_created_at_idx", + expected: true, + }, + { + name: "exact parent name matches via HasSuffix", + input: "idx_transactions_hash", + expected: true, + }, + { + name: "matches transactions toid_time (PK replacement)", + input: "_hyper_1_5_chunk_idx_transactions_toid_time", + expected: true, + }, + { + name: "matches operations id_time (PK replacement)", + input: "_hyper_3_15_chunk_idx_operations_id_time", + expected: true, + }, + { + name: "matches state_changes toid_opid_scid_time (PK replacement)", + input: "_hyper_5_20_chunk_idx_state_changes_toid_opid_scid_time", + expected: true, + }, + { + name: "empty string", + input: "", + expected: false, + }, + { + name: "unrelated index", + input: "_hyper_1_5_chunk_some_other_index", + expected: false, + }, + { + name: "partial prefix match with extra suffix should not match", + input: "idx_transactions_hash_extra_suffix", + expected: false, + }, + { + name: "substring of droppable but not suffix", + input: "prefix_idx_transactions_hash_nope", + expected: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + result := shouldDropIndex(tc.input) + assert.Equal(t, tc.expected, result) + }) + } +} + +// --- Test: dropChunkIndexes (real DB) --- + +func TestDropChunkIndexes(t *testing.T) { + pool, ctx := openChunksTestPool(t) + + // Use Jan 2025 — isolated from other tests + start := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC) + end := time.Date(2025, 1, 2, 0, 0, 0, 0, time.UTC) + chunkName := createTestChunk(t, ctx, pool, "transactions", start, end) + + t.Run("drops expected indexes on new chunk", func(t *testing.T) { + indexesBefore := countChunkIndexes(t, ctx, pool, chunkName) + droppableBefore := countDroppableIndexes(t, ctx, pool, chunkName) + require.Greater(t, droppableBefore, 0, "new chunk should have droppable indexes") + + dropped, err := dropChunkIndexes(ctx, pool, chunkName) + require.NoError(t, err) + assert.Equal(t, droppableBefore, dropped, "should drop all droppable indexes") + + indexesAfter := countChunkIndexes(t, ctx, pool, chunkName) + assert.Equal(t, indexesBefore-dropped, indexesAfter, "total indexes should decrease by dropped count") + + // No droppable indexes should remain + assert.Equal(t, 0, countDroppableIndexes(t, ctx, pool, chunkName)) + }) + + t.Run("returns zero on second call", func(t *testing.T) { + dropped, err := dropChunkIndexes(ctx, pool, chunkName) + require.NoError(t, err) + assert.Equal(t, 0, dropped) + }) +} + +// --- Test: executeChunkDDL (real DB) --- + +func TestExecuteChunkDDL(t *testing.T) { + pool, ctx := openChunksTestPool(t) + + // Use Feb 2025 + start := time.Date(2025, 2, 1, 0, 0, 0, 0, time.UTC) + end := time.Date(2025, 2, 2, 0, 0, 0, 0, time.UTC) + chunkName := createTestChunk(t, ctx, pool, "transactions", start, end) + + t.Run("SET UNLOGGED", func(t *testing.T) { + err := executeChunkDDL(ctx, pool, chunkName, "SET UNLOGGED") + require.NoError(t, err) + assert.Equal(t, "u", queryRelpersistence(t, ctx, pool, chunkName)) + }) + + t.Run("SET LOGGED", func(t *testing.T) { + err := executeChunkDDL(ctx, pool, chunkName, "SET LOGGED") + require.NoError(t, err) + assert.Equal(t, "p", queryRelpersistence(t, ctx, pool, chunkName)) + }) + + t.Run("SET autovacuum disabled", func(t *testing.T) { + err := executeChunkDDL(ctx, pool, chunkName, "SET (autovacuum_enabled = false)") + require.NoError(t, err) + assert.True(t, hasReloption(t, ctx, pool, chunkName, "autovacuum_enabled=false")) + }) + + t.Run("invalid DDL returns error", func(t *testing.T) { + err := executeChunkDDL(ctx, pool, chunkName, "INVALID_DDL") + require.Error(t, err) + assert.ErrorContains(t, err, "alter table") + }) + + t.Run("nonexistent chunk returns error", func(t *testing.T) { + err := executeChunkDDL(ctx, pool, "_timescaledb_internal.nonexistent_chunk", "SET LOGGED") + require.Error(t, err) + assert.ErrorContains(t, err, "alter table") + }) +} + +// --- Test: prepareNewChunk (real DB) --- + +func TestPrepareNewChunk(t *testing.T) { + pool, ctx := openChunksTestPool(t) + + t.Run("creates chunk with correct state", func(t *testing.T) { + // Use Mar 2025 + boundary := chunkRange{ + start: time.Date(2025, 3, 1, 0, 0, 0, 0, time.UTC), + end: time.Date(2025, 3, 2, 0, 0, 0, 0, time.UTC), + } + chunk, err := prepareNewChunk(ctx, pool, "transactions", boundary) + require.NoError(t, err) + require.NotNil(t, chunk) + + // Name should be populated + assert.NotEmpty(t, chunk.Name) + assert.Contains(t, chunk.Name, "_timescaledb_internal.") + + // Start and End should match boundary + assert.Equal(t, boundary.start, chunk.Start) + assert.Equal(t, boundary.end, chunk.End) + + // Should be UNLOGGED + assert.Equal(t, "u", queryRelpersistence(t, ctx, pool, chunk.Name)) + + // No droppable indexes should remain + assert.Equal(t, 0, countDroppableIndexes(t, ctx, pool, chunk.Name)) + + // Autovacuum should be disabled + assert.True(t, hasReloption(t, ctx, pool, chunk.Name, "autovacuum_enabled=false")) + }) + + t.Run("returns nil for existing chunk", func(t *testing.T) { + // Same boundary as above — chunk already exists + boundary := chunkRange{ + start: time.Date(2025, 3, 1, 0, 0, 0, 0, time.UTC), + end: time.Date(2025, 3, 2, 0, 0, 0, 0, time.UTC), + } + chunk, err := prepareNewChunk(ctx, pool, "transactions", boundary) + require.NoError(t, err) + assert.Nil(t, chunk, "should return nil for already-existing chunk") + }) + + t.Run("works for all hypertables", func(t *testing.T) { + tables := []string{"transactions", "transactions_accounts", "operations", "operations_accounts", "state_changes"} + // Use Apr 2025 — fresh boundary for all tables + boundary := chunkRange{ + start: time.Date(2025, 4, 1, 0, 0, 0, 0, time.UTC), + end: time.Date(2025, 4, 2, 0, 0, 0, 0, time.UTC), + } + for _, table := range tables { + t.Run(table, func(t *testing.T) { + chunk, err := prepareNewChunk(ctx, pool, table, boundary) + require.NoError(t, err) + require.NotNil(t, chunk, "should create chunk for %s", table) + assert.Equal(t, "u", queryRelpersistence(t, ctx, pool, chunk.Name)) + }) + } + }) +} + +// --- Test: PreCreateChunks (real DB) --- + +func TestPreCreateChunks(t *testing.T) { + pool, ctx := openChunksTestPool(t) + + t.Run("single-day range, 2 tables", func(t *testing.T) { + // Use May 2025 + start := time.Date(2025, 5, 1, 0, 0, 0, 0, time.UTC) + end := time.Date(2025, 5, 1, 23, 59, 59, 0, time.UTC) + tables := []string{"transactions", "operations"} + + chunks, err := PreCreateChunks(ctx, pool, tables, start, end) + require.NoError(t, err) + require.NotEmpty(t, chunks) + + // Each chunk should be UNLOGGED + for _, chunk := range chunks { + assert.Equal(t, "u", queryRelpersistence(t, ctx, pool, chunk.Name), + "chunk %s should be UNLOGGED", chunk.Name) + } + }) + + t.Run("multi-day range", func(t *testing.T) { + // Use Jun 2025 — 3-day range + start := time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC) + end := time.Date(2025, 6, 3, 23, 59, 59, 0, time.UTC) + tables := []string{"transactions"} + + chunks, err := PreCreateChunks(ctx, pool, tables, start, end) + require.NoError(t, err) + // With default 1-day chunk interval, 3-day range should yield at least 3 chunks + assert.GreaterOrEqual(t, len(chunks), 3, "3-day range should produce at least 3 chunks") + }) + + t.Run("idempotent (skips existing)", func(t *testing.T) { + // Use Jul 2025 + start := time.Date(2025, 7, 1, 0, 0, 0, 0, time.UTC) + end := time.Date(2025, 7, 1, 23, 59, 59, 0, time.UTC) + tables := []string{"transactions"} + + chunks1, err := PreCreateChunks(ctx, pool, tables, start, end) + require.NoError(t, err) + require.NotEmpty(t, chunks1) + + chunks2, err := PreCreateChunks(ctx, pool, tables, start, end) + require.NoError(t, err) + assert.Empty(t, chunks2, "second call should return empty — all chunks already exist") + }) + + t.Run("unknown table returns empty chunks (no matching dimension)", func(t *testing.T) { + start := time.Date(2025, 8, 1, 0, 0, 0, 0, time.UTC) + end := time.Date(2025, 8, 1, 23, 59, 59, 0, time.UTC) + // The first table is used to look up the dimension interval in the catalog. + // A nonexistent table has no dimension entry, so the boundary query returns + // zero rows and PreCreateChunks returns an empty slice without error. + tables := []string{"nonexistent_table"} + + chunks, err := PreCreateChunks(ctx, pool, tables, start, end) + require.NoError(t, err) + assert.Empty(t, chunks, "nonexistent table should produce no chunks") + }) + + t.Run("chunk boundaries are day-aligned to midnight UTC", func(t *testing.T) { + // Use Sep 2025 — start mid-day to verify alignment snaps to midnight + start := time.Date(2025, 9, 1, 14, 30, 0, 0, time.UTC) + end := time.Date(2025, 9, 3, 14, 30, 0, 0, time.UTC) + tables := []string{"transactions"} + + chunks, err := PreCreateChunks(ctx, pool, tables, start, end) + require.NoError(t, err) + require.GreaterOrEqual(t, len(chunks), 2, "need at least 2 chunks to verify alignment") + + // TimescaleDB aligns 1-day chunks to midnight UTC because: + // - Internal formula: range_start = (pg_epoch_usec / interval_length) * interval_length + // - The PG epoch (2000-01-01 00:00:00 UTC) is itself at midnight + // - 1 day = 86400000000 µs divides evenly into the epoch difference + for _, chunk := range chunks { + utcStart := chunk.Start.UTC() + assert.Equal(t, 0, utcStart.Hour(), "chunk %s start hour should be 0", chunk.Name) + assert.Equal(t, 0, utcStart.Minute(), "chunk %s start minute should be 0", chunk.Name) + assert.Equal(t, 0, utcStart.Second(), "chunk %s start second should be 0", chunk.Name) + } + + // Each chunk's duration (End - Start) should equal exactly 1 day + oneDayMicros := int64(24 * time.Hour / time.Microsecond) + for i, chunk := range chunks { + durationMicros := chunk.End.Sub(chunk.Start).Microseconds() + assert.Equal(t, oneDayMicros, durationMicros, + "chunk %d duration should be exactly 1 day", i) + } + }) +} + +// --- Test: DisableInsertAutovacuum / RestoreInsertAutovacuum (real DB) --- + +func TestDisableInsertAutovacuum(t *testing.T) { + pool, ctx := openChunksTestPool(t) + + t.Run("disables insert autovacuum", func(t *testing.T) { + tables := []string{"transactions", "operations"} + err := DisableInsertAutovacuum(ctx, pool, tables) + require.NoError(t, err) + + for _, table := range tables { + assert.True(t, hasReloption(t, ctx, pool, table, "autovacuum_vacuum_insert_threshold=-1"), + "table %s should have insert threshold = -1", table) + } + }) + + t.Run("nonexistent table returns error", func(t *testing.T) { + err := DisableInsertAutovacuum(ctx, pool, []string{"nonexistent_table"}) + require.Error(t, err) + assert.ErrorContains(t, err, "disabling insert autovacuum") + }) +} + +func TestRestoreInsertAutovacuum(t *testing.T) { + pool, ctx := openChunksTestPool(t) + + t.Run("restore removes threshold", func(t *testing.T) { + tables := []string{"transactions"} + + // First disable + err := DisableInsertAutovacuum(ctx, pool, tables) + require.NoError(t, err) + assert.True(t, hasReloption(t, ctx, pool, "transactions", "autovacuum_vacuum_insert_threshold=-1")) + + // Then restore + err = RestoreInsertAutovacuum(ctx, pool, tables) + require.NoError(t, err) + assert.False(t, hasReloption(t, ctx, pool, "transactions", "autovacuum_vacuum_insert_threshold=-1"), + "insert threshold should be removed after restore") + }) + + t.Run("nonexistent table returns error", func(t *testing.T) { + err := RestoreInsertAutovacuum(ctx, pool, []string{"nonexistent_table"}) + require.Error(t, err) + assert.ErrorContains(t, err, "restoring insert autovacuum") + }) +} + +// --- Test: SetChunkLogged (real DB) --- + +func TestSetChunkLogged(t *testing.T) { + pool, ctx := openChunksTestPool(t) + + t.Run("converts UNLOGGED to LOGGED and re-enables autovacuum", func(t *testing.T) { + // Use Oct 2025 + boundary := chunkRange{ + start: time.Date(2025, 10, 1, 0, 0, 0, 0, time.UTC), + end: time.Date(2025, 10, 2, 0, 0, 0, 0, time.UTC), + } + chunk, err := prepareNewChunk(ctx, pool, "transactions", boundary) + require.NoError(t, err) + require.NotNil(t, chunk) + + // Verify precondition: chunk is UNLOGGED with autovacuum disabled + assert.Equal(t, "u", queryRelpersistence(t, ctx, pool, chunk.Name)) + assert.True(t, hasReloption(t, ctx, pool, chunk.Name, "autovacuum_enabled=false")) + + // Set chunk logged + err = SetChunkLogged(ctx, pool, chunk.Name) + require.NoError(t, err) + + // Verify: now LOGGED + assert.Equal(t, "p", queryRelpersistence(t, ctx, pool, chunk.Name)) + + // Verify: autovacuum re-enabled + assert.True(t, hasReloption(t, ctx, pool, chunk.Name, "autovacuum_enabled=true")) + }) + + t.Run("nonexistent chunk returns error", func(t *testing.T) { + err := SetChunkLogged(ctx, pool, "_timescaledb_internal.nonexistent_chunk") + require.Error(t, err) + assert.ErrorContains(t, err, "alter table") + }) +} diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 7d78544b3..ebb1a68e3 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -21,6 +21,12 @@ import ( "github.com/stellar/wallet-backend/internal/utils" ) +// txResultPool reuses TransactionResult structs to reduce per-transaction +// allocation churn during ingestion (~34K allocations per gap). +var txResultPool = sync.Pool{ + New: func() any { return &TransactionResult{} }, +} + type IndexerBufferInterface interface { BatchPushTransactionResult(result *TransactionResult) BatchPushChanges(trustlines []types.TrustlineChange, accounts []types.AccountChange, sacBalances []types.SACBalanceChange, sacContracts []*data.Contract) @@ -209,16 +215,27 @@ func (i *Indexer) processTransaction(ctx context.Context, tx ingest.LedgerTransa txParticipantList = append(txParticipantList, p) } - // Push all transaction data in a single lock acquisition - buffer.BatchPushTransactionResult(&TransactionResult{ - Transaction: dataTx, - TxParticipants: txParticipantList, - Operations: operationsMap, - OpParticipants: opParticipantMap, - ContractChanges: contractChanges, - StateChanges: stateChanges, - StateChangeOpMap: operationsMap, - }) + // Push all transaction data in a single lock acquisition. + // Use pooled TransactionResult to reduce per-transaction allocation churn. + result := txResultPool.Get().(*TransactionResult) + result.Transaction = dataTx + result.TxParticipants = txParticipantList + result.Operations = operationsMap + result.OpParticipants = opParticipantMap + result.ContractChanges = contractChanges + result.StateChanges = stateChanges + result.StateChangeOpMap = operationsMap + + buffer.BatchPushTransactionResult(result) + + result.Transaction = nil + result.TxParticipants = nil + result.Operations = nil + result.OpParticipants = nil + result.ContractChanges = nil + result.StateChanges = nil + result.StateChangeOpMap = nil + txResultPool.Put(result) // Process trustline, account, and SAC balance changes from ledger changes. // Each operation's changes are pushed in a single lock acquisition. @@ -265,7 +282,7 @@ func (i *Indexer) processTransaction(ctx context.Context, tx ingest.LedgerTransa // getTransactionStateChanges processes operations of a transaction and calculates all state changes func (i *Indexer) getTransactionStateChanges(ctx context.Context, transaction ingest.LedgerTransaction, opsParticipants map[int64]processors.OperationParticipants) ([]types.StateChange, error) { - stateChanges := []types.StateChange{} + stateChanges := make([]types.StateChange, 0, len(opsParticipants)*4) // Process operations sequentially since there are only 3 processors per operation // Creating a worker pool here adds unnecessary overhead @@ -312,6 +329,28 @@ func GetLedgerTransactions(ctx context.Context, networkPassphrase string, ledger return transactions, nil } +// ProcessLedgerTransactionsSequential processes all transactions in a ledger sequentially. +// Use this in backfill mode where inter-ledger parallelism (multiple process workers) +// already saturates CPU — adding intra-ledger pond pool fan-out causes contention. +func (i *Indexer) ProcessLedgerTransactionsSequential(ctx context.Context, transactions []ingest.LedgerTransaction, ledgerBuffer IndexerBufferInterface) (int, error) { + participantCounts := make([]int, len(transactions)) + + for idx, tx := range transactions { + count, err := i.processTransaction(ctx, tx, ledgerBuffer) + if err != nil { + return 0, fmt.Errorf("processing transaction at ledger=%d tx=%d: %w", tx.Ledger.LedgerSequence(), tx.Index, err) + } + participantCounts[idx] = count + } + + totalParticipants := 0 + for _, count := range participantCounts { + totalParticipants += count + } + + return totalParticipants, nil +} + // ProcessLedger extracts transactions from a ledger and indexes them. // Returns the participant count for optional metrics recording. func ProcessLedger(ctx context.Context, networkPassphrase string, ledgerMeta xdr.LedgerCloseMeta, ledgerIndexer *Indexer, buffer *IndexerBuffer) (int, error) { @@ -328,3 +367,20 @@ func ProcessLedger(ctx context.Context, networkPassphrase string, ledgerMeta xdr return participantCount, nil } + +// ProcessLedgerSequential extracts transactions and processes them sequentially. +// Used by the backfill pipeline where multiple process workers provide parallelism. +func ProcessLedgerSequential(ctx context.Context, networkPassphrase string, ledgerMeta xdr.LedgerCloseMeta, ledgerIndexer *Indexer, buffer *IndexerBuffer) (int, error) { + ledgerSeq := ledgerMeta.LedgerSequence() + transactions, err := GetLedgerTransactions(ctx, networkPassphrase, ledgerMeta) + if err != nil { + return 0, fmt.Errorf("getting transactions for ledger %d: %w", ledgerSeq, err) + } + + participantCount, err := ledgerIndexer.ProcessLedgerTransactionsSequential(ctx, transactions, buffer) + if err != nil { + return 0, fmt.Errorf("processing transactions for ledger %d: %w", ledgerSeq, err) + } + + return participantCount, nil +} diff --git a/internal/indexer/processors/accounts.go b/internal/indexer/processors/accounts.go index 5142ae827..e5d5b50d3 100644 --- a/internal/indexer/processors/accounts.go +++ b/internal/indexer/processors/accounts.go @@ -47,7 +47,7 @@ func (p *AccountsProcessor) ProcessOperation(ctx context.Context, opWrapper *Tra } }() - changes, err := opWrapper.Transaction.GetOperationChanges(opWrapper.Index) + changes, err := opWrapper.GetChanges() if err != nil { return nil, fmt.Errorf("getting operation changes: %w", err) } diff --git a/internal/indexer/processors/contracts/sac.go b/internal/indexer/processors/contracts/sac.go index d0b1a2680..51f3155dd 100644 --- a/internal/indexer/processors/contracts/sac.go +++ b/internal/indexer/processors/contracts/sac.go @@ -75,7 +75,7 @@ func (p *SACEventsProcessor) ProcessOperation(_ context.Context, opWrapper *proc } // Get operation changes to access previous trustline flag state - changes, err := tx.GetOperationChanges(opWrapper.Index) + changes, err := opWrapper.GetChanges() if err != nil { return nil, fmt.Errorf("getting operation changes for operation %d: %w", opWrapper.ID(), err) } diff --git a/internal/indexer/processors/effects.go b/internal/indexer/processors/effects.go index 953558fdc..e47ec7e00 100644 --- a/internal/indexer/processors/effects.go +++ b/internal/indexer/processors/effects.go @@ -106,7 +106,7 @@ func (p *EffectsProcessor) ProcessOperation(_ context.Context, opWrapper *Transa } // Get operation changes to access old values when needed - changes, err := opWrapper.Transaction.GetOperationChanges(opWrapper.Index) + changes, err := opWrapper.GetChanges() if err != nil { return nil, fmt.Errorf("getting operation changes for tx: %s, opID: %d, err: %w", txHash, opWrapper.ID(), err) } diff --git a/internal/indexer/processors/effects_horizon.go b/internal/indexer/processors/effects_horizon.go index 166b3a012..ad9a6fd38 100644 --- a/internal/indexer/processors/effects_horizon.go +++ b/internal/indexer/processors/effects_horizon.go @@ -107,7 +107,7 @@ func Effects(operation *TransactionOperationWrapper) ([]EffectOutput, error) { return []EffectOutput{}, nil } - changes, err := operation.Transaction.GetOperationChanges(operation.Index) + changes, err := operation.GetChanges() if err != nil { return nil, fmt.Errorf("getting operation changes: %w", err) } @@ -418,7 +418,7 @@ func (e *effectsWrapper) addSetOptionsEffects() error { e.addMuxed(source, EffectAccountFlagsUpdated, flagDetails) } - changes, err := e.operation.Transaction.GetOperationChanges(e.operation.Index) + changes, err := e.operation.GetChanges() if err != nil { return fmt.Errorf("getting operation changes: %w", err) } @@ -490,7 +490,7 @@ func (e *effectsWrapper) addChangeTrustEffects() error { source := e.operation.SourceAccount() op := e.operation.Operation.Body.MustChangeTrustOp() - changes, err := e.operation.Transaction.GetOperationChanges(e.operation.Index) + changes, err := e.operation.GetChanges() if err != nil { return fmt.Errorf("getting operation changes: %w", err) } @@ -588,7 +588,7 @@ func (e *effectsWrapper) addManageDataEffects() error { op := e.operation.Operation.Body.MustManageDataOp() details := map[string]interface{}{"name": op.DataName} effect := EffectType(0) - changes, err := e.operation.Transaction.GetOperationChanges(e.operation.Index) + changes, err := e.operation.GetChanges() if err != nil { return fmt.Errorf("getting operation changes: %w", err) } diff --git a/internal/indexer/processors/sac_balances.go b/internal/indexer/processors/sac_balances.go index c3c347af5..0f37a9f8e 100644 --- a/internal/indexer/processors/sac_balances.go +++ b/internal/indexer/processors/sac_balances.go @@ -42,7 +42,7 @@ func (p *SACBalancesProcessor) Name() string { // Returns SACBalanceChange structs with absolute balance values for database upsert. // Only processes changes for contract addresses (C...) since G-addresses use trustlines. func (p *SACBalancesProcessor) ProcessOperation(ctx context.Context, opWrapper *TransactionOperationWrapper) ([]types.SACBalanceChange, error) { - changes, err := opWrapper.Transaction.GetOperationChanges(opWrapper.Index) + changes, err := opWrapper.GetChanges() if err != nil { return nil, fmt.Errorf("getting operation changes: %w", err) } diff --git a/internal/indexer/processors/sac_instances.go b/internal/indexer/processors/sac_instances.go index 59cc7fdc6..b9587e01f 100644 --- a/internal/indexer/processors/sac_instances.go +++ b/internal/indexer/processors/sac_instances.go @@ -38,7 +38,7 @@ func (p *SACInstanceProcessor) Name() string { // Returns Contract structs with asset code and issuer for database insertion. // Only processes contract instance entries that represent SAC tokens. func (p *SACInstanceProcessor) ProcessOperation(ctx context.Context, opWrapper *TransactionOperationWrapper) ([]*data.Contract, error) { - changes, err := opWrapper.Transaction.GetOperationChanges(opWrapper.Index) + changes, err := opWrapper.GetChanges() if err != nil { return nil, fmt.Errorf("getting operation changes: %w", err) } diff --git a/internal/indexer/processors/transaction_operation_wrapper.go b/internal/indexer/processors/transaction_operation_wrapper.go index 872b92b30..a61ca10eb 100644 --- a/internal/indexer/processors/transaction_operation_wrapper.go +++ b/internal/indexer/processors/transaction_operation_wrapper.go @@ -31,6 +31,22 @@ type TransactionOperationWrapper struct { LedgerSequence uint32 Network string LedgerClosed time.Time + + // Cached operation changes — computed once on first GetChanges() call. + // Safe without mutex: each wrapper is processed by a single goroutine. + changes []ingest.Change + changesErr error + changesDone bool +} + +// GetChanges returns the operation's ledger entry changes, caching the result +// so multiple processors can read them without redundant SDK allocations. +func (operation *TransactionOperationWrapper) GetChanges() ([]ingest.Change, error) { + if !operation.changesDone { + operation.changes, operation.changesErr = operation.Transaction.GetOperationChanges(operation.Index) + operation.changesDone = true + } + return operation.changes, operation.changesErr } // ID returns the ID for the operation. @@ -101,7 +117,7 @@ func (operation *TransactionOperationWrapper) getSignerSponsorInChange(signerKey } func (operation *TransactionOperationWrapper) getSponsor() (*xdr.AccountId, error) { - changes, err := operation.Transaction.GetOperationChanges(operation.Index) + changes, err := operation.GetChanges() if err != nil { return nil, fmt.Errorf("getting operation changes: %w", err) } diff --git a/internal/indexer/processors/trustlines.go b/internal/indexer/processors/trustlines.go index ec841be91..0e700f726 100644 --- a/internal/indexer/processors/trustlines.go +++ b/internal/indexer/processors/trustlines.go @@ -44,7 +44,7 @@ func (p *TrustlinesProcessor) ProcessOperation(ctx context.Context, opWrapper *T } }() - changes, err := opWrapper.Transaction.GetOperationChanges(opWrapper.Index) + changes, err := opWrapper.GetChanges() if err != nil { return nil, fmt.Errorf("getting operation changes: %w", err) } diff --git a/internal/ingest/backfill_fetcher.go b/internal/ingest/backfill_fetcher.go new file mode 100644 index 000000000..4e835e063 --- /dev/null +++ b/internal/ingest/backfill_fetcher.go @@ -0,0 +1,235 @@ +// backfillFetcher downloads S3 ledger files in parallel and fans out +// individual LedgerCloseMeta entries directly to an external channel. +// No ordering guarantees — designed for backfill where consumers +// handle ledgers independently and the watermark tracker handles +// out-of-order flushes. +package ingest + +import ( + "bufio" + "context" + "errors" + "fmt" + "os" + "sync" + "time" + + "github.com/klauspost/compress/zstd" + "github.com/stellar/go-stellar-sdk/support/datastore" + "github.com/stellar/go-stellar-sdk/support/log" + "github.com/stellar/go-stellar-sdk/xdr" +) + +// zstdDecoderPool reuses zstd decoders across fetch workers. +// Each decoder allocates internal buffers on first use; pooling avoids +// re-allocating them on every S3 file download (~39 GB of churn per gap). +var zstdDecoderPool = sync.Pool{ + New: func() any { + d, err := zstd.NewReader(nil, zstd.WithDecoderConcurrency(1)) + if err != nil { + panic(fmt.Sprintf("creating zstd decoder: %v", err)) + } + return d + }, +} + +// BackfillFetcherConfig configures the parallel S3 fetcher for backfill. +type BackfillFetcherConfig struct { + NumWorkers uint32 + RetryLimit uint32 + RetryWait time.Duration + GapStart uint32 + GapEnd uint32 + // Optional Prometheus observers (nil-safe — no metrics if nil). + OnFetchDuration func(seconds float64) + OnChannelWait func(channel, direction string, seconds float64) +} + +// BackfillFetchStats accumulates timing from all fetch workers. +// Returned by Run for gap summary log aggregation. +type BackfillFetchStats struct { + FetchCount int + FetchTotal time.Duration + ChannelWait map[string]time.Duration // key: "channel:direction" +} + +// backfillFetcher downloads S3 ledger files in parallel and pushes +// individual LedgerCloseMeta entries to an external channel. +type backfillFetcher struct { + dataStore datastore.DataStore + schema datastore.DataStoreSchema + config BackfillFetcherConfig + + ledgerCh chan<- xdr.LedgerCloseMeta // external, caller-owned +} + +// NewBackfillFetcher creates a fetcher that will push ledgers from the +// configured gap range to ledgerCh. Call Run to start workers. +func NewBackfillFetcher( + config BackfillFetcherConfig, + ds datastore.DataStore, + schema datastore.DataStoreSchema, + ledgerCh chan<- xdr.LedgerCloseMeta, +) *backfillFetcher { + return &backfillFetcher{ + dataStore: ds, + schema: schema, + config: config, + ledgerCh: ledgerCh, + } +} + +// Run starts fetch workers, waits for them to complete, then closes ledgerCh. +// Returns aggregated fetch stats for the gap summary log. +// On error, calls cancel with the cause; the caller checks context.Cause. +func (f *backfillFetcher) Run(ctx context.Context, cancel context.CancelCauseFunc) *BackfillFetchStats { + defer close(f.ledgerCh) + + // Compute all file-start sequences up-front (bounded range). + startBoundary := f.schema.GetSequenceNumberStartBoundary(f.config.GapStart) + endBoundary := f.schema.GetSequenceNumberStartBoundary(f.config.GapEnd) + + taskCh := make(chan uint32, f.config.NumWorkers*4) + + // Seed tasks in a goroutine to avoid blocking if taskCh fills. + go func() { + defer close(taskCh) + for seq := startBoundary; seq <= endBoundary; seq += f.schema.LedgersPerFile { + select { + case taskCh <- seq: + case <-ctx.Done(): + return + } + } + }() + + // Collect per-worker stats. + statsCh := make(chan *BackfillFetchStats, f.config.NumWorkers) + + var wg sync.WaitGroup + for i := uint32(0); i < f.config.NumWorkers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + stats := f.worker(ctx, cancel, taskCh) + statsCh <- stats + }() + } + wg.Wait() + close(statsCh) + + // Aggregate stats from all workers. + agg := &BackfillFetchStats{ChannelWait: make(map[string]time.Duration)} + for ws := range statsCh { + agg.FetchCount += ws.FetchCount + agg.FetchTotal += ws.FetchTotal + for k, v := range ws.ChannelWait { + agg.ChannelWait[k] += v + } + } + return agg +} + +// worker processes file-start sequences from taskCh until the channel +// closes or the context is cancelled. +func (f *backfillFetcher) worker(ctx context.Context, cancel context.CancelCauseFunc, taskCh <-chan uint32) *BackfillFetchStats { + stats := &BackfillFetchStats{ChannelWait: make(map[string]time.Duration)} + + for sequence := range taskCh { + if ctx.Err() != nil { + return stats + } + + fetchStart := time.Now() + batch, err := f.downloadWithRetry(ctx, cancel, sequence) + fetchDur := time.Since(fetchStart) + if err != nil { + return stats // cancel already called by downloadWithRetry + } + stats.FetchCount++ + stats.FetchTotal += fetchDur + + if f.config.OnFetchDuration != nil { + f.config.OnFetchDuration(fetchDur.Seconds()) + } + + // Fan out individual ledgers to ledgerCh, filtering to gap range. + for i := range batch.LedgerCloseMetas { + lcm := batch.LedgerCloseMetas[i] + batch.LedgerCloseMetas[i] = xdr.LedgerCloseMeta{} // allow GC to collect + seq := lcm.LedgerSequence() + if seq < f.config.GapStart || seq > f.config.GapEnd { + continue + } + + sendStart := time.Now() + select { + case f.ledgerCh <- lcm: + sendDur := time.Since(sendStart) + stats.ChannelWait["ledger:send"] += sendDur + if f.config.OnChannelWait != nil { + f.config.OnChannelWait("ledger", "send", sendDur.Seconds()) + } + case <-ctx.Done(): + return stats + } + } + } + return stats +} + +// downloadWithRetry downloads and decodes a single S3 file with retry. +// On permanent failure, calls cancel and returns the error. +func (f *backfillFetcher) downloadWithRetry(ctx context.Context, cancel context.CancelCauseFunc, sequence uint32) (xdr.LedgerCloseMetaBatch, error) { + for attempt := uint32(0); attempt <= f.config.RetryLimit; attempt++ { + batch, err := f.downloadAndDecode(ctx, sequence) + if err != nil { + if ctx.Err() != nil { + return xdr.LedgerCloseMetaBatch{}, fmt.Errorf("context cancelled for sequence %d: %w", sequence, ctx.Err()) + } + if errors.Is(err, os.ErrNotExist) { + cancel(fmt.Errorf("ledger file for sequence %d not found: %w", sequence, err)) + return xdr.LedgerCloseMetaBatch{}, err + } + if attempt == f.config.RetryLimit { + cancel(fmt.Errorf("downloading ledger file for sequence %d: maximum retries (%d) exceeded: %w", + sequence, f.config.RetryLimit, err)) + return xdr.LedgerCloseMetaBatch{}, err + } + log.WithField("sequence", sequence).WithError(err). + Warnf("Backfill fetch error (attempt %d/%d), retrying...", attempt+1, f.config.RetryLimit) + if !sleepWithContext(ctx, f.config.RetryWait) { + return xdr.LedgerCloseMetaBatch{}, fmt.Errorf("context cancelled during retry wait for sequence %d: %w", sequence, ctx.Err()) + } + continue + } + return batch, nil + } + // Unreachable, but Go requires a return. + return xdr.LedgerCloseMetaBatch{}, fmt.Errorf("unreachable: retry loop exited for sequence %d", sequence) +} + +// downloadAndDecode fetches and stream-decodes a single S3 file. +// Uses pooled zstd decoders to avoid reallocating internal buffers per file. +func (f *backfillFetcher) downloadAndDecode(ctx context.Context, sequence uint32) (xdr.LedgerCloseMetaBatch, error) { + objectKey := f.schema.GetObjectKeyFromSequenceNumber(sequence) + reader, err := f.dataStore.GetFile(ctx, objectKey) + if err != nil { + return xdr.LedgerCloseMetaBatch{}, fmt.Errorf("fetching ledger file %s: %w", objectKey, err) + } + defer reader.Close() //nolint:errcheck + + buffered := bufio.NewReaderSize(reader, 256*1024) + + dec := zstdDecoderPool.Get().(*zstd.Decoder) + defer zstdDecoderPool.Put(dec) + if err := dec.Reset(buffered); err != nil { + return xdr.LedgerCloseMetaBatch{}, fmt.Errorf("resetting zstd decoder for %s: %w", objectKey, err) + } + + var batch xdr.LedgerCloseMetaBatch + if _, err := xdr.Unmarshal(dec, &batch); err != nil { + return xdr.LedgerCloseMetaBatch{}, fmt.Errorf("decoding ledger file %s: %w", objectKey, err) + } + return batch, nil +} diff --git a/internal/ingest/backfill_fetcher_test.go b/internal/ingest/backfill_fetcher_test.go new file mode 100644 index 000000000..074a82673 --- /dev/null +++ b/internal/ingest/backfill_fetcher_test.go @@ -0,0 +1,235 @@ +package ingest + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/stellar/go-stellar-sdk/xdr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +func TestBackfillFetcher_DeliversAllLedgers(t *testing.T) { + ledgersPerFile := uint32(10) + schema := testSchema(ledgersPerFile) + ds := &mockDataStore{} + + // Register 3 files covering ledgers 100-129 + for fileStart := uint32(100); fileStart < 130; fileStart += ledgersPerFile { + objectKey := schema.GetObjectKeyFromSequenceNumber(fileStart) + ds.On("GetFile", mock.Anything, objectKey).Return( + encodeBatch(t, fileStart, fileStart+ledgersPerFile-1), nil, + ).Once() + } + + ledgerCh := make(chan xdr.LedgerCloseMeta, 100) + + fetcher := NewBackfillFetcher(BackfillFetcherConfig{ + NumWorkers: 3, + RetryLimit: 1, + RetryWait: 10 * time.Millisecond, + GapStart: 100, + GapEnd: 129, + }, ds, schema, ledgerCh) + + ctx := context.Background() + fetchCtx, fetchCancel := context.WithCancelCause(ctx) + defer fetchCancel(nil) + + stats := fetcher.Run(fetchCtx, fetchCancel) + + // ledgerCh should be closed by Run + var received []uint32 + for lcm := range ledgerCh { + received = append(received, lcm.LedgerSequence()) + } + + // All 30 ledgers delivered (order doesn't matter) + assert.Len(t, received, 30) + assert.Equal(t, 3, stats.FetchCount) // 3 files fetched, not 30 ledgers + + // Verify all sequences present + seqSet := make(map[uint32]bool) + for _, seq := range received { + seqSet[seq] = true + } + for seq := uint32(100); seq <= 129; seq++ { + assert.True(t, seqSet[seq], "missing ledger %d", seq) + } +} + +func TestBackfillFetcher_PartialFileFiltering(t *testing.T) { + // Gap 105-124 with LedgersPerFile=10 spans files 100-109, 110-119, 120-129. + // Only ledgers 105-124 should be delivered. + ledgersPerFile := uint32(10) + schema := testSchema(ledgersPerFile) + ds := &mockDataStore{} + + for fileStart := uint32(100); fileStart < 130; fileStart += ledgersPerFile { + objectKey := schema.GetObjectKeyFromSequenceNumber(fileStart) + ds.On("GetFile", mock.Anything, objectKey).Return( + encodeBatch(t, fileStart, fileStart+ledgersPerFile-1), nil, + ).Once() + } + + ledgerCh := make(chan xdr.LedgerCloseMeta, 100) + + fetcher := NewBackfillFetcher(BackfillFetcherConfig{ + NumWorkers: 2, + RetryLimit: 1, + RetryWait: 10 * time.Millisecond, + GapStart: 105, + GapEnd: 124, + }, ds, schema, ledgerCh) + + ctx := context.Background() + fetchCtx, fetchCancel := context.WithCancelCause(ctx) + defer fetchCancel(nil) + + fetcher.Run(fetchCtx, fetchCancel) + + var received []uint32 + for lcm := range ledgerCh { + received = append(received, lcm.LedgerSequence()) + } + + assert.Len(t, received, 20) + seqSet := make(map[uint32]bool) + for _, seq := range received { + seqSet[seq] = true + } + for seq := uint32(105); seq <= 124; seq++ { + assert.True(t, seqSet[seq], "missing ledger %d", seq) + } + // Verify out-of-range ledgers are NOT delivered + for _, seq := range []uint32{100, 101, 102, 103, 104, 125, 126, 127, 128, 129} { + assert.False(t, seqSet[seq], "unexpected ledger %d", seq) + } +} + +func TestBackfillFetcher_ContextCancellation(t *testing.T) { + ledgersPerFile := uint32(10) + schema := testSchema(ledgersPerFile) + ds := &mockDataStore{} + + // Use a small ledgerCh buffer so workers will block on send + ledgerCh := make(chan xdr.LedgerCloseMeta, 1) + + // Register many files so workers have work to do + for fileStart := uint32(0); fileStart < 100; fileStart += ledgersPerFile { + objectKey := schema.GetObjectKeyFromSequenceNumber(fileStart) + ds.On("GetFile", mock.Anything, objectKey).Return( + encodeBatch(t, fileStart, fileStart+ledgersPerFile-1), nil, + ).Maybe() + } + + fetcher := NewBackfillFetcher(BackfillFetcherConfig{ + NumWorkers: 3, + RetryLimit: 1, + RetryWait: 10 * time.Millisecond, + GapStart: 0, + GapEnd: 99, + }, ds, schema, ledgerCh) + + ctx := context.Background() + fetchCtx, fetchCancel := context.WithCancelCause(ctx) + + // Cancel after a short delay + go func() { + time.Sleep(50 * time.Millisecond) + fetchCancel(fmt.Errorf("test cancellation")) + }() + + done := make(chan struct{}) + go func() { + fetcher.Run(fetchCtx, fetchCancel) + close(done) + }() + + // Drain ledgerCh so workers can make progress until cancellation + go func() { + for range ledgerCh { //nolint:revive + } + }() + + // Run should complete without deadlock + select { + case <-done: + // Success + case <-time.After(5 * time.Second): + t.Fatal("Run did not exit after context cancellation — possible deadlock") + } +} + +func TestBackfillFetcher_RetryOnTransientError(t *testing.T) { + ledgersPerFile := uint32(10) + schema := testSchema(ledgersPerFile) + ds := &mockDataStore{} + + objectKey := schema.GetObjectKeyFromSequenceNumber(100) + // First call fails, second succeeds + ds.On("GetFile", mock.Anything, objectKey). + Return(nil, fmt.Errorf("transient S3 error")).Once() + ds.On("GetFile", mock.Anything, objectKey). + Return(encodeBatch(t, 100, 109), nil).Once() + + ledgerCh := make(chan xdr.LedgerCloseMeta, 100) + + fetcher := NewBackfillFetcher(BackfillFetcherConfig{ + NumWorkers: 1, + RetryLimit: 2, + RetryWait: 10 * time.Millisecond, + GapStart: 100, + GapEnd: 109, + }, ds, schema, ledgerCh) + + ctx := context.Background() + fetchCtx, fetchCancel := context.WithCancelCause(ctx) + defer fetchCancel(nil) + + stats := fetcher.Run(fetchCtx, fetchCancel) + + var received []uint32 + for lcm := range ledgerCh { + received = append(received, lcm.LedgerSequence()) + } + + assert.Len(t, received, 10) + assert.Equal(t, 1, stats.FetchCount) + assert.Nil(t, context.Cause(fetchCtx), "context should not be cancelled") +} + +func TestBackfillFetcher_MissingFile(t *testing.T) { + ledgersPerFile := uint32(10) + schema := testSchema(ledgersPerFile) + ds := &mockDataStore{} + + objectKey := schema.GetObjectKeyFromSequenceNumber(100) + ds.On("GetFile", mock.Anything, objectKey). + Return(nil, os.ErrNotExist) + + ledgerCh := make(chan xdr.LedgerCloseMeta, 100) + + fetcher := NewBackfillFetcher(BackfillFetcherConfig{ + NumWorkers: 1, + RetryLimit: 1, + RetryWait: 10 * time.Millisecond, + GapStart: 100, + GapEnd: 109, + }, ds, schema, ledgerCh) + + ctx := context.Background() + fetchCtx, fetchCancel := context.WithCancelCause(ctx) + defer fetchCancel(nil) + + fetcher.Run(fetchCtx, fetchCancel) + + // Context should be cancelled with cause + cause := context.Cause(fetchCtx) + require.NotNil(t, cause) + assert.Contains(t, cause.Error(), "not found") +} diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index b7a82cd34..7e05858c4 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -18,6 +18,7 @@ import ( "github.com/stellar/go-stellar-sdk/ingest/ledgerbackend" "github.com/stellar/go-stellar-sdk/support/datastore" "github.com/stellar/go-stellar-sdk/support/log" + "github.com/stellar/go-stellar-sdk/xdr" "github.com/stellar/wallet-backend/internal/apptracker" "github.com/stellar/wallet-backend/internal/data" @@ -69,15 +70,24 @@ type Configs struct { LedgerBackendType LedgerBackendType // DatastoreConfigPath is the path to the TOML config file for datastore backend DatastoreConfigPath string - // BackfillWorkers limits concurrent batch processing during backfill. - // Defaults to runtime.NumCPU(). Lower values reduce RAM usage. - BackfillWorkers int - // BackfillBatchSize is the number of ledgers processed per batch during backfill. - // Defaults to 250. Lower values reduce RAM usage at cost of more DB transactions. - BackfillBatchSize int - // BackfillDBInsertBatchSize is the number of ledgers to process before flushing to DB. - // Defaults to 50. Lower values reduce RAM usage at cost of more DB transactions. + // BackfillProcessWorkers is the number of Stage 2 process workers in the pipeline. + // Defaults to runtime.NumCPU(). + BackfillProcessWorkers int + // BackfillFlushWorkers is the number of Stage 3 flush workers in the pipeline. + // Defaults to 4. + BackfillFlushWorkers int + // BackfillDBInsertBatchSize is the number of ledgers per flush batch. + // Defaults to 100. Lower values reduce RAM usage at cost of more DB transactions. BackfillDBInsertBatchSize int + // BackfillLedgerChanSize is the bounded channel size between dispatcher and process workers. + // Defaults to 256. + BackfillLedgerChanSize int + // BackfillFlushChanSize is the bounded channel size between process workers and flush workers. + // Defaults to 8. + BackfillFlushChanSize int + // BackfillFetchWorkers is the number of parallel S3 download goroutines in the backfill fetcher. + // Defaults to 15. + BackfillFetchWorkers int // ChunkInterval sets the TimescaleDB chunk time interval for hypertables. // Only affects future chunks. Uses PostgreSQL INTERVAL syntax (e.g., "1 day", "7 days"). ChunkInterval string @@ -198,6 +208,43 @@ func setupDeps(cfg Configs) (services.IngestService, error) { return NewLedgerBackend(ctx, cfg) } + // Create factory for the parallel backfill fetcher. + // Each gap gets its own fetcher with fresh S3 connections. + var backfillFetcherFactory services.BackfillFetcherFactory + if cfg.LedgerBackendType == LedgerBackendTypeDatastore { + backfillFetcherFactory = func(gapStart, gapEnd uint32, ledgerCh chan<- xdr.LedgerCloseMeta) services.BackfillFetcherRunner { + return func(ctx context.Context, cancel context.CancelCauseFunc) (int, time.Duration, map[string]time.Duration) { + storageConfig, err := loadDatastoreBackendConfig(cfg.DatastoreConfigPath) + if err != nil { + cancel(fmt.Errorf("loading datastore config: %w", err)) + return 0, 0, nil + } + storageConfig.DataStoreConfig.NetworkPassphrase = cfg.NetworkPassphrase + + ds, schema, err := newBackfillDataStore(ctx, storageConfig, cfg.BackfillFetchWorkers) + if err != nil { + cancel(fmt.Errorf("creating backfill datastore: %w", err)) + return 0, 0, nil + } + fetcher := NewBackfillFetcher(BackfillFetcherConfig{ + NumWorkers: uint32(cfg.BackfillFetchWorkers), + RetryLimit: 3, + RetryWait: 5 * time.Second, + GapStart: gapStart, + GapEnd: gapEnd, + OnFetchDuration: func(s float64) { + m.Ingestion.PhaseDuration.WithLabelValues("backfill_fetch").Observe(s) + }, + OnChannelWait: func(ch, dir string, s float64) { + m.Ingestion.BackfillChannelWait.WithLabelValues(ch, dir).Observe(s) + }, + }, ds, schema, ledgerCh) + stats := fetcher.Run(ctx, cancel) + return stats.FetchCount, stats.FetchTotal, stats.ChannelWait + } + } + } + ingestService, err := services.NewIngestService(services.IngestServiceConfig{ IngestionMode: cfg.IngestionMode, Models: models, @@ -215,9 +262,13 @@ func setupDeps(cfg Configs) (services.IngestService, error) { Network: cfg.Network, NetworkPassphrase: cfg.NetworkPassphrase, Archive: archive, - BackfillWorkers: cfg.BackfillWorkers, - BackfillBatchSize: cfg.BackfillBatchSize, + BackfillProcessWorkers: cfg.BackfillProcessWorkers, + BackfillFlushWorkers: cfg.BackfillFlushWorkers, BackfillDBInsertBatchSize: cfg.BackfillDBInsertBatchSize, + BackfillLedgerChanSize: cfg.BackfillLedgerChanSize, + BackfillFlushChanSize: cfg.BackfillFlushChanSize, + BackfillFetcherFactory: backfillFetcherFactory, + BackfillFetchWorkers: cfg.BackfillFetchWorkers, }) if err != nil { return nil, fmt.Errorf("instantiating ingest service: %w", err) diff --git a/internal/ingest/ledger_backend.go b/internal/ingest/ledger_backend.go index 2795e408a..ece0c8b81 100644 --- a/internal/ingest/ledger_backend.go +++ b/internal/ingest/ledger_backend.go @@ -3,8 +3,13 @@ package ingest import ( "context" "fmt" + "net" + "net/http" "time" + "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/pelletier/go-toml" rpc "github.com/stellar/go-stellar-sdk/clients/rpcclient" "github.com/stellar/go-stellar-sdk/ingest/ledgerbackend" @@ -24,36 +29,47 @@ func NewLedgerBackend(ctx context.Context, cfg Configs) (ledgerbackend.LedgerBac } } -func newDatastoreLedgerBackend(ctx context.Context, datastoreConfigPath string, networkPassphrase string) (ledgerbackend.LedgerBackend, error) { - storageBackendConfig, err := loadDatastoreBackendConfig(datastoreConfigPath) +// newDatastoreResources creates the DataStore client and loads the schema +// from the S3 manifest. Shared by both optimizedStorageBackend (live) and +// backfillFetcher (backfill). +func newDatastoreResources(ctx context.Context, configPath string, networkPassphrase string) ( + datastore.DataStore, datastore.DataStoreSchema, ledgerbackend.BufferedStorageBackendConfig, error, +) { + storageBackendConfig, err := loadDatastoreBackendConfig(configPath) if err != nil { - return nil, fmt.Errorf("loading datastore config: %w", err) + return nil, datastore.DataStoreSchema{}, ledgerbackend.BufferedStorageBackendConfig{}, + fmt.Errorf("loading datastore config: %w", err) } - storageBackendConfig.DataStoreConfig.NetworkPassphrase = networkPassphrase - dataStore, err := datastore.NewDataStore(ctx, storageBackendConfig.DataStoreConfig) + ds, err := datastore.NewDataStore(ctx, storageBackendConfig.DataStoreConfig) if err != nil { - return nil, fmt.Errorf("creating datastore: %w", err) + return nil, datastore.DataStoreSchema{}, ledgerbackend.BufferedStorageBackendConfig{}, + fmt.Errorf("creating datastore: %w", err) } - schema, err := datastore.LoadSchema(ctx, dataStore, storageBackendConfig.DataStoreConfig) + schema, err := datastore.LoadSchema(ctx, ds, storageBackendConfig.DataStoreConfig) if err != nil { - return nil, fmt.Errorf("loading datastore schema: %w", err) + return nil, datastore.DataStoreSchema{}, ledgerbackend.BufferedStorageBackendConfig{}, + fmt.Errorf("loading datastore schema: %w", err) } - ledgerBackend, err := newOptimizedStorageBackend( - storageBackendConfig.BufferedStorageBackendConfig, - dataStore, - schema, - ) + return ds, schema, storageBackendConfig.BufferedStorageBackendConfig, nil +} + +func newDatastoreLedgerBackend(ctx context.Context, datastoreConfigPath string, networkPassphrase string) (ledgerbackend.LedgerBackend, error) { + ds, schema, bufConfig, err := newDatastoreResources(ctx, datastoreConfigPath, networkPassphrase) + if err != nil { + return nil, err + } + + ledgerBackend, err := newOptimizedStorageBackend(bufConfig, ds, schema) if err != nil { return nil, fmt.Errorf("creating optimized storage backend: %w", err) } log.Infof("Using optimized storage backend with buffer size %d, %d workers", - storageBackendConfig.BufferedStorageBackendConfig.BufferSize, - storageBackendConfig.BufferedStorageBackendConfig.NumWorkers) + bufConfig.BufferSize, bufConfig.NumWorkers) return ledgerBackend, nil } @@ -64,6 +80,69 @@ func newRPCLedgerBackend(cfg Configs) (ledgerbackend.LedgerBackend, error) { return backend, nil } +// newBackfillDataStore creates a DataStore with an HTTP transport tuned for +// high-concurrency S3 downloads. The default AWS SDK transport sets +// MaxIdleConnsPerHost=10, which forces workers beyond that count to tear down +// and re-establish TLS connections on every request. This function raises the +// pool limits to match the worker count, eliminating connection churn. +func newBackfillDataStore(ctx context.Context, cfg StorageBackendConfig, numWorkers int) (datastore.DataStore, datastore.DataStoreSchema, error) { + dsCfg := cfg.DataStoreConfig + + destinationBucketPath, ok := dsCfg.Params["destination_bucket_path"] + if !ok { + return nil, datastore.DataStoreSchema{}, fmt.Errorf("invalid S3 config, no destination_bucket_path") + } + region, ok := dsCfg.Params["region"] + if !ok { + return nil, datastore.DataStoreSchema{}, fmt.Errorf("invalid S3 config, no region") + } + endpointURL := dsCfg.Params["endpoint_url"] + + poolSize := numWorkers + 20 + transport := &http.Transport{ + MaxIdleConns: poolSize, + MaxIdleConnsPerHost: poolSize, + IdleConnTimeout: 90 * time.Second, + ForceAttemptHTTP2: true, + DialContext: (&net.Dialer{ + Timeout: 10 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + } + + awsCfg, err := awsconfig.LoadDefaultConfig(ctx, + awsconfig.WithHTTPClient(&http.Client{Transport: transport}), + ) + if err != nil { + return nil, datastore.DataStoreSchema{}, fmt.Errorf("loading AWS config: %w", err) + } + + client := s3.NewFromConfig(awsCfg, func(o *s3.Options) { + if endpointURL != "" { + o.BaseEndpoint = aws.String(endpointURL) + } + _, credErr := awsCfg.Credentials.Retrieve(ctx) + if credErr != nil { + o.Credentials = aws.AnonymousCredentials{} + } + o.Region = region + o.UsePathStyle = true + }) + + ds, err := datastore.FromS3Client(ctx, client, destinationBucketPath) + if err != nil { + return nil, datastore.DataStoreSchema{}, fmt.Errorf("creating S3 datastore: %w", err) + } + + schema, err := datastore.LoadSchema(ctx, ds, dsCfg) + if err != nil { + return nil, datastore.DataStoreSchema{}, fmt.Errorf("loading datastore schema: %w", err) + } + + log.Infof("Backfill S3 client: MaxIdleConnsPerHost=%d for %d workers", poolSize, numWorkers) + return ds, schema, nil +} + func loadDatastoreBackendConfig(configPath string) (StorageBackendConfig, error) { if configPath == "" { return StorageBackendConfig{}, fmt.Errorf("datastore config file path is required for datastore backend type") diff --git a/internal/ingest/storage_backend.go b/internal/ingest/storage_backend.go index 1d86fcc0c..f4c206f23 100644 --- a/internal/ingest/storage_backend.go +++ b/internal/ingest/storage_backend.go @@ -168,6 +168,12 @@ func (b *optimizedStorageBackend) newStorageBuffer(ledgerRange ledgerbackend.Ran ledgerRange: ledgerRange, } + // Start workers first so they can consume from taskQueue during seeding. + for i := uint32(0); i < b.config.NumWorkers; i++ { + buf.wg.Add(1) + go buf.worker() + } + // Seed task queue with initial tasks. The +1 matches the SDK's buffer invariant: // len(taskQueue) + len(batchQueue) + priorityQueue.Len() <= bufferSize, // and the extra task accounts for the one being actively processed by a worker. @@ -177,12 +183,6 @@ func (b *optimizedStorageBackend) newStorageBuffer(ledgerRange ledgerbackend.Ran } } - // Start workers. - for i := uint32(0); i < b.config.NumWorkers; i++ { - buf.wg.Add(1) - go buf.worker() - } - return buf } diff --git a/internal/loadtest/runner.go b/internal/loadtest/runner.go index 64c28ca71..f207e89f7 100644 --- a/internal/loadtest/runner.go +++ b/internal/loadtest/runner.go @@ -208,7 +208,7 @@ func runIngestionLoop( // Write to database using shared persistence logic dbStart := time.Now() - numTxs, numOps, err := ingestSvc.PersistLedgerData(ctx, currentLedger, buffer, loadtestLatestCursor) + numTxs, numOps, err := ingestSvc.PersistLedgerData(ctx, currentLedger, buffer, loadtestLatestCursor, services.NewCopyResult()) if err != nil { return fmt.Errorf("persisting ledger %d: %w", currentLedger, err) } diff --git a/internal/metrics/ingestion.go b/internal/metrics/ingestion.go index e117ff3e4..417544bdb 100644 --- a/internal/metrics/ingestion.go +++ b/internal/metrics/ingestion.go @@ -46,6 +46,32 @@ type IngestionMetrics struct { // State change metrics StateChangeProcessingDuration *prometheus.HistogramVec StateChangesTotal *prometheus.CounterVec + + // --- Backfill Pipeline Metrics --- + + // BackfillChannelWait observes time goroutines spend blocked on channel operations. + // Labels: channel ("ledger", "flush"), direction ("send", "receive"). + // PromQL: histogram_quantile(0.99, rate(wallet_ingestion_backfill_channel_wait_seconds_bucket{channel="ledger",direction="send"}[5m])) + BackfillChannelWait *prometheus.HistogramVec + // BackfillChannelUtilization reports channel fill ratio (0.0-1.0), sampled every second. + // Labels: channel ("ledger", "flush"). + // PromQL: wallet_ingestion_backfill_channel_utilization_ratio{channel="ledger"} + BackfillChannelUtilization *prometheus.GaugeVec + // BackfillLedgersFlushed counts ledgers successfully flushed to DB during backfill. + // PromQL: rate(wallet_ingestion_backfill_ledgers_flushed_total[5m]) + BackfillLedgersFlushed prometheus.Counter + // BackfillBatchSize observes the number of ledgers per flush batch. + // PromQL: histogram_quantile(0.5, rate(wallet_ingestion_backfill_batch_size_bucket[5m])) + BackfillBatchSize prometheus.Histogram + // BackfillGapProgress reports completion ratio (0.0-1.0) of the current gap. + // PromQL: wallet_ingestion_backfill_gap_progress_ratio + BackfillGapProgress prometheus.Gauge + // BackfillGapStartLedger is the start ledger of the gap currently being processed. + // PromQL: wallet_ingestion_backfill_gap_start_ledger + BackfillGapStartLedger prometheus.Gauge + // BackfillGapEndLedger is the end ledger of the gap currently being processed. + // PromQL: wallet_ingestion_backfill_gap_end_ledger + BackfillGapEndLedger prometheus.Gauge } func newIngestionMetrics(reg prometheus.Registerer) *IngestionMetrics { @@ -115,6 +141,36 @@ func newIngestionMetrics(reg prometheus.Registerer) *IngestionMetrics { Name: "wallet_ingestion_state_changes_total", Help: "Total number of state changes persisted to database by type and category.", }, []string{"type", "category"}), + BackfillChannelWait: prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "wallet_ingestion_backfill_channel_wait_seconds", + Help: "Time goroutines spend blocked on backfill pipeline channel operations.", + Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10, 30}, + }, []string{"channel", "direction"}), + BackfillChannelUtilization: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "wallet_ingestion_backfill_channel_utilization_ratio", + Help: "Fill ratio (0.0-1.0) of backfill pipeline channels, sampled every second.", + }, []string{"channel"}), + BackfillLedgersFlushed: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "wallet_ingestion_backfill_ledgers_flushed_total", + Help: "Total ledgers successfully flushed to database during backfill.", + }), + BackfillBatchSize: prometheus.NewHistogram(prometheus.HistogramOpts{ + Name: "wallet_ingestion_backfill_batch_size", + Help: "Number of ledgers per flush batch during backfill.", + Buckets: prometheus.LinearBuckets(10, 10, 10), + }), + BackfillGapProgress: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "wallet_ingestion_backfill_gap_progress_ratio", + Help: "Completion ratio (0.0-1.0) of the backfill gap currently being processed.", + }), + BackfillGapStartLedger: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "wallet_ingestion_backfill_gap_start_ledger", + Help: "Start ledger of the backfill gap currently being processed (0 when idle).", + }), + BackfillGapEndLedger: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "wallet_ingestion_backfill_gap_end_ledger", + Help: "End ledger of the backfill gap currently being processed (0 when idle).", + }), } reg.MustRegister( m.LatestLedger, @@ -132,6 +188,13 @@ func newIngestionMetrics(reg prometheus.Registerer) *IngestionMetrics { m.ErrorsTotal, m.StateChangeProcessingDuration, m.StateChangesTotal, + m.BackfillChannelWait, + m.BackfillChannelUtilization, + m.BackfillLedgersFlushed, + m.BackfillBatchSize, + m.BackfillGapProgress, + m.BackfillGapStartLedger, + m.BackfillGapEndLedger, ) return m } diff --git a/internal/metrics/ingestion_test.go b/internal/metrics/ingestion_test.go index ff47f6b9f..0a4bde630 100644 --- a/internal/metrics/ingestion_test.go +++ b/internal/metrics/ingestion_test.go @@ -183,6 +183,63 @@ func TestIngestionMetrics_ErrorsTotal(t *testing.T) { assert.Equal(t, 2.0, testutil.ToFloat64(m.ErrorsTotal.WithLabelValues("ingest_live"))) } +func TestIngestionMetrics_BackfillRegistration(t *testing.T) { + reg := prometheus.NewRegistry() + m := newIngestionMetrics(reg) + + require.NotNil(t, m.BackfillChannelWait) + require.NotNil(t, m.BackfillChannelUtilization) + require.NotNil(t, m.BackfillLedgersFlushed) + require.NotNil(t, m.BackfillBatchSize) + require.NotNil(t, m.BackfillGapProgress) + require.NotNil(t, m.BackfillGapStartLedger) + require.NotNil(t, m.BackfillGapEndLedger) +} + +func TestIngestionMetrics_BackfillChannelWait_Buckets(t *testing.T) { + reg := prometheus.NewRegistry() + m := newIngestionMetrics(reg) + + m.BackfillChannelWait.WithLabelValues("ledger", "send").Observe(0.1) + + families, err := reg.Gather() + require.NoError(t, err) + for _, f := range families { + if f.GetName() == "wallet_ingestion_backfill_channel_wait_seconds" { + h := f.GetMetric()[0].GetHistogram() + assert.Len(t, h.GetBucket(), 10) + } + } +} + +func TestIngestionMetrics_BackfillBatchSize_Buckets(t *testing.T) { + reg := prometheus.NewRegistry() + m := newIngestionMetrics(reg) + + m.BackfillBatchSize.Observe(50) + + families, err := reg.Gather() + require.NoError(t, err) + for _, f := range families { + if f.GetName() == "wallet_ingestion_backfill_batch_size" { + h := f.GetMetric()[0].GetHistogram() + assert.Len(t, h.GetBucket(), 10) // LinearBuckets(10, 10, 10) + } + } +} + +func TestIngestionMetrics_BackfillChannelWait_Labels(t *testing.T) { + reg := prometheus.NewRegistry() + m := newIngestionMetrics(reg) + + assert.NotPanics(t, func() { + m.BackfillChannelWait.WithLabelValues("ledger", "send").Observe(0.1) + m.BackfillChannelWait.WithLabelValues("ledger", "receive").Observe(0.1) + m.BackfillChannelWait.WithLabelValues("flush", "send").Observe(0.1) + m.BackfillChannelWait.WithLabelValues("flush", "receive").Observe(0.1) + }) +} + func TestIngestionMetrics_Lint(t *testing.T) { reg := prometheus.NewRegistry() m := newIngestionMetrics(reg) @@ -194,6 +251,9 @@ func TestIngestionMetrics_Lint(t *testing.T) { m.LagLedgers, m.LedgerFetchDuration, m.RetriesTotal, m.RetryExhaustionsTotal, m.ErrorsTotal, m.StateChangeProcessingDuration, m.StateChangesTotal, + m.BackfillChannelWait, m.BackfillChannelUtilization, + m.BackfillLedgersFlushed, m.BackfillBatchSize, + m.BackfillGapProgress, m.BackfillGapStartLedger, m.BackfillGapEndLedger, } { problems, err := testutil.CollectAndLint(c) require.NoError(t, err) diff --git a/internal/services/backfill_stats.go b/internal/services/backfill_stats.go new file mode 100644 index 000000000..365e45b6c --- /dev/null +++ b/internal/services/backfill_stats.go @@ -0,0 +1,96 @@ +package services + +import ( + "fmt" + "time" +) + +// backfillWorkerStats accumulates timing for a single process worker. +// Each worker owns its instance — no mutex needed. +type backfillWorkerStats struct { + fetchCount int + fetchTotal time.Duration + processCount int + processTotal time.Duration + channelWait map[string]time.Duration // key: "channel:direction" +} + +func (s *backfillWorkerStats) addFetch(d time.Duration) { + s.fetchCount++ + s.fetchTotal += d +} + +func (s *backfillWorkerStats) addProcess(d time.Duration) { + s.processCount++ + s.processTotal += d +} + +func (s *backfillWorkerStats) addChannelWait(channel, direction string, d time.Duration) { + if s.channelWait == nil { + s.channelWait = make(map[string]time.Duration) + } + s.channelWait[fmt.Sprintf("%s:%s", channel, direction)] += d +} + +// backfillFlushWorkerStats accumulates timing for a single flush worker. +type backfillFlushWorkerStats struct { + flushCount int + flushTotal time.Duration + channelWait map[string]time.Duration +} + +func (s *backfillFlushWorkerStats) addFlush(d time.Duration) { + s.flushCount++ + s.flushTotal += d +} + +func (s *backfillFlushWorkerStats) addChannelWait(channel, direction string, d time.Duration) { + if s.channelWait == nil { + s.channelWait = make(map[string]time.Duration) + } + s.channelWait[fmt.Sprintf("%s:%s", channel, direction)] += d +} + +// backfillGapStats aggregates stats from all workers for a single gap. +// Only accessed from processGap after workers complete — no mutex needed. +type backfillGapStats struct { + fetchCount int + fetchTotal time.Duration + processCount int + processTotal time.Duration + flushCount int + flushTotal time.Duration + channelWait map[string]time.Duration +} + +func newBackfillGapStats() *backfillGapStats { + return &backfillGapStats{ + channelWait: make(map[string]time.Duration), + } +} + +func (gs *backfillGapStats) mergeWorker(w *backfillWorkerStats) { + gs.fetchCount += w.fetchCount + gs.fetchTotal += w.fetchTotal + gs.processCount += w.processCount + gs.processTotal += w.processTotal + for k, v := range w.channelWait { + gs.channelWait[k] += v + } +} + +func (gs *backfillGapStats) mergeFlushWorker(w *backfillFlushWorkerStats) { + gs.flushCount += w.flushCount + gs.flushTotal += w.flushTotal + for k, v := range w.channelWait { + gs.channelWait[k] += v + } +} + +// avgOrZero returns average duration, or zero if count is 0. +func avgOrZero(total time.Duration, count int) time.Duration { + if count == 0 { + return 0 + } + return total / time.Duration(count) +} diff --git a/internal/services/backfill_stats_test.go b/internal/services/backfill_stats_test.go new file mode 100644 index 000000000..1937e7852 --- /dev/null +++ b/internal/services/backfill_stats_test.go @@ -0,0 +1,67 @@ +package services + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestBackfillWorkerStats_Add(t *testing.T) { + s := &backfillWorkerStats{} + + s.addFetch(100 * time.Millisecond) + s.addFetch(200 * time.Millisecond) + s.addProcess(50 * time.Millisecond) + s.addChannelWait("ledger", "send", 10*time.Millisecond) + + assert.Equal(t, 2, s.fetchCount) + assert.Equal(t, 300*time.Millisecond, s.fetchTotal) + assert.Equal(t, 1, s.processCount) + assert.Equal(t, 50*time.Millisecond, s.processTotal) + assert.Equal(t, 10*time.Millisecond, s.channelWait["ledger:send"]) +} + +func TestBackfillGapStats_Merge(t *testing.T) { + gs := newBackfillGapStats() + + w1 := &backfillWorkerStats{} + w1.addFetch(100 * time.Millisecond) + w1.addFetch(200 * time.Millisecond) + w1.addProcess(50 * time.Millisecond) + w1.addChannelWait("ledger", "send", 10*time.Millisecond) + + w2 := &backfillWorkerStats{} + w2.addFetch(200 * time.Millisecond) + w2.addProcess(75 * time.Millisecond) + w2.addChannelWait("ledger", "send", 20*time.Millisecond) + + gs.mergeWorker(w1) + gs.mergeWorker(w2) + + assert.Equal(t, 3, gs.fetchCount) + assert.Equal(t, 500*time.Millisecond, gs.fetchTotal) + assert.Equal(t, 2, gs.processCount) + assert.Equal(t, 125*time.Millisecond, gs.processTotal) + assert.Equal(t, 30*time.Millisecond, gs.channelWait["ledger:send"]) +} + +func TestBackfillGapStats_MergeFlush(t *testing.T) { + gs := newBackfillGapStats() + + w := &backfillFlushWorkerStats{} + w.addFlush(500 * time.Millisecond) + w.addFlush(300 * time.Millisecond) + w.addChannelWait("flush", "receive", 100*time.Millisecond) + + gs.mergeFlushWorker(w) + + assert.Equal(t, 2, gs.flushCount) + assert.Equal(t, 800*time.Millisecond, gs.flushTotal) + assert.Equal(t, 100*time.Millisecond, gs.channelWait["flush:receive"]) +} + +func TestAvgOrZero(t *testing.T) { + assert.Equal(t, time.Duration(0), avgOrZero(100*time.Millisecond, 0)) + assert.Equal(t, 50*time.Millisecond, avgOrZero(100*time.Millisecond, 2)) +} diff --git a/internal/services/backfill_watermark.go b/internal/services/backfill_watermark.go new file mode 100644 index 000000000..69e03b455 --- /dev/null +++ b/internal/services/backfill_watermark.go @@ -0,0 +1,59 @@ +package services + +import "sync" + +// backfillWatermark tracks which ledgers in a gap have been flushed to DB, +// and reports the highest contiguous flushed ledger for cursor updates. +// Thread-safe: multiple flush workers call MarkFlushed concurrently. +type backfillWatermark struct { + mu sync.Mutex + flushed []bool // indexed by (ledgerSeq - start) + start uint32 + end uint32 + cursor uint32 // highest contiguous flushed ledger (0 = none) + scanFrom uint32 // resume scan from last cursor position +} + +func newBackfillWatermark(start, end uint32) *backfillWatermark { + return &backfillWatermark{ + flushed: make([]bool, end-start+1), + start: start, + end: end, + scanFrom: start, + } +} + +// MarkFlushed records ledger sequences as flushed and advances the cursor +// through any contiguous range. Returns true if the cursor advanced. +func (w *backfillWatermark) MarkFlushed(ledgers []uint32) bool { + w.mu.Lock() + defer w.mu.Unlock() + + for _, seq := range ledgers { + if seq >= w.start && seq <= w.end { + w.flushed[seq-w.start] = true + } + } + + oldCursor := w.cursor + for w.scanFrom <= w.end && w.flushed[w.scanFrom-w.start] { + w.cursor = w.scanFrom + w.scanFrom++ + } + + return w.cursor != oldCursor +} + +// Cursor returns the highest contiguous flushed ledger, or 0 if none. +func (w *backfillWatermark) Cursor() uint32 { + w.mu.Lock() + defer w.mu.Unlock() + return w.cursor +} + +// Complete returns true if all ledgers in the gap have been flushed. +func (w *backfillWatermark) Complete() bool { + w.mu.Lock() + defer w.mu.Unlock() + return w.cursor == w.end +} diff --git a/internal/services/backfill_watermark_test.go b/internal/services/backfill_watermark_test.go new file mode 100644 index 000000000..f7996fbaf --- /dev/null +++ b/internal/services/backfill_watermark_test.go @@ -0,0 +1,72 @@ +package services + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_backfillWatermark_MarkFlushed_sequential(t *testing.T) { + w := newBackfillWatermark(100, 109) // 10 ledgers: 100-109 + + // Flush ledgers 100-104 + advanced := w.MarkFlushed([]uint32{100, 101, 102, 103, 104}) + assert.True(t, advanced) + assert.Equal(t, uint32(104), w.Cursor()) + + // Flush ledgers 105-109 + advanced = w.MarkFlushed([]uint32{105, 106, 107, 108, 109}) + assert.True(t, advanced) + assert.Equal(t, uint32(109), w.Cursor()) + assert.True(t, w.Complete()) +} + +func Test_backfillWatermark_MarkFlushed_outOfOrder(t *testing.T) { + w := newBackfillWatermark(100, 109) + + // Flush ledgers 105-109 first — cursor can't advance + advanced := w.MarkFlushed([]uint32{105, 106, 107, 108, 109}) + assert.False(t, advanced) + assert.Equal(t, uint32(0), w.Cursor()) // no contiguous range from start + + // Flush ledgers 100-104 — cursor jumps to 109 + advanced = w.MarkFlushed([]uint32{100, 101, 102, 103, 104}) + assert.True(t, advanced) + assert.Equal(t, uint32(109), w.Cursor()) + assert.True(t, w.Complete()) +} + +func Test_backfillWatermark_MarkFlushed_withGap(t *testing.T) { + w := newBackfillWatermark(100, 109) + + // Flush 100-102 + w.MarkFlushed([]uint32{100, 101, 102}) + assert.Equal(t, uint32(102), w.Cursor()) + + // Flush 105-107 (skip 103-104) + advanced := w.MarkFlushed([]uint32{105, 106, 107}) + assert.False(t, advanced) // cursor stuck at 102 + + // Flush 103-104 — cursor jumps to 107 + advanced = w.MarkFlushed([]uint32{103, 104}) + assert.True(t, advanced) + assert.Equal(t, uint32(107), w.Cursor()) + assert.False(t, w.Complete()) // 108, 109 still missing +} + +func Test_backfillWatermark_singleLedger(t *testing.T) { + w := newBackfillWatermark(500, 500) + + advanced := w.MarkFlushed([]uint32{500}) + assert.True(t, advanced) + assert.Equal(t, uint32(500), w.Cursor()) + assert.True(t, w.Complete()) +} + +func Test_backfillWatermark_emptyFlush(t *testing.T) { + w := newBackfillWatermark(100, 109) + + advanced := w.MarkFlushed([]uint32{}) + assert.False(t, advanced) + assert.Equal(t, uint32(0), w.Cursor()) +} diff --git a/internal/services/ingest.go b/internal/services/ingest.go index 5a81ebe36..decf1d0d1 100644 --- a/internal/services/ingest.go +++ b/internal/services/ingest.go @@ -2,22 +2,20 @@ package services import ( "context" - "errors" "fmt" "hash/fnv" "runtime" "strings" + "sync" "time" "github.com/alitto/pond/v2" set "github.com/deckarep/golang-set/v2" "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/pgconn" "github.com/stellar/go-stellar-sdk/historyarchive" "github.com/stellar/go-stellar-sdk/ingest/ledgerbackend" "github.com/stellar/go-stellar-sdk/support/log" "github.com/stellar/go-stellar-sdk/xdr" - "golang.org/x/sync/errgroup" "github.com/stellar/wallet-backend/internal/apptracker" "github.com/stellar/wallet-backend/internal/data" @@ -42,6 +40,15 @@ const ( // Each batch needs its own backend because LedgerBackend is not thread-safe. type LedgerBackendFactory func(ctx context.Context) (ledgerbackend.LedgerBackend, error) +// BackfillFetcherRunner starts fetch workers and returns when done. +// It must close ledgerCh before returning. +// Returns (fetchCount, fetchTotal, channelWait). +type BackfillFetcherRunner func(ctx context.Context, cancel context.CancelCauseFunc) (int, time.Duration, map[string]time.Duration) + +// BackfillFetcherFactory creates a runner for a specific gap range. +// Each gap gets its own fetcher with fresh S3 connections. +type BackfillFetcherFactory func(gapStart, gapEnd uint32, ledgerCh chan<- xdr.LedgerCloseMeta) BackfillFetcherRunner + // IngestServiceConfig holds the configuration for creating an IngestService. type IngestServiceConfig struct { // === Core === @@ -73,9 +80,13 @@ type IngestServiceConfig struct { GetLedgersLimit int // === Backfill Tuning === - BackfillWorkers int - BackfillBatchSize int - BackfillDBInsertBatchSize int + BackfillProcessWorkers int // Stage 2 process workers (default: NumCPU) + BackfillFlushWorkers int // Stage 3 flush workers (default: 4) + BackfillDBInsertBatchSize int // Ledgers per flush batch (default: 100) + BackfillLedgerChanSize int // ledgerCh buffer size (default: 256) + BackfillFlushChanSize int // flushCh buffer size (default: 8) + BackfillFetcherFactory BackfillFetcherFactory // Parallel S3 fetcher for backfill gaps + BackfillFetchWorkers int // S3 download goroutines (default: 15) } // generateAdvisoryLockID creates a deterministic advisory lock ID based on the network name. @@ -88,36 +99,41 @@ func generateAdvisoryLockID(network string) int { type IngestService interface { Run(ctx context.Context, startLedger uint32, endLedger uint32) error - // PersistLedgerData persists processed ledger data to the database in a single atomic transaction. + // PersistLedgerData persists processed ledger data to the database. // This is the shared core used by both live ingestion and loadtest. + // The CopyResult tracks per-table success across retries to prevent duplicates. // Returns the number of transactions and operations persisted. - PersistLedgerData(ctx context.Context, ledgerSeq uint32, buffer *indexer.IndexerBuffer, cursorName string) (int, int, error) + PersistLedgerData(ctx context.Context, ledgerSeq uint32, buffer *indexer.IndexerBuffer, cursorName string, result *CopyResult) (int, int, error) } var _ IngestService = (*ingestService)(nil) type ingestService struct { - ingestionMode string - models *data.Models - latestLedgerCursorName string - oldestLedgerCursorName string - advisoryLockID int - appTracker apptracker.AppTracker - rpcService RPCService - ledgerBackend ledgerbackend.LedgerBackend - ledgerBackendFactory LedgerBackendFactory - chAccStore store.ChannelAccountStore - tokenIngestionService TokenIngestionService - contractMetadataService ContractMetadataService - appMetrics *metrics.Metrics - networkPassphrase string - getLedgersLimit int - ledgerIndexer *indexer.Indexer - archive historyarchive.ArchiveInterface - backfillPool pond.Pool - backfillBatchSize uint32 - backfillDBInsertBatchSize uint32 - knownContractIDs set.Set[string] + ingestionMode string + models *data.Models + latestLedgerCursorName string + oldestLedgerCursorName string + advisoryLockID int + appTracker apptracker.AppTracker + rpcService RPCService + ledgerBackend ledgerbackend.LedgerBackend + ledgerBackendFactory LedgerBackendFactory + chAccStore store.ChannelAccountStore + tokenIngestionService TokenIngestionService + contractMetadataService ContractMetadataService + appMetrics *metrics.Metrics + networkPassphrase string + getLedgersLimit int + ledgerIndexer *indexer.Indexer + archive historyarchive.ArchiveInterface + backfillProcessWorkers int + backfillFlushWorkers int + backfillFlushBatchSize uint32 + backfillLedgerChanSize int + backfillFlushChanSize int + backfillFetcherFactory BackfillFetcherFactory + backfillFetchWorkers int + knownContractIDs set.Set[string] } func NewIngestService(cfg IngestServiceConfig) (*ingestService, error) { @@ -126,37 +142,51 @@ func NewIngestService(cfg IngestServiceConfig) (*ingestService, error) { ledgerIndexerPool := pond.NewPool(runtime.NumCPU()) cfg.Metrics.RegisterPoolMetrics("ledger_indexer", ledgerIndexerPool) - // Create backfill pool with bounded size to control memory usage. - // Default to NumCPU if not specified. - backfillWorkers := cfg.BackfillWorkers - if backfillWorkers <= 0 { - backfillWorkers = runtime.NumCPU() + // Backfill pipeline defaults — only process workers needs a runtime default (NumCPU). + // All other defaults are set via CLI flag defaults in cmd/ingest.go. + processWorkers := cfg.BackfillProcessWorkers + if processWorkers <= 0 { + processWorkers = runtime.NumCPU() + } + + // Validate connection pool can handle flush worker concurrency. + // Each flush worker runs 5 parallel COPYs, each needing its own connection. + if cfg.IngestionMode == IngestionModeBackfill && cfg.BackfillFlushWorkers > 0 { + requiredConns := int32(cfg.BackfillFlushWorkers*5 + 5) // +5 headroom for cursor updates + maxConns := cfg.Models.DB.Config().MaxConns + if maxConns > 0 && maxConns < requiredConns { + return nil, fmt.Errorf( + "pgxpool max connections (%d) too low for %d flush workers (need at least %d: %d workers × 5 COPYs + 5 headroom)", + maxConns, cfg.BackfillFlushWorkers, requiredConns, cfg.BackfillFlushWorkers) + } } - backfillPool := pond.NewPool(backfillWorkers) - cfg.Metrics.RegisterPoolMetrics("backfill", backfillPool) return &ingestService{ - ingestionMode: cfg.IngestionMode, - models: cfg.Models, - latestLedgerCursorName: cfg.LatestLedgerCursorName, - oldestLedgerCursorName: cfg.OldestLedgerCursorName, - advisoryLockID: generateAdvisoryLockID(cfg.Network), - appTracker: cfg.AppTracker, - rpcService: cfg.RPCService, - ledgerBackend: cfg.LedgerBackend, - ledgerBackendFactory: cfg.LedgerBackendFactory, - chAccStore: cfg.ChannelAccountStore, - tokenIngestionService: cfg.TokenIngestionService, - contractMetadataService: cfg.ContractMetadataService, - appMetrics: cfg.Metrics, - networkPassphrase: cfg.NetworkPassphrase, - getLedgersLimit: cfg.GetLedgersLimit, - ledgerIndexer: indexer.NewIndexer(cfg.NetworkPassphrase, ledgerIndexerPool, cfg.Metrics.Ingestion), - archive: cfg.Archive, - backfillPool: backfillPool, - backfillBatchSize: uint32(cfg.BackfillBatchSize), - backfillDBInsertBatchSize: uint32(cfg.BackfillDBInsertBatchSize), - knownContractIDs: set.NewSet[string](), + ingestionMode: cfg.IngestionMode, + models: cfg.Models, + latestLedgerCursorName: cfg.LatestLedgerCursorName, + oldestLedgerCursorName: cfg.OldestLedgerCursorName, + advisoryLockID: generateAdvisoryLockID(cfg.Network), + appTracker: cfg.AppTracker, + rpcService: cfg.RPCService, + ledgerBackend: cfg.LedgerBackend, + ledgerBackendFactory: cfg.LedgerBackendFactory, + chAccStore: cfg.ChannelAccountStore, + tokenIngestionService: cfg.TokenIngestionService, + contractMetadataService: cfg.ContractMetadataService, + appMetrics: cfg.Metrics, + networkPassphrase: cfg.NetworkPassphrase, + getLedgersLimit: cfg.GetLedgersLimit, + ledgerIndexer: indexer.NewIndexer(cfg.NetworkPassphrase, ledgerIndexerPool, cfg.Metrics.Ingestion), + archive: cfg.Archive, + backfillProcessWorkers: processWorkers, + backfillFlushWorkers: cfg.BackfillFlushWorkers, + backfillFlushBatchSize: uint32(cfg.BackfillDBInsertBatchSize), + backfillLedgerChanSize: cfg.BackfillLedgerChanSize, + backfillFlushChanSize: cfg.BackfillFlushChanSize, + backfillFetcherFactory: cfg.BackfillFetcherFactory, + backfillFetchWorkers: cfg.BackfillFetchWorkers, + knownContractIDs: set.NewSet[string](), }, nil } @@ -227,82 +257,153 @@ func (m *ingestService) processLedger(ctx context.Context, ledgerMeta xdr.Ledger return nil } -// insertAndUpsertParallel runs parallel goroutines via errgroup: 5 COPY operations (transactions, +// processLedgerSequential processes a single ledger without the pond pool. +// Used by backfill where multiple process workers already provide parallelism. +func (m *ingestService) processLedgerSequential(ctx context.Context, ledgerMeta xdr.LedgerCloseMeta, buffer *indexer.IndexerBuffer) error { + participantCount, err := indexer.ProcessLedgerSequential(ctx, m.networkPassphrase, ledgerMeta, m.ledgerIndexer, buffer) + if err != nil { + return fmt.Errorf("processing ledger %d: %w", ledgerMeta.LedgerSequence(), err) + } + m.appMetrics.Ingestion.ParticipantsCount.Observe(float64(participantCount)) + return nil +} + +// copyTable identifies one of the parallel insert/upsert targets. +type copyTable int + +const ( + copyTransactions copyTable = iota + copyTransactionsAccounts + copyOperations + copyOperationsAccounts + copyStateChanges + // Live-only balance upserts (skipped in backfill mode) + copyTrustlineBalances + copyNativeBalances + copySACBalances + copyContractTokens + numCopyTables // = 9 +) + +// CopyResult tracks per-table COPY outcomes across retry attempts. +// On retry, tables where done[i] is true are skipped — preventing duplicates +// without requiring uniqueness constraints. +type CopyResult struct { + done [numCopyTables]bool + errs [numCopyTables]error +} + +// NewCopyResult creates a fresh CopyResult for tracking parallel insert outcomes. +func NewCopyResult() *CopyResult { return &CopyResult{} } + +func (cr *CopyResult) allDone() bool { + for _, d := range cr.done { + if !d { + return false + } + } + return true +} + +func (cr *CopyResult) firstError() error { + for i, d := range cr.done { + if !d && cr.errs[i] != nil { + return cr.errs[i] + } + } + return nil +} + +// insertParallel runs parallel goroutines via sync.WaitGroup: 5 COPY operations (transactions, // transactions_accounts, operations, operations_accounts, state_changes) plus, in live mode only, // 4 balance upserts (trustline, native, SAC, account-contract tokens). Balance upserts are // skipped in backfill mode since they represent current state, not historical data. -// Each goroutine acquires its own pool connection. UniqueViolation errors are treated as -// success for idempotent retry. -func (m *ingestService) insertAndUpsertParallel(ctx context.Context, txs []*types.Transaction, buffer indexer.IndexerBufferInterface) (int, int, error) { +// +// Unlike errgroup, sync.WaitGroup does not cancel sibling goroutines on failure. Each COPY +// runs fully to completion, and result.done[i] is set only after a confirmed commit. On retry, +// already-committed tables are skipped — guaranteeing zero duplicates without uniqueness constraints. +func (m *ingestService) insertParallel(ctx context.Context, txs []*types.Transaction, buffer indexer.IndexerBufferInterface, result *CopyResult) (int, int, error) { txParticipants := buffer.GetTransactionsParticipants() ops := buffer.GetOperations() opParticipants := buffer.GetOperationsParticipants() stateChanges := buffer.GetStateChanges() - g, gCtx := errgroup.WithContext(ctx) + // Define all table operations indexed by copyTable. + type copyOp struct { + table copyTable + fn func(context.Context, pgx.Tx) error + } - // 5 COPY goroutines - g.Go(func() error { - return m.copyWithPoolConn(gCtx, func(ctx context.Context, tx pgx.Tx) error { + copyOps := []copyOp{ + {copyTransactions, func(ctx context.Context, tx pgx.Tx) error { if _, err := m.models.Transactions.BatchCopy(ctx, tx, txs); err != nil { return fmt.Errorf("copying transactions: %w", err) } return nil - }) - }) - g.Go(func() error { - return m.copyWithPoolConn(gCtx, func(ctx context.Context, tx pgx.Tx) error { + }}, + {copyTransactionsAccounts, func(ctx context.Context, tx pgx.Tx) error { return m.models.Transactions.BatchCopyAccounts(ctx, tx, txs, txParticipants) - }) - }) - g.Go(func() error { - return m.copyWithPoolConn(gCtx, func(ctx context.Context, tx pgx.Tx) error { + }}, + {copyOperations, func(ctx context.Context, tx pgx.Tx) error { if _, err := m.models.Operations.BatchCopy(ctx, tx, ops); err != nil { return fmt.Errorf("copying operations: %w", err) } return nil - }) - }) - g.Go(func() error { - return m.copyWithPoolConn(gCtx, func(ctx context.Context, tx pgx.Tx) error { + }}, + {copyOperationsAccounts, func(ctx context.Context, tx pgx.Tx) error { return m.models.Operations.BatchCopyAccounts(ctx, tx, ops, opParticipants) - }) - }) - g.Go(func() error { - return m.copyWithPoolConn(gCtx, func(ctx context.Context, tx pgx.Tx) error { + }}, + {copyStateChanges, func(ctx context.Context, tx pgx.Tx) error { if _, err := m.models.StateChanges.BatchCopy(ctx, tx, stateChanges); err != nil { return fmt.Errorf("copying state changes: %w", err) } return nil - }) - }) + }}, + } - // 4 upsert goroutines — skipped in backfill mode (balance tables represent current state) - if m.ingestionMode != IngestionModeBackfill { - g.Go(func() error { - return m.copyWithPoolConn(gCtx, func(ctx context.Context, tx pgx.Tx) error { + // 4 upsert operations — skipped in backfill mode (balance tables represent current state). + // Mark skipped tables as done so allDone() only reflects actually-attempted tables. + if m.ingestionMode == IngestionModeBackfill { + result.done[copyTrustlineBalances] = true + result.done[copyNativeBalances] = true + result.done[copySACBalances] = true + result.done[copyContractTokens] = true + } else { + copyOps = append(copyOps, + copyOp{copyTrustlineBalances, func(ctx context.Context, tx pgx.Tx) error { return m.tokenIngestionService.ProcessTrustlineChanges(ctx, tx, buffer.GetTrustlineChanges()) - }) - }) - g.Go(func() error { - return m.copyWithPoolConn(gCtx, func(ctx context.Context, tx pgx.Tx) error { + }}, + copyOp{copyNativeBalances, func(ctx context.Context, tx pgx.Tx) error { return m.tokenIngestionService.ProcessNativeBalanceChanges(ctx, tx, buffer.GetAccountChanges()) - }) - }) - g.Go(func() error { - return m.copyWithPoolConn(gCtx, func(ctx context.Context, tx pgx.Tx) error { + }}, + copyOp{copySACBalances, func(ctx context.Context, tx pgx.Tx) error { return m.tokenIngestionService.ProcessSACBalanceChanges(ctx, tx, buffer.GetSACBalanceChanges()) - }) - }) - g.Go(func() error { - return m.copyWithPoolConn(gCtx, func(ctx context.Context, tx pgx.Tx) error { + }}, + copyOp{copyContractTokens, func(ctx context.Context, tx pgx.Tx) error { return m.tokenIngestionService.ProcessContractTokenChanges(ctx, tx, buffer.GetContractChanges()) - }) - }) + }}, + ) + } + + var wg sync.WaitGroup + for _, op := range copyOps { + if result.done[op.table] { + continue // Already committed on a previous attempt + } + wg.Add(1) + go func(idx copyTable, fn func(context.Context, pgx.Tx) error) { + defer wg.Done() + if err := m.copyWithPoolConn(ctx, fn); err != nil { + result.errs[idx] = err + return + } + result.done[idx] = true + }(op.table, op.fn) } + wg.Wait() - if err := g.Wait(); err != nil { - return 0, 0, fmt.Errorf("parallel insert/upsert: %w", err) + if !result.allDone() { + return 0, 0, fmt.Errorf("parallel insert: %w", result.firstError()) } m.recordStateChangeMetrics(stateChanges) @@ -311,8 +412,7 @@ func (m *ingestService) insertAndUpsertParallel(ctx context.Context, txs []*type } // copyWithPoolConn acquires a connection from the pool, begins a transaction, runs fn, -// and commits. On UniqueViolation the insert is idempotent (prior partial insert) so we -// return nil. +// and commits. func (m *ingestService) copyWithPoolConn(ctx context.Context, fn func(context.Context, pgx.Tx) error) error { conn, err := m.models.DB.Acquire(ctx) if err != nil { @@ -327,15 +427,12 @@ func (m *ingestService) copyWithPoolConn(ctx context.Context, fn func(context.Co defer tx.Rollback(ctx) //nolint:errcheck // Ingestion is idempotent (replayed from cursor on crash), so WAL fsync durability - // is unnecessary. This eliminates fsync latency from each of the 9 parallel commits. + // is unnecessary. This eliminates fsync latency from each of the parallel commits. if _, err := tx.Exec(ctx, "SET LOCAL synchronous_commit = off"); err != nil { return fmt.Errorf("setting synchronous_commit=off: %w", err) } if err := fn(ctx, tx); err != nil { - if isUniqueViolation(err) { - return nil - } return err } @@ -345,12 +442,6 @@ func (m *ingestService) copyWithPoolConn(ctx context.Context, fn func(context.Co return nil } -// isUniqueViolation checks if an error is a PostgreSQL unique_violation (23505). -func isUniqueViolation(err error) bool { - var pgErr *pgconn.PgError - return errors.As(err, &pgErr) && pgErr.Code == "23505" -} - // recordStateChangeMetrics aggregates state changes by reason and category, then records metrics. func (m *ingestService) recordStateChangeMetrics(stateChanges []types.StateChange) { counts := make(map[string]int) // key: "reason|category" diff --git a/internal/services/ingest_backfill.go b/internal/services/ingest_backfill.go index 62f8c71de..efa033663 100644 --- a/internal/services/ingest_backfill.go +++ b/internal/services/ingest_backfill.go @@ -2,52 +2,29 @@ package services import ( "context" + "errors" "fmt" "sync" "time" "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/pgxpool" "github.com/stellar/go-stellar-sdk/ingest/ledgerbackend" "github.com/stellar/go-stellar-sdk/support/log" + "github.com/stellar/go-stellar-sdk/xdr" "github.com/stellar/wallet-backend/internal/data" "github.com/stellar/wallet-backend/internal/db" "github.com/stellar/wallet-backend/internal/indexer" ) -// BackfillBatch represents a contiguous range of ledgers to process as a unit. -type BackfillBatch struct { - StartLedger uint32 - EndLedger uint32 +// backfillBufferPool reuses IndexerBuffers across flush cycles to avoid +// re-allocating 11 maps + 2 slices per batch. Clear() preserves capacity. +var backfillBufferPool = sync.Pool{ + New: func() any { return indexer.NewIndexerBuffer() }, } -// BackfillResult tracks the outcome of processing a single batch. -type BackfillResult struct { - Batch BackfillBatch - LedgersCount int - Duration time.Duration - Error error - StartTime time.Time // First ledger close time in batch (for compression) - EndTime time.Time // Last ledger close time in batch (for compression) -} - -// analyzeBatchResults aggregates backfill batch results and logs any failures. -func analyzeBatchResults(ctx context.Context, results []BackfillResult) int { - numFailed := 0 - for _, result := range results { - if result.Error != nil { - numFailed++ - log.Ctx(ctx).Errorf("Batch [%d-%d] failed: %v", - result.Batch.StartLedger, result.Batch.EndLedger, result.Error) - } - } - log.Ctx(ctx).Infof("Backfilling completed: %d/%d batches failed", numFailed, len(results)) - return numFailed -} - -// startBackfilling processes ledgers in the specified range, identifying gaps -// and processing them in parallel batches for historical backfill. +// startBackfilling identifies gaps in the ledger range and fills them +// sequentially via a 3-stage pipeline (dispatcher → process → flush). func (m *ingestService) startBackfilling(ctx context.Context, startLedger, endLedger uint32) error { if startLedger > endLedger { return fmt.Errorf("start ledger cannot be greater than end ledger") @@ -71,27 +48,49 @@ func (m *ingestService) startBackfilling(ctx context.Context, startLedger, endLe return nil } - backfillBatches := m.splitGapsIntoBatches(gaps) - - // Create progressive recompressor that compresses chunks as contiguous batches complete. + // Must match ingest.hypertables — duplicated here to avoid import cycle + // (ingest imports services). tables := []string{ "transactions", "transactions_accounts", "operations", "operations_accounts", "state_changes", } - recompressor := newProgressiveRecompressor(ctx, m.models.DB, tables, len(backfillBatches)) - startTime := time.Now() - results := m.processBackfillBatchesParallel(ctx, backfillBatches, recompressor) - duration := time.Since(startTime) + // Fetch boundary timestamps for chunk pre-creation. + rangeStartTime, err := m.fetchLedgerCloseTime(ctx, gaps[0].GapStart) + if err != nil { + return fmt.Errorf("fetching start ledger %d timestamp: %w", gaps[0].GapStart, err) + } + rangeEndTime, err := m.fetchLedgerCloseTime(ctx, gaps[len(gaps)-1].GapEnd) + if err != nil { + return fmt.Errorf("fetching end ledger %d timestamp: %w", gaps[len(gaps)-1].GapEnd, err) + } - analyzeBatchResults(ctx, results) + // Suppress insert-triggered autovacuum on parent hypertables during backfill. + if err := db.DisableInsertAutovacuum(ctx, m.models.DB, tables); err != nil { + return fmt.Errorf("disabling insert autovacuum: %w", err) + } + defer func() { + if restoreErr := db.RestoreInsertAutovacuum(ctx, m.models.DB, tables); restoreErr != nil { + log.Ctx(ctx).Warnf("Failed to restore insert autovacuum: %v", restoreErr) + } + }() - // Wait for progressive compression to finish. - // Compression proceeds even if some batches failed — already-compressed - // chunks contain valid data and compress_chunk is idempotent. - recompressor.Wait() + // Pre-create chunks with indexes dropped, UNLOGGED, and per-chunk autovacuum disabled. + // Discard []*Chunk — progressive compression will be added separately. + if _, err := db.PreCreateChunks(ctx, m.models.DB, tables, rangeStartTime, rangeEndTime); err != nil { + return fmt.Errorf("pre-creating chunks: %w", err) + } + + overallStart := time.Now() + for i, gap := range gaps { + log.Ctx(ctx).Infof("Processing gap %d/%d [%d - %d]", i+1, len(gaps), gap.GapStart, gap.GapEnd) + if err := m.processGap(ctx, gap); err != nil { + log.Ctx(ctx).Errorf("Gap %d/%d [%d - %d] failed: %v", i+1, len(gaps), gap.GapStart, gap.GapEnd, err) + continue + } + } - log.Ctx(ctx).Infof("Backfilling completed in %v: %d batches", duration, len(backfillBatches)) + log.Ctx(ctx).Infof("Backfilling completed in %v: %d gaps", time.Since(overallStart), len(gaps)) return nil } @@ -156,360 +155,358 @@ func (m *ingestService) calculateBackfillGaps(ctx context.Context, startLedger, return newGaps, nil } -// splitGapsIntoBatches divides ledger gaps into fixed-size batches for parallel processing. -func (m *ingestService) splitGapsIntoBatches(gaps []data.LedgerRange) []BackfillBatch { - var batches []BackfillBatch - - for _, gap := range gaps { - start := gap.GapStart - for start <= gap.GapEnd { - end := min(start+m.backfillBatchSize-1, gap.GapEnd) - batches = append(batches, BackfillBatch{ - StartLedger: start, - EndLedger: end, - }) - start = end + 1 - } - } - - return batches -} +// processGap runs the 3-stage pipeline for a single contiguous gap: +// 1. Fetcher: parallel S3 downloads → ledgerCh +// 2. Process workers: N goroutines process ledgers into buffers → flushCh +// 3. Flush workers: M goroutines write buffers to DB via parallel COPYs +func (m *ingestService) processGap(ctx context.Context, gap data.LedgerRange) error { + gapStart := time.Now() + gapCtx, gapCancel := context.WithCancelCause(ctx) + defer gapCancel(nil) + + ledgerCh := make(chan xdr.LedgerCloseMeta, m.backfillLedgerChanSize) + flushCh := make(chan flushItem, m.backfillFlushChanSize) + watermark := newBackfillWatermark(gap.GapStart, gap.GapEnd) + + // Set gap boundary gauges + m.appMetrics.Ingestion.BackfillGapStartLedger.Set(float64(gap.GapStart)) + m.appMetrics.Ingestion.BackfillGapEndLedger.Set(float64(gap.GapEnd)) + m.appMetrics.Ingestion.BackfillGapProgress.Set(0) + defer func() { + m.appMetrics.Ingestion.BackfillGapStartLedger.Set(0) + m.appMetrics.Ingestion.BackfillGapEndLedger.Set(0) + }() -// processBackfillBatchesParallel processes backfill batches in parallel using a worker pool. -// Data is inserted uncompressed; the progressive compressor compresses chunks via -// compress_chunk() as contiguous batches complete. -func (m *ingestService) processBackfillBatchesParallel(ctx context.Context, batches []BackfillBatch, recompressor *progressiveRecompressor) []BackfillResult { - results := make([]BackfillResult, len(batches)) - group := m.backfillPool.NewGroupContext(ctx) - - for i, batch := range batches { - group.Submit(func() { - results[i] = m.processSingleBatch(ctx, batch, i, len(batches)) - if results[i].Error == nil { - recompressor.MarkDone(i, results[i].StartTime, results[i].EndTime) + var pipelineWg sync.WaitGroup + samplerDone := make(chan struct{}) + + // Channel utilization sampler — snapshots fill ratios every second. + // Not part of pipelineWg: its lifecycle is controlled by samplerDone, + // which is closed after the pipeline finishes. + go func() { + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + for { + select { + case <-samplerDone: + return + case <-gapCtx.Done(): + return + case <-ticker.C: + if cap(ledgerCh) > 0 { + m.appMetrics.Ingestion.BackfillChannelUtilization.WithLabelValues("ledger").Set( + float64(len(ledgerCh)) / float64(cap(ledgerCh)), + ) + } + if cap(flushCh) > 0 { + m.appMetrics.Ingestion.BackfillChannelUtilization.WithLabelValues("flush").Set( + float64(len(flushCh)) / float64(cap(flushCh)), + ) + } } - }) - } + } + }() - if err := group.Wait(); err != nil { - log.Ctx(ctx).Warnf("Backfill batch group wait returned error: %v", err) - } + // Stage 3: flush workers (start first so they're ready) + flushStatsCh := make(chan *backfillFlushWorkerStats, m.backfillFlushWorkers) + pipelineWg.Add(1) + go func() { + defer pipelineWg.Done() + m.runFlushWorkers(gapCtx, flushCh, watermark, m.backfillFlushWorkers, gap, flushStatsCh) + }() - return results -} + // Stage 2: process workers + processStatsCh := make(chan *backfillWorkerStats, m.backfillProcessWorkers) + pipelineWg.Add(1) + go func() { + defer pipelineWg.Done() + defer close(flushCh) + m.runProcessWorkers(gapCtx, gapCancel, ledgerCh, flushCh, processStatsCh) + }() -// processSingleBatch processes a single backfill batch with its own ledger backend. -func (m *ingestService) processSingleBatch(ctx context.Context, batch BackfillBatch, batchIndex, totalBatches int) BackfillResult { - start := time.Now() - result := BackfillResult{Batch: batch} + // Stage 1: parallel S3 fetcher (replaces dispatcher + backend) + var fetchCount int + var fetchTotal time.Duration + var fetchChannelWait map[string]time.Duration - // Setup backend - backend, err := m.setupBatchBackend(ctx, batch) - if err != nil { - result.Error = err - result.Duration = time.Since(start) - return result - } - defer func() { - if closeErr := backend.Close(); closeErr != nil { - log.Ctx(ctx).Warnf("Error closing ledger backend for batch [%d-%d]: %v", batch.StartLedger, batch.EndLedger, closeErr) - } + pipelineWg.Add(1) + go func() { + defer pipelineWg.Done() + runner := m.backfillFetcherFactory(gap.GapStart, gap.GapEnd, ledgerCh) + fetchCount, fetchTotal, fetchChannelWait = runner(gapCtx, gapCancel) }() - // Process all ledgers in batch (cursor is updated atomically with final flush) - ledgersCount, batchStartTime, batchEndTime, err := m.processLedgersInBatch(ctx, backend, batch) - result.LedgersCount = ledgersCount - result.StartTime = batchStartTime - result.EndTime = batchEndTime - if err != nil { - result.Error = err - result.Duration = time.Since(start) - return result + // Wait for all pipeline stages to complete. + // The fetcher closes ledgerCh → process workers drain and close flushCh → + // flush workers drain. Then close samplerDone to stop the utilization sampler. + pipelineWg.Wait() + close(samplerDone) + + // Aggregate gap stats from all pipeline stages + close(processStatsCh) + gapStats := newBackfillGapStats() + gapStats.fetchCount += fetchCount + gapStats.fetchTotal += fetchTotal + for k, v := range fetchChannelWait { + gapStats.channelWait[k] += v + } + for ws := range processStatsCh { + gapStats.mergeWorker(ws) + } + close(flushStatsCh) + for fs := range flushStatsCh { + gapStats.mergeFlushWorker(fs) + } + if cause := context.Cause(gapCtx); cause != nil && !errors.Is(cause, context.Canceled) { + return fmt.Errorf("pipeline failed: %w", cause) } - m.appMetrics.Ingestion.OldestLedger.Set(float64(batch.StartLedger)) - - result.Duration = time.Since(start) - log.Ctx(ctx).Infof("Batch %d/%d [%d - %d] completed: %d ledgers in %v", - batchIndex+1, totalBatches, batch.StartLedger, batch.EndLedger, result.LedgersCount, result.Duration) - - return result -} - -// setupBatchBackend creates and prepares a ledger backend for a batch range. -// Caller is responsible for calling Close() on the returned backend. -func (m *ingestService) setupBatchBackend(ctx context.Context, batch BackfillBatch) (ledgerbackend.LedgerBackend, error) { - backend, err := m.ledgerBackendFactory(ctx) - if err != nil { - return nil, fmt.Errorf("creating ledger backend: %w", err) + // Log gap summary with per-stage timing breakdown + total := gap.GapEnd - gap.GapStart + 1 + elapsed := time.Since(gapStart) + ledgersPerSec := float64(0) + if elapsed > 0 { + ledgersPerSec = float64(total) / elapsed.Seconds() } - ledgerRange := ledgerbackend.BoundedRange(batch.StartLedger, batch.EndLedger) - if err := backend.PrepareRange(ctx, ledgerRange); err != nil { - return nil, fmt.Errorf("preparing backend range: %w", err) + if watermark.Complete() { + log.Ctx(ctx).Infof("Gap [%d-%d] complete (%v, %.0f ledgers/sec):\n"+ + " fetch: %v total, %v avg (%d calls)\n"+ + " process: %v total, %v avg (%d calls)\n"+ + " flush: %v total, %v avg (%d batches)\n"+ + " channel_wait: ledger_send=%v ledger_recv=%v flush_send=%v flush_recv=%v", + gap.GapStart, gap.GapEnd, elapsed, ledgersPerSec, + gapStats.fetchTotal, avgOrZero(gapStats.fetchTotal, gapStats.fetchCount), gapStats.fetchCount, + gapStats.processTotal, avgOrZero(gapStats.processTotal, gapStats.processCount), gapStats.processCount, + gapStats.flushTotal, avgOrZero(gapStats.flushTotal, gapStats.flushCount), gapStats.flushCount, + gapStats.channelWait["ledger:send"], + gapStats.channelWait["ledger:receive"], + gapStats.channelWait["flush:send"], + gapStats.channelWait["flush:receive"], + ) + } else { + log.Ctx(ctx).Warnf("Gap [%d-%d] partial: cursor at %d of %d (%v)", + gap.GapStart, gap.GapEnd, watermark.Cursor(), gap.GapEnd, elapsed) } - return backend, nil + return nil } -// flushBatchBufferWithRetry persists buffered data to the database within a transaction. -// If updateCursorTo is non-nil, it also updates the oldest cursor atomically. -func (m *ingestService) flushBatchBufferWithRetry(ctx context.Context, buffer *indexer.IndexerBuffer, updateCursorTo *uint32) error { - var lastErr error - for attempt := 0; attempt < maxIngestProcessedDataRetries; attempt++ { - select { - case <-ctx.Done(): - return fmt.Errorf("context cancelled: %w", ctx.Err()) - default: - } - - err := db.RunInTransaction(ctx, m.models.DB, func(dbTx pgx.Tx) error { - // Disable synchronous commit for this transaction only — safe for backfill - // since data can be re-ingested if a crash occurs before WAL flush. - if _, txErr := dbTx.Exec(ctx, "SET LOCAL synchronous_commit = off"); txErr != nil { - return fmt.Errorf("setting synchronous_commit=off: %w", txErr) - } - txs := buffer.GetTransactions() - if _, _, err := m.insertAndUpsertParallel(ctx, txs, buffer); err != nil { - return fmt.Errorf("inserting processed data into db: %w", err) - } - // Update cursor atomically with data insertion if requested - if updateCursorTo != nil { - if err := m.models.IngestStore.UpdateMin(ctx, dbTx, m.oldestLedgerCursorName, *updateCursorTo); err != nil { - return fmt.Errorf("updating oldest cursor: %w", err) +// runProcessWorkers is Stage 2: N workers pull from ledgerCh, process ledgers +// into IndexerBuffers, and send filled buffers to flushCh. +// Each worker sends its stats to statsCh on exit for gap summary aggregation. +func (m *ingestService) runProcessWorkers( + ctx context.Context, + cancel context.CancelCauseFunc, + ledgerCh <-chan xdr.LedgerCloseMeta, + flushCh chan<- flushItem, + statsCh chan<- *backfillWorkerStats, +) { + var wg sync.WaitGroup + for range m.backfillProcessWorkers { + wg.Add(1) + go func() { + defer wg.Done() + stats := &backfillWorkerStats{} + defer func() { statsCh <- stats }() + + buffer := backfillBufferPool.Get().(*indexer.IndexerBuffer) + buffer.Clear() + var ledgers []uint32 + + flush := func() { + if len(ledgers) == 0 { + return + } + sendStart := time.Now() + select { + case flushCh <- flushItem{Buffer: buffer, Ledgers: ledgers}: + sendDur := time.Since(sendStart) + stats.addChannelWait("flush", "send", sendDur) + m.appMetrics.Ingestion.BackfillChannelWait.WithLabelValues("flush", "send").Observe(sendDur.Seconds()) + case <-ctx.Done(): + return } + buffer = backfillBufferPool.Get().(*indexer.IndexerBuffer) + buffer.Clear() + ledgers = nil } - return nil - }) - if err == nil { - return nil - } - lastErr = err - m.appMetrics.Ingestion.RetriesTotal.WithLabelValues("batch_flush").Inc() - - backoff := time.Duration(1< maxIngestProcessedDataRetryBackoff { - backoff = maxIngestProcessedDataRetryBackoff - } - log.Ctx(ctx).Warnf("Error flushing batch buffer (attempt %d/%d): %v, retrying in %v...", - attempt+1, maxIngestProcessedDataRetries, lastErr, backoff) - select { - case <-ctx.Done(): - return fmt.Errorf("context cancelled during backoff: %w", ctx.Err()) - case <-time.After(backoff): - } - } - m.appMetrics.Ingestion.RetryExhaustionsTotal.WithLabelValues("batch_flush").Inc() - return lastErr -} + for { + recvStart := time.Now() + lcm, ok := <-ledgerCh + if !ok { + break + } + recvDur := time.Since(recvStart) + stats.addChannelWait("ledger", "receive", recvDur) + m.appMetrics.Ingestion.BackfillChannelWait.WithLabelValues("ledger", "receive").Observe(recvDur.Seconds()) -// processLedgersInBatch processes all ledgers in a batch, flushing to DB periodically. -// The cursor is updated atomically with the final data flush. -// Returns the count of ledgers processed and the time range of the batch. -func (m *ingestService) processLedgersInBatch( - ctx context.Context, - backend ledgerbackend.LedgerBackend, - batch BackfillBatch, -) (int, time.Time, time.Time, error) { - batchBuffer := indexer.NewIndexerBuffer() - ledgersInBuffer := uint32(0) - ledgersProcessed := 0 - var startTime, endTime time.Time - - for ledgerSeq := batch.StartLedger; ledgerSeq <= batch.EndLedger; ledgerSeq++ { - ledgerMeta, err := m.getLedgerWithRetry(ctx, backend, ledgerSeq) - if err != nil { - return ledgersProcessed, startTime, endTime, fmt.Errorf("getting ledger %d: %w", ledgerSeq, err) - } + if ctx.Err() != nil { + return + } - // Track time range for compression - ledgerTime := ledgerMeta.ClosedAt() - if startTime.IsZero() { - startTime = ledgerTime - } - endTime = ledgerTime + processStart := time.Now() + if err := m.processLedgerSequential(ctx, lcm, buffer); err != nil { + cancel(fmt.Errorf("processing ledger %d: %w", lcm.LedgerSequence(), err)) + return + } + processDur := time.Since(processStart) + stats.addProcess(processDur) + m.appMetrics.Ingestion.PhaseDuration.WithLabelValues("backfill_process").Observe(processDur.Seconds()) - if err := m.processLedger(ctx, ledgerMeta, batchBuffer); err != nil { - return ledgersProcessed, startTime, endTime, fmt.Errorf("processing ledger %d: %w", ledgerSeq, err) - } - ledgersProcessed++ - ledgersInBuffer++ + ledgers = append(ledgers, lcm.LedgerSequence()) - // Flush buffer periodically to control memory usage (intermediate flushes, no cursor update) - if ledgersInBuffer >= m.backfillDBInsertBatchSize { - if err := m.flushBatchBufferWithRetry(ctx, batchBuffer, nil); err != nil { - return ledgersProcessed, startTime, endTime, err + if uint32(len(ledgers)) >= m.backfillFlushBatchSize { + flush() + } } - batchBuffer.Clear() - ledgersInBuffer = 0 - } - } - // Final flush with cursor update - if ledgersInBuffer > 0 { - if err := m.flushBatchBufferWithRetry(ctx, batchBuffer, &batch.StartLedger); err != nil { - return ledgersProcessed, startTime, endTime, err - } - } else { - // All data was flushed in intermediate batches, but we still need to update the cursor - // This happens when ledgersInBuffer == 0 (exact multiple of batch size) - if err := m.updateOldestCursor(ctx, batch.StartLedger); err != nil { - return ledgersProcessed, startTime, endTime, err - } + flush() + }() } - - return ledgersProcessed, startTime, endTime, nil + wg.Wait() } -// updateOldestCursor updates the oldest ledger cursor to the given ledger. -func (m *ingestService) updateOldestCursor(ctx context.Context, ledgerSeq uint32) error { - err := db.RunInTransaction(ctx, m.models.DB, func(dbTx pgx.Tx) error { - return m.models.IngestStore.UpdateMin(ctx, dbTx, m.oldestLedgerCursorName, ledgerSeq) - }) - if err != nil { - return fmt.Errorf("updating oldest ledger cursor: %w", err) - } - return nil -} - -// progressiveRecompressor compresses uncompressed TimescaleDB chunks as they become safe during backfill. -// Tracks batch completion via a watermark to determine when chunks are fully written. -type progressiveRecompressor struct { - pool *pgxpool.Pool - tables []string - ctx context.Context - - mu sync.Mutex - completed []bool - endTimes []time.Time - watermarkIdx int // index of highest contiguous completed batch (-1 = none) - globalStart time.Time // lower bound for chunk queries (batch 0's StartTime) - - triggerCh chan time.Time // safeEnd for recompression window - done chan struct{} +// flushItem is the unit of work sent from process workers to flush workers. +// Contains a filled IndexerBuffer and the ledger sequences it covers. +type flushItem struct { + Buffer *indexer.IndexerBuffer + Ledgers []uint32 } -// newProgressiveRecompressor creates a compressor that progressively compresses uncompressed chunks -// as contiguous batches complete. Starts a background goroutine for compression work. -func newProgressiveRecompressor(ctx context.Context, pool *pgxpool.Pool, tables []string, totalBatches int) *progressiveRecompressor { - r := &progressiveRecompressor{ - pool: pool, - tables: tables, - ctx: ctx, - completed: make([]bool, totalBatches), - endTimes: make([]time.Time, totalBatches), - watermarkIdx: -1, - triggerCh: make(chan time.Time, totalBatches), - done: make(chan struct{}), - } - go r.runCompression() - return r -} +// runFlushWorkers starts M flush workers that read from flushCh, +// write data to DB, and report flushed ledgers to the watermark. +// Each worker sends its stats to statsCh on exit for gap summary aggregation. +func (m *ingestService) runFlushWorkers( + ctx context.Context, + flushCh <-chan flushItem, + watermark *backfillWatermark, + numWorkers int, + gap data.LedgerRange, + statsCh chan<- *backfillFlushWorkerStats, +) { + var wg sync.WaitGroup + for i := range numWorkers { + wg.Add(1) + go func(workerID int) { + defer wg.Done() + stats := &backfillFlushWorkerStats{} + defer func() { statsCh <- stats }() + + for { + recvStart := time.Now() + item, ok := <-flushCh + if !ok { + return + } + recvDur := time.Since(recvStart) + stats.addChannelWait("flush", "receive", recvDur) + m.appMetrics.Ingestion.BackfillChannelWait.WithLabelValues("flush", "receive").Observe(recvDur.Seconds()) -// MarkDone records a batch as complete and advances the watermark if possible. -// If the watermark advances, triggers recompression of chunks in the safe window. -func (r *progressiveRecompressor) MarkDone(batchIdx int, startTime, endTime time.Time) { - var safeEnd time.Time - r.mu.Lock() - r.completed[batchIdx] = true - r.endTimes[batchIdx] = endTime - - // Record global start from batch 0 (earliest time boundary for queries) - if batchIdx == 0 { - r.globalStart = startTime - } + if ctx.Err() != nil { + return + } - // Advance watermark past contiguous completed batches - oldWatermark := r.watermarkIdx - for r.watermarkIdx+1 < len(r.completed) && r.completed[r.watermarkIdx+1] { - r.watermarkIdx++ - } + m.appMetrics.Ingestion.BackfillBatchSize.Observe(float64(len(item.Ledgers))) - sendToChannel := (r.watermarkIdx > oldWatermark) - if sendToChannel { - safeEnd = r.endTimes[r.watermarkIdx] - } - r.mu.Unlock() + flushStart := time.Now() + if err := m.flushBufferWithRetry(ctx, item.Buffer); err != nil { + log.Ctx(ctx).Errorf("Flush worker %d: %d ledgers failed: %v", + workerID, len(item.Ledgers), err) + item.Buffer.Clear() + backfillBufferPool.Put(item.Buffer) + continue + } + flushDur := time.Since(flushStart) + stats.addFlush(flushDur) + m.appMetrics.Ingestion.PhaseDuration.WithLabelValues("backfill_flush").Observe(flushDur.Seconds()) + m.appMetrics.Ingestion.BackfillLedgersFlushed.Add(float64(len(item.Ledgers))) + + if advanced := watermark.MarkFlushed(item.Ledgers); advanced { + gapSize := float64(gap.GapEnd - gap.GapStart + 1) + progress := float64(watermark.Cursor()-gap.GapStart+1) / gapSize + m.appMetrics.Ingestion.BackfillGapProgress.Set(progress) + + if err := m.updateOldestCursor(ctx, watermark.Cursor()); err != nil { + log.Ctx(ctx).Warnf("Flush worker %d: cursor update failed: %v", + workerID, err) + } + } - // If watermark advanced then we trigger recompression outside the lock - if sendToChannel { - r.triggerCh <- safeEnd + item.Buffer.Clear() + backfillBufferPool.Put(item.Buffer) + } + }(i) } + wg.Wait() } -// Wait closes the trigger channel and waits for background compression to finish. -func (r *progressiveRecompressor) Wait() { - close(r.triggerCh) - <-r.done -} - -// runCompression processes compression triggers in the background. -// For each safe window, queries and compresses uncompressed chunks per table. -func (r *progressiveRecompressor) runCompression() { - defer close(r.done) +// flushBufferWithRetry persists a buffer's data to DB via parallel COPYs +// with exponential backoff retry. A CopyResult tracks which tables have already +// committed — on retry, only failed tables are re-attempted, preventing duplicates. +func (m *ingestService) flushBufferWithRetry(ctx context.Context, buffer *indexer.IndexerBuffer) error { + txs := buffer.GetTransactions() + result := NewCopyResult() - totalCompressed := 0 - for safeEnd := range r.triggerCh { - for _, table := range r.tables { - count := r.compressTableChunks(table, safeEnd) - totalCompressed += count + var lastErr error + for attempt := range maxIngestProcessedDataRetries { + select { + case <-ctx.Done(): + return fmt.Errorf("context cancelled: %w", ctx.Err()) + default: } - } - log.Ctx(r.ctx).Infof("Progressive compression complete: %d total chunks compressed", totalCompressed) -} + if _, _, err := m.insertParallel(ctx, txs, buffer, result); err != nil { + lastErr = err + m.appMetrics.Ingestion.RetriesTotal.WithLabelValues("batch_flush").Inc() -// compressTableChunks compresses uncompressed chunks for a single table within the safe window. -// Queries chunks where range_end falls within (globalStart, safeEnd) to catch the leading -// boundary chunk that overlaps globalStart. -func (r *progressiveRecompressor) compressTableChunks(table string, safeEnd time.Time) int { - rows, err := r.pool.Query(r.ctx, - `SELECT c.chunk_schema || '.' || c.chunk_name - FROM timescaledb_information.chunks c - WHERE c.hypertable_name = $1 - AND NOT c.is_compressed - AND c.range_end < $2::timestamptz - AND c.range_end > $3::timestamptz`, - table, safeEnd, r.globalStart) - if err != nil { - log.Ctx(r.ctx).Warnf("Failed to get chunks for %s: %v", table, err) - return 0 - } - defer rows.Close() + backoff := min(time.Duration(1< 0 { - log.Ctx(r.ctx).Infof("Compressed %d chunks for table %s (window end: %s)", - compressed, table, safeEnd.Format(time.RFC3339)) + meta, err := backend.GetLedger(ctx, ledger) + if err != nil { + return time.Time{}, fmt.Errorf("getting ledger %d: %w", ledger, err) } + return meta.ClosedAt(), nil +} - return compressed +// updateOldestCursor updates the oldest ledger cursor to the given ledger. +func (m *ingestService) updateOldestCursor(ctx context.Context, ledgerSeq uint32) error { + err := db.RunInTransaction(ctx, m.models.DB, func(dbTx pgx.Tx) error { + return m.models.IngestStore.UpdateMin(ctx, dbTx, m.oldestLedgerCursorName, ledgerSeq) + }) + if err != nil { + return fmt.Errorf("updating oldest ledger cursor: %w", err) + } + return nil } diff --git a/internal/services/ingest_backfill_test.go b/internal/services/ingest_backfill_test.go deleted file mode 100644 index afbcf4c73..000000000 --- a/internal/services/ingest_backfill_test.go +++ /dev/null @@ -1,176 +0,0 @@ -// Tests for progressive recompression watermark logic during historical backfill. -package services - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// newTestRecompressor creates a progressiveRecompressor for testing watermark logic. -// No background goroutine is started; triggerCh is buffered for direct inspection. -func newTestRecompressor(totalBatches int) *progressiveRecompressor { - return &progressiveRecompressor{ - completed: make([]bool, totalBatches), - endTimes: make([]time.Time, totalBatches), - watermarkIdx: -1, - triggerCh: make(chan time.Time, totalBatches), - } -} - -// drainWindows reads all available safeEnd values from triggerCh without blocking. -func drainWindows(r *progressiveRecompressor) []time.Time { - var windows []time.Time - for { - select { - case w := <-r.triggerCh: - windows = append(windows, w) - default: - return windows - } - } -} - -func Test_progressiveRecompressor_MarkDone_sequential(t *testing.T) { - r := newTestRecompressor(5) - base := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC) - - // Complete batches 0-4 in order - for i := 0; i < 5; i++ { - startTime := base.Add(time.Duration(i) * time.Hour) - endTime := base.Add(time.Duration(i+1) * time.Hour) - r.MarkDone(i, startTime, endTime) - } - - windows := drainWindows(r) - // Each call advances the watermark — expect 5 windows - require.Len(t, windows, 5) - - // First window: safeEnd = batch 0 endTime - assert.Equal(t, base.Add(1*time.Hour), windows[0]) - - // Second window: safeEnd = batch 1 endTime - assert.Equal(t, base.Add(2*time.Hour), windows[1]) - - // Last window: safeEnd = batch 4 endTime - assert.Equal(t, base.Add(5*time.Hour), windows[4]) - - // Verify globalStart set from batch 0 - assert.Equal(t, base, r.globalStart) -} - -func Test_progressiveRecompressor_MarkDone_outOfOrder(t *testing.T) { - r := newTestRecompressor(5) - base := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC) - endTime := func(i int) time.Time { return base.Add(time.Duration(i+1) * time.Hour) } - startTime := func(i int) time.Time { return base.Add(time.Duration(i) * time.Hour) } - - // Complete batch 2 first — watermark can't advance (batch 0 not done) - r.MarkDone(2, startTime(2), endTime(2)) - windows := drainWindows(r) - assert.Empty(t, windows) - assert.Equal(t, -1, r.watermarkIdx) - - // Complete batch 4 — still no advancement - r.MarkDone(4, startTime(4), endTime(4)) - windows = drainWindows(r) - assert.Empty(t, windows) - - // Complete batch 0 — watermark advances to 0 only (batch 1 missing) - r.MarkDone(0, startTime(0), endTime(0)) - windows = drainWindows(r) - require.Len(t, windows, 1) - assert.Equal(t, endTime(0), windows[0]) - assert.Equal(t, 0, r.watermarkIdx) - - // Complete batch 1 — watermark jumps from 0 to 2 (batch 2 was already done) - r.MarkDone(1, startTime(1), endTime(1)) - windows = drainWindows(r) - require.Len(t, windows, 1) - assert.Equal(t, endTime(2), windows[0]) // safeEnd = batch 2's endTime - assert.Equal(t, 2, r.watermarkIdx) - - // Complete batch 3 — watermark jumps from 2 to 4 (batch 4 was already done) - r.MarkDone(3, startTime(3), endTime(3)) - windows = drainWindows(r) - require.Len(t, windows, 1) - assert.Equal(t, endTime(4), windows[0]) - assert.Equal(t, 4, r.watermarkIdx) -} - -func Test_progressiveRecompressor_MarkDone_failedBatchBlocksWatermark(t *testing.T) { - r := newTestRecompressor(5) - base := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC) - endTime := func(i int) time.Time { return base.Add(time.Duration(i+1) * time.Hour) } - startTime := func(i int) time.Time { return base.Add(time.Duration(i) * time.Hour) } - - // Complete batches 0 and 1 - r.MarkDone(0, startTime(0), endTime(0)) - r.MarkDone(1, startTime(1), endTime(1)) - _ = drainWindows(r) // consume windows - - // Batch 2 fails (never call MarkDone for it) - - // Complete batches 3 and 4 - r.MarkDone(3, startTime(3), endTime(3)) - r.MarkDone(4, startTime(4), endTime(4)) - - // No new windows — watermark stuck at 1 - windows := drainWindows(r) - assert.Empty(t, windows) - assert.Equal(t, 1, r.watermarkIdx) -} - -func Test_progressiveRecompressor_MarkDone_singleBatch(t *testing.T) { - r := newTestRecompressor(1) - start := time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC) - end := time.Date(2025, 6, 1, 1, 0, 0, 0, time.UTC) - - r.MarkDone(0, start, end) - - windows := drainWindows(r) - require.Len(t, windows, 1) - assert.Equal(t, end, windows[0]) - assert.Equal(t, 0, r.watermarkIdx) - assert.Equal(t, start, r.globalStart) -} - -func Test_progressiveRecompressor_MarkDone_globalStartSetFromBatchZero(t *testing.T) { - r := newTestRecompressor(3) - base := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC) - - // Complete batch 1 first — globalStart should NOT be set - r.MarkDone(1, base.Add(1*time.Hour), base.Add(2*time.Hour)) - assert.True(t, r.globalStart.IsZero()) - - // Complete batch 0 — globalStart should be set to batch 0's startTime - r.MarkDone(0, base, base.Add(1*time.Hour)) - assert.Equal(t, base, r.globalStart) - - // Complete batch 2 — globalStart unchanged - r.MarkDone(2, base.Add(2*time.Hour), base.Add(3*time.Hour)) - assert.Equal(t, base, r.globalStart) -} - -func Test_progressiveRecompressor_MarkDone_allSimultaneous(t *testing.T) { - r := newTestRecompressor(4) - base := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC) - endTime := func(i int) time.Time { return base.Add(time.Duration(i+1) * time.Hour) } - startTime := func(i int) time.Time { return base.Add(time.Duration(i) * time.Hour) } - - // Complete all batches in reverse order except batch 0 - r.MarkDone(3, startTime(3), endTime(3)) - r.MarkDone(2, startTime(2), endTime(2)) - r.MarkDone(1, startTime(1), endTime(1)) - windows := drainWindows(r) - assert.Empty(t, windows) // Nothing yet — batch 0 missing - - // Complete batch 0 — watermark jumps from -1 to 3 in one step - r.MarkDone(0, startTime(0), endTime(0)) - windows = drainWindows(r) - require.Len(t, windows, 1) - assert.Equal(t, endTime(3), windows[0]) // safeEnd = last batch - assert.Equal(t, 3, r.watermarkIdx) -} diff --git a/internal/services/ingest_live.go b/internal/services/ingest_live.go index d7ed45c7a..511da2f7e 100644 --- a/internal/services/ingest_live.go +++ b/internal/services/ingest_live.go @@ -28,13 +28,13 @@ const ( // Phase 1: Insert FK prerequisites (trustline assets, contract tokens) and commit, // making parent rows visible for phase 2's balance upserts. // -// Phase 2: Parallel COPYs (5 tables) + parallel upserts (4 balance tables) via errgroup, -// each on its own pool connection. UniqueViolation errors are treated as success for -// idempotent crash recovery. +// Phase 2: Parallel COPYs (5 tables) + parallel upserts (4 balance tables), +// each on its own pool connection. Per-table success is tracked in result so +// retries skip already-committed tables (zero duplicates without PK constraints). // // Phase 3: Finalize — unlock channel accounts + advance cursor. The cursor update is the // idempotency marker; if we crash before it, everything replays safely. -func (m *ingestService) PersistLedgerData(ctx context.Context, ledgerSeq uint32, buffer *indexer.IndexerBuffer, cursorName string) (int, int, error) { +func (m *ingestService) PersistLedgerData(ctx context.Context, ledgerSeq uint32, buffer *indexer.IndexerBuffer, cursorName string, result *CopyResult) (int, int, error) { // Phase 1: FK prerequisites — commit so parent rows are visible to phase 2. if err := m.persistFKPrerequisites(ctx, ledgerSeq, buffer); err != nil { return 0, 0, fmt.Errorf("persisting FK prerequisites for ledger %d: %w", ledgerSeq, err) @@ -42,7 +42,7 @@ func (m *ingestService) PersistLedgerData(ctx context.Context, ledgerSeq uint32, // Phase 2: Parallel COPYs + upserts on separate pool connections. txs := buffer.GetTransactions() - numTxs, numOps, err := m.insertAndUpsertParallel(ctx, txs, buffer) + numTxs, numOps, err := m.insertParallel(ctx, txs, buffer, result) if err != nil { return 0, 0, fmt.Errorf("parallel insert/upsert for ledger %d: %w", ledgerSeq, err) } @@ -234,7 +234,11 @@ func (m *ingestService) ingestLiveLedgers(ctx context.Context, startLedger uint3 } // ingestProcessedDataWithRetry wraps PersistLedgerData with retry logic. +// A CopyResult is created once and passed across retries so that already-committed +// tables are skipped on subsequent attempts — preventing duplicates. func (m *ingestService) ingestProcessedDataWithRetry(ctx context.Context, currentLedger uint32, buffer *indexer.IndexerBuffer) (int, int, error) { + result := NewCopyResult() + var lastErr error for attempt := 0; attempt < maxIngestProcessedDataRetries; attempt++ { select { @@ -243,7 +247,7 @@ func (m *ingestService) ingestProcessedDataWithRetry(ctx context.Context, curren default: } - numTxs, numOps, err := m.PersistLedgerData(ctx, currentLedger, buffer, m.latestLedgerCursorName) + numTxs, numOps, err := m.PersistLedgerData(ctx, currentLedger, buffer, m.latestLedgerCursorName, result) if err == nil { return numTxs, numOps, nil } diff --git a/internal/services/ingest_test.go b/internal/services/ingest_test.go index 81b0b21fd..7aac867ad 100644 --- a/internal/services/ingest_test.go +++ b/internal/services/ingest_test.go @@ -48,6 +48,60 @@ const ( ledgerMetadataWith1Tx = "AAAAAQAAAAD8G2qemHnBKFkbq90RTagxAypNnA7DXDc63Giipq9mNwAAABYLEZ5DrTv6njXTOAFEdOO0yeLtJjCRyH4ryJkgpRh7VPJvwbisrc9A0yzFxxCdkICgB3Gv7qHOi8ZdsK2CNks2AAAAAGhTTAsAAAAAAAAAAQAAAACoJM0YvJ11Bk0pmltbrKQ7w6ovMmk4FT2ML5u1y23wMwAAAEAunZtorOSbnRpgnykoDe4kzAvLwNXefncy1R/1ynBWyDv0DfdnqJ6Hcy/0AJf6DkBZlRayg775h3HjV0GKF/oPua7l8wkLlJBtSk1kRDt55qSf6btSrgcupB/8bnpJfUUgZJ76saUrj29HukYHS1bq7SyuoCAY+5F9iBYTmW1G9QAAEX4N4Lazp2QAAAAAAAMtS3veAAAAAAAAAAAAAAAMAAAAZABMS0AAAADIXukLfWC53MCmzxKd/+LBbaYxQkgxATFDLI3hWj7EqWgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAELEZ5DrTv6njXTOAFEdOO0yeLtJjCRyH4ryJkgpRh7VAAAAAIAAAAAAAAAAQAAAAAAAAABAAAAAAAAAGQAAAABAAAAAgAAAADg4mtiLKjJVgrmOpO9+Ff3XAmnycHyNUKu/v9KhHevAAAAAGQAAA7FAAAAGgAAAAAAAAAAAAAAAQAAAAAAAAABAAAAALvqzdVyRxgBMcLzbw1wNWcJYHPNPok1GdVSgmy4sjR2AAAAAVVTREMAAAAA4OJrYiyoyVYK5jqTvfhX91wJp8nB8jVCrv7/SoR3rwAAAAACVAvkAAAAAAAAAAABhHevAAAAAEDq2yIDzXUoLboBHQkbr8U2oKqLzf0gfpwXbmRPLB6Ek3G8uCEYyry1vt5Sb+LCEd81fefFQcQN0nydr1FmiXcDAAAAAAAAAAAAAAABXFSiWcxpDRa8frBs1wbEaMUw4hMe7ctFtdw3Ci73IEwAAAAAAAAAZAAAAAAAAAABAAAAAAAAAAEAAAAAAAAAAAAAAAIAAAADAAARfQAAAAAAAAAA4OJrYiyoyVYK5jqTvfhX91wJp8nB8jVCrv7/SoR3rwAAAAAukO3GPAAADsUAAAAZAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAwAAAAAAABF9AAAAAGhTTAYAAAAAAAAAAQAAEX4AAAAAAAAAAODia2IsqMlWCuY6k734V/dcCafJwfI1Qq7+/0qEd68AAAAALpDtxdgAAA7FAAAAGQAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAMAAAAAAAARfQAAAABoU0wGAAAAAAAAAAMAAAAAAAAAAgAAAAMAABF+AAAAAAAAAADg4mtiLKjJVgrmOpO9+Ff3XAmnycHyNUKu/v9KhHevAAAAAC6Q7cXYAAAOxQAAABkAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAADAAAAAAAAEX0AAAAAaFNMBgAAAAAAAAABAAARfgAAAAAAAAAA4OJrYiyoyVYK5jqTvfhX91wJp8nB8jVCrv7/SoR3rwAAAAAukO3F2AAADsUAAAAaAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAwAAAAAAABF+AAAAAGhTTAsAAAAAAAAAAQAAAAIAAAADAAARcwAAAAEAAAAAu+rN1XJHGAExwvNvDXA1Zwlgc80+iTUZ1VKCbLiyNHYAAAABVVNEQwAAAADg4mtiLKjJVgrmOpO9+Ff3XAmnycHyNUKu/v9KhHevAAAAAAlQL5AAf/////////8AAAABAAAAAAAAAAAAAAABAAARfgAAAAEAAAAAu+rN1XJHGAExwvNvDXA1Zwlgc80+iTUZ1VKCbLiyNHYAAAABVVNEQwAAAADg4mtiLKjJVgrmOpO9+Ff3XAmnycHyNUKu/v9KhHevAAAAAAukO3QAf/////////8AAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA8RxEAAAAAAAAAAA==" ) +func Test_NewIngestService_poolValidation(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + ctx := context.Background() + + // Open a pool with a very small max connections (3) + dsn := dbt.DSN + "&pool_max_conns=3" + dbConnectionPool, err := db.OpenDBConnectionPool(ctx, dsn) + require.NoError(t, err) + defer dbConnectionPool.Close() + + m := metrics.NewMetrics(prometheus.NewRegistry()) + models, err := data.NewModels(dbConnectionPool, m.DB) + require.NoError(t, err) + + mockRPCService := &RPCServiceMock{} + mockRPCService.On("NetworkPassphrase").Return(network.TestNetworkPassphrase).Maybe() + + // 4 flush workers × 5 COPYs + 5 = 25 required, but pool only has 3 + _, err = NewIngestService(IngestServiceConfig{ + IngestionMode: IngestionModeBackfill, + Models: models, + AppTracker: &apptracker.MockAppTracker{}, + RPCService: mockRPCService, + LedgerBackend: &LedgerBackendMock{}, + Metrics: m, + Network: network.TestNetworkPassphrase, + NetworkPassphrase: network.TestNetworkPassphrase, + Archive: &HistoryArchiveMock{}, + BackfillFlushWorkers: 4, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "pgxpool max connections") + + // Same config in live mode should NOT error (validation only applies to backfill) + m2 := metrics.NewMetrics(prometheus.NewRegistry()) + models2, err := data.NewModels(dbConnectionPool, m2.DB) + require.NoError(t, err) + + _, err = NewIngestService(IngestServiceConfig{ + IngestionMode: IngestionModeLive, + Models: models2, + AppTracker: &apptracker.MockAppTracker{}, + RPCService: mockRPCService, + LedgerBackend: &LedgerBackendMock{}, + Metrics: m2, + Network: network.TestNetworkPassphrase, + NetworkPassphrase: network.TestNetworkPassphrase, + Archive: &HistoryArchiveMock{}, + BackfillFlushWorkers: 4, + }) + require.NoError(t, err) +} + func Test_generateAdvisoryLockID(t *testing.T) { testCases := []struct { name string @@ -88,99 +142,6 @@ func Test_generateAdvisoryLockID(t *testing.T) { } } -func Test_ingestService_splitGapsIntoBatches(t *testing.T) { - svc := &ingestService{} - - testCases := []struct { - name string - gaps []data.LedgerRange - batchSize uint32 - expected []BackfillBatch - }{ - { - name: "empty_gaps", - gaps: []data.LedgerRange{}, - batchSize: 100, - expected: nil, - }, - { - name: "single_gap_smaller_than_batch", - gaps: []data.LedgerRange{ - {GapStart: 100, GapEnd: 150}, - }, - batchSize: 200, - expected: []BackfillBatch{ - {StartLedger: 100, EndLedger: 150}, - }, - }, - { - name: "single_gap_larger_than_batch", - gaps: []data.LedgerRange{ - {GapStart: 100, GapEnd: 399}, - }, - batchSize: 100, - expected: []BackfillBatch{ - {StartLedger: 100, EndLedger: 199}, - {StartLedger: 200, EndLedger: 299}, - {StartLedger: 300, EndLedger: 399}, - }, - }, - { - name: "single_gap_exact_batch_size", - gaps: []data.LedgerRange{ - {GapStart: 100, GapEnd: 199}, - }, - batchSize: 100, - expected: []BackfillBatch{ - {StartLedger: 100, EndLedger: 199}, - }, - }, - { - name: "multiple_gaps", - gaps: []data.LedgerRange{ - {GapStart: 100, GapEnd: 149}, - {GapStart: 300, GapEnd: 349}, - }, - batchSize: 100, - expected: []BackfillBatch{ - {StartLedger: 100, EndLedger: 149}, - {StartLedger: 300, EndLedger: 349}, - }, - }, - { - name: "multiple_gaps_with_splits", - gaps: []data.LedgerRange{ - {GapStart: 100, GapEnd: 249}, - {GapStart: 500, GapEnd: 599}, - }, - batchSize: 100, - expected: []BackfillBatch{ - {StartLedger: 100, EndLedger: 199}, - {StartLedger: 200, EndLedger: 249}, - {StartLedger: 500, EndLedger: 599}, - }, - }, - { - name: "single_ledger_gap", - gaps: []data.LedgerRange{ - {GapStart: 100, GapEnd: 100}, - }, - batchSize: 100, - expected: []BackfillBatch{ - {StartLedger: 100, EndLedger: 100}, - }, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - svc.backfillBatchSize = tc.batchSize - result := svc.splitGapsIntoBatches(tc.gaps) - assert.Equal(t, tc.expected, result) - }) - } -} - func Test_ingestService_calculateBackfillGaps(t *testing.T) { dbt := dbtest.Open(t) defer dbt.Close() @@ -336,6 +297,7 @@ func Test_ingestService_calculateBackfillGaps(t *testing.T) { Network: network.TestNetworkPassphrase, NetworkPassphrase: network.TestNetworkPassphrase, Archive: mockArchive, + BackfillFlushWorkers: 1, }) require.NoError(t, err) @@ -430,7 +392,7 @@ func Test_Backfill_Validation(t *testing.T) { Network: network.TestNetworkPassphrase, NetworkPassphrase: network.TestNetworkPassphrase, Archive: mockArchive, - BackfillBatchSize: 100, + BackfillFlushWorkers: 1, }) require.NoError(t, err) @@ -439,8 +401,8 @@ func Test_Backfill_Validation(t *testing.T) { require.Error(t, err) assert.Contains(t, err.Error(), tc.errorContains) } else { - // For valid cases, validation passes but batch processing fails. - // Historical mode logs failures but returns nil (continues processing) + // For valid cases, validation passes but gap processing fails + // (mock factory returns error). Pipeline logs failures and continues. require.NoError(t, err) } }) @@ -463,9 +425,6 @@ func setupDBCursors(t *testing.T, ctx context.Context, pool *pgxpool.Pool, lates } } -// ptrUint32 returns a pointer to the given uint32 value -func ptrUint32(v uint32) *uint32 { return &v } - // createTestTransaction creates a transaction with required fields for testing. func createTestTransaction(hash string, toID int64) types.Transaction { now := time.Now() @@ -510,56 +469,6 @@ func createTestStateChange(toID int64, accountID string, opID int64) types.State } } -// ==================== New Tests ==================== - -func Test_analyzeBatchResults(t *testing.T) { - ctx := context.Background() - - testCases := []struct { - name string - results []BackfillResult - wantFailures int - }{ - { - name: "empty_results", - results: []BackfillResult{}, - wantFailures: 0, - }, - { - name: "all_success", - results: []BackfillResult{ - {Batch: BackfillBatch{StartLedger: 100, EndLedger: 199}, LedgersCount: 100}, - {Batch: BackfillBatch{StartLedger: 200, EndLedger: 299}, LedgersCount: 100}, - }, - wantFailures: 0, - }, - { - name: "some_failures", - results: []BackfillResult{ - {Batch: BackfillBatch{StartLedger: 100, EndLedger: 199}, LedgersCount: 100}, - {Batch: BackfillBatch{StartLedger: 200, EndLedger: 299}, Error: fmt.Errorf("failed")}, - {Batch: BackfillBatch{StartLedger: 300, EndLedger: 399}, LedgersCount: 100}, - }, - wantFailures: 1, - }, - { - name: "all_failures", - results: []BackfillResult{ - {Batch: BackfillBatch{StartLedger: 100, EndLedger: 199}, Error: fmt.Errorf("failed1")}, - {Batch: BackfillBatch{StartLedger: 200, EndLedger: 299}, Error: fmt.Errorf("failed2")}, - }, - wantFailures: 2, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - numFailed := analyzeBatchResults(ctx, tc.results) - assert.Equal(t, tc.wantFailures, numFailed) - }) - } -} - func Test_ingestService_getLedgerWithRetry(t *testing.T) { dbt := dbtest.Open(t) defer dbt.Close() @@ -646,6 +555,7 @@ func Test_ingestService_getLedgerWithRetry(t *testing.T) { Network: network.TestNetworkPassphrase, NetworkPassphrase: network.TestNetworkPassphrase, Archive: &HistoryArchiveMock{}, + BackfillFlushWorkers: 1, }) require.NoError(t, err) @@ -670,100 +580,6 @@ func Test_ingestService_getLedgerWithRetry(t *testing.T) { } } -func Test_ingestService_setupBatchBackend(t *testing.T) { - dbt := dbtest.Open(t) - defer dbt.Close() - ctx := context.Background() - dbConnectionPool, err := db.OpenDBConnectionPool(ctx, dbt.DSN) - require.NoError(t, err) - defer dbConnectionPool.Close() - - testCases := []struct { - name string - batch BackfillBatch - setupFactory func() LedgerBackendFactory - wantErr bool - wantErrContains string - }{ - { - name: "success", - batch: BackfillBatch{StartLedger: 100, EndLedger: 199}, - setupFactory: func() LedgerBackendFactory { - return func(ctx context.Context) (ledgerbackend.LedgerBackend, error) { - mockBackend := &LedgerBackendMock{} - mockBackend.On("PrepareRange", mock.Anything, ledgerbackend.BoundedRange(100, 199)).Return(nil) - return mockBackend, nil - } - }, - wantErr: false, - }, - { - name: "factory_error", - batch: BackfillBatch{StartLedger: 100, EndLedger: 199}, - setupFactory: func() LedgerBackendFactory { - return func(ctx context.Context) (ledgerbackend.LedgerBackend, error) { - return nil, fmt.Errorf("factory failed") - } - }, - wantErr: true, - wantErrContains: "creating ledger backend", - }, - { - name: "prepare_range_error", - batch: BackfillBatch{StartLedger: 100, EndLedger: 199}, - setupFactory: func() LedgerBackendFactory { - return func(ctx context.Context) (ledgerbackend.LedgerBackend, error) { - mockBackend := &LedgerBackendMock{} - mockBackend.On("PrepareRange", mock.Anything, mock.Anything).Return(fmt.Errorf("prepare failed")) - return mockBackend, nil - } - }, - wantErr: true, - wantErrContains: "preparing backend range", - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - m := metrics.NewMetrics(prometheus.NewRegistry()) - - models, err := data.NewModels(dbConnectionPool, m.DB) - require.NoError(t, err) - - mockRPCService := &RPCServiceMock{} - mockRPCService.On("NetworkPassphrase").Return(network.TestNetworkPassphrase).Maybe() - - svc, err := NewIngestService(IngestServiceConfig{ - IngestionMode: IngestionModeBackfill, - Models: models, - LatestLedgerCursorName: "latest_ledger_cursor", - OldestLedgerCursorName: "oldest_ledger_cursor", - AppTracker: &apptracker.MockAppTracker{}, - RPCService: mockRPCService, - LedgerBackend: &LedgerBackendMock{}, - LedgerBackendFactory: tc.setupFactory(), - Metrics: m, - GetLedgersLimit: defaultGetLedgersLimit, - Network: network.TestNetworkPassphrase, - NetworkPassphrase: network.TestNetworkPassphrase, - Archive: &HistoryArchiveMock{}, - }) - require.NoError(t, err) - - backend, err := svc.setupBatchBackend(ctx, tc.batch) - if tc.wantErr { - require.Error(t, err) - if tc.wantErrContains != "" { - assert.Contains(t, err.Error(), tc.wantErrContains) - } - } else { - require.NoError(t, err) - require.NotNil(t, backend) - } - }) - } -} - func Test_ingestService_updateOldestCursor(t *testing.T) { dbt := dbtest.Open(t) defer dbt.Close() @@ -824,6 +640,7 @@ func Test_ingestService_updateOldestCursor(t *testing.T) { Network: network.TestNetworkPassphrase, NetworkPassphrase: network.TestNetworkPassphrase, Archive: &HistoryArchiveMock{}, + BackfillFlushWorkers: 1, }) require.NoError(t, err) @@ -966,7 +783,7 @@ func Test_ingestService_Run(t *testing.T) { } } -func Test_ingestService_flushBatchBufferWithRetry(t *testing.T) { +func Test_ingestService_flushBufferWithRetry(t *testing.T) { dbt := dbtest.Open(t) defer dbt.Close() ctx := context.Background() @@ -975,26 +792,18 @@ func Test_ingestService_flushBatchBufferWithRetry(t *testing.T) { defer dbConnectionPool.Close() testCases := []struct { - name string - setupBuffer func() *indexer.IndexerBuffer - updateCursorTo *uint32 - initialCursor uint32 - wantCursor uint32 - wantTxCount int - wantOpCount int - wantStateChangeCount int - txHashes []string // For verification queries + name string + setupBuffer func() *indexer.IndexerBuffer + wantTxCount int + wantOpCount int + txHashes []string }{ { - name: "flush_empty_buffer_no_cursor_update", - setupBuffer: func() *indexer.IndexerBuffer { return indexer.NewIndexerBuffer() }, - updateCursorTo: nil, - initialCursor: 100, - wantCursor: 100, - wantTxCount: 0, - wantOpCount: 0, - wantStateChangeCount: 0, - txHashes: []string{}, + name: "flush_empty_buffer", + setupBuffer: func() *indexer.IndexerBuffer { return indexer.NewIndexerBuffer() }, + wantTxCount: 0, + wantOpCount: 0, + txHashes: []string{}, }, { name: "flush_with_data_inserts_to_database", @@ -1015,63 +824,25 @@ func Test_ingestService_flushBatchBufferWithRetry(t *testing.T) { buf.PushStateChange(&tx2, &op2, sc2) return buf }, - updateCursorTo: nil, - initialCursor: 100, - wantCursor: 100, - wantTxCount: 2, - wantOpCount: 2, - wantStateChangeCount: 2, - txHashes: []string{flushTxHash1, flushTxHash2}, - }, - { - name: "flush_with_cursor_update_to_lower_value", - setupBuffer: func() *indexer.IndexerBuffer { - buf := indexer.NewIndexerBuffer() - tx1 := createTestTransaction(flushTxHash3, 3) - buf.PushTransaction(testAddr1, &tx1) - return buf - }, - updateCursorTo: ptrUint32(50), - initialCursor: 100, - wantCursor: 50, - wantTxCount: 1, - wantOpCount: 0, - wantStateChangeCount: 0, - txHashes: []string{flushTxHash3}, - }, - { - name: "flush_with_cursor_update_to_higher_value_keeps_existing", - setupBuffer: func() *indexer.IndexerBuffer { - buf := indexer.NewIndexerBuffer() - tx1 := createTestTransaction(flushTxHash4, 4) - buf.PushTransaction(testAddr1, &tx1) - return buf - }, - updateCursorTo: ptrUint32(150), - initialCursor: 100, - wantCursor: 100, // UpdateMin keeps lower - wantTxCount: 1, - wantOpCount: 0, - wantStateChangeCount: 0, - txHashes: []string{flushTxHash4}, + wantTxCount: 2, + wantOpCount: 2, + txHashes: []string{flushTxHash1, flushTxHash2}, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - // Clean up test data from previous runs (using HashBytea for BYTEA column) + // Clean up test data for _, hash := range []string{flushTxHash1, flushTxHash2, flushTxHash3, flushTxHash4} { _, err = dbConnectionPool.Exec(ctx, `DELETE FROM state_changes WHERE to_id IN (SELECT to_id FROM transactions WHERE hash = $1)`, types.HashBytea(hash)) require.NoError(t, err) _, err = dbConnectionPool.Exec(ctx, `DELETE FROM transactions WHERE hash = $1`, types.HashBytea(hash)) require.NoError(t, err) } - // Also clean up any orphan operations _, err = dbConnectionPool.Exec(ctx, `TRUNCATE operations, operations_accounts CASCADE`) require.NoError(t, err) - // Set up initial cursor - setupDBCursors(t, ctx, dbConnectionPool, 200, tc.initialCursor) + setupDBCursors(t, ctx, dbConnectionPool, 200, 100) m := metrics.NewMetrics(prometheus.NewRegistry()) @@ -1094,21 +865,15 @@ func Test_ingestService_flushBatchBufferWithRetry(t *testing.T) { Network: network.TestNetworkPassphrase, NetworkPassphrase: network.TestNetworkPassphrase, Archive: &HistoryArchiveMock{}, + BackfillFlushWorkers: 1, }) require.NoError(t, err) buffer := tc.setupBuffer() - - // Call flushBatchBuffer - err = svc.flushBatchBufferWithRetry(ctx, buffer, tc.updateCursorTo) + err = svc.flushBufferWithRetry(ctx, buffer) require.NoError(t, err) - // Verify the cursor value - cursor, err := models.IngestStore.Get(ctx, "oldest_ledger_cursor") - require.NoError(t, err) - assert.Equal(t, tc.wantCursor, cursor) - - // Verify transaction count in database + // Verify transaction count if len(tc.txHashes) > 0 { hashBytes := make([][]byte, len(tc.txHashes)) for i, h := range tc.txHashes { @@ -1124,7 +889,7 @@ func Test_ingestService_flushBatchBufferWithRetry(t *testing.T) { assert.Equal(t, tc.wantTxCount, txCount, "transaction count mismatch") } - // Verify operation count in database + // Verify operation count if tc.wantOpCount > 0 { var opCount int err = dbConnectionPool.QueryRow(ctx, @@ -1132,158 +897,14 @@ func Test_ingestService_flushBatchBufferWithRetry(t *testing.T) { require.NoError(t, err) assert.Equal(t, tc.wantOpCount, opCount, "operation count mismatch") } - - // Verify state change count in database - if tc.wantStateChangeCount > 0 { - scHashBytes := make([][]byte, len(tc.txHashes)) - for i, h := range tc.txHashes { - val, err := types.HashBytea(h).Value() - require.NoError(t, err) - scHashBytes[i] = val.([]byte) - } - var scCount int - err = dbConnectionPool.QueryRow(ctx, - `SELECT COUNT(*) FROM state_changes WHERE to_id IN (SELECT to_id FROM transactions WHERE hash = ANY($1))`, - scHashBytes).Scan(&scCount) - require.NoError(t, err) - assert.Equal(t, tc.wantStateChangeCount, scCount, "state change count mismatch") - } }) } } -// ==================== Backfill Failure Scenario Tests ==================== - -// Test_ingestService_processBackfillBatchesParallel_PartialFailure verifies that when one -// batch fails during parallel processing, other batches still complete successfully. -func Test_ingestService_processBackfillBatchesParallel_PartialFailure(t *testing.T) { - dbt := dbtest.Open(t) - defer dbt.Close() - ctx := context.Background() - dbConnectionPool, err := db.OpenDBConnectionPool(ctx, dbt.DSN) - require.NoError(t, err) - defer dbConnectionPool.Close() - - testCases := []struct { - name string - batches []BackfillBatch - failBatchIndex int // which batch index should fail (0-based) - wantFailedCount int - }{ - { - name: "middle_batch_fails", - batches: []BackfillBatch{ - {StartLedger: 100, EndLedger: 109}, - {StartLedger: 110, EndLedger: 119}, // This one fails - {StartLedger: 120, EndLedger: 129}, - }, - failBatchIndex: 1, - wantFailedCount: 1, - }, - { - name: "first_batch_fails", - batches: []BackfillBatch{ - {StartLedger: 100, EndLedger: 109}, // This one fails - {StartLedger: 110, EndLedger: 119}, - {StartLedger: 120, EndLedger: 129}, - }, - failBatchIndex: 0, - wantFailedCount: 1, - }, - { - name: "last_batch_fails", - batches: []BackfillBatch{ - {StartLedger: 100, EndLedger: 109}, - {StartLedger: 110, EndLedger: 119}, - {StartLedger: 120, EndLedger: 129}, // This one fails - }, - failBatchIndex: 2, - wantFailedCount: 1, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - m := metrics.NewMetrics(prometheus.NewRegistry()) - - models, modelsErr := data.NewModels(dbConnectionPool, m.DB) - require.NoError(t, modelsErr) - - mockRPCService := &RPCServiceMock{} - mockRPCService.On("NetworkPassphrase").Return(network.TestNetworkPassphrase).Maybe() - - failBatch := tc.batches[tc.failBatchIndex] - - // Factory that returns a backend that fails PrepareRange for the specified batch - factory := func(ctx context.Context) (ledgerbackend.LedgerBackend, error) { - mockBackend := &LedgerBackendMock{} - // Use MatchedBy to check if this is the failing batch - mockBackend.On("PrepareRange", mock.Anything, mock.MatchedBy(func(r ledgerbackend.Range) bool { - return r.From() == failBatch.StartLedger - })).Return(fmt.Errorf("simulated failure for batch starting at %d", failBatch.StartLedger)) - // All other batches succeed - return error to prevent processing - // This avoids nil pointer issues with empty LedgerCloseMeta - mockBackend.On("PrepareRange", mock.Anything, mock.Anything).Return(nil).Maybe() - // Return proper empty ledger meta to avoid nil pointer issues - mockBackend.On("GetLedger", mock.Anything, mock.Anything).Return(xdr.LedgerCloseMeta{ - V: 0, - V0: &xdr.LedgerCloseMetaV0{ - LedgerHeader: xdr.LedgerHeaderHistoryEntry{ - Header: xdr.LedgerHeader{ - LedgerSeq: xdr.Uint32(100), - }, - }, - }, - }, nil).Maybe() - mockBackend.On("Close").Return(nil).Maybe() - return mockBackend, nil - } - - svc, svcErr := NewIngestService(IngestServiceConfig{ - IngestionMode: IngestionModeBackfill, - Models: models, - LatestLedgerCursorName: "latest_ledger_cursor", - OldestLedgerCursorName: "oldest_ledger_cursor", - AppTracker: &apptracker.MockAppTracker{}, - RPCService: mockRPCService, - LedgerBackend: &LedgerBackendMock{}, - LedgerBackendFactory: factory, - Metrics: m, - GetLedgersLimit: defaultGetLedgersLimit, - Network: network.TestNetworkPassphrase, - NetworkPassphrase: network.TestNetworkPassphrase, - Archive: &HistoryArchiveMock{}, - BackfillBatchSize: 10, - }) - require.NoError(t, svcErr) - - results := svc.processBackfillBatchesParallel(ctx, tc.batches, nil) - - // Verify results - require.Len(t, results, len(tc.batches)) - - // Count actual failures - actualFailed := 0 - for i, result := range results { - if result.Error != nil { - actualFailed++ - assert.Equal(t, tc.failBatchIndex, i, "expected batch at index %d to fail", tc.failBatchIndex) - assert.Contains(t, result.Error.Error(), "preparing backend range") - } - } - assert.Equal(t, tc.wantFailedCount, actualFailed) - - // Verify analyzeBatchResults returns correct count - numFailed := analyzeBatchResults(ctx, results) - assert.Equal(t, tc.wantFailedCount, numFailed) - }) - } -} +// Old batch-based backfill tests removed — the pipeline now processes gaps +// sequentially with 3-stage pipeline (dispatcher → process → flush). -// Test_ingestService_startBackfilling_HistoricalMode_PartialFailure_CursorUpdate verifies -// that the oldest_ingest_ledger cursor is updated to the minimum of successful batches -// when some batches fail during historical backfill. -func Test_ingestService_startBackfilling_HistoricalMode_PartialFailure_CursorUpdate(t *testing.T) { +func Test_ingestService_startBackfilling_AllGapsFail_CursorUnchanged(t *testing.T) { dbt := dbtest.Open(t) defer dbt.Close() ctx := context.Background() @@ -1291,257 +912,10 @@ func Test_ingestService_startBackfilling_HistoricalMode_PartialFailure_CursorUpd require.NoError(t, err) defer dbConnectionPool.Close() - testCases := []struct { - name string - initialOldest uint32 - initialLatest uint32 - startLedger uint32 - endLedger uint32 - batchSize uint32 - failBatchStart uint32 // start ledger of batch to fail - wantFinalOldest uint32 - wantError bool - wantErrContains string - }{ - { - name: "middle_batch_fails_cursor_updates_to_minimum", - initialOldest: 100, - initialLatest: 100, - startLedger: 50, - endLedger: 99, - batchSize: 10, - failBatchStart: 70, // Batch [70-79] fails - wantFinalOldest: 50, // Minimum of successful batches (50, 60, 80, 90) - wantError: false, - }, - { - name: "first_batch_fails_cursor_updates_to_second_batch", - initialOldest: 100, - initialLatest: 100, - startLedger: 50, - endLedger: 79, - batchSize: 10, - failBatchStart: 50, // Batch [50-59] fails - wantFinalOldest: 60, // Next successful batch - wantError: false, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - // Clean up database - _, err = dbConnectionPool.Exec(ctx, `DELETE FROM transactions`) - require.NoError(t, err) - _, err = dbConnectionPool.Exec(ctx, `DELETE FROM operations`) - require.NoError(t, err) - _, err = dbConnectionPool.Exec(ctx, `DELETE FROM state_changes`) - require.NoError(t, err) - - // Set up initial cursors - setupDBCursors(t, ctx, dbConnectionPool, tc.initialLatest, tc.initialOldest) - - // Insert anchor transaction so GetOldestLedger() returns initialOldest - _, err = dbConnectionPool.Exec(ctx, - `INSERT INTO transactions (hash, to_id, fee_charged, result_code, ledger_number, ledger_created_at) - VALUES ('anchor_hash', 1, 100, 'TransactionResultCodeTxSuccess', $1, NOW())`, - tc.initialOldest) - require.NoError(t, err) - - m := metrics.NewMetrics(prometheus.NewRegistry()) - - models, modelsErr := data.NewModels(dbConnectionPool, m.DB) - require.NoError(t, modelsErr) - - mockRPCService := &RPCServiceMock{} - mockRPCService.On("NetworkPassphrase").Return(network.TestNetworkPassphrase).Maybe() - - // Factory that fails for the specified batch - factory := func(ctx context.Context) (ledgerbackend.LedgerBackend, error) { - mockBackend := &LedgerBackendMock{} - mockBackend.On("PrepareRange", mock.Anything, mock.MatchedBy(func(r ledgerbackend.Range) bool { - return r.From() == tc.failBatchStart - })).Return(fmt.Errorf("simulated network failure")) - mockBackend.On("PrepareRange", mock.Anything, mock.Anything).Return(nil).Maybe() - // For successful batches, return empty ledgers (no transactions) - mockBackend.On("GetLedger", mock.Anything, mock.Anything).Return(xdr.LedgerCloseMeta{ - V: 0, - V0: &xdr.LedgerCloseMetaV0{ - LedgerHeader: xdr.LedgerHeaderHistoryEntry{ - Header: xdr.LedgerHeader{ - LedgerSeq: xdr.Uint32(50), - }, - }, - }, - }, nil).Maybe() - mockBackend.On("Close").Return(nil).Maybe() - return mockBackend, nil - } - - svc, svcErr := NewIngestService(IngestServiceConfig{ - IngestionMode: IngestionModeBackfill, - Models: models, - LatestLedgerCursorName: "latest_ledger_cursor", - OldestLedgerCursorName: "oldest_ledger_cursor", - AppTracker: &apptracker.MockAppTracker{}, - RPCService: mockRPCService, - LedgerBackend: &LedgerBackendMock{}, - LedgerBackendFactory: factory, - Metrics: m, - GetLedgersLimit: defaultGetLedgersLimit, - Network: network.TestNetworkPassphrase, - NetworkPassphrase: network.TestNetworkPassphrase, - Archive: &HistoryArchiveMock{}, - BackfillBatchSize: int(tc.batchSize), - }) - require.NoError(t, svcErr) - - // Run backfilling - backfillErr := svc.startBackfilling(ctx, tc.startLedger, tc.endLedger) - - if tc.wantError { - require.Error(t, backfillErr) - if tc.wantErrContains != "" { - assert.Contains(t, backfillErr.Error(), tc.wantErrContains) - } - } else { - // Historical mode should not return error even with partial failures - require.NoError(t, backfillErr) - } - - // Verify cursor was updated correctly using IngestStore.Get - finalOldest, getErr := models.IngestStore.Get(ctx, "oldest_ledger_cursor") - require.NoError(t, getErr) - assert.Equal(t, tc.wantFinalOldest, finalOldest, - "oldest cursor should be updated to minimum of successful batches") - }) - } -} - -// Test_ingestService_processBackfillBatches_PartialFailure_OnlySuccessfulBatchPersisted verifies -// that when one batch fails and another succeeds: -// - The failed batch does not persist any data -// - The successful batch persists its transactions -// - Proper error handling for both cases -func Test_ingestService_processBackfillBatches_PartialFailure_OnlySuccessfulBatchPersisted(t *testing.T) { - dbt := dbtest.Open(t) - defer dbt.Close() - ctx := context.Background() - dbConnectionPool, err := db.OpenDBConnectionPool(ctx, dbt.DSN) - require.NoError(t, err) - defer dbConnectionPool.Close() - - // Clean up database + // Clean up _, err = dbConnectionPool.Exec(ctx, `DELETE FROM transactions`) require.NoError(t, err) - _, err = dbConnectionPool.Exec(ctx, `DELETE FROM operations`) - require.NoError(t, err) - _, err = dbConnectionPool.Exec(ctx, `DELETE FROM state_changes`) - require.NoError(t, err) - - // Set up initial cursors - setupDBCursors(t, ctx, dbConnectionPool, 200, 200) - - m := metrics.NewMetrics(prometheus.NewRegistry()) - - models, modelsErr := data.NewModels(dbConnectionPool, m.DB) - require.NoError(t, modelsErr) - - mockRPCService := &RPCServiceMock{} - mockRPCService.On("NetworkPassphrase").Return(network.TestNetworkPassphrase).Maybe() - // Parse ledger metadata with a transaction for the successful batch - var metaWithTx xdr.LedgerCloseMeta - err = xdr.SafeUnmarshalBase64(ledgerMetadataWith1Tx, &metaWithTx) - require.NoError(t, err) - - // Two batches: first fails, second succeeds - // Use single-ledger batches to avoid duplicate key issues with test fixtures - batches := []BackfillBatch{ - {StartLedger: 100, EndLedger: 100}, // Will fail - {StartLedger: 110, EndLedger: 110}, // Will succeed - single ledger to avoid duplicate tx hash - } - - // Factory that returns a mock backend configured based on the batch range - factory := func(ctx context.Context) (ledgerbackend.LedgerBackend, error) { - mockBackend := &LedgerBackendMock{} - // Fail batch 1 (100-109) at PrepareRange - mockBackend.On("PrepareRange", mock.Anything, mock.MatchedBy(func(r ledgerbackend.Range) bool { - return r.From() == 100 - })).Return(fmt.Errorf("ledger range unavailable")) - // Succeed batch 2 (110-119) - mockBackend.On("PrepareRange", mock.Anything, mock.Anything).Return(nil).Maybe() - mockBackend.On("GetLedger", mock.Anything, mock.Anything).Return(metaWithTx, nil).Maybe() - mockBackend.On("Close").Return(nil).Maybe() - return mockBackend, nil - } - - svc, svcErr := NewIngestService(IngestServiceConfig{ - IngestionMode: IngestionModeBackfill, - Models: models, - LatestLedgerCursorName: "latest_ledger_cursor", - OldestLedgerCursorName: "oldest_ledger_cursor", - AppTracker: &apptracker.MockAppTracker{}, - RPCService: mockRPCService, - LedgerBackend: &LedgerBackendMock{}, - LedgerBackendFactory: factory, - Metrics: m, - GetLedgersLimit: defaultGetLedgersLimit, - Network: network.TestNetworkPassphrase, - NetworkPassphrase: network.TestNetworkPassphrase, - Archive: &HistoryArchiveMock{}, - BackfillBatchSize: 10, - BackfillDBInsertBatchSize: 50, - }) - require.NoError(t, svcErr) - - // Process both batches in parallel - results := svc.processBackfillBatchesParallel(ctx, batches, nil) - - // Verify we got results for both batches - require.Len(t, results, 2) - - // Verify batch 1 (100-109) failed - require.Error(t, results[0].Error, "batch 1 should have failed") - assert.Contains(t, results[0].Error.Error(), "preparing backend range") - assert.Contains(t, results[0].Error.Error(), "ledger range unavailable") - - // Verify batch 2 (110-119) succeeded - require.NoError(t, results[1].Error, "batch 2 should have succeeded") - - // Verify no transactions were persisted for failed batch (ledger 100) - var failedBatchTxCount int - err = dbConnectionPool.QueryRow(ctx, - `SELECT COUNT(*) FROM transactions WHERE ledger_number BETWEEN $1 AND $2`, - 100, 109).Scan(&failedBatchTxCount) - require.NoError(t, err) - assert.Equal(t, 0, failedBatchTxCount, "no transactions should be persisted for failed batch") - - // Verify transactions were persisted for successful batch - // The ledgerMetadataWith1Tx fixture has a fixed ledger sequence of 4478 - // so we query for that ledger number (the XDR metadata determines the stored ledger) - var successBatchTxCount int - err = dbConnectionPool.QueryRow(ctx, - `SELECT COUNT(*) FROM transactions WHERE ledger_number = $1`, - 4478).Scan(&successBatchTxCount) - require.NoError(t, err) - assert.Equal(t, 1, successBatchTxCount, "1 transaction should be persisted for successful batch") -} - -// Test_ingestService_startBackfilling_HistoricalMode_AllBatchesFail_CursorUnchanged -// verifies that the cursor remains unchanged when all batches fail during historical backfill. -func Test_ingestService_startBackfilling_HistoricalMode_AllBatchesFail_CursorUnchanged(t *testing.T) { - dbt := dbtest.Open(t) - defer dbt.Close() - ctx := context.Background() - dbConnectionPool, err := db.OpenDBConnectionPool(ctx, dbt.DSN) - require.NoError(t, err) - defer dbConnectionPool.Close() - - // Clean up database - _, err = dbConnectionPool.Exec(ctx, `DELETE FROM transactions`) - require.NoError(t, err) - - // Set up initial cursors initialOldest := uint32(100) initialLatest := uint32(100) setupDBCursors(t, ctx, dbConnectionPool, initialLatest, initialOldest) @@ -1554,7 +928,7 @@ func Test_ingestService_startBackfilling_HistoricalMode_AllBatchesFail_CursorUnc mockRPCService := &RPCServiceMock{} mockRPCService.On("NetworkPassphrase").Return(network.TestNetworkPassphrase).Maybe() - // Factory that always fails + // Factory that always fails — every gap will fail at backend creation factory := func(ctx context.Context) (ledgerbackend.LedgerBackend, error) { return nil, fmt.Errorf("all backends unavailable") } @@ -1573,26 +947,26 @@ func Test_ingestService_startBackfilling_HistoricalMode_AllBatchesFail_CursorUnc Network: network.TestNetworkPassphrase, NetworkPassphrase: network.TestNetworkPassphrase, Archive: &HistoryArchiveMock{}, - BackfillBatchSize: 10, + BackfillFlushWorkers: 1, }) require.NoError(t, svcErr) - // Run backfilling with all batches failing + // Run backfilling with all gaps failing backfillErr := svc.startBackfilling(ctx, 50, 99) - // Historical mode should NOT return error even when all batches fail - require.NoError(t, backfillErr, "historical mode should not return error even when all batches fail") + // Pipeline logs failures but returns nil + require.NoError(t, backfillErr) // Verify cursor remains unchanged finalOldest, getErr := models.IngestStore.Get(ctx, "oldest_ledger_cursor") require.NoError(t, getErr) assert.Equal(t, initialOldest, finalOldest, - "oldest cursor should remain unchanged when all batches fail") + "oldest cursor should remain unchanged when all gaps fail") finalLatest, getErr := models.IngestStore.Get(ctx, "latest_ledger_cursor") require.NoError(t, getErr) assert.Equal(t, initialLatest, finalLatest, - "latest cursor should remain unchanged when all batches fail") + "latest cursor should remain unchanged when all gaps fail") } // Test_ingestProcessedDataWithRetry tests the ingestProcessedDataWithRetry function covering success, failure, and retry scenarios. @@ -1626,7 +1000,7 @@ func Test_ingestProcessedDataWithRetry(t *testing.T) { mockChAccStore := &store.ChannelAccountStoreMock{} - // Mock token ingestion methods (called in parallel by insertAndUpsertParallel) + // Mock token ingestion methods (called in parallel by insertParallel) mockTokenIngestionService := NewTokenIngestionServiceMock(t) mockTokenIngestionService.On("ProcessTrustlineChanges", mock.Anything, mock.Anything, mock.Anything).Return(nil) mockTokenIngestionService.On("ProcessNativeBalanceChanges", mock.Anything, mock.Anything, mock.Anything).Return(nil) @@ -1838,11 +1212,3 @@ func Test_ingestProcessedDataWithRetry(t *testing.T) { mockTokenIngestionService.AssertExpectations(t) }) } - -// Catchup tests removed — optimized catchup was removed in favor of natural live ingestion loop catchup. -// The following tests were deleted: -// - Test_ingestService_processBatchChanges -// - Test_ingestService_flushBatchBuffer_batchChanges -// - Test_ingestService_processLedgersInBatch_catchupMode -// - Test_ingestService_startBackfilling_CatchupMode_ProcessesBatchChanges -// - Test_ingestService_processBackfillBatchesParallel_BothModes diff --git a/services.test b/services.test new file mode 100755 index 000000000..da9fec2fa Binary files /dev/null and b/services.test differ