diff --git a/.github/workflows/CI-tsdb-sizing.yml b/.github/workflows/CI-tsdb-sizing.yml new file mode 100644 index 0000000000..31f4d6d8d2 --- /dev/null +++ b/.github/workflows/CI-tsdb-sizing.yml @@ -0,0 +1,193 @@ +name: CI-tsdb-sizing + +# Nightly measurement job for the TSDB sizing lab (test/tsdb-lab/). This is +# NOT a per-PR gate: it builds ProxySQL, expands the committed real-metric +# fixture into a realistically sized proxysql_stats.db (raw + rollups + a +# 3-node cluster tier), measures storage/query characteristics with +# measure.py, and uploads the printed report as a job artifact. The job +# goes red only when measure.py detects drift beyond baseline.json's +# +/-25% default threshold on EITHER of two different-purpose gates: +# bytes/row (guards fixture/tooling consistency -- it is fixture-text +# derived and does NOT see product-side label/metric growth until a human +# re-runs capture.bash and commits a refreshed fixture, see +# test/tsdb-lab/README.md's "Maintenance" section) and the whole-file +# overhead ratio (guards schema/index bloat -- sensitive to a new column or +# index even when payload bytes don't change). Total DB size and query +# latency are reported only, never gated. That two-gate split is the whole +# point of running it nightly on a schedule instead of on every push. +# +# IMPORTANT: this workflow only starts running once merged to the default +# branch (schedule/workflow_dispatch don't fire off a branch) -- see +# test/tsdb-lab/README.md's "After merge: first-run validation checklist" +# for the owner actions that are still outstanding as of this commit +# (manual dispatch, confirming the 4h/7d profile, replacing the analogy- +# derived build-time term below, confirming runner disk headroom). +# +# Profile sizing (2026-08-13 measurement, see +# docs/superpowers/specs/2026-08-13-tsdb-sizing-lab-design.md "Measured +# results"): the originally planned CI profile (--raw-window 24h --span 14d +# --nodes 3) was actually run once against a local release (PROXYSQL31) +# build to check disk feasibility. It produced ~28.7M rows and a 6.0 GB +# proxysql_stats.db in 113s -- on top of the >1.4 GB the repo build itself +# consumes, that is too tight a margin for a standard GitHub-hosted runner's +# disk. bytes/row was confirmed scale-invariant (0.0% drift vs the +# small-profile baseline.json), so a much smaller profile below gives the +# same sizing signal -- and still exercises multi-hour rollup catch-up and +# the 3-node cluster leader cost -- at a projected ~1 GB DB / ~20-30s +# expand time (linearly scaled from the measured 24h/14d/3-node rate: +# 7,144,960 raw rows/24h/node, 10,008 hourly rows/day, same rate for the +# cluster tier per source node). IMPORTANT: this 4h/7d/3-node profile has +# NOT itself been run end-to-end -- only the 24h/14d/3-node profile above +# was actually executed. The ~1 GB / ~20-30s figures are a projection, not +# a measurement of this exact profile; the first nightly/dispatched run of +# this workflow is that validation -- check its printed report against the +# projection above. +# +# Security note: every 'run:' step below uses only static, repo-controlled +# values (no untrusted user input is interpolated into a shell command). + +on: + workflow_dispatch: + schedule: + # 03:22 UTC daily -- off the top-of-hour to avoid the scheduled-workflow + # stampede on GitHub's infrastructure. + - cron: '22 3 * * *' + +jobs: + measure: + runs-on: ubuntu-latest + # The lab-step terms below are grounded in this session's measurements: + # rollup wait is hard-bounded at 10 min in that step's own poll loop, + # and expand/measure/checkout/apt/artifact overhead is measured (small + # profile) or linearly projected (see the profile-sizing comment above) + # at under ~4 min total. The build term (120 min) is NOT a measurement + # of this job's build step -- it's taken by analogy from comparable + # from-scratch full builds elsewhere in this repo (CI-package-*-v31.yml, + # build+package, same order of magnitude of work minus packaging). A + # local `PROXYSQL31=1 make clean && make` this session ran in ~55s, but + # `make clean` doesn't clean deps/ (only `make cleanall` does) and + # deps/ was already built beforehand, so that only timed a lib+src + # recompile, not a from-scratch build of the 25+ vendored dependencies + # -- the dominant, slow part of a real CI build -- so it isn't usable + # as the build-time term. 120 (build, analogy) + 10 (rollup, bounded) + + # ~4 (everything else, measured/projected) = ~134 min; 150 keeps a + # deliberate margin over that without carrying forward the previous + # unexamined 180-minute guess -- but the build term itself should be + # replaced with a real measurement after this workflow's first run. + timeout-minutes: 150 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + + - name: Install build dependencies + # Package list per INSTALL.md's "Debian / Ubuntu based" from-source + # build section. + run: | + sudo apt-get update -qq + sudo apt-get install -y --no-install-recommends \ + automake bzip2 cmake make g++ gcc git openssl libssl-dev \ + libgnutls28-dev libmysqlclient-dev libunwind8 libunwind-dev \ + uuid-dev libncurses-dev libicu-dev libevent-dev libtirpc-dev + + - name: Build ProxySQL (PROXYSQL31) + # Release build: the TSDB sizing lab only issues normal admin SQL + # (PROXYSQL SHUTDOWN, SET tsdb-*, LOAD TSDB VARIABLES TO RUNTIME), + # none of which requires a debug-only admin command. + run: PROXYSQL31=1 make -j$(nproc) + + - name: Create TSDB schema (start once, then stop) + run: | + mkdir -p /tmp/tsdb-lab-ci + cat > /tmp/tsdb-lab-ci/n.cnf <<'EOF' + datadir="/tmp/tsdb-lab-ci" + admin_variables = { admin_credentials="admin:admin"; mysql_ifaces="0.0.0.0:16392" } + mysql_variables = { threads=2; interfaces="0.0.0.0:16393" } + EOF + src/proxysql --initial -f -c /tmp/tsdb-lab-ci/n.cnf -D /tmp/tsdb-lab-ci & + echo "started pid $!" + for i in $(seq 1 30); do + mysql -uadmin -padmin -h127.0.0.1 -P16392 -e "SELECT 1" >/dev/null 2>&1 && break + sleep 1 + done + mysql -uadmin -padmin -h127.0.0.1 -P16392 -e "PROXYSQL SHUTDOWN" + sleep 2 + test -f /tmp/tsdb-lab-ci/proxysql_stats.db + + - name: Expand fixture into raw/hourly/cluster tiers + # CI profile, sized to fit a standard GitHub-hosted runner's disk + # (see the top-of-file comment and the design spec's "Measured + # results" section): 4h of raw (5s) metrics, 7d total retention, + # 3-node cluster tier. Projected ~1 GB DB / ~20-30s expand time, + # scaled from the measured 24h/14d/3-node run (28.7M rows, 6.0 GB, + # 113s) using the confirmed-scale-invariant bytes/row. + run: | + python3 test/tsdb-lab/expand.py \ + --db /tmp/tsdb-lab-ci/proxysql_stats.db \ + --raw-window 4h --span 7d --nodes 3 + + - name: Start ProxySQL and wait for hourly rollup catch-up + # tsdb_downsample_metrics() fires once immediately after + # tsdb-enabled is switched on (its internal timer starts at 0), then + # not again for an hour -- so tsdb_metrics_hour grows exactly once + # here as it catches up the raw window we just wrote via expand.py, + # then goes flat. Poll stats_history.tsdb_metrics_hour's row count + # until it holds steady across 3 consecutive samples, bounded so a + # stuck rollup can't hang the job forever. + run: | + src/proxysql -f -c /tmp/tsdb-lab-ci/n.cnf -D /tmp/tsdb-lab-ci & + echo "started pid $!" + for i in $(seq 1 30); do + mysql -uadmin -padmin -h127.0.0.1 -P16392 -e "SELECT 1" >/dev/null 2>&1 && break + sleep 1 + done + mysql -uadmin -padmin -h127.0.0.1 -P16392 -e \ + "SET tsdb-enabled='1'; LOAD TSDB VARIABLES TO RUNTIME;" + + wait_start=$(date +%s) + prev=-1 + stable=0 + max_iterations=120 # bounded wait: up to 120 * 5s = 10 minutes + iter=0 + for iter in $(seq 1 "$max_iterations"); do + cur=$(mysql -uadmin -padmin -h127.0.0.1 -P16392 -N -B \ + -e "SELECT COUNT(*) FROM stats_history.tsdb_metrics_hour" 2>/dev/null || echo "") + if [ "$cur" = "$prev" ] && [ -n "$cur" ]; then + stable=$((stable + 1)) + else + stable=0 + fi + prev="$cur" + if [ "$stable" -ge 3 ]; then + break + fi + sleep 5 + done + wait_end=$(date +%s) + echo "rollup catch-up: $((wait_end - wait_start))s wall time, " \ + "tsdb_metrics_hour final row count=${prev} " \ + "(stable=${stable}/3, iterations=${iter}/${max_iterations})" \ + | tee /tmp/tsdb-lab-ci/rollup-catchup.log + + mysql -uadmin -padmin -h127.0.0.1 -P16392 -e "PROXYSQL SHUTDOWN" + sleep 2 + + - name: Measure + run: | + python3 test/tsdb-lab/measure.py \ + --db /tmp/tsdb-lab-ci/proxysql_stats.db \ + --baseline test/tsdb-lab/baseline.json \ + | tee /tmp/tsdb-lab-ci/report.txt + + - name: Assemble report artifact + if: always() + run: | + mkdir -p /tmp/tsdb-lab-ci/artifact + cp /tmp/tsdb-lab-ci/report.txt /tmp/tsdb-lab-ci/artifact/ 2>/dev/null || true + cp /tmp/tsdb-lab-ci/rollup-catchup.log /tmp/tsdb-lab-ci/artifact/ 2>/dev/null || true + + - name: Upload report + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: tsdb-sizing-report-${{ github.run_id }} + path: /tmp/tsdb-lab-ci/artifact/ + if-no-files-found: warn diff --git a/.gitignore b/.gitignore index 61433d248d..76fabcb13b 100644 --- a/.gitignore +++ b/.gitignore @@ -241,4 +241,10 @@ deps/protobuf/protobuf-*/ # accidentally committed once with an absolute /home path). test/scripts/deps/mysqlbinlog +# superpowers scratch (SDD workspaces, review packages) +.superpowers/ + +# tsdb-lab capture.bash working dir: per-node datadirs/logs for the 3 +# ephemeral ProxySQL instances it spawns; never the committed fixture. +test/tsdb-lab/.capture/ # Temporary ASAN CI end-to-end validation trigger; this branch will not merge. diff --git a/doc/tsdb/embedded_tsdb_overview.md b/doc/tsdb/embedded_tsdb_overview.md index d35b8b5882..3bc012b5ff 100644 --- a/doc/tsdb/embedded_tsdb_overview.md +++ b/doc/tsdb/embedded_tsdb_overview.md @@ -45,6 +45,6 @@ REST API endpoints are available under `/api/tsdb/` for external integrations. ## Retention -- Raw metrics (`tsdb_metrics`): `tsdb-retention_days` +- Raw metrics (`tsdb_metrics`): `tsdb-retention_days` (default 2 days) - Backend probes (`tsdb_backend_health`): `tsdb-retention_days` -- Hourly rollups (`tsdb_metrics_hour`): fixed 365 days +- Hourly rollups (`tsdb_metrics_hour`): `tsdb-hourly_retention_days` (default 365 days) diff --git a/doc/tsdb/embedded_tsdb_reference.md b/doc/tsdb/embedded_tsdb_reference.md index fe8abd4383..468e851f20 100644 --- a/doc/tsdb/embedded_tsdb_reference.md +++ b/doc/tsdb/embedded_tsdb_reference.md @@ -8,7 +8,7 @@ The behavior of the TSDB subsystem is controlled by the following global variabl |---|---|---:|---|---| | `tsdb-enabled` | int | `0` | `0/1` | Master switch | | `tsdb-sample_interval` | int | `5` | `1..3600` | Prometheus sampling interval (seconds) | -| `tsdb-retention_days` | int | `7` | `1..3650` | Raw/probe retention in days | +| `tsdb-retention_days` | int | `2` | `1..3650` | Raw/probe retention in days | | `tsdb-monitor_enabled` | int | `0` | `0/1` | Backend probe switch | | `tsdb-monitor_interval` | int | `10` | `1..3600` | Probe interval (seconds) | diff --git a/doc/tsdb/embedded_tsdb_specs.md b/doc/tsdb/embedded_tsdb_specs.md index 9cd414c494..9021eb95ed 100644 --- a/doc/tsdb/embedded_tsdb_specs.md +++ b/doc/tsdb/embedded_tsdb_specs.md @@ -30,9 +30,9 @@ Embedded time-series storage in SQLite for ProxySQL runtime metrics and backend ## Retention -- Raw metrics retention: `tsdb-retention_days` +- Raw metrics retention: `tsdb-retention_days` (default 2 days) - Backend probe retention: `tsdb-retention_days` -- Hourly rollup retention: 365 days +- Hourly rollup retention: `tsdb-hourly_retention_days` (default 365 days) ## Variable Semantics diff --git a/docs/superpowers/plans/2026-08-11-cluster-leader-election.md b/docs/superpowers/plans/2026-08-11-cluster-leader-election.md new file mode 100644 index 0000000000..47e65b8e66 --- /dev/null +++ b/docs/superpowers/plans/2026-08-11-cluster-leader-election.md @@ -0,0 +1,1505 @@ +# Cluster Leader Election Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Deterministic, ballot-free leader election for ProxySQL Cluster (liveness + election + read-only config steering), per spec `docs/superpowers/specs/2026-08-11-cluster-leader-election-design.md`. + +**Architecture:** Pure election logic lives in new files (`ProxySQL_Cluster_Leader.{h,cpp}`); liveness is bookkeeping on the existing per-peer `GLOBAL_CHECKSUM()` poll loop; peer UUIDs are learned via a new `SELECT GLOBAL_UUID()` admin intercept; an election tick in the Admin main loop drives a tri-state read-only mode (`AUTO`/`FORCED_RO`/`FORCED_RW`) that gates admin SQL writes plus `LOAD … TO RUNTIME`/`SAVE … TO DISK`. + +**Tech Stack:** C++17, pthreads, GCC atomic builtins (`__sync_*`) matching surrounding code, SQLite3 (admin), libmariadb (cluster transport), prometheus-cpp, TAP tests. + +## Global Constraints + +- Build with the tier flag on EVERY make invocation: `PROXYSQL31=1 make debug` (never bare `make`; `make clean` first if the tree was built under a different tier). See CLAUDE.md. +- All election/tri-state code compiles **unconditionally in every tier**. The ONLY `#ifdef PROXYSQL31` allowed is around the registration of the `admin-cluster_leader_election` variable (name list, get_variable, set_variable). No other new `#ifdef`s. +- The feature must be a no-op when `admin-cluster_leader_election=false` (the default): bit-for-bit today's behavior. +- Cluster-initiated syncs (direct C++ calls `GloAdmin->load_*_to_runtime()` from `lib/ProxySQL_Cluster.cpp`) must NEVER be blocked by read-only mode. +- Naming: classes `PascalCase` with `ProxySQL_`/`Cluster_` prefixes, members `snake_case`, macros `UPPER_SNAKE_CASE`. Tabs for indentation (match surrounding code). +- New admin variables and defaults (exact values from spec): `admin-cluster_leader_election` = `false`; `admin-cluster_leader_node_timeout_ms` = `3000` (range 1000–600000); `admin-cluster_leader_grace_ms` = `3000` (range 0–600000). +- Election rule (exact): among candidates that are `alive` **and** have a known (non-empty) UUID, highest `weight` wins; ties broken by lexicographically smallest UUID; no electable candidate → no leader. +- Commit after every task. Branch: `feat/cluster-leader-election` (already exists, spec committed there). + +--- + +### Task 1: Pure election engine + unit test + +**Files:** +- Create: `include/ProxySQL_Cluster_Leader.h` +- Create: `lib/ProxySQL_Cluster_Leader.cpp` +- Modify: `lib/Makefile:91` (add `ProxySQL_Cluster_Leader.oo` to `_OBJ_CXX`) +- Create: `test/tap/tests/unit/cluster_leader_election_unit-t.cpp` +- Modify: `test/tap/tests/unit/Makefile:387+` (add to `UNIT_TESTS`) +- Modify: `test/tap/groups/groups.json` (register unit test in `unit-tests-g1`) + +**Interfaces:** +- Consumes: nothing (pure logic, standalone header). +- Produces (used by Tasks 2, 4, 6): + - `struct Cluster_Leader_Candidate { std::string uuid; std::string hostname; uint16_t port; uint64_t weight; bool alive; }` + - `int cluster_elect_leader(const std::vector& candidates)` → index into `candidates`, or `-1` + - `class Cluster_Leader_State` with `bool update(const std::string& computed_uuid, unsigned long long now_ms, unsigned long long grace_ms)` (returns true when effective leader changed), `void reset()`, public members `current_leader_uuid`, `pending_leader_uuid`, `pending_since_ms`. + +- [ ] **Step 1: Write the failing unit test** + +Create `test/tap/tests/unit/cluster_leader_election_unit-t.cpp` (pure-logic test, no `test_init_minimal()`, same style as `cluster_sync_unit-t.cpp`): + +```cpp +/** + * @file cluster_leader_election_unit-t.cpp + * @brief Unit tests for the pure cluster leader election logic + * (cluster_elect_leader + Cluster_Leader_State grace-window machine). + */ + +#include +#include + +#include "tap.h" +#include "test_globals.h" +#include "test_init.h" +#include "ProxySQL_Cluster_Leader.h" + +static Cluster_Leader_Candidate mk(const char* uuid, uint64_t weight, bool alive) { + Cluster_Leader_Candidate c; + c.uuid = uuid; + c.hostname = "host"; + c.port = 6032; + c.weight = weight; + c.alive = alive; + return c; +} + +// 7 oks +static void test_elect_leader() { + std::vector v; + ok(cluster_elect_leader(v) == -1, "empty candidate set elects nobody"); + + v = { mk("aaa", 0, false), mk("bbb", 0, false) }; + ok(cluster_elect_leader(v) == -1, "no alive candidate elects nobody"); + + v = { mk("aaa", 0, true) }; + ok(cluster_elect_leader(v) == 0, "single alive candidate is leader"); + + v = { mk("aaa", 100, true), mk("bbb", 300, true), mk("ccc", 200, true) }; + ok(cluster_elect_leader(v) == 1, "highest weight wins"); + + v = { mk("ccc", 100, true), mk("aaa", 100, true), mk("bbb", 100, true) }; + ok(cluster_elect_leader(v) == 1, "equal weight: lexicographically smallest uuid wins"); + + v = { mk("aaa", 300, false), mk("bbb", 100, true) }; + ok(cluster_elect_leader(v) == 1, "dead high-weight candidate is skipped"); + + v = { mk("", 300, true), mk("bbb", 100, true) }; + ok(cluster_elect_leader(v) == 1, "candidate with unknown uuid is not electable"); +} + +// 10 oks +static void test_grace_state() { + Cluster_Leader_State s; + ok(s.current_leader_uuid.empty(), "initial state has no leader"); + + // First observation enters pending, no change yet (grace 1000ms) + ok(s.update("aaa", 1000, 1000) == false, "new leader is pending, not applied"); + ok(s.current_leader_uuid.empty(), "leader unchanged during grace"); + + // Still pending, grace not elapsed + ok(s.update("aaa", 1500, 1000) == false, "grace not elapsed yet"); + + // Grace elapsed -> applied + ok(s.update("aaa", 2100, 1000) == true, "leader applied after grace"); + ok(s.current_leader_uuid == "aaa", "current leader is aaa"); + + // Stable: no change + ok(s.update("aaa", 3000, 1000) == false, "stable leader: no change"); + + // Flap within grace: bbb appears then aaa returns before grace elapses + s.update("bbb", 4000, 1000); + ok(s.update("aaa", 4500, 1000) == false && s.current_leader_uuid == "aaa", + "flap within grace window is ignored"); + + // Loss of leader ("" computed) also honors grace + s.update("", 5000, 1000); + ok(s.update("", 6100, 1000) == true && s.current_leader_uuid.empty(), + "leader loss applied after grace"); + + // grace_ms == 0 applies on the same update + Cluster_Leader_State z; + ok(z.update("ccc", 100, 0) == true && z.current_leader_uuid == "ccc", + "zero grace applies immediately"); +} + +// 2 oks +static void test_reset() { + Cluster_Leader_State s; + s.update("aaa", 100, 0); + s.update("bbb", 200, 5000); + s.reset(); + ok(s.current_leader_uuid.empty() && s.pending_leader_uuid.empty() && s.pending_since_ms == 0, + "reset clears all state"); + ok(s.update("aaa", 300, 0) == true, "state machine works again after reset"); +} + +int main() { + plan(19); + test_elect_leader(); // 7 + test_grace_state(); // 10 + test_reset(); // 2 + return exit_status(); +} +``` + +- [ ] **Step 2: Create the header and a stub, verify test FAILS** + +Create `include/ProxySQL_Cluster_Leader.h`: + +```cpp +#ifndef __CLASS_PROXYSQL_CLUSTER_LEADER_H +#define __CLASS_PROXYSQL_CLUSTER_LEADER_H + +#include +#include +#include + +struct Cluster_Leader_Candidate { + std::string uuid; // empty = unknown (not electable) + std::string hostname; + uint16_t port = 0; + uint64_t weight = 0; + bool alive = false; +}; + +// Deterministic, ballot-free election over a locally-observed candidate set. +// Electable = alive && uuid non-empty. Highest weight wins; ties broken by +// lexicographically smallest uuid. Returns index into candidates, or -1. +int cluster_elect_leader(const std::vector& candidates); + +// Grace-window state machine: a computed leader (or leader loss, "") must be +// observed continuously for grace_ms before it becomes effective. +class Cluster_Leader_State { + public: + std::string current_leader_uuid; // empty = no leader + std::string pending_leader_uuid; + unsigned long long pending_since_ms = 0; + // Returns true when the effective leader changed. + bool update(const std::string& computed_uuid, unsigned long long now_ms, unsigned long long grace_ms); + void reset(); +}; + +#endif // __CLASS_PROXYSQL_CLUSTER_LEADER_H +``` + +Create `lib/ProxySQL_Cluster_Leader.cpp` as a failing stub: + +```cpp +#include "ProxySQL_Cluster_Leader.h" + +int cluster_elect_leader(const std::vector& candidates) { + (void)candidates; + return -1; +} + +bool Cluster_Leader_State::update(const std::string& computed_uuid, unsigned long long now_ms, unsigned long long grace_ms) { + (void)computed_uuid; (void)now_ms; (void)grace_ms; + return false; +} + +void Cluster_Leader_State::reset() { +} +``` + +Edit `lib/Makefile:91`: in the `_OBJ_CXX :=` list, insert `ProxySQL_Cluster_Leader.oo` immediately after `ProxySQL_Cluster.oo`. + +Edit `test/tap/tests/unit/Makefile`: add `cluster_leader_election_unit-t` to the `UNIT_TESTS :=` list (starts at line 387; keep alphabetical-ish placement near `cluster_sync_unit-t`). The generic `%-t:` pattern rule (line 862) handles the build — no explicit rule needed. + +Run: +```bash +cd /data/rene/proxysql7/proxysql && PROXYSQL31=1 make debug -j$(nproc) 2>&1 | tail -5 +cd test/tap/tests/unit && PROXYSQL31=1 make cluster_leader_election_unit-t && ./cluster_leader_election_unit-t +``` +Expected: builds, test runs, multiple `not ok` lines (stub returns -1/false). If the build fails on the new files, fix before proceeding. + +- [ ] **Step 3: Implement the real logic** + +Replace `lib/ProxySQL_Cluster_Leader.cpp` body: + +```cpp +#include "ProxySQL_Cluster_Leader.h" + +int cluster_elect_leader(const std::vector& candidates) { + int best = -1; + for (size_t i = 0; i < candidates.size(); i++) { + const Cluster_Leader_Candidate& c = candidates[i]; + if (c.alive == false || c.uuid.empty()) { + continue; + } + if (best == -1) { + best = (int)i; + continue; + } + const Cluster_Leader_Candidate& b = candidates[best]; + if (c.weight > b.weight || (c.weight == b.weight && c.uuid < b.uuid)) { + best = (int)i; + } + } + return best; +} + +bool Cluster_Leader_State::update(const std::string& computed_uuid, unsigned long long now_ms, unsigned long long grace_ms) { + if (computed_uuid == current_leader_uuid) { + pending_leader_uuid.clear(); + pending_since_ms = 0; + return false; + } + if (pending_since_ms == 0 || pending_leader_uuid != computed_uuid) { + pending_leader_uuid = computed_uuid; + pending_since_ms = now_ms; + } + if (now_ms - pending_since_ms >= grace_ms) { + current_leader_uuid = pending_leader_uuid; + pending_leader_uuid.clear(); + pending_since_ms = 0; + return true; + } + return false; +} + +void Cluster_Leader_State::reset() { + current_leader_uuid.clear(); + pending_leader_uuid.clear(); + pending_since_ms = 0; +} +``` + +Note on the grace semantics the test encodes: a *brand new* pending observation at time T becomes effective at the first `update()` call with `now_ms >= T + grace_ms`; with `grace_ms == 0` it is effective on the same call. `pending_since_ms == 0` is the "no pending" sentinel — `update()` is never called with a real `now_ms` of 0 in production (monotonic µs / 1000), and the unit test uses now_ms ≥ 100 everywhere. + +- [ ] **Step 4: Run the unit test — expect PASS** + +```bash +cd test/tap/tests/unit && PROXYSQL31=1 make cluster_leader_election_unit-t && ./cluster_leader_election_unit-t +``` +Expected: `1..19`, all `ok`, exit 0. + +- [ ] **Step 5: Register in groups.json and commit** + +In `test/tap/groups/groups.json` add (one line, compact array, keep alphabetical order with neighbors — same format as `"cluster_sync_unit-t"`): +```json +"cluster_leader_election_unit-t" : [ "unit-tests-g1" ], +``` + +```bash +python3 test/tap/groups/lint_groups_json.py +git add include/ProxySQL_Cluster_Leader.h lib/ProxySQL_Cluster_Leader.cpp lib/Makefile \ + test/tap/tests/unit/cluster_leader_election_unit-t.cpp test/tap/tests/unit/Makefile \ + test/tap/groups/groups.json +git commit -m "feat(cluster): pure leader election engine with grace-window state machine" +``` + +--- + +### Task 2: Liveness bookkeeping + peer UUID exchange + +**Files:** +- Modify: `include/ProxySQL_Cluster.hpp` (node entry fields ~:270-280, accessors ~:286-311; `ProxySQL_Cluster_Nodes` methods ~:403-414; `ProxySQL_Cluster` forwarders ~:667-672) +- Modify: `lib/ProxySQL_Cluster.cpp` (ctor ~:400-480 region where entry members init; `Update_Global_Checksum` :4051-4073; monitor thread :236-244 and failure branch :331-342; new methods near :4088) +- Modify: `lib/Admin_Handler.cpp` (new `SELECT GLOBAL_UUID()` intercept after :3795) + +**Interfaces:** +- Consumes: nothing from Task 1 yet. +- Produces (used by Tasks 4, 6): + - `ProxySQL_Node_Entry` new fields + inline accessors: `const char* get_uuid()` (NULL until known), `unsigned long long get_last_success_at_us()`, `uint64_t get_global_version()`, `uint64_t get_checks_ok()`, `uint64_t get_checks_err()` + - `ProxySQL_Cluster::Update_Node_UUID(char* hostname, uint16_t port, const char* uuid)` (public forwarder into nodes, takes nodes mutex) + - `ProxySQL_Cluster::Update_Node_Failure(char* hostname, uint16_t port)` (increments `checks_err` under nodes mutex) + - Admin intercept: `SELECT GLOBAL_UUID()` returns one row/one column `UUID` = `GloVars.uuid`. + +- [ ] **Step 1: Add fields to `ProxySQL_Node_Entry`** + +In `include/ProxySQL_Cluster.hpp`, in the private section of `ProxySQL_Node_Entry` (after `char* ip_addr;` ~:275): + +```cpp + char *uuid; // learned via SELECT GLOBAL_UUID(); NULL until known + unsigned long long last_success_at_us; // monotonic_time() of last successful GLOBAL_CHECKSUM poll; 0 = never + uint64_t global_version; // number of observed global checksum changes on this peer + uint64_t checks_ok; + uint64_t checks_err; +``` + +In the public section (near `get_hostname()` ~:301), add inline accessors and a setter: + +```cpp + const char * get_uuid() { return uuid; } + void set_uuid(const char* u); // strdup, frees previous + unsigned long long get_last_success_at_us() { return last_success_at_us; } + uint64_t get_global_version() { return global_version; } + uint64_t get_checks_ok() { return checks_ok; } + uint64_t get_checks_err() { return checks_err; } +``` + +In `lib/ProxySQL_Cluster.cpp`: initialize all five in BOTH `ProxySQL_Node_Entry` constructors (find them near :400-480; every other pointer member like `ip_addr` is NULLed there — mirror that): `uuid = NULL; last_success_at_us = 0; global_version = 0; checks_ok = 0; checks_err = 0;`. In the destructor (frees `hostname`/`comment`/`ip_addr`), add `if (uuid) { free(uuid); uuid = NULL; }`. Implement: + +```cpp +void ProxySQL_Node_Entry::set_uuid(const char* u) { + if (uuid) { + free(uuid); + uuid = NULL; + } + if (u) { + uuid = strdup(u); + } +} +``` + +- [ ] **Step 2: Record success/failure in the poll loop** + +`lib/ProxySQL_Cluster.cpp`, `ProxySQL_Cluster_Nodes::Update_Global_Checksum` (:4051-4073) — it already locks `mutex`, finds the node entry, and compares the fetched checksum. Inside the "entry found" branch add, before returning: + +```cpp + node->last_success_at_us = monotonic_time(); + node->checks_ok++; +``` +and in the sub-branch where the fetched global checksum **differs** from `node->global_checksum` (the `update_checksum = true` path): `node->global_version++;`. + +Add the failure counterpart on `ProxySQL_Cluster_Nodes` (implementation next to `Update_Node_Metrics` ~:4137) plus declaration in the hpp (~:408) and a public forwarder on `ProxySQL_Cluster` (declare near the other forwarders ~:667-672): + +```cpp +void ProxySQL_Cluster_Nodes::Update_Node_Failure(char * _hostname, uint16_t _port) { + uint64_t hash_ = generate_hash(_hostname, _port); + pthread_mutex_lock(&mutex); + auto ite = umap_proxy_nodes.find(hash_); + if (ite != umap_proxy_nodes.end()) { + ite->second->checks_err++; + } + pthread_mutex_unlock(&mutex); +} +``` +Forwarder: `void ProxySQL_Cluster::Update_Node_Failure(char* h, uint16_t p) { nodes.Update_Node_Failure(h, p); }`. + +Call it from the monitor thread's query-failure branch (:331-342, where the error for a failed `rc_query` is handled): `GloProxyCluster->Update_Node_Failure(node->hostname, node->port);`. + +- [ ] **Step 3: `SELECT GLOBAL_UUID()` admin intercept** + +`lib/Admin_Handler.cpp`, immediately after the `GLOBAL_CHECKSUM()` block (after :3795, before the `PROXYSQL ` section at :3798), add the mirror (string column instead of longlong): + +```cpp + if ((query_no_space_length == strlen("SELECT GLOBAL_UUID()")) && (!strncasecmp("SELECT GLOBAL_UUID()", query_no_space, strlen("SELECT GLOBAL_UUID()")))) { + const char *uuid_val = (GloVars.uuid ? GloVars.uuid : ""); + uint16_t setStatus = 0; + auto *myds=sess->client_myds; + auto *myprot=&sess->client_myds->myprot; + myds->DSS=STATE_QUERY_SENT_DS; + int sid=1; + myprot->generate_pkt_column_count(true,NULL,NULL,sid,1); sid++; + myprot->generate_pkt_field(true,NULL,NULL,sid,(char *)"",(char *)"",(char *)"",(char *)"UUID",(char *)"",33,36,MYSQL_TYPE_VAR_STRING,0,0,false,0,NULL); sid++; + myds->DSS=STATE_COLUMN_DEFINITION; + myprot->generate_pkt_EOF(true,NULL,NULL,sid,0, setStatus); sid++; + char **p=(char **)malloc(sizeof(char*)*1); + unsigned long *l=(unsigned long *)malloc(sizeof(unsigned long *)*1); + l[0]=strlen(uuid_val); + p[0]=(char *)uuid_val; + myprot->generate_pkt_row(true,NULL,NULL,sid,1,l,p); sid++; + myds->DSS=STATE_ROW; + myprot->generate_pkt_EOF(true,NULL,NULL,sid,0, setStatus); sid++; + myds->DSS=STATE_SLEEP; + run_query=false; + free(l); + free(p); + goto __run_query; + } +``` + +- [ ] **Step 4: Monitor thread queries the peer's UUID** + +`lib/ProxySQL_Cluster.cpp`, monitor thread body: right after the `PROXYSQL CLUSTER_NODE_UUID` announce block (:236-244, i.e. once per successful (re)connection, inside the same connected branch), add: + +```cpp + rc_query = mysql_query(conn, (char *)"SELECT GLOBAL_UUID()"); + if (rc_query == 0) { + MYSQL_RES *uuid_res = mysql_store_result(conn); + if (uuid_res) { + MYSQL_ROW urow = mysql_fetch_row(uuid_res); + if (urow && urow[0] && strlen(urow[0]) > 0) { + GloProxyCluster->Update_Node_UUID(node->hostname, node->port, urow[0]); + } + mysql_free_result(uuid_res); + } + } +``` + +Add `Update_Node_UUID` on `ProxySQL_Cluster_Nodes` (same shape as `Update_Node_Failure`, but calls `ite->second->set_uuid(_uuid);`), declaration in hpp ~:408, and public forwarder `void ProxySQL_Cluster::Update_Node_UUID(char* h, uint16_t p, const char* u) { nodes.Update_Node_UUID(h, p, u); }`. + +- [ ] **Step 5: Build, smoke-check, commit** + +```bash +PROXYSQL31=1 make debug -j$(nproc) 2>&1 | tail -5 +``` +Expected: clean build. Manual smoke check of the intercept (no infra needed): +```bash +src/proxysql --idle-threads -f -c /etc/proxysql.cnf -D /tmp/claude-1004/-data-rene-proxysql7-proxysql/dc518716-1df1-4e06-9608-25ec703bd0b2/scratchpad/px-smoke -M & +sleep 3 +mysql -uadmin -padmin -h127.0.0.1 -P6032 -e "SELECT GLOBAL_UUID()" +mysql -uadmin -padmin -h127.0.0.1 -P6032 -e "PROXYSQL SHUTDOWN" || true +``` +(If no default config exists, generate a minimal one in the scratchpad with `datadir` + `admin_variables.mysql_ifaces="0.0.0.0:6032"`.) Expected: one row, a 36-char UUID. + +```bash +git add include/ProxySQL_Cluster.hpp lib/ProxySQL_Cluster.cpp lib/Admin_Handler.cpp +git commit -m "feat(cluster): per-node liveness bookkeeping and GLOBAL_UUID() peer identity exchange" +``` + +--- + +### Task 3: Tri-state admin read-only mode + `PROXYSQL READONLY AUTO` + +**Files:** +- Modify: `include/proxysql_admin.h` (:646-647 replace inline get/set; add enum + atomics near the `variables` struct) +- Modify: `lib/ProxySQL_Admin.cpp` (:2900 default init; :4816-4826 `set_variable` for `read_only`) +- Modify: `lib/Admin_Handler.cpp` (:745-760 PROXYSQL READONLY/READWRITE handlers + new AUTO command; :3697, :3722, :5369 enforcement call sites) + +**Interfaces:** +- Consumes: nothing yet (the follower flag is fed by Task 4). +- Produces (used by Tasks 4, 5): + - `enum admin_ro_mode_t { ADMIN_RO_MODE_AUTO = 0, ADMIN_RO_MODE_FORCED_RO = 1, ADMIN_RO_MODE_FORCED_RW = 2 };` + - `bool ProxySQL_Admin::effective_read_only()` + - `void ProxySQL_Admin::set_ro_mode(admin_ro_mode_t m)` + - `void ProxySQL_Admin::set_cluster_follower(bool f)` + - Admin commands: `PROXYSQL READONLY` → FORCED_RO, `PROXYSQL READWRITE` → FORCED_RW, `PROXYSQL READONLY AUTO` → AUTO. + +- [ ] **Step 1: Replace the boolean API in the header** + +`include/proxysql_admin.h`: above `class ProxySQL_Admin` (or right before it), add: + +```cpp +enum admin_ro_mode_t { + ADMIN_RO_MODE_AUTO = 0, // read-only iff this node is a cluster follower (leader election) + ADMIN_RO_MODE_FORCED_RO = 1, // operator-forced read-only (PROXYSQL READONLY) + ADMIN_RO_MODE_FORCED_RW = 2, // operator-forced read-write (PROXYSQL READWRITE) +}; +``` + +Keep the `bool admin_read_only;` field at :373 (it remains the storage for the `admin-read_only` boot variable). Replace the two inlines at :646-647 with: + +```cpp + bool effective_read_only() { + int m = ro_mode.load(std::memory_order_relaxed); + if (m == ADMIN_RO_MODE_FORCED_RO) return true; + if (m == ADMIN_RO_MODE_FORCED_RW) return false; + return cluster_follower.load(std::memory_order_relaxed); + } + void set_ro_mode(admin_ro_mode_t m) { ro_mode.store((int)m, std::memory_order_relaxed); } + admin_ro_mode_t get_ro_mode() { return (admin_ro_mode_t)ro_mode.load(std::memory_order_relaxed); } + void set_cluster_follower(bool f) { cluster_follower.store(f, std::memory_order_relaxed); } +``` + +and add the two members next to other member declarations (e.g. near `SerialExposer` :358): + +```cpp + std::atomic ro_mode { ADMIN_RO_MODE_AUTO }; + std::atomic cluster_follower { false }; +``` + +(`` is already available in this header's include set; add `#include ` at the top if the build says otherwise.) + +- [ ] **Step 2: Update all former `get_read_only()` / `set_read_only()` call sites** + +There are exactly five (verified by grep): +1. `lib/Admin_Handler.cpp:5369` — `if (SPA->get_read_only())` → `if (SPA->effective_read_only())` (the `PRAGMA query_only` wrapper). +2. `lib/Admin_Handler.cpp:3697` — `bool ro=SPA->get_read_only();` → `bool ro=SPA->effective_read_only();` (`SHOW GLOBAL VARIABLES LIKE 'read_only'` canned response — external HA tooling now sees follower state). +3. `lib/Admin_Handler.cpp:3722` — same change (`SELECT @@global.read_only` canned response). +4. `lib/Admin_Handler.cpp:749` — `SPA->set_read_only(true);` → `SPA->set_ro_mode(ADMIN_RO_MODE_FORCED_RO);` +5. `lib/Admin_Handler.cpp:757` — `SPA->set_read_only(false);` → `SPA->set_ro_mode(ADMIN_RO_MODE_FORCED_RW);` + +Add the new command as a separate exact-length block adjacent to the two existing ones (after :760; ordering is irrelevant because matching is exact-length): + +```cpp + if (query_no_space_length==strlen("PROXYSQL READONLY AUTO") && !strncasecmp("PROXYSQL READONLY AUTO",query_no_space, query_no_space_length)) { + // returns read-only control to the cluster leader election (AUTO mode) + proxy_info("Received PROXYSQL READONLY AUTO command\n"); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + SPA->set_ro_mode(ADMIN_RO_MODE_AUTO); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } +``` + +Also add a `proxy_info` on each mode change by extending the two existing handlers' log lines (they already `proxy_info` the command; that satisfies the spec's transition logging together with Task 4's election logs). + +- [ ] **Step 3: Map the `admin-read_only` boot variable onto the tri-state** + +`lib/ProxySQL_Admin.cpp` `set_variable`, the `read_only` branch (:4816-4826): after `variables.admin_read_only` is assigned, add: + +```cpp + set_ro_mode(variables.admin_read_only ? ADMIN_RO_MODE_FORCED_RO : ADMIN_RO_MODE_AUTO); +``` + +(Default init at :2900 stays `variables.admin_read_only=false;`; the atomic already defaults to AUTO. With election disabled the follower flag is always false, so AUTO ⇒ read-write ⇒ today's behavior exactly.) + +- [ ] **Step 4: Build and behavior-check, commit** + +```bash +PROXYSQL31=1 make debug -j$(nproc) 2>&1 | tail -5 +``` +Manual check against a scratch instance (same spawn recipe as Task 2 Step 5): +- `INSERT INTO mysql_servers (hostgroup_id,hostname) VALUES (1,'127.0.0.1');` succeeds (AUTO, no election). +- `PROXYSQL READONLY` → the same INSERT now fails with `attempt to write a readonly database`. +- `PROXYSQL READONLY AUTO` → INSERT succeeds again. +- `SELECT @@global.read_only` reflects each state (0/1/0). + +```bash +git add include/proxysql_admin.h lib/ProxySQL_Admin.cpp lib/Admin_Handler.cpp +git commit -m "feat(admin): tri-state read-only mode (AUTO/FORCED_RO/FORCED_RW) with PROXYSQL READONLY AUTO" +``` + +--- + +### Task 4: Admin variables, election tick, and leader state on ProxySQL_Cluster + +**Files:** +- Modify: `include/proxysql_admin.h` (:379 region — 3 new fields in `variables` struct) +- Modify: `lib/ProxySQL_Admin.cpp` (name list :390-461; defaults :2900-2930; `get_variable` :3731 region; `set_variable` :4236 region) +- Modify: `include/ProxySQL_Cluster.hpp` (`ProxySQL_Cluster` members :607-631 region + method declarations) +- Modify: `lib/ProxySQL_Cluster.cpp` (ctor defaults :5494-5526; new methods; `#include "ProxySQL_Cluster_Leader.h"`) +- Modify: `lib/ProxySQL_Admin.cpp` `admin_main_loop` (:2624-2640 region — tick call) +- Modify: `test/tap/tests/proxysql_reference_select_config_file.cnf`, `test/tap/tests/test_cluster_sync-t.cpp`, `test/tap/tests/reg_test_3847_admin_lock-t.cpp` (reference lists of admin variables — see Step 5) + +**Interfaces:** +- Consumes: Task 1 (`cluster_elect_leader`, `Cluster_Leader_State`), Task 2 (`get_uuid()`, `get_last_success_at_us()`), Task 3 (`set_cluster_follower()`). +- Produces (used by Tasks 5, 6, 7): + - `ProxySQL_Cluster` fields: `int cluster_leader_election;` (0/1), `int cluster_leader_node_timeout_ms;`, `int cluster_leader_grace_ms;` (all `__sync_*` access) + - `void ProxySQL_Cluster::leader_election_tick(unsigned long long curtime_us)` + - `bool ProxySQL_Cluster::is_leader()` — true iff current effective leader uuid == `GloVars.uuid` + - `void ProxySQL_Cluster::get_leader_info(std::string& hostname, int& port, std::string& uuid)` — all empty/0 when no leader + - `std::vector ProxySQL_Cluster_Nodes::get_leader_candidates(unsigned long long alive_timeout_us)` + +- [ ] **Step 1: Wire the three admin variables** + +`include/proxysql_admin.h`, in the `variables` struct next to `cluster_check_interval_ms` (:379): + +```cpp + bool cluster_leader_election; + int cluster_leader_node_timeout_ms; + int cluster_leader_grace_ms; +``` + +`lib/ProxySQL_Admin.cpp` `admin_variables_names[]` (insert after `(char *)"cluster_check_interval_ms",` at :412): + +```cpp +#ifdef PROXYSQL31 + (char *)"cluster_leader_election", +#endif /* PROXYSQL31 */ + (char *)"cluster_leader_node_timeout_ms", + (char *)"cluster_leader_grace_ms", +``` + +Defaults (next to :2904): + +```cpp + variables.cluster_leader_election=false; + variables.cluster_leader_node_timeout_ms=3000; + variables.cluster_leader_grace_ms=3000; +``` + +`get_variable` (next to the `cluster_check_interval_ms` block at :3731): + +```cpp +#ifdef PROXYSQL31 + if (!strcasecmp(name,"cluster_leader_election")) { + return strdup((variables.cluster_leader_election ? "true" : "false")); + } +#endif /* PROXYSQL31 */ + if (!strcasecmp(name,"cluster_leader_node_timeout_ms")) { + sprintf(intbuf,"%d",variables.cluster_leader_node_timeout_ms); + return strdup(intbuf); + } + if (!strcasecmp(name,"cluster_leader_grace_ms")) { + sprintf(intbuf,"%d",variables.cluster_leader_grace_ms); + return strdup(intbuf); + } +``` + +`set_variable` (next to the `cluster_check_interval_ms` block at :4236; same `__sync_lock_test_and_set` push pattern): + +```cpp +#ifdef PROXYSQL31 + if (!strcasecmp(name,"cluster_leader_election")) { + if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { + variables.cluster_leader_election=true; + __sync_lock_test_and_set(&GloProxyCluster->cluster_leader_election, 1); + // Spec: with election enabled a node is effective-RO until the first + // election settles. Assume follower immediately; the next tick corrects + // it (the elected leader flips back to RW within tick+grace). + set_cluster_follower(true); + return true; + } + if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { + variables.cluster_leader_election=false; + __sync_lock_test_and_set(&GloProxyCluster->cluster_leader_election, 0); + set_cluster_follower(false); // immediate, don't wait for the next tick + return true; + } + return false; + } +#endif /* PROXYSQL31 */ + if (!strcasecmp(name,"cluster_leader_node_timeout_ms")) { + int intv=atoi(value); + if (intv >= 1000 && intv <= 600000) { + variables.cluster_leader_node_timeout_ms=intv; + __sync_lock_test_and_set(&GloProxyCluster->cluster_leader_node_timeout_ms, intv); + return true; + } else { + return false; + } + } + if (!strcasecmp(name,"cluster_leader_grace_ms")) { + int intv=atoi(value); + if (intv >= 0 && intv <= 600000) { + variables.cluster_leader_grace_ms=intv; + __sync_lock_test_and_set(&GloProxyCluster->cluster_leader_grace_ms, intv); + return true; + } else { + return false; + } + } +``` + +- [ ] **Step 2: Leader state on `ProxySQL_Cluster` + candidate collection** + +`include/ProxySQL_Cluster.hpp`: add `#include "ProxySQL_Cluster_Leader.h"` at the top (after the existing includes). On `ProxySQL_Cluster_Nodes` (public, near :403): + +```cpp + std::vector get_leader_candidates(unsigned long long alive_timeout_us); +``` + +On `ProxySQL_Cluster` (public, near :607): + +```cpp + int cluster_leader_election; // 0/1, __sync access + int cluster_leader_node_timeout_ms; + int cluster_leader_grace_ms; + pthread_mutex_t leader_mutex; // guards leader_state + leader_hostname/leader_port + Cluster_Leader_State leader_state; + char * leader_hostname; // NULL = no leader + int leader_port; + unsigned long long leader_next_check_at; // monotonic us, 0 initially + void leader_election_tick(unsigned long long curtime_us); + bool is_leader(); + void get_leader_info(std::string& hostname, int& port, std::string& uuid); +``` + +`lib/ProxySQL_Cluster.cpp` ctor (:5494-5526 region): `cluster_leader_election = 0; cluster_leader_node_timeout_ms = 3000; cluster_leader_grace_ms = 3000; leader_hostname = NULL; leader_port = 0; leader_next_check_at = 0; pthread_mutex_init(&leader_mutex, NULL);`. + +Candidate collection (implementation near the other stats builders, ~:4640): + +```cpp +std::vector ProxySQL_Cluster_Nodes::get_leader_candidates(unsigned long long alive_timeout_us) { + std::vector candidates; + unsigned long long now = monotonic_time(); + pthread_mutex_lock(&mutex); + for (auto it = umap_proxy_nodes.begin(); it != umap_proxy_nodes.end(); it++) { + ProxySQL_Node_Entry * node = it->second; + Cluster_Leader_Candidate c; + c.uuid = (node->get_uuid() ? node->get_uuid() : ""); + c.hostname = node->get_hostname(); + c.port = node->get_port(); + c.weight = node->get_weight(); + bool is_self = (GloVars.uuid && node->get_uuid() && strcmp(node->get_uuid(), GloVars.uuid) == 0); + unsigned long long last = node->get_last_success_at_us(); + c.alive = is_self || (last != 0 && (now - last) < alive_timeout_us); + candidates.push_back(c); + } + pthread_mutex_unlock(&mutex); + return candidates; +} +``` + +(Self-identification is UUID equality; a node's own entry gets its uuid from the node polling its own admin port, which every standard cluster deployment does. A node that cannot reach even its own admin interface has no electable self — acceptable and noted in the spec's edge cases.) + +- [ ] **Step 3: The election tick** + +`lib/ProxySQL_Cluster.cpp` (near `p_update_metrics` ~:5543): + +```cpp +void ProxySQL_Cluster::leader_election_tick(unsigned long long curtime_us) { + if (curtime_us < leader_next_check_at) return; + leader_next_check_at = curtime_us + 500000; // evaluate at most every 500ms + int enabled = __sync_fetch_and_add(&cluster_leader_election, 0); + char *c_user = NULL; char *c_pass = NULL; + get_credentials(&c_user, &c_pass); + bool clustering_active = (c_user && strlen(c_user) > 0); + free(c_user); free(c_pass); + bool am_leader_or_standalone = true; + if (enabled == 0 || clustering_active == false) { + pthread_mutex_lock(&leader_mutex); + leader_state.reset(); + if (leader_hostname) { free(leader_hostname); leader_hostname = NULL; } + leader_port = 0; + pthread_mutex_unlock(&leader_mutex); + } else { + unsigned long long timeout_us = (unsigned long long)__sync_fetch_and_add(&cluster_leader_node_timeout_ms, 0) * 1000ULL; + unsigned long long grace_ms = (unsigned long long)__sync_fetch_and_add(&cluster_leader_grace_ms, 0); + std::vector candidates = nodes.get_leader_candidates(timeout_us); + if (candidates.empty()) { + // proxysql_servers is empty: standalone behavior + pthread_mutex_lock(&leader_mutex); + leader_state.reset(); + if (leader_hostname) { free(leader_hostname); leader_hostname = NULL; } + leader_port = 0; + pthread_mutex_unlock(&leader_mutex); + } else { + int idx = cluster_elect_leader(candidates); + std::string computed = (idx >= 0 ? candidates[idx].uuid : ""); + pthread_mutex_lock(&leader_mutex); + bool changed = leader_state.update(computed, curtime_us / 1000, grace_ms); + if (changed) { + if (leader_hostname) { free(leader_hostname); leader_hostname = NULL; } + leader_port = 0; + if (idx >= 0 && leader_state.current_leader_uuid == candidates[idx].uuid) { + leader_hostname = strdup(candidates[idx].hostname.c_str()); + leader_port = candidates[idx].port; + } + proxy_info("Cluster leader changed: new leader is %s (%s:%d)\n", + (leader_state.current_leader_uuid.empty() ? "NONE" : leader_state.current_leader_uuid.c_str()), + (leader_hostname ? leader_hostname : ""), leader_port); + metrics.p_counter_array[p_cluster_counter::cluster_leader_changes]->Increment(); + } + am_leader_or_standalone = (GloVars.uuid && leader_state.current_leader_uuid == GloVars.uuid); + pthread_mutex_unlock(&leader_mutex); + } + } + GloAdmin->set_cluster_follower(enabled != 0 && clustering_active && am_leader_or_standalone == false); +} +``` + +Note: `p_cluster_counter::cluster_leader_changes` is added in Step 4 of THIS task (so this task compiles standalone); Task 7 adds only the gauge and the per-node metric. + +`is_leader()` / `get_leader_info()`: + +```cpp +bool ProxySQL_Cluster::is_leader() { + pthread_mutex_lock(&leader_mutex); + bool r = (GloVars.uuid && leader_state.current_leader_uuid.empty() == false && leader_state.current_leader_uuid == GloVars.uuid); + pthread_mutex_unlock(&leader_mutex); + return r; +} + +void ProxySQL_Cluster::get_leader_info(std::string& hostname, int& port, std::string& uuid) { + pthread_mutex_lock(&leader_mutex); + hostname = (leader_hostname ? leader_hostname : ""); + port = leader_port; + uuid = leader_state.current_leader_uuid; + pthread_mutex_unlock(&leader_mutex); +} +``` + +Semantics encoded above (matches spec): election enabled + clustering active + this node is not the effective leader (including "no leader yet") ⇒ follower ⇒ AUTO means read-only. Election disabled, or clustering unconfigured, or `proxysql_servers` empty ⇒ not a follower ⇒ AUTO means read-write. + +- [ ] **Step 4: Counter enum + tick call in the Admin main loop** + +`include/ProxySQL_Cluster.hpp` `p_cluster_counter` enum (:433-510): add `cluster_leader_changes,` before `SIZE_`. In `lib/ProxySQL_Cluster.cpp` `cluster_counter_vector` (starts ~:4872), add: + +```cpp + std::make_tuple ( + p_cluster_counter::cluster_leader_changes, + "proxysql_cluster_leader_changes_total", + "Number of times this node observed an effective cluster leader change.", + metric_tags {} + ), +``` + +`lib/ProxySQL_Admin.cpp` `admin_main_loop`, right after the `#endif` of the TSDB block (:2640), **outside** any `#ifdef`: + +```cpp + if (GloProxyCluster) { + GloProxyCluster->leader_election_tick(curtime); + } +``` + +(`curtime` is the monotonic µs captured at :2486; worst-case tick resolution is ~500ms — fine against a 3000ms default grace.) + +- [ ] **Step 5: Update admin-variable reference fixtures** + +Some tests enumerate admin variables. Run: +```bash +grep -n "cluster_check_interval_ms" test/tap/tests/proxysql_reference_select_config_file.cnf \ + test/tap/tests/test_cluster_sync-t.cpp test/tap/tests/reg_test_3847_admin_lock-t.cpp +``` +For **each** occurrence found, mirror the same entry style for `cluster_leader_node_timeout_ms` and `cluster_leader_grace_ms` (the two ungated variables). Do NOT add `cluster_leader_election` to fixtures that a stable-tier CI build would also exercise — it only exists under PROXYSQL31; add it only where the fixture is tier-aware (if unclear, leave it out; the TAP test in Task 8 covers it). + +- [ ] **Step 6: Build, verify, commit** + +```bash +PROXYSQL31=1 make debug -j$(nproc) 2>&1 | tail -5 +``` +Scratch-instance check: `SELECT * FROM global_variables WHERE variable_name LIKE 'admin-cluster_leader%'` shows all three with defaults; `SET admin-cluster_leader_election='true'; LOAD ADMIN VARIABLES TO RUNTIME;` then (with empty `proxysql_servers`) verify writes still work (standalone short-circuit). Also build once WITHOUT the tier flag to prove the ifdef discipline: +```bash +make clean && make -j$(nproc) 2>&1 | tail -3 && make clean && PROXYSQL31=1 make debug -j$(nproc) 2>&1 | tail -3 +``` +Expected: both build; in the stable build `admin-cluster_leader_election` does not exist. + +```bash +git add include/proxysql_admin.h lib/ProxySQL_Admin.cpp include/ProxySQL_Cluster.hpp lib/ProxySQL_Cluster.cpp \ + test/tap/tests/proxysql_reference_select_config_file.cnf test/tap/tests/test_cluster_sync-t.cpp \ + test/tap/tests/reg_test_3847_admin_lock-t.cpp +git commit -m "feat(cluster): leader election tick, admin variables (PROXYSQL31-gated switch), follower steering" +``` + +--- + +### Task 5: Gate `LOAD … TO RUNTIME` / `SAVE … TO DISK` under effective read-only + +**Files:** +- Modify: `lib/Admin_Handler.cpp` (`admin_handler_command_load_or_save()`, top of function ~:1416) + +**Interfaces:** +- Consumes: Task 3 `effective_read_only()`, Task 4 `get_leader_info()`. +- Produces: operator-facing refusal with leader identity; nothing consumed by later tasks. + +- [ ] **Step 1: Add the gate at the single choke point** + +Every `LOAD *`/`SAVE *` admin command flows through `admin_handler_command_load_or_save()` (dispatch at `lib/Admin_Handler.cpp:3836-3840`). Cluster syncs bypass it (direct C++ calls) — exactly what we want. At the top of the function (~:1416, after `SPA` is available; add the cast if the function derives it later): + +```cpp + { + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + if (SPA->effective_read_only()) { + bool is_load = (!strncasecmp("LOAD ", query_no_space, 5)); + bool is_save = (!strncasecmp("SAVE ", query_no_space, 5)); + bool refuse = false; + if (query_no_space_length > 11 && !strncasecmp(" TO RUNTIME", query_no_space+query_no_space_length-11, 11)) { + refuse = true; // LOAD ... TO RUNTIME + } + if (query_no_space_length > 8 && is_save && !strncasecmp(" TO DISK", query_no_space+query_no_space_length-8, 8)) { + refuse = true; // SAVE ... TO DISK + } + if (query_no_space_length > 12 && (is_load || is_save) && !strncasecmp(" FROM MEMORY", query_no_space+query_no_space_length-12, 12)) { + refuse = true; // aliases: LOAD x FROM MEMORY == LOAD x TO RUNTIME ; SAVE x FROM MEMORY == SAVE x TO DISK + } + if (refuse) { + std::string l_host; int l_port = 0; std::string l_uuid; + GloProxyCluster->get_leader_info(l_host, l_port, l_uuid); + char msg[512]; + if (l_host.length()) { + snprintf(msg, sizeof(msg), + "Admin is in read-only mode (cluster follower). Current leader is %s:%d (%s). Use PROXYSQL READWRITE to override.", + l_host.c_str(), l_port, l_uuid.c_str()); + } else { + snprintf(msg, sizeof(msg), + "Admin is in read-only mode. Use PROXYSQL READWRITE to override."); + } + proxy_warning("Refused '%s' : %s\n", query_no_space, msg); + SPA->send_error_msg_to_client(sess, msg); + return false; + } + } + } +``` + +Check the actual signature of `send_error_msg_to_client` at its declaration in `include/proxysql_admin.h` and match it (it is used at `lib/Admin_Handler.cpp:734` as `SPA->send_error_msg_to_client(sess, (char *)"...")` — cast `msg` to `(char *)` if required). `GloProxyCluster` is already an `extern` visible in `Admin_Handler.cpp` (used at :686-745); add the extern declaration at the top of the file if the compiler disagrees. + +- [ ] **Step 2: Build, verify, commit** + +```bash +PROXYSQL31=1 make debug -j$(nproc) 2>&1 | tail -5 +``` +Scratch-instance check: `PROXYSQL READONLY`, then `LOAD MYSQL SERVERS TO RUNTIME` → error containing "read-only mode"; `SAVE MYSQL SERVERS TO DISK` → same; `LOAD MYSQL SERVERS FROM DISK` → still allowed; `PROXYSQL READWRITE` → all allowed again. + +```bash +git add lib/Admin_Handler.cpp +git commit -m "feat(admin): refuse LOAD ... TO RUNTIME / SAVE ... TO DISK in effective read-only mode" +``` + +--- + +### Task 6: Implement `stats_proxysql_servers_status` + +**Files:** +- Modify: `include/ProxySQL_Admin_Tables_Definitions.h:289` (add `uuid` column) +- Modify: `include/ProxySQL_Cluster.hpp` (declare producer + forwarder) +- Modify: `lib/ProxySQL_Cluster.cpp` (producer implementation) +- Modify: `include/proxysql_admin.h:809` region (declare `stats___proxysql_servers_status()`) +- Modify: `lib/ProxySQL_Admin_Stats.cpp` (implementation, modeled on :1516-1563) +- Modify: `lib/ProxySQL_Admin.cpp` (:1345, :1492-1496, :1724-1727 — re-enable the three commented blocks) + +**Interfaces:** +- Consumes: Task 2 node-entry accessors, Task 4 `get_leader_info()` + `cluster_leader_node_timeout_ms`. +- Produces: queryable `stats_proxysql_servers_status` table (used by Task 8's TAP test); `SQLite3_result * ProxySQL_Cluster::get_stats_proxysql_servers_status()`. + +- [ ] **Step 1: Schema — append the uuid column** + +`include/ProxySQL_Admin_Tables_Definitions.h:289`, change the define to (only addition: `uuid`): + +```c +#define STATS_SQLITE_TABLE_PROXYSQL_SERVERS_STATUS "CREATE TABLE stats_proxysql_servers_status (hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 6032 , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 0 , master VARCHAR NOT NULL , global_version INT NOT NULL , check_age_us INT NOT NULL , ping_time_us INT NOT NULL, checks_OK INT NOT NULL , checks_ERR INT NOT NULL , uuid VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostname, port) )" +``` + +- [ ] **Step 2: Producer on the cluster side** + +`include/ProxySQL_Cluster.hpp`: on `ProxySQL_Cluster_Nodes` (near the other stats declarations ~:413): +```cpp + SQLite3_result * stats_proxysql_servers_status(const std::string& leader_uuid, unsigned long long alive_timeout_us); +``` +On `ProxySQL_Cluster` (forwarders ~:667-672): +```cpp + SQLite3_result * get_stats_proxysql_servers_status(); +``` + +`lib/ProxySQL_Cluster.cpp` (next to `stats_proxysql_servers_metrics` ~:4581; same 100%-strdup/SQLITE_TEXT pattern as `dump_table_proxysql_servers` :4635): + +```cpp +SQLite3_result * ProxySQL_Cluster_Nodes::stats_proxysql_servers_status(const std::string& leader_uuid, unsigned long long alive_timeout_us) { + const int colnum = 10; + SQLite3_result *result = new SQLite3_result(colnum); + result->add_column_definition(SQLITE_TEXT,"hostname"); + result->add_column_definition(SQLITE_TEXT,"port"); + result->add_column_definition(SQLITE_TEXT,"weight"); + result->add_column_definition(SQLITE_TEXT,"master"); + result->add_column_definition(SQLITE_TEXT,"global_version"); + result->add_column_definition(SQLITE_TEXT,"check_age_us"); + result->add_column_definition(SQLITE_TEXT,"ping_time_us"); + result->add_column_definition(SQLITE_TEXT,"checks_OK"); + result->add_column_definition(SQLITE_TEXT,"checks_ERR"); + result->add_column_definition(SQLITE_TEXT,"uuid"); + (void)alive_timeout_us; // liveness is derivable from check_age_us; kept for future use + unsigned long long now = monotonic_time(); + char buf[64]; + pthread_mutex_lock(&mutex); + for (auto it = umap_proxy_nodes.begin(); it != umap_proxy_nodes.end(); it++) { + ProxySQL_Node_Entry * node = it->second; + char **pta = (char **)malloc(sizeof(char *)*colnum); + pta[0] = strdup(node->get_hostname()); + sprintf(buf, "%d", node->get_port()); pta[1] = strdup(buf); + sprintf(buf, "%lu", node->get_weight()); pta[2] = strdup(buf); + const char *nuuid = node->get_uuid(); + bool is_master = (nuuid && leader_uuid.empty() == false && leader_uuid == nuuid); + pta[3] = strdup(is_master ? "YES" : "NO"); + sprintf(buf, "%lu", (unsigned long)node->get_global_version()); pta[4] = strdup(buf); + unsigned long long last = node->get_last_success_at_us(); + if (last == 0) { + pta[5] = strdup("-1"); + } else { + sprintf(buf, "%llu", now - last); pta[5] = strdup(buf); + } + ProxySQL_Node_Metrics *curr = node->get_metrics_curr(); + sprintf(buf, "%llu", (curr ? curr->response_time_us : 0)); pta[6] = strdup(buf); + sprintf(buf, "%lu", (unsigned long)node->get_checks_ok()); pta[7] = strdup(buf); + sprintf(buf, "%lu", (unsigned long)node->get_checks_err()); pta[8] = strdup(buf); + pta[9] = strdup(nuuid ? nuuid : ""); + result->add_row(pta); + for (int k = 0; k < colnum; k++) { + if (pta[k]) free(pta[k]); + } + free(pta); + } + pthread_mutex_unlock(&mutex); + return result; +} + +SQLite3_result * ProxySQL_Cluster::get_stats_proxysql_servers_status() { + std::string l_host; int l_port = 0; std::string l_uuid; + get_leader_info(l_host, l_port, l_uuid); + unsigned long long timeout_us = (unsigned long long)__sync_fetch_and_add(&cluster_leader_node_timeout_ms, 0) * 1000ULL; + return nodes.stats_proxysql_servers_status(l_uuid, timeout_us); +} +``` + +(Check the exact field name for response time in `ProxySQL_Node_Metrics` at `include/ProxySQL_Cluster.hpp:233-249` — the member measured in `set_metrics` — and use that name; if the metrics ring has never been filled, `get_metrics_curr()` returns a zeroed entry, which yields 0.) + +- [ ] **Step 3: Admin side — `stats___proxysql_servers_status()` + re-enable interception** + +`include/proxysql_admin.h`: next to `:809` (`stats___proxysql_servers_metrics`), declare `void stats___proxysql_servers_status();`. + +`lib/ProxySQL_Admin_Stats.cpp`: implement modeled on `stats___proxysql_servers_checksums` (:1516-1563) **including the `sql_query_global_mutex` unlock/relock dance** (the producer takes the cluster nodes mutex — same deadlock hazard documented at :1517-1529): + +```cpp +void ProxySQL_Admin::stats___proxysql_servers_status() { + // Same deadlock avoidance as stats___proxysql_servers_checksums: + // release sql_query_global_mutex while calling into the cluster nodes mutex. + pthread_mutex_unlock(&pa->sql_query_global_mutex); + SQLite3_result *resultset = GloProxyCluster->get_stats_proxysql_servers_status(); + pthread_mutex_lock(&pa->sql_query_global_mutex); + if (resultset == NULL) return; + statsdb->execute("BEGIN"); + statsdb->execute("DELETE FROM stats_proxysql_servers_status"); + char *query = (char *)"INSERT INTO stats_proxysql_servers_status VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)"; + sqlite3_stmt *statement = NULL; + int rc = statsdb->prepare_v2(query, &statement); + ASSERT_SQLITE_OK(rc, statsdb); + for (std::vector::iterator it = resultset->rows.begin(); it != resultset->rows.end(); ++it) { + SQLite3_row *r = *it; + rc = (*proxy_sqlite3_bind_text)(statement, 1, r->fields[0], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc = (*proxy_sqlite3_bind_int64)(statement, 2, atoll(r->fields[1])); ASSERT_SQLITE_OK(rc, statsdb); + rc = (*proxy_sqlite3_bind_int64)(statement, 3, atoll(r->fields[2])); ASSERT_SQLITE_OK(rc, statsdb); + rc = (*proxy_sqlite3_bind_text)(statement, 4, r->fields[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc = (*proxy_sqlite3_bind_int64)(statement, 5, atoll(r->fields[4])); ASSERT_SQLITE_OK(rc, statsdb); + rc = (*proxy_sqlite3_bind_int64)(statement, 6, atoll(r->fields[5])); ASSERT_SQLITE_OK(rc, statsdb); + rc = (*proxy_sqlite3_bind_int64)(statement, 7, atoll(r->fields[6])); ASSERT_SQLITE_OK(rc, statsdb); + rc = (*proxy_sqlite3_bind_int64)(statement, 8, atoll(r->fields[7])); ASSERT_SQLITE_OK(rc, statsdb); + rc = (*proxy_sqlite3_bind_int64)(statement, 9, atoll(r->fields[8])); ASSERT_SQLITE_OK(rc, statsdb); + rc = (*proxy_sqlite3_bind_text)(statement, 10, r->fields[9], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + SAFE_SQLITE3_STEP2(statement); + rc = (*proxy_sqlite3_clear_bindings)(statement); ASSERT_SQLITE_OK(rc, statsdb); + rc = (*proxy_sqlite3_reset)(statement); ASSERT_SQLITE_OK(rc, statsdb); + } + (*proxy_sqlite3_finalize)(statement); + statsdb->execute("COMMIT"); + delete resultset; +} +``` + +Open the template at `lib/ProxySQL_Admin_Stats.cpp:1516-1563` first and copy its EXACT idioms (the `pa->` handle, bind function pointer names, `ASSERT_SQLITE_OK`, finalize call) — the snippet above shows structure and column order; the template file is authoritative for helper spellings. + +`lib/ProxySQL_Admin.cpp` — re-enable the three commented blocks exactly: +- `:1345` → `bool stats_proxysql_servers_status = false;` +- `:1492-1496` → +```cpp + if (strstr(query_no_space,"stats_proxysql_servers_status")) + { stats_proxysql_servers_status = true; refresh = true; } +``` +- `:1724-1727` → +```cpp + if (stats_proxysql_servers_status) { + stats___proxysql_servers_status(); + } +``` + +- [ ] **Step 4: Build, verify, commit** + +```bash +PROXYSQL31=1 make debug -j$(nproc) 2>&1 | tail -5 +``` +Scratch check: with empty `proxysql_servers`, `SELECT * FROM stats_proxysql_servers_status` returns 0 rows (not an error). (Populated-path verification happens in Task 8's multi-node TAP test.) + +```bash +git add include/ProxySQL_Admin_Tables_Definitions.h include/ProxySQL_Cluster.hpp lib/ProxySQL_Cluster.cpp \ + include/proxysql_admin.h lib/ProxySQL_Admin_Stats.cpp lib/ProxySQL_Admin.cpp +git commit -m "feat(cluster): implement stats_proxysql_servers_status with leader flag and liveness data" +``` + +--- + +### Task 7: Prometheus metrics (leader gauge + per-node alive gauge) + +**Files:** +- Modify: `include/ProxySQL_Cluster.hpp` (`p_cluster_gauge` :512-516; `p_cluster_nodes_dyn_gauge` :352-364; map member :386-401) +- Modify: `lib/ProxySQL_Cluster.cpp` (`cluster_gauge_vector` :5491; `cluster_nodes_dyn_gauge_vector` ~:3890; `update_prometheus_nodes_metrics` :4681+ update/cleanup lists; `p_update_metrics` :5543) + +**Interfaces:** +- Consumes: Task 4 `is_leader()`, node-entry liveness from Task 2. (`proxysql_cluster_leader_changes_total` was already added in Task 4.) +- Produces: `/metrics` families `proxysql_cluster_leader_status` (0/1) and per-node `proxysql_servers_alive{hostname,port}`. + +- [ ] **Step 1: Static leader gauge** + +`include/ProxySQL_Cluster.hpp` `p_cluster_gauge` (:512-516, currently only `SIZE_`): add `cluster_leader_status,` before `SIZE_`. In `lib/ProxySQL_Cluster.cpp`, populate the empty `cluster_gauge_vector {}` (:5491): + +```cpp + cluster_gauge_vector { + std::make_tuple ( + p_cluster_gauge::cluster_leader_status, + "proxysql_cluster_leader_status", + "1 when this node is the elected cluster leader, 0 otherwise.", + metric_tags {} + ), + } +``` + +In `ProxySQL_Cluster::p_update_metrics()` (:5543-5545), add: + +```cpp + metrics.p_gauge_array[p_cluster_gauge::cluster_leader_status]->Set(is_leader() ? 1 : 0); +``` + +- [ ] **Step 2: Per-node alive gauge** + +Four edits, following the existing per-node dyn-gauge pattern exactly (use `proxysql_servers_checksums_updated_at` or a neighbor as the template): +1. `include/ProxySQL_Cluster.hpp` `p_cluster_nodes_dyn_gauge` (:352-364): add `proxysql_servers_alive,` before `SIZE_`. +2. Same header, map members (:386-401): add `std::map p_proxysql_servers_alive {};`. +3. `lib/ProxySQL_Cluster.cpp` `cluster_nodes_dyn_gauge_vector` (~:3890): add +```cpp + std::make_tuple ( + p_cluster_nodes_dyn_gauge::proxysql_servers_alive, + "proxysql_servers_alive", + "1 when the peer answered the cluster liveness poll within admin-cluster_leader_node_timeout_ms, 0 otherwise.", + metric_tags {} + ), +``` +4. `update_prometheus_nodes_metrics()` (:4681+): inside the per-node loop (where `m_common_labels`/`m_id` are built, :4696-4697), compute and publish: +```cpp + unsigned long long alive_timeout_us = (unsigned long long)__sync_fetch_and_add(&GloProxyCluster->cluster_leader_node_timeout_ms, 0) * 1000ULL; + unsigned long long last_ok = entry->get_last_success_at_us(); + bool node_alive = (last_ok != 0 && (monotonic_time() - last_ok) < alive_timeout_us); + p_update_map_gauge(p_proxysql_servers_alive, + gauge_array[p_cluster_nodes_dyn_gauge::proxysql_servers_alive], + m_id, m_common_labels, node_alive ? 1 : 0); +``` +(Match the local variable names actually used in that function — the entry pointer and the gauge family array are already in scope; hoist the `alive_timeout_us` computation above the loop.) Then add `p_proxysql_servers_alive` to the `gauge_maps` cleanup list (:4793-4803) and to the `metric_gauges` update vector (:4750-4754) following how the neighboring gauge maps appear in each. + +- [ ] **Step 3: Build, verify, commit** + +```bash +PROXYSQL31=1 make debug -j$(nproc) 2>&1 | tail -5 +``` +Scratch check: `curl -s http://127.0.0.1:6070/metrics | grep -E "proxysql_cluster_leader_status|proxysql_cluster_leader_changes"` shows the gauge at 0 and the counter at 0. + +```bash +git add include/ProxySQL_Cluster.hpp lib/ProxySQL_Cluster.cpp +git commit -m "feat(cluster): prometheus leader-status gauge and per-node alive gauge" +``` + +--- + +### Task 8: End-to-end TAP test (3-node self-spawned cluster) + +**Files:** +- Create: `test/tap/tests/test_cluster_leader_election-t.cpp` +- Modify: `test/tap/groups/groups.json` (register) + +**Interfaces:** +- Consumes: everything above, through the admin protocol only. +- Produces: CI coverage. No code consumers. + +**Design:** self-spawned nodes (pattern of `test_cluster_sync-t.cpp:1288-1330`), NOT the shared infra cluster nodes — this test kills its leader and force-flips read-only modes, which must not disturb other tests sharing the infra cluster. All 3 nodes run on 127.0.0.1 inside the test-runner container with bespoke generated configs. Weights 300/200/100 ⇒ deterministic leader order node1 → node2 → node3. Fast timings: `cluster_check_interval_ms=200`, `cluster_leader_node_timeout_ms=1000`, `cluster_leader_grace_ms=500`. + +One deliberate deviation from the spec's illustrative test list: after the killed ex-leader rejoins, it has the highest weight and therefore **retakes** leadership (weight is priority, keepalived-style — the deterministic-election design implies this). The test asserts retake, not rejoin-as-follower. + +- [ ] **Step 1: Write the test** + +Create `test/tap/tests/test_cluster_leader_election-t.cpp`: + +```cpp +/** + * @file test_cluster_leader_election-t.cpp + * @brief E2E test for ProxySQL Cluster leader election (PROXYSQL31 feature). + * + * Spawns a self-contained 3-node ProxySQL cluster on 127.0.0.1 (bespoke + * configs, weights 300/200/100), then verifies: convergence to a single + * leader, follower write refusal (SQL + LOAD TO RUNTIME + SAVE TO DISK), + * FORCED_RW stickiness, leader failover on kill, leadership retake on + * rejoin, and full-RW behavior with election disabled. + * Skips (plan 1) on non-PROXYSQL31 builds where the master switch is absent. + */ + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "mysql.h" +#include "tap.h" +#include "command_line.h" +#include "utils.h" + +using std::string; + +struct node_t { + int idx; // 1..3 + int admin_port; // 16062/16072/16082 + int weight; // 300/200/100 + string datadir; + string cnf_path; + std::atomic pid { -1 }; + std::thread* runner = nullptr; +}; + +static node_t nodes_def[3]; + +static string workdir_path; + +static void write_node_config(node_t& n) { + FILE* f = fopen(n.cnf_path.c_str(), "w"); + if (f == NULL) { return; } + fprintf(f, "datadir=\"%s\"\n", n.datadir.c_str()); + fprintf(f, "admin_variables = {\n"); + fprintf(f, "\tadmin_credentials=\"admin:admin;cluster1:secret1pass\"\n"); + fprintf(f, "\tmysql_ifaces=\"0.0.0.0:%d\"\n", n.admin_port); + fprintf(f, "\tcluster_username=\"cluster1\"\n"); + fprintf(f, "\tcluster_password=\"secret1pass\"\n"); + fprintf(f, "\tcluster_check_interval_ms=200\n"); + fprintf(f, "\tcluster_leader_election=\"true\"\n"); + fprintf(f, "\tcluster_leader_node_timeout_ms=1000\n"); + fprintf(f, "\tcluster_leader_grace_ms=500\n"); + fprintf(f, "}\n"); + fprintf(f, "mysql_variables = {\n"); + fprintf(f, "\tthreads=2\n"); + fprintf(f, "\tinterfaces=\"0.0.0.0:%d\"\n", n.admin_port + 1); + fprintf(f, "}\n"); + fprintf(f, "proxysql_servers = (\n"); + for (int i = 0; i < 3; i++) { + fprintf(f, "\t{ hostname=\"127.0.0.1\"; port=%d; weight=%d; comment=\"node%d\"; }%s\n", + nodes_def[i].admin_port, nodes_def[i].weight, nodes_def[i].idx, (i < 2 ? "," : "")); + } + fprintf(f, ")\n"); + fclose(f); +} + +static void spawn_node(node_t& n, bool initial) { + const string binary = workdir_path + "../../../src/proxysql"; + const string stderr_f = n.datadir + "/node_stderr.txt"; + string cmd = binary + " -f -M -c " + n.cnf_path + " -D " + n.datadir; + if (initial) { cmd += " --initial"; } + cmd += " >> " + stderr_f + " 2>&1"; + n.runner = new std::thread([&n, cmd]() { + pid_t p = fork(); + if (p == 0) { execl("/bin/sh", "sh", "-c", cmd.c_str(), (char*)nullptr); _exit(127); } + n.pid.store(p); + int status = 0; + waitpid(p, &status, 0); + }); + // give fork+exec a moment before callers start polling the admin port + usleep(500 * 1000); +} + +static void join_node(node_t& n) { + if (n.runner) { n.runner->join(); delete n.runner; n.runner = nullptr; } + n.pid.store(-1); +} + +static MYSQL* admin_conn(int port) { + conn_opts_t opts {}; + opts.host = "127.0.0.1"; + opts.user = "admin"; + opts.pass = "admin"; + opts.port = port; + return wait_for_proxysql(opts, 15); +} + +// Returns the admin_port of the row with master='YES' as seen by `conn`, +// or -1 if there is not exactly one leader row. +static int observed_leader(MYSQL* conn) { + if (mysql_query(conn, "SELECT port, master FROM stats_proxysql_servers_status")) { return -1; } + MYSQL_RES* res = mysql_store_result(conn); + if (res == NULL) { return -1; } + int leader = -1; int leaders = 0; + MYSQL_ROW row; + while ((row = mysql_fetch_row(res))) { + if (row[1] && strcasecmp(row[1], "YES") == 0) { leaders++; leader = atoi(row[0]); } + } + mysql_free_result(res); + return (leaders == 1 ? leader : -1); +} + +// Polls until `conn` observes `expected_port` as the unique leader. +static bool wait_leader(MYSQL* conn, int expected_port, int timeout_s) { + for (int i = 0; i < timeout_s * 2; i++) { + if (observed_leader(conn) == expected_port) { return true; } + usleep(500 * 1000); + } + return false; +} + +static bool query_ok(MYSQL* conn, const char* q) { + return mysql_query(conn, q) == 0; +} + +// true if the query FAILED (as expected for a follower); stores the error. +static bool query_refused(MYSQL* conn, const char* q, string& err) { + if (mysql_query(conn, q) == 0) { err = ""; return false; } + err = mysql_error(conn); + return true; +} + +static const char* Q_INSERT = "INSERT INTO mysql_servers (hostgroup_id, hostname, port) VALUES (9999, '127.0.0.1', 13306)"; +static const char* Q_DELETE = "DELETE FROM mysql_servers WHERE hostgroup_id=9999"; +static const char* Q_LOAD = "LOAD MYSQL SERVERS TO RUNTIME"; +static const char* Q_SAVE = "SAVE MYSQL SERVERS TO DISK"; + +int main(int argc, char** argv) { + CommandLine cl; + if (cl.getEnv()) { diag("Failed to get the required environmental variables."); return -1; } + workdir_path = cl.workdir; + + const string base = workdir_path + "test_cluster_leader_election_config"; + mkdir(base.c_str(), 0777); + for (int i = 0; i < 3; i++) { + nodes_def[i].idx = i + 1; + nodes_def[i].admin_port = 16062 + i * 10; + nodes_def[i].weight = 300 - i * 100; + nodes_def[i].datadir = base + "/node" + std::to_string(i + 1); + nodes_def[i].cnf_path = nodes_def[i].datadir + "/node.cnf"; + mkdir(nodes_def[i].datadir.c_str(), 0777); + write_node_config(nodes_def[i]); + spawn_node(nodes_def[i], true); + } + + MYSQL* a1 = admin_conn(nodes_def[0].admin_port); + MYSQL* a2 = admin_conn(nodes_def[1].admin_port); + MYSQL* a3 = admin_conn(nodes_def[2].admin_port); + if (a1 == NULL || a2 == NULL || a3 == NULL) { + fprintf(stderr, "File %s, line %d, Error: failed to start the 3 cluster nodes\n", __FILE__, __LINE__); + for (int i = 0; i < 3; i++) { + pid_t p = nodes_def[i].pid.load(); + if (p > 0) { kill(p, SIGKILL); } + join_node(nodes_def[i]); + } + return -1; + } + + // Feature detection: skip everything on non-PROXYSQL31 builds. + bool feature = false; + if (mysql_query(a1, "SELECT count(*) FROM global_variables WHERE variable_name='admin-cluster_leader_election'") == 0) { + MYSQL_RES* res = mysql_store_result(a1); + MYSQL_ROW row = mysql_fetch_row(res); + feature = (row && row[0] && atoi(row[0]) == 1); + mysql_free_result(res); + } + if (feature == false) { + plan(1); + ok(1, "admin-cluster_leader_election not present (non-PROXYSQL31 build) - skipping"); + } else { + plan(27); + string err; + + // --- Convergence: all 3 nodes agree node1 (16062) is leader --- (3) + ok(wait_leader(a1, 16062, 15), "node1 observes node1 as the unique leader"); + ok(wait_leader(a2, 16062, 15), "node2 observes node1 as the unique leader"); + ok(wait_leader(a3, 16062, 15), "node3 observes node1 as the unique leader"); + + // --- Leader accepts writes --- (3) + ok(query_ok(a1, Q_INSERT), "leader accepts INSERT: %s", mysql_error(a1)); + ok(query_ok(a1, Q_LOAD), "leader accepts LOAD TO RUNTIME: %s", mysql_error(a1)); + ok(query_ok(a1, Q_SAVE), "leader accepts SAVE TO DISK: %s", mysql_error(a1)); + query_ok(a1, Q_DELETE); query_ok(a1, Q_LOAD); query_ok(a1, Q_SAVE); // cleanup + + // --- Follower refuses writes --- (3 + 1) + ok(query_refused(a2, Q_INSERT, err), "follower refuses INSERT (%s)", err.c_str()); + ok(query_refused(a2, Q_LOAD, err), "follower refuses LOAD TO RUNTIME (%s)", err.c_str()); + ok(strstr(err.c_str(), "16062") != NULL, "refusal error names the leader: %s", err.c_str()); + ok(query_refused(a2, Q_SAVE, err), "follower refuses SAVE TO DISK (%s)", err.c_str()); + + // --- FORCED_RW override is sticky across election ticks --- (6) + ok(query_ok(a2, "PROXYSQL READWRITE"), "PROXYSQL READWRITE accepted on follower"); + ok(query_ok(a2, Q_INSERT), "FORCED_RW follower accepts INSERT: %s", mysql_error(a2)); + ok(query_ok(a2, Q_LOAD), "FORCED_RW follower accepts LOAD TO RUNTIME: %s", mysql_error(a2)); + sleep(3); // several election ticks + grace periods + ok(query_ok(a2, "DELETE FROM mysql_servers WHERE hostgroup_id=9999"), "FORCED_RW sticks across election ticks: %s", mysql_error(a2)); + query_ok(a2, Q_LOAD); // cleanup runtime on node2 + ok(query_ok(a2, "PROXYSQL READONLY AUTO"), "PROXYSQL READONLY AUTO accepted"); + ok(query_refused(a2, Q_INSERT, err), "AUTO follower refuses INSERT again (%s)", err.c_str()); + + // --- Leader failover on kill --- (3) + { + pid_t p1 = nodes_def[0].pid.load(); + if (p1 > 0) { kill(p1, SIGKILL); } + join_node(nodes_def[0]); + mysql_close(a1); a1 = NULL; + } + ok(wait_leader(a2, 16072, 15), "after leader kill, node2 observes node2 as leader"); + ok(wait_leader(a3, 16072, 15), "after leader kill, node3 observes node2 as leader"); + ok(query_ok(a2, Q_INSERT), "new leader accepts INSERT: %s", mysql_error(a2)); + query_ok(a2, Q_DELETE); query_ok(a2, Q_LOAD); // cleanup + + // --- Ex-leader rejoins and retakes leadership (highest weight) --- (5) + spawn_node(nodes_def[0], false); // keep datadir: same uuid, config from db + a1 = admin_conn(nodes_def[0].admin_port); + ok(a1 != NULL, "ex-leader restarted and reachable"); + ok(a1 != NULL && wait_leader(a1, 16062, 20), "rejoined node1 observes itself as leader again"); + ok(wait_leader(a2, 16062, 20), "node2 observes node1 as leader again"); + ok(wait_leader(a3, 16062, 20), "node3 observes node1 as leader again"); + ok(query_refused(a2, Q_INSERT, err), "node2 is follower again (%s)", err.c_str()); + + // --- Election disabled: everyone read-write --- (3) + MYSQL* conns[3] = { a1, a2, a3 }; + for (int i = 0; i < 3; i++) { + if (conns[i] == NULL) { continue; } + query_ok(conns[i], "PROXYSQL READWRITE"); + query_ok(conns[i], "SET admin-cluster_leader_election='false'"); + query_ok(conns[i], "LOAD ADMIN VARIABLES TO RUNTIME"); + query_ok(conns[i], "PROXYSQL READONLY AUTO"); + } + sleep(2); // let ticks observe the disable + ok(query_ok(a1, Q_INSERT), "election disabled: node1 accepts writes: %s", mysql_error(a1)); + ok(query_ok(a2, Q_INSERT), "election disabled: node2 accepts writes: %s", mysql_error(a2)); + ok(query_ok(a3, Q_INSERT), "election disabled: node3 accepts writes: %s", mysql_error(a3)); + } + + // Teardown: shut all nodes down. + MYSQL* conns[3] = { a1, a2, a3 }; + for (int i = 0; i < 3; i++) { + if (conns[i]) { + mysql_query(conns[i], "PROXYSQL SHUTDOWN"); + mysql_close(conns[i]); + } + pid_t p = nodes_def[i].pid.load(); + if (p > 0) { + for (int w = 0; w < 10 && kill(p, 0) == 0; w++) { usleep(500 * 1000); } + if (kill(p, 0) == 0) { kill(p, SIGKILL); } + } + join_node(nodes_def[i]); + } + return exit_status(); +} +``` + +- [ ] **Step 2: Build the test binary** + +```bash +cd test/tap/tests && make test_cluster_leader_election-t +``` +(The `%-t:` wildcard rule at `test/tap/tests/Makefile:266` picks it up — no Makefile edit needed.) Expected: clean compile. Fix any `conn_opts_t`/`wait_for_proxysql` signature mismatches against `test/tap/tap/utils.h:550/567`. + +- [ ] **Step 3: Register in groups.json** + +Add (same groups as `test_cluster_sync-t`'s core set): +```json +"test_cluster_leader_election-t" : [ "legacy-g5", "mysql84-g5", "mysql90-g5", "mysql95-g5" ], +``` +Run the linters: +```bash +python3 test/tap/groups/lint_groups_json.py && python3 test/tap/groups/lint_group_coverage.py +``` + +- [ ] **Step 4: Run the test through the isolated harness** + +The binary under test must be the debug build (`PROXYSQL31=1 make debug` already done in earlier tasks) and the test needs no backends beyond the runner: +```bash +WORKSPACE=$(pwd) INFRA_ID=dev-$USER TAP_GROUP=legacy-g5 test/infra/control/ensure-infras.bash +WORKSPACE=$(pwd) INFRA_ID=dev-$USER TAP_GROUP=legacy-g5 \ + TEST_PY_TAP_INCL="test_cluster_leader_election-t" \ + test/infra/control/run-tests-isolated.bash +``` +Expected: `1..27`, all ok. On failure, read the per-node `node_stderr.txt` files under the test's config dir and the test output — do not retry blindly. + +- [ ] **Step 5: Run the full unit group + commit** + +```bash +WORKSPACE=$(pwd) INFRA_ID=dev-$USER TAP_GROUP=unit-tests-g1 test/infra/control/run-tests-isolated.bash 2>&1 | tail -20 +git add test/tap/tests/test_cluster_leader_election-t.cpp test/tap/groups/groups.json +git commit -m "test(cluster): E2E leader election test with 3-node self-spawned cluster" +``` + +--- + +### Task 9: Final verification sweep + +**Files:** none (verification only). + +- [ ] **Step 1: Tier-discipline build matrix** + +```bash +make clean && make -j$(nproc) 2>&1 | tail -3 # stable tier: must build, feature unenableable +make clean && PROXYSQL31=1 make debug -j$(nproc) 2>&1 | tail -3 # target tier, debug +PROXYSQL31=1 make build_tap_test_debug 2>&1 | tail -3 +``` + +- [ ] **Step 2: Re-run both new tests** + +```bash +cd test/tap/tests/unit && PROXYSQL31=1 make cluster_leader_election_unit-t && ./cluster_leader_election_unit-t +cd ../../../.. || cd /data/rene/proxysql7/proxysql +WORKSPACE=$(pwd) INFRA_ID=dev-$USER TAP_GROUP=legacy-g5 \ + TEST_PY_TAP_INCL="test_cluster_leader_election-t" test/infra/control/run-tests-isolated.bash +``` + +- [ ] **Step 3: Regression spot-check on cluster suite** + +```bash +WORKSPACE=$(pwd) INFRA_ID=dev-$USER TAP_GROUP=legacy-g5 \ + TEST_PY_TAP_INCL="test_cluster_sync-t|test_cluster1-t" test/infra/control/run-tests-isolated.bash +``` +Expected: both pass (election defaults to off ⇒ no behavior change). Per CLAUDE.md: any failure here gets a root-cause analysis, not a "flaky" label. + +- [ ] **Step 4: Commit anything outstanding; leave branch ready for PR** + +```bash +git status --short && git log --oneline v3.0..HEAD +``` diff --git a/docs/superpowers/plans/2026-08-11-cluster-stats-aggregation.md b/docs/superpowers/plans/2026-08-11-cluster-stats-aggregation.md new file mode 100644 index 0000000000..c3506686c0 --- /dev/null +++ b/docs/superpowers/plans/2026-08-11-cluster-stats-aggregation.md @@ -0,0 +1,1104 @@ +# Cluster Stats Aggregation Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** The cluster leader replicates every node's TSDB samples into its own `tsdb_metrics_cluster` table (pull + watermark over the admin channel), per spec `docs/superpowers/specs/2026-08-11-cluster-stats-aggregation-design.md`. + +**Architecture:** Pure watermark/fetch planning in new files (`TSDB_Cluster_Aggregator.{h,cpp}`); an aggregator worker thread owned by `ProxySQL_Statistics`, started/stopped by a cheap check in the Admin main loop on leadership transitions; peers queried via `stats_history.tsdb_metrics` over MySQL connections using cluster credentials; self replicated via local `INSERT OR IGNORE ... SELECT`. Query surface: `node=` param on `/api/tsdb/query`, new `/api/tsdb/nodes`, status extension, minimal dashboard node selector. + +**Tech Stack:** C++17, pthreads + `std::atomic`, SQLite3 (`statsdb_disk`), libmariadb, libhttpserver, TAP tests. + +## Global Constraints + +- Branch: `feat/cluster-stats-aggregation` (stacked on `feat/cluster-leader-election`). Build with `PROXYSQL31=1` on EVERY make (never bare `make`); after editing any header, `touch src/*.cpp` before an incremental build (src/Makefile lacks header-dep tracking). Before running cluster TAP tests after making commits, do a full `make clean && PROXYSQL31=1 make debug` (per-TU version-string skew otherwise makes nodes refuse to cluster — see memory `build-version-skew`). +- ALL new TSDB code lives inside the existing `#ifdef PROXYSQLTSDB` regions. No new tier flags. The pure planner files compile unconditionally (like `ProxySQL_Cluster_Leader.cpp`). +- New variables (TSDB family, exact names/defaults/ranges from spec — defaults are provisional placeholders pending sizing data): `tsdb-cluster_aggregation`=1 (0/1), `tsdb-cluster_interval`=10 (5–300 s), `tsdb-cluster_backfill_hours`=24 (0–168), `tsdb-cluster_retention_days`=3 (1–30), `tsdb-cluster_batch_rows`=10000 (1000–100000). +- Node identity in the cluster table: `hostname:port` exactly as in `proxysql_servers`. UUID is NOT used. +- Ingest uses `INSERT OR IGNORE` (replicated samples immutable; PK makes replication idempotent). +- The aggregator writes only via internal `statsdb_disk` handles — admin read-only mode never applies to it. Cluster-monitor connection options are copied verbatim (1s connect timeout, SSL enforce, keylog callback); no read/write timeouts. +- Tabs for indentation; match surrounding idioms (this module uses `strtol` validation, `SAFE_SQLITE3_STEP2`, explicit BEGIN/COMMIT batching). +- Commit after every task. + +--- + +### Task 1: Pure watermark/fetch planner + unit test + +**Files:** +- Create: `include/TSDB_Cluster_Aggregator.h` +- Create: `lib/TSDB_Cluster_Aggregator.cpp` +- Modify: `lib/Makefile:91` (add `TSDB_Cluster_Aggregator.oo` to `_OBJ_CXX`, after `ProxySQL_Cluster_Leader.oo`) +- Create: `test/tap/tests/unit/tsdb_cluster_aggregator_unit-t.cpp` +- Modify: `test/tap/tests/unit/Makefile` `UNIT_TESTS :=` list (line ~387) +- Modify: `test/tap/groups/groups.json` (register in `unit-tests-g1`, alphabetical) + +**Interfaces:** +- Consumes: nothing. +- Produces (used by Task 3): + - `long tsdb_agg_effective_watermark(long existing_max_ts, long now, int backfill_hours)` — start-point for a node: `max(existing_max_ts, now - backfill_hours*3600)`; `existing_max_ts <= 0` means "none yet". + - `struct Tsdb_Agg_Fetch_Result { long new_watermark; bool caught_up; };` + - `Tsdb_Agg_Fetch_Result tsdb_agg_apply_fetch(long prev_watermark, int rows_fetched, long last_row_ts, int limit)` — 0 rows → watermark unchanged, caught_up; `rows < limit` → watermark=last_row_ts, caught_up; `rows == limit` → watermark=last_row_ts, NOT caught_up. + +- [ ] **Step 1: Write the failing unit test** + +Create `test/tap/tests/unit/tsdb_cluster_aggregator_unit-t.cpp`: + +```cpp +/** + * @file tsdb_cluster_aggregator_unit-t.cpp + * @brief Unit tests for the pure TSDB cluster aggregation planner + * (effective watermark + fetch-result bookkeeping). + */ + +#include "tap.h" +#include "test_globals.h" +#include "test_init.h" +#include "TSDB_Cluster_Aggregator.h" + +int main() { + plan(11); + + const long now = 1000000000L; + + // --- tsdb_agg_effective_watermark (5 oks) --- + ok(tsdb_agg_effective_watermark(0, now, 24) == now - 24*3600, + "no existing data: watermark = now - backfill horizon"); + ok(tsdb_agg_effective_watermark(-1, now, 24) == now - 24*3600, + "negative existing max treated as none"); + ok(tsdb_agg_effective_watermark(now - 100, now, 24) == now - 100, + "recent existing max wins over horizon"); + ok(tsdb_agg_effective_watermark(now - 200000, now, 24) == now - 24*3600, + "stale existing max (deposed leader) clamped forward to horizon"); + ok(tsdb_agg_effective_watermark(now - 100, now, 0) == now, + "zero backfill hours: horizon is now (no history pulled)"); + + // --- tsdb_agg_apply_fetch (6 oks) --- + Tsdb_Agg_Fetch_Result r = tsdb_agg_apply_fetch(500, 0, 0, 1000); + ok(r.new_watermark == 500 && r.caught_up == true, + "empty fetch: watermark unchanged, caught up"); + + r = tsdb_agg_apply_fetch(500, 999, 750, 1000); + ok(r.new_watermark == 750, "partial fetch advances watermark to last row ts"); + ok(r.caught_up == true, "partial fetch (rows < limit) means caught up"); + + r = tsdb_agg_apply_fetch(500, 1000, 800, 1000); + ok(r.new_watermark == 800, "full fetch advances watermark to last row ts"); + ok(r.caught_up == false, "full fetch (rows == limit) means more to pull"); + + r = tsdb_agg_apply_fetch(500, 1, 501, 1000); + ok(r.new_watermark == 501 && r.caught_up == true, + "single-row fetch advances and completes"); + + return exit_status(); +} +``` + +- [ ] **Step 2: Header + failing stub, verify FAIL** + +Create `include/TSDB_Cluster_Aggregator.h`: + +```cpp +#ifndef __CLASS_TSDB_CLUSTER_AGGREGATOR_H +#define __CLASS_TSDB_CLUSTER_AGGREGATOR_H + +// Pure planning logic for TSDB cluster aggregation (leader pulls peers' +// tsdb_metrics with a per-node watermark). Kept dependency-free so it is +// unit-testable in every tier. + +struct Tsdb_Agg_Fetch_Result { + long new_watermark = 0; + bool caught_up = false; +}; + +// Start-point for replicating a node: the max timestamp already replicated, +// clamped forward to the backfill horizon (now - backfill_hours). +// existing_max_ts <= 0 means "nothing replicated yet". +long tsdb_agg_effective_watermark(long existing_max_ts, long now, int backfill_hours); + +// Bookkeeping after one fetch of up to `limit` rows ordered by timestamp, +// where `last_row_ts` is the max timestamp among the fetched rows +// (ignored when rows_fetched == 0). +Tsdb_Agg_Fetch_Result tsdb_agg_apply_fetch(long prev_watermark, int rows_fetched, long last_row_ts, int limit); + +#endif // __CLASS_TSDB_CLUSTER_AGGREGATOR_H +``` + +Create `lib/TSDB_Cluster_Aggregator.cpp` as a failing stub: + +```cpp +#include "TSDB_Cluster_Aggregator.h" + +long tsdb_agg_effective_watermark(long existing_max_ts, long now, int backfill_hours) { + (void)existing_max_ts; (void)now; (void)backfill_hours; + return 0; +} + +Tsdb_Agg_Fetch_Result tsdb_agg_apply_fetch(long prev_watermark, int rows_fetched, long last_row_ts, int limit) { + (void)prev_watermark; (void)rows_fetched; (void)last_row_ts; (void)limit; + return Tsdb_Agg_Fetch_Result {}; +} +``` + +Edit `lib/Makefile` `_OBJ_CXX` (line ~91): insert `TSDB_Cluster_Aggregator.oo` right after `ProxySQL_Cluster_Leader.oo`. Edit `test/tap/tests/unit/Makefile`: add `tsdb_cluster_aggregator_unit-t` to `UNIT_TESTS` (generic `%-t:` rule handles the build). + +Run: +```bash +cd /data/rene/proxysql7/proxysql && PROXYSQL31=1 make debug -j$(nproc) 2>&1 | tail -3 +cd test/tap/tests/unit && PROXYSQL31=1 make tsdb_cluster_aggregator_unit-t && ./tsdb_cluster_aggregator_unit-t +``` +Expected: builds; multiple `not ok`. + +- [ ] **Step 3: Implement** + +```cpp +#include "TSDB_Cluster_Aggregator.h" + +long tsdb_agg_effective_watermark(long existing_max_ts, long now, int backfill_hours) { + long horizon = now - (long)backfill_hours * 3600L; + if (existing_max_ts > horizon) { + return existing_max_ts; + } + return horizon; +} + +Tsdb_Agg_Fetch_Result tsdb_agg_apply_fetch(long prev_watermark, int rows_fetched, long last_row_ts, int limit) { + Tsdb_Agg_Fetch_Result r; + if (rows_fetched == 0) { + r.new_watermark = prev_watermark; + r.caught_up = true; + return r; + } + r.new_watermark = last_row_ts; + r.caught_up = (rows_fetched < limit); + return r; +} +``` + +- [ ] **Step 4: Run test — expect PASS (1..11 all ok)** + +- [ ] **Step 5: Register + lint + commit** + +groups.json (alphabetical, compact one-line style): `"tsdb_cluster_aggregator_unit-t" : [ "unit-tests-g1" ],` + +```bash +python3 test/tap/groups/lint_groups_json.py +git add include/TSDB_Cluster_Aggregator.h lib/TSDB_Cluster_Aggregator.cpp lib/Makefile \ + test/tap/tests/unit/tsdb_cluster_aggregator_unit-t.cpp test/tap/tests/unit/Makefile test/tap/groups/groups.json +git commit -m "feat(tsdb): pure watermark/fetch planner for cluster aggregation" +``` + +--- + +### Task 2: Schema, variables, retention + +**Files:** +- Modify: `include/ProxySQL_Statistics.hpp` (schema define after :111; variables struct :158-165) +- Modify: `lib/ProxySQL_Statistics.cpp` (defaults :147-153; meta table :157-168; `set_variable` :170-191; `get_variable` :193-213; table registration ~:297; index ~:327; `tsdb_retention_cleanup` :1590-1615) +- Modify: `test/tap/tests/test_tsdb_variables-t.cpp:119` (runtime `tsdb-%` count 5 → 10; scan the file for other hardcoded counts/lists of tsdb variables and update all of them) + +**Interfaces:** +- Consumes: nothing. +- Produces (used by Tasks 3, 4, 6): table `tsdb_metrics_cluster(node, timestamp, metric_name, labels, value)` in `statsdb_disk` AND `statsdb_mem` (both are built from `tables_defs_statsdb_disk`); `variables.tsdb_cluster_aggregation`, `variables.tsdb_cluster_interval`, `variables.tsdb_cluster_backfill_hours`, `variables.tsdb_cluster_retention_days`, `variables.tsdb_cluster_batch_rows` (all `int`). + +- [ ] **Step 1: Schema define + registration + index** + +`include/ProxySQL_Statistics.hpp` after the `STATSDB_SQLITE_TABLE_TSDB_BACKEND_HEALTH` define (:111): + +```c +#define STATSDB_SQLITE_TABLE_TSDB_METRICS_CLUSTER "CREATE TABLE IF NOT EXISTS tsdb_metrics_cluster (node VARCHAR NOT NULL , timestamp INTEGER NOT NULL , metric_name VARCHAR NOT NULL , labels VARCHAR NOT NULL DEFAULT '{}' , value REAL , PRIMARY KEY (node, timestamp, metric_name, labels)) WITHOUT ROWID" +``` + +(Match the exact `CREATE TABLE` prefix style of the sibling defines — check whether they use `IF NOT EXISTS`; mirror them.) + +`lib/ProxySQL_Statistics.cpp` in `init()` next to the other three (`~:297`): +```cpp + insert_into_tables_defs(tables_defs_statsdb_disk,"tsdb_metrics_cluster", STATSDB_SQLITE_TABLE_TSDB_METRICS_CLUSTER); +``` +Index next to the others (~:327): +```cpp + statsdb_disk->execute("CREATE INDEX IF NOT EXISTS idx_tsdb_metrics_cluster_node_metric_time ON tsdb_metrics_cluster (node, metric_name, timestamp)"); +``` +No schema-upgrade block (brand-new table). + +- [ ] **Step 2: Variables** + +`include/ProxySQL_Statistics.hpp` variables struct (:158-165), add: +```cpp + int tsdb_cluster_aggregation; + int tsdb_cluster_interval; + int tsdb_cluster_backfill_hours; + int tsdb_cluster_retention_days; + int tsdb_cluster_batch_rows; +``` + +`lib/ProxySQL_Statistics.cpp` ctor defaults (:147-153): +```cpp + variables.tsdb_cluster_aggregation = 1; + variables.tsdb_cluster_interval = 10; + variables.tsdb_cluster_backfill_hours = 24; + variables.tsdb_cluster_retention_days = 3; + variables.tsdb_cluster_batch_rows = 10000; +``` + +Meta table (:157-168) — append BEFORE the `{NULL,0,0}` terminator (names are WITHOUT the `tsdb-` prefix): +```c + {"cluster_aggregation", 0, 1}, + {"cluster_interval", 5, 300}, + {"cluster_backfill_hours", 0, 168}, + {"cluster_retention_days", 1, 30}, + {"cluster_batch_rows", 1000, 100000}, +``` + +`set_variable` (:170-191) dispatches by POSITIONAL index — existing entries are `i==0..4`; append: +```cpp + } else if (i == 5) { + variables.tsdb_cluster_aggregation = (int)v; + } else if (i == 6) { + variables.tsdb_cluster_interval = (int)v; + } else if (i == 7) { + variables.tsdb_cluster_backfill_hours = (int)v; + } else if (i == 8) { + variables.tsdb_cluster_retention_days = (int)v; + } else if (i == 9) { + variables.tsdb_cluster_batch_rows = (int)v; + } +``` +(Match the exact local variable names in the function — the parsed value variable may not be called `v`; anchor on the `i == 4` branch and continue the pattern.) + +`get_variable` (:193-213) — add the five `strcasecmp` branches following the existing shape, e.g.: +```cpp + if (!strcasecmp(name, "cluster_aggregation")) { + sprintf(buf, "%d", variables.tsdb_cluster_aggregation); + return strdup(buf); + } +``` +(one per variable; match the actual buffer name in the function). + +`get_variables_list`/`has_variable`/Admin flush paths are auto-derived from the meta table — no other edits. + +- [ ] **Step 3: Retention** + +`tsdb_retention_cleanup()` (:1590-1615) — add after the existing three DELETEs, same `snprintf`+`execute` pattern: +```cpp + const int cluster_retention_days = std::max(1, variables.tsdb_cluster_retention_days); + snprintf(delete_buf, sizeof(delete_buf), "DELETE FROM tsdb_metrics_cluster WHERE timestamp < %ld", ts - 86400L*cluster_retention_days); + statsdb_disk->execute(delete_buf); +``` + +- [ ] **Step 4: Update the tsdb variables test** + +`test/tap/tests/test_tsdb_variables-t.cpp`: line ~119 asserts the runtime `tsdb-%` variable count is `"5"` — change to `"10"`. Grep the file for `5` in variable-count contexts, any enumerated variable-name lists, and any plan-count that depends on the variable count; update consistently. + +- [ ] **Step 5: Build, verify, commit** + +```bash +PROXYSQL31=1 make debug -j$(nproc) 2>&1 | tail -3 +``` +Scratch instance (ports 16032/16033, `--initial`; recipe: minimal cnf with datadir + admin_variables { admin_credentials="admin:admin" mysql_ifaces="0.0.0.0:16032" } + mysql_variables { interfaces="0.0.0.0:16033" }): +- `SELECT variable_name, variable_value FROM global_variables WHERE variable_name LIKE 'tsdb-cluster%'` → 5 rows with the defaults. +- `SET tsdb-cluster_interval='3'; LOAD TSDB VARIABLES TO RUNTIME;` → value rejected (below range floor 5, stays 10); `SET tsdb-cluster_interval='30'` → accepted. +- `SELECT COUNT(*) FROM stats_history.tsdb_metrics_cluster` → 0 rows, no error. + +```bash +git add include/ProxySQL_Statistics.hpp lib/ProxySQL_Statistics.cpp test/tap/tests/test_tsdb_variables-t.cpp +git commit -m "feat(tsdb): tsdb_metrics_cluster table, cluster aggregation variables, retention" +``` + +--- + +### Task 3: Aggregator engine (worker thread + replication cycle) + +**Files:** +- Modify: `include/ProxySQL_Statistics.hpp` (members + method declarations, `#ifdef PROXYSQLTSDB` region; includes ``, `` if missing) +- Modify: `lib/ProxySQL_Statistics.cpp` (thread fn, lifecycle check, cycle, self/peer replication; `#include "TSDB_Cluster_Aggregator.h"`, `#include "ProxySQL_Cluster.hpp"`, `extern ProxySQL_Cluster* GloProxyCluster;`; also `#include "proxysql_sslkeylog.h"` if `proxysql_keylog_write_line_callback` needs it — check how `ProxySQL_Cluster.cpp` gets it) +- Modify: `lib/ProxySQL_Admin.cpp:2637` (invoke the lifecycle check inside the PROXYSQLTSDB block, before `#endif`) + +**Interfaces:** +- Consumes: Task 1 planner functions; Task 2 table + variables; from the leader-election branch: `GloProxyCluster->is_leader()`, `get_leader_info(std::string&,int&,std::string&)`, `dump_table_proxysql_servers()` (SQLite3_result*, 4 TEXT cols hostname/port/weight/comment, caller deletes), `get_credentials()` → `cluster_creds_t{ std::string user, pass; }` (empty user = clustering off). +- Produces (used by Task 4): `std::atomic tsdb_agg_active`, `std::atomic tsdb_agg_rows_total`, `std::atomic tsdb_agg_last_cycle_ts`, `std::atomic tsdb_agg_cap_hit_last_cycle` (public members, read by status/REST); method `void tsdb_cluster_aggregation_check(unsigned long long curtime)` (called from admin loop). + +- [ ] **Step 1: Declarations** + +`include/ProxySQL_Statistics.hpp`, inside the class's `#ifdef PROXYSQLTSDB` region — private members (next to `next_timer_tsdb_*`, :132-141): + +```cpp + unsigned long long next_timer_tsdb_cluster_check = 0; + pthread_t tsdb_agg_thread; + bool tsdb_agg_thread_started = false; // only touched by the admin main loop thread + std::atomic tsdb_agg_stop { false }; + sqlite3_stmt *stmt_insert_tsdb_cluster_metric = NULL; +``` + +Public members + declarations (next to the loop declarations, :254-262): + +```cpp + std::atomic tsdb_agg_active { false }; // thread running (read by REST) + std::atomic tsdb_agg_rows_total { 0 }; // rows replicated since start + std::atomic tsdb_agg_last_cycle_ts { 0 }; // unix ts of last completed cycle + std::atomic tsdb_agg_cap_hit_last_cycle { false }; + void tsdb_cluster_aggregation_check(unsigned long long curtime); + void tsdb_cluster_aggregation_thread_loop(); // thread body (public for the C trampoline) + SQLite3_result * get_tsdb_cluster_nodes(); // implemented in Task 4 +private: + void tsdb_cluster_aggregation_cycle(); + long tsdb_cluster_node_max_ts(const std::string& node); + void tsdb_cluster_replicate_self(const std::string& node, long watermark, int limit); + bool tsdb_cluster_replicate_peer(const std::string& host, int port, const std::string& node, long watermark, int limit, const std::string& user, const std::string& pass); +``` + +(Adjust access-section placement to fit the class's existing public/private layout; keep the trampoline-callable pieces public.) + +- [ ] **Step 2: Lifecycle (check + thread body + teardown)** + +`lib/ProxySQL_Statistics.cpp` (inside `#ifdef PROXYSQLTSDB`): + +```cpp +static void * tsdb_cluster_agg_thread_fn(void *arg) { + set_thread_name("TSDBClusterAgg"); // only if a set_thread_name helper exists in this codebase; otherwise omit + ((ProxySQL_Statistics *)arg)->tsdb_cluster_aggregation_thread_loop(); + return NULL; +} + +void ProxySQL_Statistics::tsdb_cluster_aggregation_check(unsigned long long curtime) { + if (curtime < next_timer_tsdb_cluster_check) return; + next_timer_tsdb_cluster_check = curtime + 1000000ULL; // evaluate at most every 1s + bool desired = false; + if (variables.tsdb_enabled && variables.tsdb_cluster_aggregation) { + if (GloProxyCluster && GloProxyCluster->is_leader()) { + desired = true; + } + } + if (desired == true && tsdb_agg_thread_started == false) { + tsdb_agg_stop.store(false); + if (pthread_create(&tsdb_agg_thread, NULL, tsdb_cluster_agg_thread_fn, this) == 0) { + tsdb_agg_thread_started = true; + tsdb_agg_active.store(true); + proxy_info("TSDB cluster aggregation: started (this node is the cluster leader)\n"); + } else { + proxy_error("TSDB cluster aggregation: failed to create worker thread\n"); + } + } else if (desired == false && tsdb_agg_thread_started == true) { + tsdb_agg_stop.store(true); + pthread_join(tsdb_agg_thread, NULL); + tsdb_agg_thread_started = false; + tsdb_agg_active.store(false); + proxy_info("TSDB cluster aggregation: stopped\n"); + } +} + +void ProxySQL_Statistics::tsdb_cluster_aggregation_thread_loop() { + while (tsdb_agg_stop.load() == false) { + tsdb_cluster_aggregation_cycle(); + tsdb_agg_last_cycle_ts.store((long long)time(NULL)); + int sleep_s = variables.tsdb_cluster_interval; + if (sleep_s < 5) sleep_s = 5; + for (int i = 0; i < sleep_s * 10 && tsdb_agg_stop.load() == false; i++) { + usleep(100000); + } + } +} +``` + +Notes: `is_leader()` takes `leader_mutex` — cheap, once per second. `variables.*` reads from the worker thread are unsynchronized plain-int reads, consistent with existing module style (sampler reads them the same way from the admin thread). Add teardown to `~ProxySQL_Statistics()` (:235-250): if `tsdb_agg_thread_started`, `tsdb_agg_stop.store(true); pthread_join(...);` then finalize `stmt_insert_tsdb_cluster_metric` next to the other two stmt finalizations. + +`lib/ProxySQL_Admin.cpp` — inside the TSDB block, after the retention line (:2635-2637), before `#endif`: +```cpp + GloProxyStats->tsdb_cluster_aggregation_check(curtime); +``` +(Unlike the other four, this is called every loop iteration — it self-throttles to 1s and must react to leadership changes promptly.) + +- [ ] **Step 3: The cycle** + +```cpp +void ProxySQL_Statistics::tsdb_cluster_aggregation_cycle() { + if (GloProxyCluster == NULL) return; + if (GloProxyCluster->is_leader() == false) return; // deposed between checks + std::string self_host; int self_port = 0; std::string self_uuid; + GloProxyCluster->get_leader_info(self_host, self_port, self_uuid); + if (self_host.length() == 0) return; + std::string self_node = self_host + ":" + std::to_string(self_port); + cluster_creds_t creds = GloProxyCluster->get_credentials(); + SQLite3_result *servers = GloProxyCluster->dump_table_proxysql_servers(); + if (servers == NULL) return; + long now = (long)time(NULL); + int limit = variables.tsdb_cluster_batch_rows; + if (limit < 1000) limit = 1000; + bool cap_hit = false; + for (std::vector::iterator it = servers->rows.begin(); it != servers->rows.end(); ++it) { + if (tsdb_agg_stop.load()) break; + SQLite3_row *r = *it; + std::string node = std::string(r->fields[0]) + ":" + std::string(r->fields[1]); + long wm = tsdb_agg_effective_watermark(tsdb_cluster_node_max_ts(node), now, variables.tsdb_cluster_backfill_hours); + if (node == self_node) { + tsdb_cluster_replicate_self(node, wm, limit); + } else { + if (creds.user.length() == 0) continue; // clustering unconfigured + bool hit = tsdb_cluster_replicate_peer(r->fields[0], atoi(r->fields[1]), node, wm, limit, creds.user, creds.pass); + if (hit) cap_hit = true; + } + } + tsdb_agg_cap_hit_last_cycle.store(cap_hit); + delete servers; +} + +long ProxySQL_Statistics::tsdb_cluster_node_max_ts(const std::string& node) { + char *error = NULL; int cols = 0; int affected_rows = 0; + SQLite3_result *res = NULL; + std::string q = "SELECT COALESCE(MAX(timestamp),0) FROM tsdb_metrics_cluster WHERE node='" + escape_sql_string_literal(node) + "'"; + statsdb_disk->execute_statement(q.c_str(), &error, &cols, &affected_rows, &res); + long max_ts = 0; + if (error == NULL && res != NULL && res->rows_count > 0) { + max_ts = atol(res->rows[0]->fields[0]); + } + if (error) free(error); + if (res) delete res; + return max_ts; +} +``` + +(`escape_sql_string_literal` is an anon-namespace helper in this file at ~:28 — confirm its exact signature and adapt; if it is not visible at the new code's position, move the new functions below it.) + +- [ ] **Step 4: Self + peer replication** + +```cpp +void ProxySQL_Statistics::tsdb_cluster_replicate_self(const std::string& node, long watermark, int limit) { + char buf[512]; + std::string esc_node = escape_sql_string_literal(node); + snprintf(buf, sizeof(buf), + "INSERT OR IGNORE INTO tsdb_metrics_cluster (node, timestamp, metric_name, labels, value) " + "SELECT '%s', timestamp, metric_name, labels, value FROM tsdb_metrics " + "WHERE timestamp > %ld ORDER BY timestamp LIMIT %d", + esc_node.c_str(), watermark, limit); + statsdb_disk->execute(buf); +} + +bool ProxySQL_Statistics::tsdb_cluster_replicate_peer(const std::string& host, int port, const std::string& node, long watermark, int limit, const std::string& user, const std::string& pass) { + MYSQL *conn = mysql_init(NULL); + if (conn == NULL) return false; + // Same options as the cluster monitor threads (lib/ProxySQL_Cluster.cpp:207-217) + unsigned int timeout = 1; + mysql_options(conn, MYSQL_OPT_CONNECT_TIMEOUT, &timeout); + { + unsigned char val = 1; + mysql_options(conn, MYSQL_OPT_SSL_ENFORCE, &val); + mysql_options(conn, MARIADB_OPT_SSL_KEYLOG_CALLBACK, (void *)proxysql_keylog_write_line_callback); + } + if (mysql_real_connect(conn, host.c_str(), user.c_str(), pass.c_str(), NULL, port, NULL, 0) == NULL) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "TSDB cluster aggregation: cannot connect to %s : %s\n", node.c_str(), mysql_error(conn)); + mysql_close(conn); + return false; + } + char q[512]; + snprintf(q, sizeof(q), + "SELECT timestamp, metric_name, labels, value FROM stats_history.tsdb_metrics " + "WHERE timestamp > %ld ORDER BY timestamp LIMIT %d", + watermark, limit); + bool cap_hit = false; + if (mysql_query(conn, q) == 0) { + MYSQL_RES *res = mysql_store_result(conn); + if (res) { + int rc = 0; + if (stmt_insert_tsdb_cluster_metric == NULL) { + rc = statsdb_disk->prepare_v2( + "INSERT OR IGNORE INTO tsdb_metrics_cluster (node, timestamp, metric_name, labels, value) VALUES (?1, ?2, ?3, ?4, ?5)", + &stmt_insert_tsdb_cluster_metric); + // on failure: log once and bail (match the error idiom of stmt_insert_tsdb_metric at :1478-1485) + } + int rows = 0; + long last_row_ts = watermark; + statsdb_disk->execute("BEGIN"); + MYSQL_ROW row; + while ((row = mysql_fetch_row(res))) { + (*proxy_sqlite3_bind_text)(stmt_insert_tsdb_cluster_metric, 1, node.c_str(), -1, SQLITE_TRANSIENT); + (*proxy_sqlite3_bind_int64)(stmt_insert_tsdb_cluster_metric, 2, atoll(row[0])); + (*proxy_sqlite3_bind_text)(stmt_insert_tsdb_cluster_metric, 3, row[1], -1, SQLITE_TRANSIENT); + (*proxy_sqlite3_bind_text)(stmt_insert_tsdb_cluster_metric, 4, (row[2] ? row[2] : "{}"), -1, SQLITE_TRANSIENT); + (*proxy_sqlite3_bind_double)(stmt_insert_tsdb_cluster_metric, 5, (row[3] ? atof(row[3]) : 0.0)); + SAFE_SQLITE3_STEP2(stmt_insert_tsdb_cluster_metric); + (*proxy_sqlite3_clear_bindings)(stmt_insert_tsdb_cluster_metric); + (*proxy_sqlite3_reset)(stmt_insert_tsdb_cluster_metric); + last_row_ts = atol(row[0]); + rows++; + } + statsdb_disk->execute("COMMIT"); + tsdb_agg_rows_total.fetch_add(rows); + Tsdb_Agg_Fetch_Result fr = tsdb_agg_apply_fetch(watermark, rows, last_row_ts, limit); + cap_hit = (fr.caught_up == false); + // fr.new_watermark is informational here: the watermark is re-derived + // from MAX(timestamp) in the table each cycle (restart-safe by design). + mysql_free_result(res); + } + } else { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "TSDB cluster aggregation: query failed on %s : %s\n", node.c_str(), mysql_error(conn)); + } + mysql_close(conn); + return cap_hit; +} +``` + +Also track per-peer progress for the spec's visibility requirement: a `std::map tsdb_agg_peer_last_wm` member (worker-thread-only) — when a peer's watermark hasn't advanced for 10+ consecutive cycles while the node is reachable, `proxy_info` once ("TSDB cluster aggregation: no new samples from %s — peer TSDB likely disabled"), and clear the once-flag when it advances again. + +Copy the EXACT bind/step/reset helper spellings from `insert_tsdb_metric` (:1478-1503) — the snippet above shows structure; that function is authoritative for the `proxy_sqlite3_*` function-pointer names and error handling, including the `prepare_v2` overload actually available on `SQLite3DB` for a cached raw `sqlite3_stmt*` (Task 6 of the leader-election plan used the RAII overload; for a CACHED statement follow `stmt_insert_tsdb_metric`'s form instead). Add a `proxy_warning` when `cap_hit` was true for the same peer on 3+ consecutive cycles (simple `std::map` member, admin-thread-free since only the worker touches it). + +- [ ] **Step 5: Build, verify with a scratch 2-node setup, commit** + +```bash +PROXYSQL31=1 make debug -j$(nproc) 2>&1 | tail -3 # header changed → touch src/*.cpp first +``` +Manual verification (two scratch instances A=16032/16033 and B=16042/16043, both `--initial`, same recipe as Task 2 plus `cluster_username`/`cluster_password` in admin_variables and a `proxysql_servers` block listing both on 127.0.0.1): on both — `SET tsdb-enabled='1'; SET admin-cluster_leader_election='true'; LOAD TSDB VARIABLES TO RUNTIME; LOAD ADMIN VARIABLES TO RUNTIME;` (order matters: do all SETs while still RW). Wait ~10s for election + a cycle, then on the leader: `SELECT node, COUNT(*) FROM stats_history.tsdb_metrics_cluster GROUP BY node` → rows for BOTH nodes growing; on the follower → 0 rows. `PROXYSQL SHUTDOWN` both. + +```bash +git add include/ProxySQL_Statistics.hpp lib/ProxySQL_Statistics.cpp lib/ProxySQL_Admin.cpp +git commit -m "feat(tsdb): cluster aggregation worker - leader replicates peers' TSDB via pull+watermark" +``` + +--- + +### Task 4: Query surface (query node param, /api/tsdb/nodes, status) + +**Files:** +- Modify: `include/ProxySQL_Statistics.hpp` (`query_tsdb_metrics` signature :238-242; `get_tsdb_cluster_nodes` already declared in Task 3) +- Modify: `lib/ProxySQL_Statistics.cpp` (`query_tsdb_metrics` :1735-1797; new `get_tsdb_cluster_nodes`) +- Modify: `lib/ProxySQL_RESTAPI_Server.cpp` (`tsdb_resource::render_GET` :356-451; registration :536-550) + +**Interfaces:** +- Consumes: Task 2 table, Task 3 atomics (`tsdb_agg_active`, `tsdb_agg_rows_total`, `tsdb_agg_last_cycle_ts`, `tsdb_agg_cap_hit_last_cycle`). +- Produces (used by Tasks 5, 6): + - `SQLite3_result* query_tsdb_metrics(const std::string& metric_name, const std::map& label_filters, time_t from, time_t to, const std::string& aggregation = "", const std::string& node = "")` — `node` empty: existing local behavior unchanged; `node` non-empty: query `tsdb_metrics_cluster` (always raw, never the hourly table), filtered `AND node=''` unless `node == "*"`. + - `SQLite3_result* get_tsdb_cluster_nodes()` — columns `node`, `last_timestamp`, `datapoints` (`SELECT node, MAX(timestamp), COUNT(*) FROM tsdb_metrics_cluster GROUP BY node ORDER BY node`). + - REST: `/api/tsdb/query?...&node=X`; `/api/tsdb/nodes` → `[{node, last_timestamp, watermark_age_s, datapoints}]`; `/api/tsdb/status` gains `cluster_aggregation_active`, `cluster_rows_replicated`, `cluster_last_cycle`, `cluster_cap_hit_last_cycle`. + +- [ ] **Step 1: Extend `query_tsdb_metrics`** + +Add the trailing defaulted `node` parameter (declaration + definition). In the implementation (:1735-1797): when `node.length() > 0`, force the raw path (`use_hourly = false`), set the table name to `tsdb_metrics_cluster`, and when `node != "*"` append `" AND node='" + escape_sql_string_literal(node) + "'"` to the WHERE clause. Everything else (label filters via `json_extract`, ordering, column list) is unchanged — the cluster table has the same 4 queried columns plus `node`. + +- [ ] **Step 2: `get_tsdb_cluster_nodes`** + +```cpp +SQLite3_result * ProxySQL_Statistics::get_tsdb_cluster_nodes() { + char *error = NULL; int cols = 0; int affected_rows = 0; + SQLite3_result *res = NULL; + statsdb_disk->execute_statement( + "SELECT node, MAX(timestamp) AS last_timestamp, COUNT(*) AS datapoints FROM tsdb_metrics_cluster GROUP BY node ORDER BY node", + &error, &cols, &affected_rows, &res); + if (error) { + proxy_error("get_tsdb_cluster_nodes: %s\n", error); + free(error); + } + return res; // may be NULL on error; callers must handle +} +``` + +- [ ] **Step 3: REST wiring** + +In `tsdb_resource::render_GET` (`lib/ProxySQL_RESTAPI_Server.cpp:356`): +- `/api/tsdb/query` branch: before the label-filter loop add `std::string node = req.get_arg("node");` and add `"node"` to the exclusion condition at :414 (`key != "metric" && key != "from" && key != "to" && key != "agg" && key != "node"`); pass `node` as the new last argument to `query_tsdb_metrics`. +- New branch after the status branch (:449): +```cpp + } else if (req_path == "/api/tsdb/nodes") { + nlohmann::json nodes_arr = nlohmann::json::array(); + SQLite3_result *res = GloProxyStats->get_tsdb_cluster_nodes(); + time_t now = time(NULL); + if (res) { + for (std::vector::iterator it = res->rows.begin(); it != res->rows.end(); ++it) { + SQLite3_row *r = *it; + nlohmann::json jn; + jn["node"] = r->fields[0]; + long last_ts = atol(r->fields[1]); + jn["last_timestamp"] = last_ts; + jn["watermark_age_s"] = (long)now - last_ts; + jn["datapoints"] = atoll(r->fields[2]); + nodes_arr.push_back(jn); + } + delete res; + } + j_resp = nodes_arr; + } +``` +(Match the file's actual json type alias — it may use `json` unqualified; mirror the neighboring branches.) +- `/api/tsdb/status` branch (:437-449): add +```cpp + j_resp["cluster_aggregation_active"] = GloProxyStats->tsdb_agg_active.load(); + j_resp["cluster_rows_replicated"] = GloProxyStats->tsdb_agg_rows_total.load(); + j_resp["cluster_last_cycle"] = GloProxyStats->tsdb_agg_last_cycle_ts.load(); + j_resp["cluster_cap_hit_last_cycle"] = GloProxyStats->tsdb_agg_cap_hit_last_cycle.load(); +``` +- Registration (:541 area): `ws->register_resource("/api/tsdb/nodes", tsdb_endpoint.get(), true);` + +- [ ] **Step 4: Build, verify, commit** + +Rebuild (`touch src/*.cpp` — header changed). Reuse the Task 3 two-node scratch setup with `restapi_enabled="true"; restapi_port=16070` in node A's admin_variables: +- `curl -s http://127.0.0.1:16070/api/tsdb/nodes` → JSON array with both nodes, sane `watermark_age_s`. +- `curl -s "http://127.0.0.1:16070/api/tsdb/query?metric=proxysql_uptime_seconds_total&node=127.0.0.1:16042"` → rows only from node B. +- `curl -s http://127.0.0.1:16070/api/tsdb/status` → the four new fields present. +- Regression: same query WITHOUT `node=` → identical behavior to before (local table). + +```bash +git add include/ProxySQL_Statistics.hpp lib/ProxySQL_Statistics.cpp lib/ProxySQL_RESTAPI_Server.cpp +git commit -m "feat(tsdb): node-scoped queries, /api/tsdb/nodes, aggregator status fields" +``` + +--- + +### Task 5: Dashboard node selector + +**Files:** +- Modify: `lib/TSDB_Dashboard_html.cpp` (embedded HTML/JS string) + +**Interfaces:** +- Consumes: Task 4 REST endpoints. Produces: UI only; nothing downstream. + +- [ ] **Step 1: Read the embedded dashboard source and add the selector** + +Read `lib/TSDB_Dashboard_html.cpp` first — it is one large C string containing the page. Behavioral contract (exact): +1. Next to the existing metric selector control, add ``. +2. On page load, `fetch('/api/tsdb/nodes')` and append one `` per entry; if the fetch fails or returns an empty array, leave only "local" (feature dormant → dashboard unchanged). +3. Wherever the page builds the `/api/tsdb/query?...` URL, append `&node=` + `encodeURIComponent(sel.value)` when `nodeSel.value` is non-empty. +4. Changing the selector triggers the same refresh path as changing the metric. +No other layout changes. Keep the JS style of the surrounding code (plain ES5/ES6, no new libraries — CSP note: the page must stay self-contained). + +- [ ] **Step 2: Verify manually, commit** + +Rebuild; on the Task 4 scratch setup open `http://127.0.0.1:16070/tsdb` (or `curl` the HTML and grep for `nodeSel` + the fetch call). Verify: selector present, populated with both nodes, switching changes the plotted series (or at minimum the query URL — check the browser network tab or add a temporary `console.log`; remove it before committing). + +```bash +git add lib/TSDB_Dashboard_html.cpp +git commit -m "feat(tsdb): dashboard node selector for cluster view" +``` + +--- + +### Task 6: E2E TAP test with synthetic backfill history + +**Files:** +- Create: `test/tap/tests/test_cluster_tsdb_aggregation-t.cpp` +- Modify: `test/tap/groups/groups.json` + +**Interfaces:** +- Consumes: everything, via admin protocol + REST. Produces: CI coverage + the storage-sizing diagnostic. + +**Test design (from spec §7):** 3 self-spawned nodes (pattern of `test_cluster_leader_election-t.cpp` — reuse its `node_t`/`write_node_config`/`spawn_node`/`wait_leader` shapes, DIFFERENT ports: admin 16162/16172/16182, mysql +1, weights 300/200/100). Config adds `restapi_enabled`/`restapi_port` (16165/16175/16185). TSDB and aggregation vars are set at runtime under `PROXYSQL READWRITE` (no `tsdb_variables` cnf section exists). Synthetic seed: 3 metrics × 6h at 5s spacing = 12,960 rows/node, distinct values per node, inserted via multi-row INSERTs (500 rows/statement) into `stats_history.tsdb_metrics`. Aggregation tuned for the test: `tsdb-cluster_interval=5`, `tsdb-cluster_batch_rows=1000`, `tsdb-cluster_backfill_hours=2` (horizon = 1,440 rows/metric/node; total in-horizon synthetic per node = 4,320 → ≥5 cycles to catch up, exercising the cap path). + +- [ ] **Step 1: Write the test** + +Create `test/tap/tests/test_cluster_tsdb_aggregation-t.cpp`: + +```cpp +/** + * @file test_cluster_tsdb_aggregation-t.cpp + * @brief E2E test for TSDB cluster aggregation (leader pulls peers' tsdb_metrics). + * + * Spawns a self-contained 3-node cluster (127.0.0.1:16162/16172/16182, + * weights 300/200/100), seeds 6h of synthetic history per node, then + * verifies: leader-only aggregation of all 3 nodes (incl. itself), + * backfill-horizon trimming, multi-cycle batch-cap catch-up, per-node + * exactness, /api/tsdb/nodes, failover backfill from peers' retention, + * and watermark resume after the old leader rejoins. + * Skips (plan 1) on builds without TSDB or leader election. + */ + +#include +#include +#include +#include + +#include +#include +#include + +#include "mysql.h" +#include "tap.h" +#include "command_line.h" +#include "utils.h" + +using std::string; + +struct node_t { + int idx; + int admin_port; + int restapi_port; + int weight; + string datadir; + string cnf_path; + std::atomic pid { -1 }; + std::thread* runner = nullptr; +}; + +static node_t nodes_def[3]; +static string workdir_path; + +static const int SEED_METRICS = 3; +static const long SEED_SPAN_S = 6 * 3600; // 6h of history +static const long SEED_STEP_S = 5; // 5s grid +static const int BACKFILL_HOURS = 2; // horizon << seeded span + +static void write_node_config(node_t& n) { + FILE* f = fopen(n.cnf_path.c_str(), "w"); + if (f == NULL) { return; } + fprintf(f, "datadir=\"%s\"\n", n.datadir.c_str()); + fprintf(f, "admin_variables = {\n"); + fprintf(f, "\tadmin_credentials=\"admin:admin;cluster1:secret1pass\"\n"); + fprintf(f, "\tmysql_ifaces=\"0.0.0.0:%d\"\n", n.admin_port); + fprintf(f, "\tcluster_username=\"cluster1\"\n"); + fprintf(f, "\tcluster_password=\"secret1pass\"\n"); + fprintf(f, "\tcluster_check_interval_ms=200\n"); + fprintf(f, "\tcluster_leader_election=\"true\"\n"); + fprintf(f, "\tcluster_leader_node_timeout_ms=1000\n"); + fprintf(f, "\tcluster_leader_grace_ms=500\n"); + fprintf(f, "\trestapi_enabled=\"true\"\n"); + fprintf(f, "\trestapi_port=%d\n", n.restapi_port); + fprintf(f, "}\n"); + fprintf(f, "mysql_variables = {\n"); + fprintf(f, "\tthreads=2\n"); + fprintf(f, "\tinterfaces=\"0.0.0.0:%d\"\n", n.admin_port + 1); + fprintf(f, "}\n"); + fprintf(f, "proxysql_servers = (\n"); + for (int i = 0; i < 3; i++) { + fprintf(f, "\t{ hostname=\"127.0.0.1\"; port=%d; weight=%d; comment=\"node%d\"; }%s\n", + nodes_def[i].admin_port, nodes_def[i].weight, nodes_def[i].idx, (i < 2 ? "," : "")); + } + fprintf(f, ")\n"); + fclose(f); +} + +static void spawn_node(node_t& n, bool initial) { + const string binary = workdir_path + "../../../src/proxysql"; + const string stderr_f = n.datadir + "/node_stderr.txt"; + // 'exec' is load-bearing: without it sh forks and the recorded pid is the + // shell, so SIGKILL would not kill proxysql (see test_cluster_leader_election-t). + string cmd = "exec " + binary + " -f -M -c " + n.cnf_path + " -D " + n.datadir; + if (initial) { cmd += " --initial"; } + cmd += " >> " + stderr_f + " 2>&1"; + n.runner = new std::thread([&n, cmd]() { + pid_t p = fork(); + if (p == 0) { execl("/bin/sh", "sh", "-c", cmd.c_str(), (char*)nullptr); _exit(127); } + n.pid.store(p); + int status = 0; + waitpid(p, &status, 0); + }); + usleep(500 * 1000); +} + +static void join_node(node_t& n) { + if (n.runner) { n.runner->join(); delete n.runner; n.runner = nullptr; } + n.pid.store(-1); +} + +static MYSQL* admin_conn(int port) { + conn_opts_t opts {}; + opts.host = "127.0.0.1"; + opts.user = "admin"; + opts.pass = "admin"; + opts.port = port; + return wait_for_proxysql(opts, 15); +} + +static bool query_ok(MYSQL* conn, const char* q) { + return mysql_query(conn, q) == 0; +} + +static long long single_ll(MYSQL* conn, const string& q, long long defval = -1) { + if (mysql_query(conn, q.c_str())) { return defval; } + MYSQL_RES* res = mysql_store_result(conn); + if (res == NULL) { return defval; } + MYSQL_ROW row = mysql_fetch_row(res); + long long v = (row && row[0]) ? atoll(row[0]) : defval; + mysql_free_result(res); + return v; +} + +static bool var_exists(MYSQL* conn, const char* name) { + string q = "SELECT count(*) FROM global_variables WHERE variable_name='" + string(name) + "'"; + return single_ll(conn, q, 0) == 1; +} + +// Seeds SEED_METRICS synthetic series covering [now-SEED_SPAN_S, now] on the +// node behind `conn`. Values encode the node index for cross-checks. +static bool seed_synthetic(MYSQL* conn, int node_idx, long now) { + const int rows_per_stmt = 500; + for (int m = 1; m <= SEED_METRICS; m++) { + long n_rows = SEED_SPAN_S / SEED_STEP_S; + long done = 0; + while (done < n_rows) { + string q = "INSERT INTO stats_history.tsdb_metrics (timestamp, metric_name, labels, value) VALUES "; + int in_stmt = 0; + while (in_stmt < rows_per_stmt && done < n_rows) { + long ts = now - SEED_SPAN_S + done * SEED_STEP_S; + if (in_stmt) { q += ","; } + q += "(" + std::to_string(ts) + ",'synthetic_metric_" + std::to_string(m) + + "','{}'," + std::to_string(node_idx * 1000000 + done) + ")"; + in_stmt++; + done++; + } + if (mysql_query(conn, q.c_str())) { + diag("seed failed on node %d metric %d: %s", node_idx, m, mysql_error(conn)); + return false; + } + } + } + return true; +} + +static string node_id(int i) { + return "127.0.0.1:" + std::to_string(nodes_def[i].admin_port); +} + +static long long cluster_count(MYSQL* conn, const string& node, const string& metric) { + return single_ll(conn, + "SELECT COUNT(*) FROM stats_history.tsdb_metrics_cluster WHERE node='" + node + + "' AND metric_name='" + metric + "'", -1); +} + +// Polls until the count is stable (two equal consecutive reads) or timeout. +static long long wait_stable_count(MYSQL* conn, const string& node, const string& metric, int timeout_s) { + long long prev = -2; + for (int i = 0; i < timeout_s; i++) { + long long c = cluster_count(conn, node, metric); + if (c >= 0 && c == prev) { return c; } + prev = c; + sleep(1); + } + return prev; +} + +int main(int argc, char** argv) { + CommandLine cl; + if (cl.getEnv()) { diag("Failed to get the required environmental variables."); return -1; } + workdir_path = cl.workdir; + + const string base = workdir_path + "test_cluster_tsdb_aggregation_config"; + mkdir(base.c_str(), 0777); + for (int i = 0; i < 3; i++) { + nodes_def[i].idx = i + 1; + nodes_def[i].admin_port = 16162 + i * 10; + nodes_def[i].restapi_port = 16165 + i * 10; + nodes_def[i].weight = 300 - i * 100; + nodes_def[i].datadir = base + "/node" + std::to_string(i + 1); + nodes_def[i].cnf_path = nodes_def[i].datadir + "/node.cnf"; + } + for (int i = 0; i < 3; i++) { + mkdir(nodes_def[i].datadir.c_str(), 0777); + write_node_config(nodes_def[i]); + spawn_node(nodes_def[i], true); + } + + MYSQL* a[3] = { admin_conn(nodes_def[0].admin_port), admin_conn(nodes_def[1].admin_port), admin_conn(nodes_def[2].admin_port) }; + if (a[0] == NULL || a[1] == NULL || a[2] == NULL) { + fprintf(stderr, "File %s, line %d, Error: failed to start the 3 cluster nodes\n", __FILE__, __LINE__); + for (int i = 0; i < 3; i++) { + pid_t p = nodes_def[i].pid.load(); + if (p > 0) { kill(p, SIGKILL); } + join_node(nodes_def[i]); + } + return -1; + } + + bool feature = var_exists(a[0], "admin-cluster_leader_election") && var_exists(a[0], "tsdb-enabled"); + if (feature == false) { + plan(1); + ok(1, "TSDB or leader election not compiled in this build - skipping"); + } else { + plan(27); + long seed_now = (long)time(NULL); + + // --- Setup under FORCED_RW: tsdb vars + synthetic seed --- (3+3+3) + for (int i = 0; i < 3; i++) { + ok(query_ok(a[i], "PROXYSQL READWRITE"), "node%d: PROXYSQL READWRITE", i + 1); + } + for (int i = 0; i < 3; i++) { + bool vars_ok = + query_ok(a[i], "SET tsdb-enabled='1'") && + query_ok(a[i], "SET tsdb-sample_interval='1'") && + query_ok(a[i], "SET tsdb-cluster_interval='5'") && + query_ok(a[i], "SET tsdb-cluster_batch_rows='1000'") && + query_ok(a[i], ("SET tsdb-cluster_backfill_hours='" + std::to_string(BACKFILL_HOURS) + "'").c_str()) && + query_ok(a[i], "LOAD TSDB VARIABLES TO RUNTIME"); + ok(vars_ok, "node%d: tsdb variables configured: %s", i + 1, mysql_error(a[i])); + } + for (int i = 0; i < 3; i++) { + ok(seed_synthetic(a[i], i + 1, seed_now), "node%d: synthetic history seeded (%ld rows)", + i + 1, (long)SEED_METRICS * (SEED_SPAN_S / SEED_STEP_S)); + } + + // --- Release to election --- (3 + 1) + for (int i = 0; i < 3; i++) { + ok(query_ok(a[i], "PROXYSQL READONLY AUTO"), "node%d: PROXYSQL READONLY AUTO", i + 1); + } + bool leader_ok = false; + for (int i = 0; i < 30; i++) { + if (single_ll(a[0], "SELECT COUNT(*) FROM stats_proxysql_servers_status WHERE master='YES' AND port=16162", 0) == 1) { leader_ok = true; break; } + usleep(500 * 1000); + } + ok(leader_ok, "node1 (highest weight) becomes leader"); + + // --- Incremental catch-up (batch cap forces multiple cycles) --- (1) + long long c1 = -1, c2 = -1; + for (int i = 0; i < 60; i++) { + long long c = cluster_count(a[0], node_id(1), "synthetic_metric_1"); + if (c > 0 && c1 < 0) { c1 = c; } + else if (c1 >= 0 && c > c1) { c2 = c; break; } + sleep(1); + } + ok(c1 > 0 && c2 > c1, "replication progresses incrementally across cycles (%lld -> %lld)", c1, c2); + + // --- Exactness within the horizon --- (3) + // Horizon = 2h at 5s grid = 1440 synthetic rows/metric/node. The leader's + // first cycle ran within ~60s of seed_now; allow generous slack. + for (int i = 0; i < 3; i++) { + long long c = wait_stable_count(a[0], node_id(i), "synthetic_metric_1", 90); + ok(c >= 1380 && c <= 1470, "node%d synthetic rows within horizon replicated exactly (got %lld, expect ~1440)", i + 1, c); + } + + // --- Horizon trim: nothing older than ~2h replicated --- (1) + long long min_ts = single_ll(a[0], + "SELECT MIN(timestamp) FROM stats_history.tsdb_metrics_cluster WHERE metric_name LIKE 'synthetic_%'", -1); + ok(min_ts >= seed_now - SEED_SPAN_S + 3 * 3600, + "backfill horizon trimmed 6h of history to ~2h (min replicated ts %lld)", min_ts); + + // --- Followers do not aggregate --- (2) + ok(single_ll(a[1], "SELECT COUNT(*) FROM stats_history.tsdb_metrics_cluster", -1) == 0, "node2 (follower) cluster table empty"); + ok(single_ll(a[2], "SELECT COUNT(*) FROM stats_history.tsdb_metrics_cluster", -1) == 0, "node3 (follower) cluster table empty"); + + // --- REST /api/tsdb/nodes on the leader --- (1) + { + string curl_cmd = "curl -s --max-time 5 http://127.0.0.1:" + std::to_string(nodes_def[0].restapi_port) + "/api/tsdb/nodes"; + FILE* p = popen(curl_cmd.c_str(), "r"); + string out; + if (p) { + char buf[4096]; size_t n; + while ((n = fread(buf, 1, sizeof(buf), p)) > 0) { out.append(buf, n); } + int rc = pclose(p); + if (rc != 0 && out.empty()) { + skip(1, "curl unavailable in the runner"); + } else { + ok(out.find(node_id(0)) != string::npos && out.find(node_id(1)) != string::npos && out.find(node_id(2)) != string::npos, + "/api/tsdb/nodes lists all three nodes: %s", out.substr(0, 200).c_str()); + } + } else { + skip(1, "popen failed for curl"); + } + } + + // Storage sizing diagnostic (spec: defaults are placeholders until sized) + { + long long rows = single_ll(a[0], "SELECT COUNT(*) FROM stats_history.tsdb_metrics_cluster", -1); + long long pc = single_ll(a[0], "PRAGMA stats_history.page_count", -1); + long long ps = single_ll(a[0], "PRAGMA stats_history.page_size", -1); + diag("SIZING: tsdb_metrics_cluster rows=%lld stats_db_bytes=%lld", rows, (pc > 0 && ps > 0) ? pc * ps : -1); + } + + // --- Failover: node2 takes over and backfills history it never observed --- (3) + long long node3_hist_before = cluster_count(a[0], node_id(2), "synthetic_metric_1"); + { + pid_t p1 = nodes_def[0].pid.load(); + if (p1 > 0) { kill(p1, SIGKILL); } + join_node(nodes_def[0]); + mysql_close(a[0]); a[0] = NULL; + } + bool failover_ok = false; + for (int i = 0; i < 30; i++) { + if (single_ll(a[1], "SELECT COUNT(*) FROM stats_proxysql_servers_status WHERE master='YES' AND port=16172", 0) == 1) { failover_ok = true; break; } + usleep(500 * 1000); + } + ok(failover_ok, "after leader kill, node2 becomes leader"); + long long c_n3 = wait_stable_count(a[1], node_id(2), "synthetic_metric_1", 90); + ok(c_n3 >= 1380, "new leader backfilled node3 synthetic history predating its leadership (got %lld)", c_n3); + long long c_self = wait_stable_count(a[1], node_id(1), "synthetic_metric_1", 30); + ok(c_self >= 1380, "new leader backfilled its own synthetic history (got %lld)", c_self); + + // --- Old leader rejoins, retakes leadership, resumes from its table --- (3) + spawn_node(nodes_def[0], false); + a[0] = admin_conn(nodes_def[0].admin_port); + ok(a[0] != NULL, "node1 restarted and reachable"); + bool retake_ok = false; + for (int i = 0; i < 40; i++) { + if (a[0] && single_ll(a[0], "SELECT COUNT(*) FROM stats_proxysql_servers_status WHERE master='YES' AND port=16162", 0) == 1) { retake_ok = true; break; } + usleep(500 * 1000); + } + ok(retake_ok, "rejoined node1 retakes leadership (highest weight)"); + long long c_resume = wait_stable_count(a[0], node_id(2), "synthetic_metric_1", 90); + ok(a[0] != NULL && c_resume >= node3_hist_before, "node1 resumed aggregation from its surviving watermarks (%lld >= %lld)", c_resume, node3_hist_before); + } + + // Teardown + for (int i = 0; i < 3; i++) { + if (a[i]) { + mysql_query(a[i], "PROXYSQL SHUTDOWN"); + mysql_close(a[i]); + } + pid_t p = nodes_def[i].pid.load(); + if (p > 0) { + for (int w = 0; w < 10 && kill(p, 0) == 0; w++) { usleep(500 * 1000); } + if (kill(p, 0) == 0) { kill(p, SIGKILL); } + } + join_node(nodes_def[i]); + } + return exit_status(); +} +``` + +Assertion count check (feature path): 3 (RW) + 3 (vars) + 3 (seed) + 3 (AUTO) + 1 (leader) + 1 (incremental) + 3 (exact) + 1 (horizon) + 2 (followers) + 1 (REST, or skip) + 1 (failover leader) + 1 (node3 backfill) + 1 (self backfill) + 1 (restart) + 1 (retake) + 1 (resume) = **27** = `plan(27)`. + +Note on `PRAGMA stats_history.page_count` through the admin interface: if the PRAGMA is rejected or returns nothing, `single_ll` returns -1 and the diag prints -1 — the sizing diagnostic is best-effort, never an assertion. + +- [ ] **Step 2: Build and run locally** + +```bash +cd test/tap/tests && make test_cluster_tsdb_aggregation-t +# kill any leftovers on 16162/16172/16182 first (ss -ltn | grep 1616) +TAP_WORKDIR="$(pwd)/" LD_LIBRARY_PATH=../tap ./test_cluster_tsdb_aggregation-t +``` +Expected: `1..27` all ok. Iterate on timing constants (poll timeouts, not sleeps) if the box is slow. If a failure implicates feature code rather than test timing, STOP and report it — do not adjust the test to mask it. Run twice consecutively clean. + +- [ ] **Step 3: Register + lint + commit** + +groups.json: `"test_cluster_tsdb_aggregation-t" : [ "legacy-g5", "mysql84-g5", "mysql90-g5", "mysql95-g5" ],` + +```bash +python3 test/tap/groups/lint_groups_json.py && python3 test/tap/groups/lint_group_coverage.py +git add test/tap/tests/test_cluster_tsdb_aggregation-t.cpp test/tap/groups/groups.json +git commit -m "test(tsdb): E2E cluster aggregation test with synthetic backfill history" +``` + +--- + +### Task 7: Final verification sweep + +**Files:** none (verification only). + +- [ ] **Step 1: Full clean rebuild (version-string consistency) + build matrix** + +```bash +make clean && make -j$(nproc) 2>&1 | tail -3 # stable tier +make clean && PROXYSQL31=1 make debug -j$(nproc) 2>&1 | tail -3 # target tier — tree must END here +``` +Note: `make clean` wipes TAP binaries; rebuild the ones needed below (`cd test/tap/tests && make test_cluster_tsdb_aggregation-t test_cluster_leader_election-t test_cluster_sync-t test_cluster1-t`; unit binaries via `cd test/tap/tests/unit && PROXYSQL31=1 make tsdb_cluster_aggregator_unit-t cluster_leader_election_unit-t`). + +- [ ] **Step 2: Unit + E2E on the final tree** + +```bash +cd test/tap/tests/unit && ./tsdb_cluster_aggregator_unit-t && ./cluster_leader_election_unit-t +cd ../ && TAP_WORKDIR="$(pwd)/" LD_LIBRARY_PATH=../tap ./test_cluster_tsdb_aggregation-t +TAP_WORKDIR="$(pwd)/" LD_LIBRARY_PATH=../tap ./test_cluster_leader_election-t +``` +Expected: all green (1..11, 1..19, 1..27, 1..30). + +- [ ] **Step 3: Harness runs (E2E + regressions)** + +Long harness invocations MUST run via background execution (a mid-run kill poisons the shared primary and orphans replicas — memory `tap-infra-gotchas` item 7). Known workaround if `ensure-infras` errors on running backends: `export COMPOSE_PROJECT=placeholder`. + +```bash +WORKSPACE=$(pwd) INFRA_ID=dev-$USER TAP_GROUP=legacy-g5 test/infra/control/start-proxysql-isolated.bash +WORKSPACE=$(pwd) INFRA_ID=dev-$USER TAP_GROUP=legacy-g5 test/infra/control/ensure-infras.bash +WORKSPACE=$(pwd) INFRA_ID=dev-$USER TAP_GROUP=legacy-g5 \ + TEST_PY_TAP_INCL="test_cluster_tsdb_aggregation-t|test_cluster_leader_election-t" test/infra/control/run-tests-isolated.bash +WORKSPACE=$(pwd) INFRA_ID=dev-$USER TAP_GROUP=legacy-g5 \ + TEST_PY_TAP_INCL="test_cluster_sync-t|test_cluster1-t|test_tsdb_variables-t" test/infra/control/run-tests-isolated.bash +``` +Expected: all PASS. Per CLAUDE.md: any failure gets root-cause analysis (test log + proxysql log + code path), never a "flaky" label. + +- [ ] **Step 4: Final state** + +```bash +git status --short && git log --oneline v3.0..HEAD +``` +Clean except expected untracked `*_config/` run debris; commit list matches the tasks. diff --git a/docs/superpowers/plans/2026-08-13-tsdb-sizing-lab.md b/docs/superpowers/plans/2026-08-13-tsdb-sizing-lab.md new file mode 100644 index 0000000000..cb39960b3a --- /dev/null +++ b/docs/superpowers/plans/2026-08-13-tsdb-sizing-lab.md @@ -0,0 +1,675 @@ +# TSDB Sizing Lab Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Produce a realistically-shaped multi-day TSDB from a small fixture of REAL captured metrics, so storage/rollup/query behaviour can be measured repeatably, per spec `docs/superpowers/specs/2026-08-13-tsdb-sizing-lab-design.md`. + +**Architecture:** Capture once (human-run lab → committed gzipped CSV fixture of real metrics), expand anywhere (`expand.py` writes raw + hourly + cluster rows straight into a stopped instance's `proxysql_stats.db`), measure in CI (nightly workflow prints a sizing table and guards only against gross bytes/row drift). + +**Tech Stack:** Python 3 (stdlib only — `sqlite3`, `gzip`, `csv`, `argparse`), bash, Docker via existing `test/infra`, GitHub Actions. + +## Global Constraints + +- Branch: `feat/tsdb-sizing-lab`, created from `feat/cluster-leader-election` (the `tsdb_metrics_cluster` table exists only there). Rebase onto `v3.0` after #6034 merges. +- **Python: standard library only.** No pip installs — CI must run the tool with the runner's stock `python3`. Target 3.8+ (no match statements, no `datetime.UTC`). +- **Fidelity is structural, not analytical** (explicit user decision): real metric names/labels/cardinality and realistic volume; counters restart at each duplicated block seam. Every tool that duplicates data must say so in its header comment. +- Measured baseline that motivates this work (idle node, 5s sampling): **268 series/tick ≈ 4.6M rows/day/node ≈ 450 MB/day/node**. Do not re-derive; it is the spec's stated basis. +- Current retention defaults (set in #6034): `tsdb-retention_days=2`, `tsdb-cluster_retention_days=1`, `tsdb-hourly_retention_days=365`. +- Table schemas the tool writes (exact, from `include/ProxySQL_Statistics.hpp`): + - `tsdb_metrics(timestamp, metric_name, labels, value)` PK `(timestamp, metric_name, labels)` + - `tsdb_metrics_hour(bucket, metric_name, labels, avg_value, max_value, min_value, count)` + - `tsdb_metrics_cluster(node, timestamp, metric_name, labels, value)` PK `(node, timestamp, metric_name, labels)` +- All inserts use `INSERT OR IGNORE` (idempotent re-runs; PKs dedupe). +- The tool **never creates tables**. If a target table is missing it skips that tier with a warning (so it still works against a pre-aggregation DB). +- If the compressed fixture would exceed **2 MB**, stop and report instead of committing it (spec's abort condition). +- Commit after every task. + +--- + +### Task 1: Expansion tool core (raw tier) + tests + +**Files:** +- Create: `test/tsdb-lab/expand.py` +- Create: `test/tsdb-lab/test_expand.py` +- Create: `test/tsdb-lab/README.md` + +**Interfaces:** +- Consumes: nothing. +- Produces (used by Tasks 2, 3, 4): + - `read_seed(path) -> (rows, block_start, block_end)` where `rows` is a list of `(timestamp:int, metric_name:str, labels:str, value:float)` sorted by timestamp, and the two bounds are ints (min/max timestamp in the fixture). + - `expand_raw(conn, rows, block_start, block_end, window_start, window_end) -> int` — tiles the block forward over `[window_start, window_end)`, inserting into `tsdb_metrics`; returns rows inserted (attempted, i.e. `len(rows) * tiles`). + - `table_exists(conn, name) -> bool` + - CLI: `python3 expand.py --db PATH [--seed PATH] [--raw-window 24h] [--span 14d] [--nodes N]` + - Duration parser `parse_duration(s) -> int` accepting `Nm`, `Nh`, `Nd` (minutes/hours/days) → seconds. + +- [ ] **Step 1: Write the failing tests** + +Create `test/tsdb-lab/test_expand.py` (stdlib `unittest`, no pytest dependency): + +```python +#!/usr/bin/env python3 +"""Tests for the TSDB sizing-lab expansion tool.""" + +import gzip +import os +import sqlite3 +import tempfile +import unittest + +import expand + +RAW_DDL = ( + "CREATE TABLE tsdb_metrics (timestamp INTEGER NOT NULL, metric_name VARCHAR NOT NULL, " + "labels VARCHAR NOT NULL DEFAULT '{}', value REAL, " + "PRIMARY KEY (timestamp, metric_name, labels)) WITHOUT ROWID" +) + +# A 3-tick block, 2 series, 5s apart: timestamps 1000, 1005, 1010. +SEED_LINES = [ + "timestamp,metric_name,labels,value", + "1000,metric_a,{},1.0", + "1000,metric_b,{\"hg\":\"1\"},10.0", + "1005,metric_a,{},2.0", + "1005,metric_b,{\"hg\":\"1\"},20.0", + "1010,metric_a,{},3.0", + "1010,metric_b,{\"hg\":\"1\"},30.0", +] + + +def write_seed(path): + with gzip.open(path, "wt", newline="") as f: + f.write("\n".join(SEED_LINES) + "\n") + + +class TestParseDuration(unittest.TestCase): + def test_units(self): + self.assertEqual(expand.parse_duration("30m"), 1800) + self.assertEqual(expand.parse_duration("24h"), 86400) + self.assertEqual(expand.parse_duration("14d"), 1209600) + + def test_rejects_garbage(self): + with self.assertRaises(ValueError): + expand.parse_duration("14") + with self.assertRaises(ValueError): + expand.parse_duration("2w") + + +class TestReadSeed(unittest.TestCase): + def test_reads_rows_and_bounds(self): + with tempfile.TemporaryDirectory() as d: + p = os.path.join(d, "seed.csv.gz") + write_seed(p) + rows, start, end = expand.read_seed(p) + self.assertEqual(len(rows), 6) + self.assertEqual(start, 1000) + self.assertEqual(end, 1010) + self.assertEqual(rows[0][1], "metric_a") + + +class TestExpandRaw(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.db = os.path.join(self.tmp.name, "stats.db") + self.conn = sqlite3.connect(self.db) + self.conn.execute(RAW_DDL) + self.seed = os.path.join(self.tmp.name, "seed.csv.gz") + write_seed(self.seed) + self.rows, self.start, self.end = expand.read_seed(self.seed) + + def tearDown(self): + self.conn.close() + self.tmp.cleanup() + + def test_tiles_fill_the_window(self): + # Block spans 1000..1010 -> stride = (1010-1000) + 5 = 15s per tile. + # Window of 60s therefore holds 4 tiles = 24 rows. + expand.expand_raw(self.conn, self.rows, self.start, self.end, 100000, 100060) + n = self.conn.execute("SELECT COUNT(*) FROM tsdb_metrics").fetchone()[0] + self.assertEqual(n, 24) + + def test_timestamps_stay_inside_the_window(self): + expand.expand_raw(self.conn, self.rows, self.start, self.end, 100000, 100060) + lo, hi = self.conn.execute("SELECT MIN(timestamp), MAX(timestamp) FROM tsdb_metrics").fetchone() + self.assertGreaterEqual(lo, 100000) + self.assertLess(hi, 100060) + + def test_no_gap_larger_than_the_sample_interval(self): + expand.expand_raw(self.conn, self.rows, self.start, self.end, 100000, 100060) + ts = [r[0] for r in self.conn.execute( + "SELECT DISTINCT timestamp FROM tsdb_metrics ORDER BY timestamp")] + gaps = {b - a for a, b in zip(ts, ts[1:])} + self.assertTrue(max(gaps) <= 5, "seam gap exceeds the 5s sample interval: %s" % sorted(gaps)) + + def test_idempotent(self): + expand.expand_raw(self.conn, self.rows, self.start, self.end, 100000, 100060) + first = self.conn.execute("SELECT COUNT(*) FROM tsdb_metrics").fetchone()[0] + expand.expand_raw(self.conn, self.rows, self.start, self.end, 100000, 100060) + second = self.conn.execute("SELECT COUNT(*) FROM tsdb_metrics").fetchone()[0] + self.assertEqual(first, second) + + +class TestTableExists(unittest.TestCase): + def test_detects_presence_and_absence(self): + with tempfile.TemporaryDirectory() as d: + conn = sqlite3.connect(os.path.join(d, "x.db")) + conn.execute(RAW_DDL) + self.assertTrue(expand.table_exists(conn, "tsdb_metrics")) + self.assertFalse(expand.table_exists(conn, "tsdb_metrics_cluster")) + conn.close() + + +if __name__ == "__main__": + unittest.main() +``` + +Note the tiling arithmetic the tests pin down: **stride = (block_end - block_start) + sample_interval**, where `sample_interval` is inferred from the fixture as the smallest positive difference between consecutive distinct timestamps (5s here). Using the raw span without adding one interval would duplicate the seam timestamp and leave a visible gap. + +- [ ] **Step 2: Run tests, verify they fail** + +```bash +cd /data/rene/proxysql7/proxysql/test/tsdb-lab && python3 -m unittest test_expand -v +``` +Expected: `ModuleNotFoundError: No module named 'expand'` (or AttributeError once the file exists but is empty). + +- [ ] **Step 3: Implement the raw tier** + +Create `test/tsdb-lab/expand.py`: + +```python +#!/usr/bin/env python3 +"""Expand a small fixture of REAL captured TSDB metrics into a realistically +shaped stats database. + +Fidelity is STRUCTURAL, not analytical: metric names, label sets, cardinality +and volume are real, but the fixture block is repeated, so counters restart at +every block seam. That is fine for storage sizing, replication load and query +cost; it is NOT suitable for rate()/dashboard realism. + +Run against a STOPPED ProxySQL instance's proxysql_stats.db. +""" + +import argparse +import gzip +import csv +import os +import sqlite3 +import sys +import time + +SEED_DEFAULT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures", "seed-10min.csv.gz") + + +def parse_duration(s): + """'30m' / '24h' / '14d' -> seconds. Raises ValueError on anything else.""" + if not s or len(s) < 2: + raise ValueError("bad duration: %r" % s) + unit = s[-1] + mult = {"m": 60, "h": 3600, "d": 86400}.get(unit) + if mult is None: + raise ValueError("bad duration unit in %r (use m/h/d)" % s) + return int(s[:-1]) * mult + + +def read_seed(path): + """Returns (rows, block_start, block_end); rows sorted by timestamp.""" + rows = [] + with gzip.open(path, "rt", newline="") as f: + for rec in csv.DictReader(f): + rows.append((int(rec["timestamp"]), rec["metric_name"], rec["labels"], float(rec["value"]))) + if not rows: + raise ValueError("seed fixture %s has no rows" % path) + rows.sort(key=lambda r: r[0]) + return rows, rows[0][0], rows[-1][0] + + +def sample_interval(rows): + """Smallest positive gap between consecutive distinct timestamps.""" + ts = sorted({r[0] for r in rows}) + gaps = [b - a for a, b in zip(ts, ts[1:]) if b > a] + return min(gaps) if gaps else 1 + + +def table_exists(conn, name): + row = conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (name,)).fetchone() + return row is not None + + +def expand_raw(conn, rows, block_start, block_end, window_start, window_end): + stride = (block_end - block_start) + sample_interval(rows) + inserted = 0 + offset = window_start - block_start + batch = [] + while True: + tile_start = block_start + offset + if tile_start >= window_end: + break + for (ts, name, labels, value) in rows: + new_ts = ts + offset + if new_ts < window_start or new_ts >= window_end: + continue + batch.append((new_ts, name, labels, value)) + if len(batch) >= 20000: + conn.executemany( + "INSERT OR IGNORE INTO tsdb_metrics (timestamp, metric_name, labels, value) VALUES (?,?,?,?)", + batch) + inserted += len(batch) + batch = [] + offset += stride + if batch: + conn.executemany( + "INSERT OR IGNORE INTO tsdb_metrics (timestamp, metric_name, labels, value) VALUES (?,?,?,?)", + batch) + inserted += len(batch) + conn.commit() + return inserted + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--db", required=True, help="target proxysql_stats.db (instance must be stopped)") + ap.add_argument("--seed", default=SEED_DEFAULT) + ap.add_argument("--raw-window", default="24h") + ap.add_argument("--span", default="14d") + ap.add_argument("--nodes", type=int, default=0) + args = ap.parse_args(argv) + + if not os.path.exists(args.db): + sys.exit("target db does not exist: %s (start ProxySQL once to create the schema)" % args.db) + rows, block_start, block_end = read_seed(args.seed) + now = int(time.time()) + raw_window = parse_duration(args.raw_window) + span = parse_duration(args.span) + if raw_window > span: + sys.exit("--raw-window (%s) cannot exceed --span (%s)" % (args.raw_window, args.span)) + + conn = sqlite3.connect(args.db) + conn.execute("PRAGMA synchronous=OFF") + conn.execute("PRAGMA journal_mode=MEMORY") + t0 = time.time() + if table_exists(conn, "tsdb_metrics"): + n = expand_raw(conn, rows, block_start, block_end, now - raw_window, now) + print("raw: %d rows over %s" % (n, args.raw_window)) + else: + print("WARNING: tsdb_metrics missing, skipping raw tier") + print("elapsed: %.1fs" % (time.time() - t0)) + conn.close() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) +``` + +- [ ] **Step 4: Run tests — expect PASS** + +```bash +cd /data/rene/proxysql7/proxysql/test/tsdb-lab && python3 -m unittest test_expand -v +``` +Expected: all tests OK. + +- [ ] **Step 5: README + commit** + +Create `test/tsdb-lab/README.md` covering: what the lab is for (sizing/rollup/query measurement with real metric shapes), the structural-fidelity caveat, how to regenerate the fixture (points at `capture.bash`, added in Task 3), example invocations, and the warning that the target instance must be stopped. + +```bash +git add test/tsdb-lab/ +git commit -m "feat(tsdb-lab): expansion tool core with raw-tier tiling" +``` + +--- + +### Task 2: Hourly and cluster tiers + +**Files:** +- Modify: `test/tsdb-lab/expand.py` +- Modify: `test/tsdb-lab/test_expand.py` + +**Interfaces:** +- Consumes: Task 1 (`read_seed`, `sample_interval`, `table_exists`, `expand_raw`, `parse_duration`). +- Produces (used by Tasks 3, 4): + - `expand_hourly(conn, rows, block_start, block_end, span_start, span_end) -> int` — writes aggregated buckets into `tsdb_metrics_hour` for whole hours in `[span_start, span_end)`; returns rows inserted. + - `expand_cluster(conn, rows, block_start, block_end, window_start, window_end, nodes) -> int` — same tiling as raw, into `tsdb_metrics_cluster`, under node identities `10.0.0.:6032` for `i` in `1..nodes`. + +- [ ] **Step 1: Write the failing tests** + +Append to `test/tsdb-lab/test_expand.py`: + +```python +HOUR_DDL = ( + "CREATE TABLE tsdb_metrics_hour (bucket INTEGER NOT NULL, metric_name VARCHAR NOT NULL, " + "labels VARCHAR NOT NULL DEFAULT '{}', avg_value REAL, max_value REAL, min_value REAL, " + "count INTEGER, PRIMARY KEY (bucket, metric_name, labels)) WITHOUT ROWID" +) +CLUSTER_DDL = ( + "CREATE TABLE tsdb_metrics_cluster (node VARCHAR NOT NULL, timestamp INTEGER NOT NULL, " + "metric_name VARCHAR NOT NULL, labels VARCHAR NOT NULL DEFAULT '{}', value REAL, " + "PRIMARY KEY (node, timestamp, metric_name, labels)) WITHOUT ROWID" +) + + +class TestExpandHourly(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.conn = sqlite3.connect(os.path.join(self.tmp.name, "stats.db")) + self.conn.execute(HOUR_DDL) + seed = os.path.join(self.tmp.name, "seed.csv.gz") + write_seed(seed) + self.rows, self.start, self.end = expand.read_seed(seed) + + def tearDown(self): + self.conn.close() + self.tmp.cleanup() + + def test_one_row_per_series_per_bucket(self): + # 3 whole hours, 2 series -> 6 rows. + expand.expand_hourly(self.conn, self.rows, self.start, self.end, 3600 * 100, 3600 * 103) + n = self.conn.execute("SELECT COUNT(*) FROM tsdb_metrics_hour").fetchone()[0] + self.assertEqual(n, 6) + + def test_buckets_are_hour_aligned(self): + expand.expand_hourly(self.conn, self.rows, self.start, self.end, 3600 * 100, 3600 * 103) + buckets = [r[0] for r in self.conn.execute("SELECT DISTINCT bucket FROM tsdb_metrics_hour")] + self.assertTrue(all(b % 3600 == 0 for b in buckets), buckets) + + def test_aggregates_match_the_block(self): + expand.expand_hourly(self.conn, self.rows, self.start, self.end, 3600 * 100, 3600 * 101) + row = self.conn.execute( + "SELECT avg_value, max_value, min_value, count FROM tsdb_metrics_hour " + "WHERE metric_name='metric_a'").fetchone() + # metric_a values in the block are 1.0, 2.0, 3.0 + self.assertAlmostEqual(row[0], 2.0) + self.assertAlmostEqual(row[1], 3.0) + self.assertAlmostEqual(row[2], 1.0) + self.assertEqual(row[3], 3) + + def test_idempotent(self): + expand.expand_hourly(self.conn, self.rows, self.start, self.end, 3600 * 100, 3600 * 103) + a = self.conn.execute("SELECT COUNT(*) FROM tsdb_metrics_hour").fetchone()[0] + expand.expand_hourly(self.conn, self.rows, self.start, self.end, 3600 * 100, 3600 * 103) + b = self.conn.execute("SELECT COUNT(*) FROM tsdb_metrics_hour").fetchone()[0] + self.assertEqual(a, b) + + +class TestExpandCluster(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.conn = sqlite3.connect(os.path.join(self.tmp.name, "stats.db")) + self.conn.execute(CLUSTER_DDL) + seed = os.path.join(self.tmp.name, "seed.csv.gz") + write_seed(seed) + self.rows, self.start, self.end = expand.read_seed(seed) + + def tearDown(self): + self.conn.close() + self.tmp.cleanup() + + def test_rows_scale_with_node_count(self): + expand.expand_cluster(self.conn, self.rows, self.start, self.end, 100000, 100060, 3) + n = self.conn.execute("SELECT COUNT(*) FROM tsdb_metrics_cluster").fetchone()[0] + self.assertEqual(n, 24 * 3) + + def test_distinct_node_identities(self): + expand.expand_cluster(self.conn, self.rows, self.start, self.end, 100000, 100060, 3) + nodes = sorted(r[0] for r in self.conn.execute( + "SELECT DISTINCT node FROM tsdb_metrics_cluster")) + self.assertEqual(nodes, ["10.0.0.1:6032", "10.0.0.2:6032", "10.0.0.3:6032"]) +``` + +- [ ] **Step 2: Run tests, verify the new ones fail** + +```bash +cd /data/rene/proxysql7/proxysql/test/tsdb-lab && python3 -m unittest test_expand -v +``` +Expected: the Task-1 tests still pass; the new classes fail with `AttributeError: module 'expand' has no attribute 'expand_hourly'`. + +- [ ] **Step 3: Implement both tiers** + +Add to `test/tsdb-lab/expand.py`: + +```python +def _block_aggregates(rows): + """(metric_name, labels) -> (avg, max, min, count) over the fixture block.""" + acc = {} + for (_ts, name, labels, value) in rows: + key = (name, labels) + cur = acc.get(key) + if cur is None: + acc[key] = [value, value, value, 1] # sum, max, min, count + else: + cur[0] += value + if value > cur[1]: + cur[1] = value + if value < cur[2]: + cur[2] = value + cur[3] += 1 + return {k: (v[0] / v[3], v[1], v[2], v[3]) for k, v in acc.items()} + + +def expand_hourly(conn, rows, block_start, block_end, span_start, span_end): + """Write one aggregated bucket per series per whole hour in [span_start, span_end).""" + aggs = _block_aggregates(rows) + first_bucket = ((span_start + 3599) // 3600) * 3600 + batch = [] + inserted = 0 + bucket = first_bucket + while bucket < span_end: + for (name, labels), (avg_v, max_v, min_v, cnt) in aggs.items(): + batch.append((bucket, name, labels, avg_v, max_v, min_v, cnt)) + if len(batch) >= 20000: + conn.executemany( + "INSERT OR IGNORE INTO tsdb_metrics_hour (bucket, metric_name, labels, " + "avg_value, max_value, min_value, count) VALUES (?,?,?,?,?,?,?)", batch) + inserted += len(batch) + batch = [] + bucket += 3600 + if batch: + conn.executemany( + "INSERT OR IGNORE INTO tsdb_metrics_hour (bucket, metric_name, labels, " + "avg_value, max_value, min_value, count) VALUES (?,?,?,?,?,?,?)", batch) + inserted += len(batch) + conn.commit() + return inserted + + +def expand_cluster(conn, rows, block_start, block_end, window_start, window_end, nodes): + stride = (block_end - block_start) + sample_interval(rows) + inserted = 0 + batch = [] + for i in range(1, nodes + 1): + node = "10.0.0.%d:6032" % i + offset = window_start - block_start + while True: + if block_start + offset >= window_end: + break + for (ts, name, labels, value) in rows: + new_ts = ts + offset + if new_ts < window_start or new_ts >= window_end: + continue + batch.append((node, new_ts, name, labels, value)) + if len(batch) >= 20000: + conn.executemany( + "INSERT OR IGNORE INTO tsdb_metrics_cluster (node, timestamp, metric_name, " + "labels, value) VALUES (?,?,?,?,?)", batch) + inserted += len(batch) + batch = [] + offset += stride + if batch: + conn.executemany( + "INSERT OR IGNORE INTO tsdb_metrics_cluster (node, timestamp, metric_name, " + "labels, value) VALUES (?,?,?,?,?)", batch) + inserted += len(batch) + conn.commit() + return inserted +``` + +Wire both into `main()` after the raw block, each guarded by `table_exists` (warn-and-skip when absent), printing the row counts: + +```python + if table_exists(conn, "tsdb_metrics_hour"): + n = expand_hourly(conn, rows, block_start, block_end, now - span, now - raw_window) + print("hourly: %d rows over %s" % (n, args.span)) + else: + print("WARNING: tsdb_metrics_hour missing, skipping hourly tier") + if args.nodes > 0: + if table_exists(conn, "tsdb_metrics_cluster"): + n = expand_cluster(conn, rows, block_start, block_end, now - raw_window, now, args.nodes) + print("cluster: %d rows across %d nodes" % (n, args.nodes)) + else: + print("WARNING: tsdb_metrics_cluster missing (pre-aggregation build?), skipping cluster tier") +``` + +- [ ] **Step 4: Run tests — expect PASS (all classes)** + +- [ ] **Step 5: Commit** + +```bash +git add test/tsdb-lab/ +git commit -m "feat(tsdb-lab): hourly and cluster tier expansion" +``` + +--- + +### Task 3: Seed capture script + fixture + +**Files:** +- Create: `test/tsdb-lab/capture.bash` +- Create: `test/tsdb-lab/fixtures/seed-10min.csv.gz` (generated artifact) +- Modify: `test/tsdb-lab/README.md` + +**Interfaces:** +- Consumes: nothing from earlier tasks at runtime (it produces the fixture Task 1's `read_seed` consumes: gzipped CSV with header `timestamp,metric_name,labels,value`). +- Produces: the committed fixture used by Tasks 4 and by every future expansion. + +- [ ] **Step 1: Write the capture script** + +Create `test/tsdb-lab/capture.bash` — a human-run script (not CI). It must: + +1. Take `WORKSPACE` (repo root, default `$(git rev-parse --show-toplevel)`) and `DURATION_S` (default 600). +2. Bring up a MySQL backend using the existing harness rather than hand-rolled Docker: `WORKSPACE=$WORKSPACE INFRA_ID=tsdb-lab TAP_GROUP=legacy-g5 test/infra/control/ensure-infras.bash` (per CLAUDE.md, never create containers manually). Export `COMPOSE_PROJECT=placeholder` first (known `ensure-infras` bug on already-running backends). +3. Spawn **3 ProxySQL instances** from `src/proxysql` on 127.0.0.1 admin ports 16362/16372/16382 (mysql +1), each with its own datadir under `test/tsdb-lab/.capture/nodeN`, cluster credentials `cluster1/secret1pass`, all three listed in `proxysql_servers`, `admin-cluster_leader_election="true"`. Use the `exec`-prefixed `sh -c` spawn form so signals reach proxysql (see `test/tap/tests/test_cluster_leader_election-t.cpp`). +4. Register the backend in **two hostgroups** (e.g. 0 and 1) on each node, add a `testuser` mysql user, and `LOAD MYSQL SERVERS/USERS TO RUNTIME` — doing this while nodes are still `PROXYSQL READWRITE`, then `PROXYSQL READONLY AUTO` (followers refuse writes once election converges). +5. Enable TSDB on all three (`SET tsdb-enabled='1'; LOAD TSDB VARIABLES TO RUNTIME;`). +6. Drive **variable** sysbench load through node 1's mysql port for `DURATION_S`, alternating rate every 60s (e.g. `--rate=20` / `--rate=200` / `--rate=60`) — the point is series/value variety, not stress. If `sysbench` is not installed, print a clear message and fall back to a simple mysql client loop issuing mixed SELECT/INSERT statements, so the script still produces a usable fixture. +7. After the run, dump the leader's window: + `SELECT timestamp, metric_name, labels, value FROM stats_history.tsdb_metrics WHERE timestamp >= ORDER BY timestamp` → CSV → gzip → `fixtures/seed-10min.csv.gz`. Include a `#` comment line? **No** — `read_seed` uses `csv.DictReader` with a plain header, so write only the header plus data rows. Record provenance (ProxySQL version, capture date, node/series counts) in `fixtures/seed-10min.README` instead. +8. Print the resulting fixture size and **abort with a clear message if it exceeds 2 MB** (spec's abort condition) rather than committing it. +9. Tear down: `PROXYSQL SHUTDOWN` all three, verify no listener remains on 16362/16372/16382, and leave the backend infra running (the harness owns it). + +- [ ] **Step 2: Run the capture** + +```bash +cd /data/rene/proxysql7/proxysql && bash test/tsdb-lab/capture.bash +``` +Expected: a fixture at `test/tsdb-lab/fixtures/seed-10min.csv.gz`, well under 2 MB, plus the provenance file. Record in the report: series count, row count, compressed size. + +- [ ] **Step 3: Verify the fixture round-trips through the tool** + +```bash +cd test/tsdb-lab && python3 -c " +import expand +rows, s, e = expand.read_seed('fixtures/seed-10min.csv.gz') +print('rows', len(rows), 'span', e - s, 'series', len({(r[1], r[2]) for r in rows}), 'interval', expand.sample_interval(rows)) +" +``` +Expected: several tens of thousands of rows, span ≈ 600s, a few hundred series, interval 5. + +- [ ] **Step 4: Commit** + +```bash +git add test/tsdb-lab/capture.bash test/tsdb-lab/fixtures/ test/tsdb-lab/README.md +git commit -m "feat(tsdb-lab): capture script and real-metric seed fixture" +``` + +--- + +### Task 4: Measurement script + CI workflow + +**Files:** +- Create: `test/tsdb-lab/measure.py` +- Create: `test/tsdb-lab/baseline.json` +- Create: `.github/workflows/CI-tsdb-sizing.yml` +- Modify: `test/tsdb-lab/README.md` + +**Interfaces:** +- Consumes: Tasks 1–3 (`expand.py` CLI, the committed fixture). +- Produces: `measure.py --db PATH [--baseline PATH] [--drift-pct 25]` printing a table and exiting non-zero only on bytes/row drift beyond the threshold. + +- [ ] **Step 1: Write the measurement script** + +Create `test/tsdb-lab/measure.py` (stdlib only). It must: + +1. Open the DB read-only (`file:...?mode=ro` URI). +2. For each of `tsdb_metrics`, `tsdb_metrics_hour`, `tsdb_metrics_cluster` (skipping absent tables): row count, and total bytes of the stored payload as a portable proxy — + `SELECT SUM(LENGTH(metric_name) + LENGTH(labels) + 16) FROM ` (plus `LENGTH(node)` for the cluster table) — reported alongside `page_count * page_size` for the whole file, since `dbstat` may not be compiled in. Derive **bytes/row** per table from the payload sum, and note in the output that the file total includes all tables plus index/page overhead. +3. Time two representative queries with `time.perf_counter()`: + - raw last-1h for a single metric: `SELECT COUNT(*), AVG(value) FROM tsdb_metrics WHERE metric_name=? AND timestamp >= ?` + - hourly full-span for the same metric: `SELECT COUNT(*), AVG(avg_value) FROM tsdb_metrics_hour WHERE metric_name=?` + Pick the metric as the most frequent `metric_name` in the raw table. +4. Print a fixed-width table (table, rows, bytes/row, total payload MB) plus the two query timings and the file size. +5. Compare bytes/row per table against `baseline.json` (`{"tsdb_metrics": {"bytes_per_row": N}, ...}`); exit 1 if any exceeds `±drift-pct`; exit 0 otherwise. If a table is missing from the baseline, print it as `NEW` and do not fail. + +- [ ] **Step 2: Create the baseline** + +Run the full local flow once to generate real numbers, then write `baseline.json` from the measured values: + +```bash +cd /data/rene/proxysql7/proxysql +rm -rf /tmp/tsdb-lab-check && mkdir -p /tmp/tsdb-lab-check +printf 'datadir="/tmp/tsdb-lab-check"\nadmin_variables = { admin_credentials="admin:admin"; mysql_ifaces="0.0.0.0:16392" }\nmysql_variables = { threads=2; interfaces="0.0.0.0:16393" }\n' > /tmp/tsdb-lab-check/n.cnf +src/proxysql --initial -f -c /tmp/tsdb-lab-check/n.cnf -D /tmp/tsdb-lab-check & +sleep 4 && mysql -uadmin -padmin -h127.0.0.1 -P16392 -e "PROXYSQL SHUTDOWN"; sleep 2 +python3 test/tsdb-lab/expand.py --db /tmp/tsdb-lab-check/proxysql_stats.db --raw-window 1h --span 1d --nodes 3 +python3 test/tsdb-lab/measure.py --db /tmp/tsdb-lab-check/proxysql_stats.db +``` +(Use the small `1h`/`1d` profile locally; the CI profile is bigger.) Record the printed bytes/row values into `baseline.json`. + +- [ ] **Step 3: Write the CI workflow** + +Create `.github/workflows/CI-tsdb-sizing.yml`, modeled on the simple structure of `CI-lint-groups-json.yml` (checkout, ubuntu-latest, explicit timeout), with `on: [workflow_dispatch, schedule (nightly cron)]` — **not** `pull_request`, since it is a measurement job. Steps: checkout → build ProxySQL (`PROXYSQL31=1 make -j$(nproc)`; reuse whatever build action the other workflows use if one exists, else plain make) → start once to create the schema and stop → `python3 test/tsdb-lab/expand.py --db /proxysql_stats.db --raw-window 24h --span 14d --nodes 3` → start ProxySQL, wait until `tsdb_metrics_hour` stops growing (bounded wait, report the duration — this is the rollup catch-up measurement) → stop → `python3 test/tsdb-lab/measure.py --db ... --baseline test/tsdb-lab/baseline.json` → upload the printed report as a job artifact. + +- [ ] **Step 4: Validate the workflow file** + +```bash +python3 -c "import sys,yaml" 2>/dev/null && python3 -c " +import yaml, sys +d = yaml.safe_load(open('.github/workflows/CI-tsdb-sizing.yml')) +print('jobs:', list(d['jobs'])) +" || echo "PyYAML unavailable — validate by eye against CI-lint-groups-json.yml structure" +``` +Expected: parses, one job. (If PyYAML is absent, do the structural comparison manually — do not add a dependency.) + +- [ ] **Step 5: Commit** + +```bash +git add test/tsdb-lab/measure.py test/tsdb-lab/baseline.json .github/workflows/CI-tsdb-sizing.yml test/tsdb-lab/README.md +git commit -m "feat(tsdb-lab): measurement script, baseline and nightly CI workflow" +``` + +--- + +### Task 5: Full-scale local run and findings + +**Files:** +- Modify: `docs/superpowers/specs/2026-08-13-tsdb-sizing-lab-design.md` (append a "Measured results" section) + +**Interfaces:** consumes everything; produces the numbers that inform projects 2 and 3. + +- [ ] **Step 1: Run the CI profile locally** + +Same flow as Task 4 Step 2 but with the real profile (`--raw-window 24h --span 14d --nodes 3`) against a fresh datadir. Record: expansion wall-clock, raw/hourly/cluster row counts, bytes/row per table, total DB size, and `measure.py`'s query timings. + +- [ ] **Step 2: Measure rollup catch-up under load** + +Start ProxySQL on the expanded datadir with `tsdb-enabled=1` and time how long `tsdb_metrics_hour` keeps growing (poll `SELECT COUNT(*)` every second until stable for 5s). This is the unbounded first-pass downsample holding `wrlock` — the reproduction for project 3. Record the duration. + +- [ ] **Step 3: Append findings to the spec** + +Add a "Measured results (YYYY-MM-DD, ProxySQL )" section with a table of the recorded numbers, and one short paragraph per implication: whether the new retention defaults hold up, how much the cluster tier costs per node, and whether the rollup catch-up duration justifies chunking (project 3). + +- [ ] **Step 4: Commit** + +```bash +git add docs/superpowers/specs/2026-08-13-tsdb-sizing-lab-design.md +git commit -m "docs(tsdb-lab): record first full-scale measurement results" +``` diff --git a/docs/superpowers/specs/2026-08-11-cluster-leader-election-design.md b/docs/superpowers/specs/2026-08-11-cluster-leader-election-design.md new file mode 100644 index 0000000000..8f81a752aa --- /dev/null +++ b/docs/superpowers/specs/2026-08-11-cluster-leader-election-design.md @@ -0,0 +1,218 @@ +# ProxySQL Cluster Leader Election — Design + +Date: 2026-08-11 +Status: approved for planning +Scope: first deliverable of the "cluster as a single entity" effort — liveness, +deterministic leader election, and read-only config steering. + +## Motivation + +Users increasingly want a ProxySQL cluster to behave as a single entity: +query stats for all nodes in one place, configure all nodes at once, and have +nodes be aware of each other (e.g. split a cluster-wide `max_connections` +budget across N proxies). + +Most building blocks already exist: + +- **ProxySQL Cluster** propagates configuration (pull-based, checksum/epoch, + highest-epoch-wins) and gives nodes awareness of each other + (`proxysql_servers`, per-peer monitor threads). +- **TSDB** stores time-series stats per node; every node exposes a + Prometheus `/metrics` endpoint. +- **Admin read-only mode** (`admin_read_only`, `PROXYSQL READONLY` / + `PROXYSQL READWRITE`) refuses admin writes while cluster sync — which + writes through `GloAdmin->admindb` internally, not through an admin + session — is unaffected by construction. It was designed for exactly this + role and only needed the glue. + +The missing glue is a **leader**: one node that is by convention the +configuration write point and (in a follow-up) the stats aggregator. + +## Decision: no Raft / no consensus + +The cluster stays AP (available under partition), by explicit requirement: +an operator must always be able to log into any reachable node and repair or +tear down a broken cluster. Quorum-based consensus (Raft) would refuse +writes on a minority side — hostile to the recovery story — and would break +the very common 2-node deployment (no quorum after a single failure). + +None of the target problems needs consensus: + +- Config propagation already tolerates forks and reconciles via epochs. +- Stats aggregation is read-only fan-in. +- Quota division needs membership/liveness, not agreement. + +Therefore: **deterministic, ballot-free, locally-computed leader election**, +advisory in nature, with epochs remaining the reconciliation mechanism. +During partitions, transient multi-leader (or no-leader) states are accepted; +the consequences are bounded to today's semantics (both sides writable) or a +safe state (all read-only, operator can override). + +## Design + +### 1. Liveness / membership + +- Each node already polls every peer with `SELECT GLOBAL_CHECKSUM()` every + `admin-cluster_check_interval_ms` from its per-peer monitor thread + (`lib/ProxySQL_Cluster.cpp`, monitor loop). Record the outcome: + `ProxySQL_Node_Entry` gains a `last_success_at` timestamp (monotonic, + updated on every successful poll). +- A peer is **alive** iff `now - last_success_at < + admin-cluster_leader_node_timeout_ms`. Self is always alive. +- Membership candidates = rows of `runtime_proxysql_servers` (already a + cluster-synced module, so the candidate set converges). A node not listed + there can never become leader. +- No new network traffic, threads, or protocol messages, except for one + extra round-trip per connection establishment: after the version/announce + handshake, the monitor thread issues `SELECT GLOBAL_UUID()` on the peer to + learn its UUID (used as the election tiebreaker). + +### 2. Election + +- **Leader = the alive candidate with the highest `weight` in + `proxysql_servers`; ties broken by lowest UUID.** The `weight` column + (synced, exposed, currently semantics-free) finally gets meaning. Existing + deployments have weight 0 everywhere, degenerating to UUID ordering — + deterministic, if arbitrary; operators who care set weights. +- Each node computes the leader **locally** from its own membership view. + No ballots, no votes, no election messages, no persistent election state. +- An election evaluation tick runs from the Admin main loop (same pattern as + the TSDB loops). Hysteresis: a leadership *change* takes effect only after + the new result has been stable for `admin-cluster_leader_grace_ms` + (protects against poll blips and monitor-thread scheduling jitter). +- Transient disagreement between nodes during churn is accepted. Bounded + consequences: two nodes briefly effective-RW (today's permanent state), or + all nodes briefly effective-RO (safe; operator override exists). +- **Opt-in**: `admin-cluster_leader_election` defaults to `false`. Off means + bit-for-bit today's behavior. Rolling upgrade story: upgrade all nodes, + then enable the variable cluster-wide (it is part of the synced + `admin_variables` module). + +### 3. Read-only steering + +Admin read-only becomes a tri-state `admin_ro_mode`: `AUTO` / `FORCED_RO` / +`FORCED_RW`. + +- **Effective read-only** = (`AUTO` && election enabled && not leader) + || `FORCED_RO`. +- With election disabled, `AUTO` means read-write — existing semantics + unchanged (today's boolean default is read-write). +- Election transitions only influence the effective value while in `AUTO`; + they never touch a `FORCED_*` state. An operator's override therefore + survives election ticks — required for partition-recovery scenarios. +- Commands: + - `PROXYSQL READWRITE` → `FORCED_RW` (the recovery escape hatch) + - `PROXYSQL READONLY` → `FORCED_RO` + - `PROXYSQL READONLY AUTO` (new) → `AUTO` (return control to election) + - Restart resets to `AUTO`, unless `admin-read_only=true` maps the boot to + `FORCED_RO` (see below). +- The existing `admin-read_only` boot variable maps onto the tri-state: + `true` → boot in `FORCED_RO`, `false` (default) → boot in `AUTO`. Its + current semantics (boot read-only until an operator lifts it) are + preserved exactly. +- Boot with election enabled: start in `AUTO` (effective-RO) until the first + election settles; a standalone or election-disabled node boots effective-RW + as today. + +**Enforcement gap closed.** Today `get_read_only()` is only enforced on the +generic admin-session SQL path (`PRAGMA query_only = ON` wrapper in +`lib/Admin_Handler.cpp`); `LOAD ... TO RUNTIME` and `SAVE ... TO DISK` are +not gated — yet `LOAD ... TO RUNTIME` is precisely the epoch-bumping +operation that creates sync conflicts. Effective read-only additionally +refuses all `LOAD TO RUNTIME` and `SAVE TO DISK` admin +commands with an error naming the current leader (hostname:port), e.g.: + + ERROR 1045: Admin is in read-only mode (follower). Current leader is + 10.0.0.5:6032. Use PROXYSQL READWRITE to override. + +Cluster-initiated pulls are unaffected: they write via `GloAdmin->admindb` +directly from `pull_*_from_peer()` and never traverse the admin session +handler. + +### 4. Observability + +- **Implement `stats_proxysql_servers_status`** (schema defined since v1.4, + never populated; `include/ProxySQL_Admin_Tables_Definitions.h:289`). One + row per candidate node, from this node's local view: `hostname`, `port`, + `weight`, `master` (`'YES'`/`'NO'` — the long-dormant column becomes the + leader flag), `global_version`, `check_age_us` (time since last successful + poll), `ping_time_us`, `checks_OK`, `checks_ERR`. Add a `uuid VARCHAR` + column (stats tables are not persisted; schema change is safe) since UUID + is the election tiebreaker. +- Prometheus: gauge `proxysql_cluster_leader_status` (1 if this node + considers itself leader, else 0); per-peer `alive` gauge on the existing + dynamic cluster-node families; counter + `proxysql_cluster_leader_changes_total`. +- `proxy_info` on every leadership transition and every state change of + `admin_ro_mode`; `proxy_warning` on every refused write/LOAD/SAVE in + effective-RO (rate-limited). + +### 5. New variables and gating + +| Variable | Default | Notes | +|---|---|---| +| `admin-cluster_leader_election` | `false` | Master switch. Registered only under `#ifdef PROXYSQL31`. | +| `admin-cluster_leader_node_timeout_ms` | `3000` | Liveness horizon; floor 1000. Should be ≥ 3× `cluster_check_interval_ms` in practice (documented, not enforced). | +| `admin-cluster_leader_grace_ms` | `3000` | Stability window before acting on a leadership change. | + +**Tier gating strategy:** all election/liveness/tri-state code compiles +unconditionally in every tier; only the **registration of +`admin-cluster_leader_election`** (and its config-file/default handling) is +wrapped in `#ifdef PROXYSQL31`. In the Stable tier the variable does not +exist, so the feature cannot be enabled. Rationale: keeps the `#ifdef` +surface to a few lines, avoids the FFTO-style stale-object/tier-mismatch +link failures, and keeps the election logic unit-testable in all tiers. +The two tuning variables are registered unconditionally (harmless without +the master switch). + +### 6. Edge cases + +- **Fully isolated node**: sees only itself alive; if it is a candidate it + elects itself → effective-RW. Deliberate: a partitioned node degrades to + standalone ProxySQL behavior. Two partitioned halves each elect a leader; + healed by epoch reconciliation exactly as today. +- **All peers alive but node is not in `proxysql_servers`**: it cannot be + leader; it follows whichever candidate it computes as leader. +- **`proxysql_servers` empty or cluster credentials unset**: election + short-circuits; node behaves as standalone (`AUTO` → effective-RW). +- **Mixed versions during rolling upgrade**: old nodes ignore the new + variables; the cluster already refuses to sync across differing versions + (`SELECT @@version` gate), so no compatibility machinery is needed. +- **Operator changes `weight` at runtime**: takes effect on the next + election tick after the `proxysql_servers` change syncs; grace window + applies, so a planned leader move is: raise weight on the target, load to + runtime on the current leader, wait one grace period. + +### 7. Testing + +- **Unit test** (`test/tap/tests/unit/`, against `libproxysql.a`): the pure + election function — (candidate set, liveness map, weights, UUIDs) → + leader — including ties, empty candidate set, self-not-candidate, and + timeout boundary conditions. +- **TAP test** (isolated infra, 3-node ProxySQL cluster): + 1. Enable election; assert exactly one leader converges and + `stats_proxysql_servers_status` agrees on all three nodes. + 2. Followers refuse `INSERT`, `LOAD MYSQL SERVERS TO RUNTIME`, + `SAVE MYSQL SERVERS TO DISK`; leader accepts all three. + 3. Kill the leader container; next-ranked node becomes leader within + `node_timeout + grace`; config writes succeed there. + 4. `PROXYSQL READWRITE` on a follower sticks across ≥ 2 election ticks; + `PROXYSQL READONLY AUTO` restores follower-RO. + 5. Restart the old leader; since it retains the highest `weight`, it + retakes leadership on rejoin (weight-priority retake) — the previous + leader's replacement steps back down to follower, no flapping. + 6. Election disabled: all nodes effective-RW (regression guard). + +## Out of scope (each needs its own design round) + +1. **Stats aggregation into the leader's TSDB** — implemented and shipped + alongside this PR; see + `2026-08-11-cluster-stats-aggregation-design.md` for the design (leader + pulls peers' `stats_*`/TSDB tables over the existing admin connections + and ingests into TSDB with a per-node label). +2. **Distributed quotas** (cluster-wide `max_connections` split by the + alive-count) — needs hysteresis/flapping design on the admission path. +3. **Leader-only backend monitoring** (semantic checks on the leader, + local reachability checks everywhere) — has availability-path + implications (correlated failures, vantage-point loss, cold-start of a + new leader's monitor) that require dedicated design. diff --git a/docs/superpowers/specs/2026-08-11-cluster-stats-aggregation-design.md b/docs/superpowers/specs/2026-08-11-cluster-stats-aggregation-design.md new file mode 100644 index 0000000000..ddfa6ae51c --- /dev/null +++ b/docs/superpowers/specs/2026-08-11-cluster-stats-aggregation-design.md @@ -0,0 +1,216 @@ +# Cluster Stats Aggregation into the Leader's TSDB — Design + +Date: 2026-08-11 +Status: approved for planning +Depends on: cluster leader election (PR #6034, `docs/superpowers/specs/2026-08-11-cluster-leader-election-design.md`). Branch stacked on `feat/cluster-leader-election`; rebase onto `v3.0` after #6034 merges. + +## Motivation + +Second deliverable of the "cluster as a single entity" effort: query statistics +for all proxies in one place. The leader (from the election feature) becomes +the stats aggregator — a single pane of glass over the whole cluster — without +making any node's local metrics depend on who leads. + +## Decision: TSDB replication over the admin channel (pull + watermark) + +Chosen over (a) HTTP scraping of peers' `/metrics` (live-only view: history +starts at leadership acquisition, partitions leave permanent gaps, requires +the REST API enabled everywhere and a new HTTP-client path) and (b) follower +push (multi-writer leader DB, follower-side buffering for catch-up, deposed +leader keeps receiving writes). Pull + watermark is idempotent and +restart-safe: leadership changes, restarts, and network blips all reduce to +"catch up from where the table says I am." + +Load-bearing verified fact: a peer's TSDB tables are directly queryable over +the existing authenticated admin channel as `stats_history.tsdb_metrics` +(the `proxysql_stats.db` file is attached to admin sessions as the +`stats_history` schema). + +**Durability promise (user requirement):** replicated history. Every node +keeps sampling its own TSDB locally (7-day retention, leader-independent); +the leader's aggregated view backfills from peers' local retention, so a +leader failover produces no gaps in the cluster view. + +## Design + +### 1. Architecture + +- Local TSDB pipeline (5s sampler → `tsdb_metrics`, hourly rollups, + retention) is untouched on every node and remains the durable source of + truth. +- New component: **TSDB aggregator** in `ProxySQL_Statistics`, active only + while `GloProxyCluster->is_leader()` AND election enabled AND + `tsdb-enabled` AND `tsdb-cluster_aggregation=true`. +- Runs on a **dedicated thread** (pulls can take seconds during backfill). + The Admin main-loop tick only starts/stops the thread on leadership or + variable transitions (same `*_timetoget` pattern as the other TSDB loops + for the transition checks). +- All code inside the existing `#ifdef PROXYSQLTSDB` region — no new tier + surface. (PROXYSQL31 implies PROXYSQLTSDB.) + +### 2. Data flow + +Per cycle (every `tsdb-cluster_interval` seconds), for each node listed in +`runtime_proxysql_servers`: + +- **Watermark** = `MAX(timestamp)` already replicated for that node in + `tsdb_metrics_cluster`; if none, initialized to + `now − tsdb-cluster_backfill_hours`. Recovered from the table itself on + restart/re-election — no separate persistent state. +- **Peers**: dedicated MySQL connection per cycle (cluster credentials, same + SSL-enforce and connect-timeout settings as the cluster monitor threads): + `SELECT timestamp, metric_name, labels, value FROM + stats_history.tsdb_metrics WHERE timestamp >= ? ORDER BY timestamp LIMIT + `, looped until caught up or the per-cycle cap is + reached (bounds leader load; catch-up resumes next cycle). `>=` (not `>`): + a previous cycle's `LIMIT` can cut in the middle of a same-timestamp + group, so the boundary group is re-fetched every cycle until fully + replicated; `INSERT OR IGNORE` plus the `(node, timestamp, metric_name, + labels)` primary key make the re-fetch idempotent and loss-free. +- **Self**: no self-connection — a local `INSERT ... SELECT` from the + leader's own `tsdb_metrics` through the same watermark logic, so the + leader appears in the cluster view identically to every other node. +- Rows inserted in batched transactions, tagged with the node's + `hostname:port` exactly as it appears in `proxysql_servers` (operator- + facing identity, stable across `--initial` re-inits; UUID is not used). + +### 3. Storage + +New table in `statsdb_disk` (sibling of `tsdb_metrics`): + +```sql +CREATE TABLE tsdb_metrics_cluster ( + node VARCHAR NOT NULL, + timestamp INT NOT NULL, + metric_name VARCHAR NOT NULL, + labels VARCHAR NOT NULL DEFAULT '', + value REAL, + PRIMARY KEY (node, timestamp, metric_name, labels) +) WITHOUT ROWID +``` + +plus an index mirroring the local table's query pattern +(`(node, metric_name, timestamp)`). + +Rationale for a separate table (vs folding a node label into `labels`): no +double-counting against the local sampler's rows, uniform treatment of the +leader itself, and independent retention. Pruned by the existing retention +loop using `tsdb-cluster_retention_days`. v1 replicates raw samples only, +not peers' hourly rollups (follow-up if long-horizon cluster trends are +wanted). The PK makes replication idempotent — duplicates are structurally +impossible (`INSERT OR IGNORE` on ingest — replicated samples are immutable). + +### 4. Variables (TSDB family, `LOAD TSDB VARIABLES TO RUNTIME`) + +| Variable | Default | Range | | +|---|---|---|---| +| `tsdb-retention_days` | `2` | 1–3650 | local raw-sample (`tsdb_metrics`) retention | +| `tsdb-cluster_aggregation` | `true` | bool | master switch (election is the real opt-in) | +| `tsdb-cluster_interval` | `10` | 5–300 s | pull cadence | +| `tsdb-cluster_backfill_hours` | `24` | 0–168 | horizon for a fresh watermark | +| `tsdb-cluster_retention_days` | `1` | 1–30 | cluster-table retention | +| `tsdb-cluster_batch_rows` | `10000` | 1000–100000 | per-cycle per-peer cap | +| `tsdb-hourly_retention_days` | `365` | 1–3650 | `tsdb_metrics_hour` rollup retention | + +**Defaults are set from a measured floor, not a placeholder.** On an idle +single node at the default 5s sample interval, the sampler emits 268 +distinct series/tick, i.e. ~4.6M rows/day/node (~450 MB/day/node) into the +raw `tsdb_metrics` table. At that rate the old defaults implied multi-GB +embedded stats DBs (7-day raw ≈ 3 GB/node; a 3-node cluster's leader-side +`tsdb_metrics_cluster` at the old 3-day retention ≈ 4 GB). Since the raw +tiers dominate storage while carrying the least query value beyond a couple +of days, they get short retention — local raw down to 2 days, cluster raw +down to 1 day (the leader already multiplies that cost by N nodes) — and +long-horizon trending is pushed onto the hourly rollup tier, which is now +governed by its own variable, `tsdb-hourly_retention_days` (default `365`, +unchanged from the previous hardcoded 1-year prune). + +This figure is a floor, not a ceiling: it was measured on an idle node with +no backends, hostgroups, or connection pools generating additional +per-object series, so a loaded production node will emit more series/tick +and correspondingly more rows/day. The forthcoming duplication-tool lab +(replaying realistic multi-hostgroup traffic) will refine these defaults +with loaded-node numbers before GA. + +### 5. Query surface + +- `/api/tsdb/query` gains `node=` — routes the query to + `tsdb_metrics_cluster`; `node=*` spans all nodes (existing `agg` + semantics apply across them). +- New `/api/tsdb/nodes`: nodes present in the cluster table, each with its + watermark age — an instant health view of the aggregation itself. +- `/api/tsdb/status` extended with aggregator state: leader or not, per-node + watermark lag, rows replicated, whether the batch cap was hit on the last + cycle. +- Dashboard: minimal v1 — a node selector fed by `/api/tsdb/nodes`; no + layout redesign. + +### 6. Edge cases + +- **Not leader / election off**: aggregator thread idle; behavior bit-for-bit + as today. A deposed leader stops pulling; its cluster table goes stale and + ages out via retention — and is correct again if re-elected (watermarks + resume from the table). +- **Peer with TSDB disabled**: query returns 0 rows; logged once per peer + state-change; watermark simply doesn't advance. +- **Mixed versions**: impossible — the cluster already refuses to exchange + with a different-version peer, so the remote schema always matches. +- **Clock skew**: timestamps are peer-local. In steady state, per-peer + watermarks are immune to skew — they're re-derived from `MAX(timestamp)` + of already-replicated (peer-stamped) rows, not from the leader's clock. + The one exception is cold start: a never-replicated peer's initial horizon + is `now - backfill_hours` computed on the *leader's* local clock, so + leader-peer clock offset affects only the initial backfill depth for that + peer, not steady-state behavior. Cross-node comparison quality still + depends on NTP — documented, not compensated. +- **Volume / overload**: worst case ~N× the leader's own sample write rate. + Bounded by the batch cap and separate retention. The aggregator logs a + warning when it cannot keep up (cap hit on consecutive cycles for the + same peer). +- **Follower write-refusal interaction**: the aggregator writes only to the + leader's own `statsdb_disk` via internal handles — admin read-only mode + (a follower concern anyway) never applies to it. + +### 7. Testing + +**Synthetic history is mandatory** (user requirement): a few seconds of live +sampling proves the loop, not the promise. The E2E test pre-seeds each +node's `stats_history.tsdb_metrics` with generated history — on the order of +3 metric names × 6h at 5s spacing ≈ 13k rows/node with distinct per-node +values — inserted over the admin connection before election converges (or +under `PROXYSQL READWRITE`), since followers refuse writes once election +engages. + +- **Unit test** (pure logic, against `libproxysql.a`): the watermark/batch + planner — given (current watermark, backfill horizon, fetched row + timestamps, batch cap) → (rows to insert, new watermark, caught-up flag), + covering: fresh-watermark horizon computation, cap-hit continuation, + empty fetch, and idempotent re-fetch. +- **E2E TAP** (extends the 3-node self-spawned pattern from #6034; election + + tsdb enabled, 1s local sampling, `tsdb-cluster_interval` minimal, + `tsdb-cluster_batch_rows` lowered to force multi-cycle catch-up, + `tsdb-cluster_backfill_hours=2` against 6h of synthetic data): + 1. Leader's `tsdb_metrics_cluster` gains rows for all 3 nodes including + itself; followers' cluster tables stay empty. + 2. **Horizon**: no replicated row older than the backfill horizon — + asserts the window trim against the deeper synthetic set. + 3. **Batch cap**: replication completes incrementally across multiple + cycles (observed row count strictly increases over ≥2 polls before + reaching the final value). + 4. **Exactness**: per-node replicated count equals the synthetic count + within the horizon (plus live samples) — proves no loss, no dupes. + 5. **Failover**: kill the leader; the new leader backfills rows predating + its leadership (synthetic rows present in its cluster table) — the + durability promise asserted directly. + 6. `node=` query API returns per-node series; `/api/tsdb/nodes` lists all + three with sane watermark ages. + 7. Diagnostic (non-assert): log `tsdb_metrics_cluster` size via + `page_count × page_size` for future default-sizing work. + +## Out of scope + +- Replicating hourly rollups / long-horizon cluster trends (follow-up). +- Dashboard redesign beyond a node selector. +- Distributed quotas and leader-only monitoring (later deliverables of the + roadmap). +- Default tuning — deliberately deferred until sizing data exists (§4). diff --git a/docs/superpowers/specs/2026-08-13-tsdb-sizing-lab-design.md b/docs/superpowers/specs/2026-08-13-tsdb-sizing-lab-design.md new file mode 100644 index 0000000000..023af3e54c --- /dev/null +++ b/docs/superpowers/specs/2026-08-13-tsdb-sizing-lab-design.md @@ -0,0 +1,269 @@ +# TSDB Sizing Lab — Seed Capture, Duplication Tool, CI — Design + +Date: 2026-08-13 +Status: approved for planning +Depends on: cluster leader election + TSDB cluster aggregation (PR #6034). +Branch: stacked on `feat/cluster-leader-election`; rebase onto `v3.0` after #6034 merges. + +## Motivation + +TSDB defaults were chosen without storage data. A measurement on an idle +single node (default 5s sampling) found **268 distinct series per tick** ≈ +**4.6M rows/day/node ≈ 450 MB/day/node** — a floor, since a loaded node with +backends, hostgroups and pools has more series. That measurement already +drove retention changes in #6034 (`tsdb-retention_days` 7→2, +`tsdb-cluster_retention_days` 3→1, hourly prune made configurable). + +What is still missing is an instrument: a repeatable way to produce a +**realistically shaped** TSDB at multi-day depth, so we can measure +bytes/row, growth rate, rollup behaviour and query latency — and re-measure +when the metric set changes. + +**Core requirement (explicit):** do not fabricate synthetic metrics. +Duplicate *real* ones. Real metric names and label sets are what determine +row cost, because SQLite stores both verbatim in every row (no interning); +synthetic 3-metric fixtures understate storage badly. + +## Decision: capture once, expand anywhere + +A one-off lab produces a small fixture of real metrics; a duplication tool +expands that fixture into a full-size database on demand. CI runs the +expansion, never the lab. Alternatives rejected: generating the seed inside +CI (needs sysbench + backends per run, slow, non-deterministic), and +measure-and-extrapolate (says nothing about rollup or query latency at +depth). + +**Fidelity: structural, not analytical.** Row shapes, names, label +cardinality and volume are real; counters restart at each duplicated block +seam (they are not offset to remain monotonic). This is sufficient for +sizing, replication load and query cost. Analytical fidelity (monotonic +counters across seams, for dashboard/rate realism) is a possible later +layer on the same insert path — out of scope here. + +## Design + +### 1. Seed capture (`test/tsdb-lab/capture.bash`) — run rarely, by a human + +- Stands up 3 ProxySQL instances forming a cluster, one backend registered + in two hostgroups, and sysbench at *variable* load (alternating rates and + think-times; load variety matters for series/value realism, stress does + not). +- TSDB enabled (`tsdb-enabled=1`, default 5s sampling), leader election + enabled so the cluster table is populated too. +- After ~10 minutes, dumps one node's `tsdb_metrics` rows for the window to + `test/tsdb-lab/fixtures/seed-10min.csv.gz` + (columns: `timestamp,metric_name,labels,value`). +- Expected size: ~400 series × 120 ticks ≈ 48k rows; gzip compresses the + repeated names/labels heavily — expected low hundreds of KB, small enough + to commit. **If the compressed fixture exceeds 2 MB, stop and reconsider** + (options: shorter window, or generate in CI) rather than committing a + large binary. +- The script stays in-repo so the fixture can be regenerated whenever the + metric set changes; the fixture header records the ProxySQL version and + capture date. + +### 2. Duplication tool (`test/tsdb-lab/expand.py`) + +Writes directly into a target `proxysql_stats.db` **with the instance +stopped** — no admin round-trips, no `PRAGMA query_only` gate, orders of +magnitude faster than SQL over the admin port. + +- `--raw-window ` (default `24h`): tiles the 10-minute block + forward (`timestamp + k*600`) until the window ending at "now" is filled. + At measured density this is ~4.6M rows. +- `--span ` (default `14d`): for the period older than the raw + window, writes **hourly buckets directly** into `tsdb_metrics_hour` + (bucket, metric_name, labels, avg/max/min/count aggregated from the + block). ~90k rows. Raw is deliberately not materialized there — retention + would prune it immediately, and the tiered shape is what a real node looks + like under the new defaults. +- `--nodes N` (default `0` = skip): writes the same expansion into + `tsdb_metrics_cluster` under N synthetic node identities + (`10.0.0.:6032`), so the leader-side footprint is measurable. +- `--db `: target stats DB (must exist with schema; the tool refuses + to create tables — it is a data tool, not a schema tool). +- Idempotence: inserts use `INSERT OR IGNORE` so a re-run over the same + target is safe (the PKs already dedupe). +- Header comment documents the counter-sawtooth caveat. + +### 3. CI workflow (`.github/workflows/CI-tsdb-sizing.yml`) + +Nightly / manual dispatch — **not** per-PR (it is a measurement job, not a +gate on every change). + +1. Build ProxySQL (debug or release, `PROXYSQL31=1`). +2. Create a datadir, start ProxySQL once to materialize the stats schema, + stop it. +3. Run `expand.py` with the CI profile (`--raw-window 24h --span 14d + --nodes 3`). +4. Start ProxySQL; wait for it to settle. +5. Measure and print a table: + - bytes per row (per table: `tsdb_metrics`, `tsdb_metrics_hour`, + `tsdb_metrics_cluster`), computed from row counts and page usage; + - the whole-file overhead ratio (page_count*page_size / summed tier + payload bytes); + - total DB size; + - rollup catch-up duration (time from start until `tsdb_metrics_hour` + stops growing — i.e. the first downsample pass, which holds `wrlock`); + - latency of two representative queries: raw last-1h for one metric, and + hourly 14d for one metric. +6. **Report always; fail only on two coarse guards**, each covering a + different failure mode, neither covering everything: + - **bytes/row** drifting more than 25% from `test/tsdb-lab/baseline.json` + (committed, updated deliberately). This is a near-pure function of the + committed fixture's text (`LENGTH(metric_name)+LENGTH(labels)+16`, + averaged) — it guards fixture/tooling consistency, not the product: a + product change that adds or lengthens labels does not move this number + until a human re-runs `capture.bash` and commits a refreshed fixture + (see `test/tsdb-lab/README.md`'s "Maintenance" section). + - **the whole-file overhead ratio** drifting more than 25% from the same + baseline file. This IS sensitive to schema/index bloat — a new column + or index inflates `page_count` while tier payload stays flat, moving + the ratio even though bytes/row does not (verified scale-invariant at + 2.17 across both the small and full-scale profiles, so a real drift in + it is a schema-shape signal, not a scale artifact). + - Total DB size and query latency are reported only, never gated (file + size is a derived total of the already-gated numbers plus non-tsdb + overhead; latency is hardware-dependent, see the non-goals below). + - Detecting product-side metric/label growth is **not** automatic under + either gate — it requires the maintenance step above. + +### 4. Tests for the tool itself + +`test/tsdb-lab/test_expand.py` (or a TAP test if the repo prefers): expand a +tiny inline fixture (2 series × 3 ticks) into a temp DB and assert: +exact raw row count for a given `--raw-window`; timestamp bounds inside the +window; no gap larger than the sample interval at block seams; hourly bucket +count and aggregate arithmetic (avg/max/min/count) for a known block; +`--nodes N` produces N× rows in the cluster table under distinct node ids; +re-running is a no-op (idempotence). + +### 5. Deliberate non-goals + +- Analytical fidelity (monotonic counters across seams). +- Running the lab in CI. +- Asserting query-latency thresholds (reported, not gated — hardware-dependent). +- Changing product defaults: this instrument *informs* default choices; any + change lands as its own reviewed commit. + +## Follow-ups this instrument enables + +- Cluster rollup tier (`tsdb_metrics_cluster_hour`) — needs a 14-day cluster + table to test against. +- Chunked downsample catch-up — the CI job's rollup-duration measurement is + the reproduction of the unbounded first pass under `wrlock`. +- Refining `tsdb-*` defaults with loaded-node series counts rather than the + idle-node floor. + +## Measured results (2026-08-13, ProxySQL 3.1.11-579-geaa0c9b) + +First full-scale local run of the CI-sized profile, against a release +(`PROXYSQL31=1`, no debug flags) build at commit `eaa0c9bdd`. Flow: start +once (`--initial`) to create the schema, stop, `expand.py --raw-window 24h +--span 14d --nodes 3` (the profile originally planned for +`CI-tsdb-sizing.yml`), start again, `measure.py`, then `SET tsdb-enabled='1'; +LOAD TSDB VARIABLES TO RUNTIME` while polling +`stats_history.tsdb_metrics_hour`'s row count every 1s until 5 consecutive +reads agreed. Datadir was scratch space under `/data` (341 GB free +beforehand), ports 16392/16393; the 5.9 GB scratch datadir was removed +afterward and `df -h /data` confirmed the space was released (345G → 351G +avail — the extra 6G includes some unrelated background churn on the shared +host, consistent with the ~5.9 GB DB). + +| Metric | Value | +|---|---| +| Expansion wall-clock (`expand.py`) | 113.1s (86.82s user + 26.12s sys, 99% CPU) | +| Raw rows (`tsdb_metrics`) | 7,144,960 over 24h (1 node) | +| Hourly rows (`tsdb_metrics_hour`) | 130,104 over the 13d preceding the raw window (10,008/day = 24×417 series) | +| Cluster rows (`tsdb_metrics_cluster`) | 21,434,880 across 3 nodes (7,144,960/node, same 24h window as raw) | +| Bytes/row — raw / hourly / cluster | 91.36 / 91.38 / 104.36 (0.0% drift vs `baseline.json`, i.e. confirmed scale-invariant between the 1h/1d and 24h/14d profiles) | +| Tiered payload (raw+hourly+cluster) | 2,767.29 MB | +| Whole-file DB size | 6,006.27 MB (page_count=1,537,605 × page_size=4096) ≈ 5.87 GB | +| File-size / tiered-payload overhead ratio | ≈2.17× (indexes + page/journal overhead) | +| Raw last-1h query | 15.3ms (rows=27,600) — was 13.5ms at the small profile | +| Hourly full-span query | 5.6ms (rows=12,480) — was 0.4ms at the small profile | +| Rollup catch-up: `tsdb_metrics_hour` growth | 130,104 → 139,695 rows (+9,591 = 23 complete hourly buckets × 417 series — the entire 24h raw backlog, caught up in one pass) | +| Rollup catch-up: single blocking pass | ≈38.6s (the poll issued immediately after enabling TSDB blocked for that long before returning, i.e. `wrlock` held that whole time) | +| Rollup catch-up: poll-detected total | 43.8s (1s poll interval, 5 consecutive stable reads required) | + +**(a) Do the new retention defaults (raw 2d local, cluster 1d, hourly 365d — +`lib/ProxySQL_Statistics.cpp`) hold up against measured per-day cost?** Yes, +comfortably, for the local (non-cluster) tiers: raw costs 622.55 MB/day/node +payload, so 2 days retained is ≈1.22 GB/node; hourly costs only ≈0.87 +MB/day (24 buckets/day vs 17,280 raw samples/day — a ~720× row-count +reduction), so even 365 days retained is ≈318 MB — the long hourly window is +essentially free. Projecting onto real on-disk bytes with the measured +≈2.17× payload-to-file-size overhead, a non-leader node's raw+hourly +footprint is ≈(1245+318)×2.17 ≈ 3.4 GB. That is a real but tractable +footprint, and a large improvement over the pre-#6034 defaults (7d raw / 3d +cluster), which this same math scales up ≈3.5× for the raw tier alone. (This +loaded-fixture 622.55 MB/day/node is itself ≈38% above the Motivation +section's idle-node floor of ≈450 MB/day/node — consistent with the loaded +fixture's higher series count, 417 vs the idle measurement's 268, which is +exactly the gap the "refine `tsdb-*` defaults with loaded-node series +counts" follow-up below anticipates closing further.) + +**(b) What does the cluster tier cost the leader per node?** The measured +3-node cluster tier is 2,133.4 MB payload for one day (the cluster window +tracked equals the raw window, 24h, which is exactly the default +`tsdb-cluster_retention_days=1`), i.e. 711.1 MB/day of payload *per tracked +node* (2133.4/3), retained on the leader only. That is the dominant single +cost on a leader: a leader tracking 3 peers carries ≈2.08 GB of cluster +payload (≈4.5 GB projected on-disk) on top of its own raw+hourly tiers, +versus a follower's ≈1.53 GB payload (≈3.3 GB projected on-disk). **The +leader's combined total — the actual capacity-planning number — is its own +raw+hourly footprint plus the cluster tier: ≈3.3-3.4 GB + ≈4.5 GB ≈ 7.8 GB, +roughly 2.3× a follower's footprint**, not the ≈4.5 GB cluster-only figure +in isolation. The cluster cost scales linearly with cluster size at ≈711 +MB/day/node retained — a 10-node cluster would put ≈7.1 GB of cluster-tier +payload alone on the leader at the 1-day default (on top of its own ≈3.3-3.4 +GB), which is worth remembering before growing cluster sizes past what's +been measured here. + +**(c) Does the rollup catch-up duration justify chunking (project 3)?** Yes. +A single, unbounded first-pass downsample — the exact scenario project 3 +targets — held `wrlock` for ≈38.6s while aggregating one node's full 24h raw +backlog (7.14M rows, 417 series, 23 hourly buckets) into 9,591 hourly rows, +on a 32-core dev machine with no contention. Every other read or write that +shares `wrlock` blocks for the same interval; in production this pass runs +after any restart or `tsdb-enabled` toggle that needs to catch up whatever +raw window has accumulated, and a loaded node has materially more series +than this fixture's idle-node-derived 417 (see Motivation), so the real +worst case is plausibly minutes, not tens of seconds. That is a real, +reproduced availability cost — this measurement is the concrete justification +for chunking the downsample pass rather than running it as one unbounded +pass. + +**CI workflow adjustment.** The originally planned CI profile +(`--raw-window 24h --span 14d --nodes 3`, measured above) produces a 6.0 GB +`proxysql_stats.db`, which combined with the repo's own build artifacts +(>1.4 GB) leaves too little headroom on a standard GitHub-hosted runner's +disk. `CI-tsdb-sizing.yml` was changed to `--raw-window 4h --span 7d --nodes +3` instead — since bytes/row is confirmed scale-invariant, this gives the +same sizing signal (still exercises multi-hour rollup catch-up and the +3-node cluster leader cost) at a projected ≈1 GB DB and ≈20-30s expand time +(linearly scaled from the measured 24h/14d/3-node rates: 7,144,960 raw +rows/24h/node, 10,008 hourly rows/day, and the same per-node rate for the +cluster tier). **The 4h/7d/3-node profile itself has not been run +end-to-end** — only the 24h/14d/3-node profile documented in the table +above was actually executed; the ≈1 GB / ≈20-30s figures are a linear +projection, justified by the confirmed bytes/row invariance but not +independently verified at this smaller size. The first nightly (or +`workflow_dispatch`) run of `CI-tsdb-sizing.yml` is that validation, and its +printed report should be checked against this projection. + +`timeout-minutes` was set to 150 (down from the previous unexamined 180). +The rollup-wait bound (10 minutes) and the lab steps' own wall-clock +(expand ≈20-30s projected, measure <5s, checkout/apt/artifact overhead ≈3 +min) are grounded in this session's measurements. The build-time term (120 +minutes) is not: it is taken by analogy from comparable from-scratch full +builds elsewhere in this repo (`CI-package-*-v31.yml`, build+package, same +order of magnitude of work minus packaging), not a measurement of this +job's actual build step. This session did run `PROXYSQL31=1 make clean && +PROXYSQL31=1 make -j32` locally (≈55s wall), but `make clean` does not clean +`deps/` (only `make cleanall` does) — `deps/` was already built from a +prior session, so that 55s measures only the lib+src recompile, not a +from-scratch build including the 25+ vendored dependencies, which is the +dominant, slow part of a real CI build. That number is therefore not usable +as a build-time measurement here and was not used; 120 minutes remains an +analogy-derived upper bound, not a measured one, pending a real timed run. diff --git a/include/ProxySQL_Admin_Tables_Definitions.h b/include/ProxySQL_Admin_Tables_Definitions.h index 4514853bd0..551b716aff 100644 --- a/include/ProxySQL_Admin_Tables_Definitions.h +++ b/include/ProxySQL_Admin_Tables_Definitions.h @@ -286,7 +286,7 @@ #define STATS_SQLITE_TABLE_PROXYSQL_SERVERS_CLIENTS_STATUS "CREATE TABLE stats_proxysql_servers_clients_status (uuid VARCHAR NOT NULL , hostname VARCHAR NOT NULL , port INT NOT NULL , admin_mysql_ifaces VARCHAR NOT NULL , last_seen_at INT NOT NULL , PRIMARY KEY (uuid, hostname, port) )" -#define STATS_SQLITE_TABLE_PROXYSQL_SERVERS_STATUS "CREATE TABLE stats_proxysql_servers_status (hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 6032 , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 0 , master VARCHAR NOT NULL , global_version INT NOT NULL , check_age_us INT NOT NULL , ping_time_us INT NOT NULL, checks_OK INT NOT NULL , checks_ERR INT NOT NULL , PRIMARY KEY (hostname, port) )" +#define STATS_SQLITE_TABLE_PROXYSQL_SERVERS_STATUS "CREATE TABLE stats_proxysql_servers_status (hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 6032 , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 0 , master VARCHAR NOT NULL , global_version INT NOT NULL , check_age_us INT NOT NULL , ping_time_us INT NOT NULL, checks_OK INT NOT NULL , checks_ERR INT NOT NULL , uuid VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostname, port) )" #define STATS_SQLITE_TABLE_PROXYSQL_SERVERS_METRICS "CREATE TABLE stats_proxysql_servers_metrics (hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 6032 , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , response_time_ms INT NOT NULL , Uptime_s INT NOT NULL , last_check_ms INT NOT NULL , Queries INT NOT NULL , Client_Connections_connected INT NOT NULL , Client_Connections_created INT NOT NULL , PRIMARY KEY (hostname, port) )" diff --git a/include/ProxySQL_Cluster.hpp b/include/ProxySQL_Cluster.hpp index eaccc4bbb6..22b9c34e5d 100644 --- a/include/ProxySQL_Cluster.hpp +++ b/include/ProxySQL_Cluster.hpp @@ -10,6 +10,8 @@ #include "prometheus/counter.h" #include "prometheus/gauge.h" +#include "ProxySQL_Cluster_Leader.h" + #define PROXYSQL_NODE_METRICS_LEN 5 /** @@ -266,6 +268,7 @@ class ProxySQL_Node_Address { }; class ProxySQL_Node_Entry { + friend class ProxySQL_Cluster_Nodes; private: uint64_t hash; char *hostname; @@ -273,6 +276,11 @@ class ProxySQL_Node_Entry { uint64_t weight; char *comment; char* ip_addr; + char *uuid; // learned via SELECT GLOBAL_UUID(); NULL until known + unsigned long long last_success_at_us; // monotonic_time() of last successful GLOBAL_CHECKSUM poll; 0 = never + uint64_t global_version; // number of observed global checksum changes on this peer + uint64_t checks_ok; + uint64_t checks_err; uint64_t generate_hash(); bool active; int metrics_idx_prev; @@ -307,6 +315,12 @@ class ProxySQL_Node_Entry { uint16_t get_port() { return port; } + const char * get_uuid() { return uuid; } + void set_uuid(const char* u); // strdup, frees previous + unsigned long long get_last_success_at_us() { return last_success_at_us; } + uint64_t get_global_version() { return global_version; } + uint64_t get_checks_ok() { return checks_ok; } + uint64_t get_checks_err() { return checks_err; } ProxySQL_Node_Metrics * get_metrics_curr(); ProxySQL_Node_Metrics * get_metrics_prev(); struct { @@ -359,6 +373,7 @@ struct p_cluster_nodes_dyn_gauge { proxysql_servers_metrics_response_time_ms, proxysql_servers_metrics_last_check_ms, proxysql_servers_metrics_client_conns_connected, + proxysql_servers_alive, SIZE_ }; }; @@ -399,6 +414,7 @@ class ProxySQL_Cluster_Nodes { std::map p_proxysql_servers_metrics_response_time_ms {}; std::map p_proxysql_servers_metrics_last_check_ms {}; std::map p_proxysql_servers_metrics_client_conns_connected {}; + std::map p_proxysql_servers_alive {}; } metrics; public: ProxySQL_Cluster_Nodes(); @@ -407,11 +423,14 @@ class ProxySQL_Cluster_Nodes { bool Update_Node_Metrics(char * _h, uint16_t _p, MYSQL_RES *_r, unsigned long long _response_time); bool Update_Global_Checksum(char * _h, uint16_t _p, MYSQL_RES *_r); bool Update_Node_Checksums(char * _h, uint16_t _p, MYSQL_RES *_r); + void Update_Node_UUID(char * _hostname, uint16_t _port, const char * _uuid); + void Update_Node_Failure(char * _hostname, uint16_t _port); void Reset_Global_Checksums(bool lock); void update_prometheus_nodes_metrics(); SQLite3_result * dump_table_proxysql_servers(); SQLite3_result * stats_proxysql_servers_checksums(); SQLite3_result * stats_proxysql_servers_metrics(); + SQLite3_result * stats_proxysql_servers_status(const std::string& leader_uuid, unsigned long long alive_timeout_us); void get_peer_to_sync_mysql_query_rules(char **host, uint16_t *port, char** ip_address); void get_peer_to_sync_runtime_mysql_servers(char **host, uint16_t *port, char **peer_checksum, char** ip_address); void get_peer_to_sync_mysql_servers_v2(char** host, uint16_t* port, char** peer_mysql_servers_v2_checksum, @@ -428,6 +447,7 @@ class ProxySQL_Cluster_Nodes { void get_peer_to_sync_pgsql_servers_v2(char** host, uint16_t* port, char** peer_pgsql_servers_v2_checksum, char** peer_runtime_pgsql_servers_checksum, char** ip_address); void get_peer_to_sync_pgsql_users(char **host, uint16_t *port, char** ip_address); + std::vector get_leader_candidates(unsigned long long alive_timeout_us); }; struct p_cluster_counter { @@ -505,12 +525,15 @@ struct p_cluster_counter { sync_delayed_pgsql_users_version_one, sync_delayed_pgsql_variables_version_one, + cluster_leader_changes, + SIZE_ }; }; struct p_cluster_gauge { enum metric : uint8_t { + cluster_leader_status, SIZE_ }; }; @@ -605,6 +628,17 @@ class ProxySQL_Cluster { char* admin_mysql_ifaces; int cluster_check_interval_ms; + int cluster_leader_election; // 0/1, __sync access + int cluster_leader_node_timeout_ms; + int cluster_leader_grace_ms; + pthread_mutex_t leader_mutex; // guards leader_state + leader_hostname/leader_port + Cluster_Leader_State leader_state; + char * leader_hostname; // NULL = no leader + int leader_port; + unsigned long long leader_next_check_at; // monotonic us, 0 initially + void leader_election_tick(unsigned long long curtime_us); + bool is_leader(); + void get_leader_info(std::string& hostname, int& port, std::string& uuid); int cluster_check_status_frequency; std::atomic cluster_mysql_query_rules_diffs_before_sync; std::atomic cluster_mysql_servers_diffs_before_sync; @@ -670,6 +704,13 @@ class ProxySQL_Cluster { SQLite3_result* get_stats_proxysql_servers_metrics() { return nodes.stats_proxysql_servers_metrics(); } + SQLite3_result* get_stats_proxysql_servers_status(); + void Update_Node_UUID(char* h, uint16_t p, const char* u) { + nodes.Update_Node_UUID(h, p, u); + } + void Update_Node_Failure(char* h, uint16_t p) { + nodes.Update_Node_Failure(h, p); + } void p_update_metrics(); void thread_ending(pthread_t); void join_term_thread(); diff --git a/include/ProxySQL_Cluster_Leader.h b/include/ProxySQL_Cluster_Leader.h new file mode 100644 index 0000000000..97f9aa0934 --- /dev/null +++ b/include/ProxySQL_Cluster_Leader.h @@ -0,0 +1,33 @@ +#ifndef __CLASS_PROXYSQL_CLUSTER_LEADER_H +#define __CLASS_PROXYSQL_CLUSTER_LEADER_H + +#include +#include +#include + +struct Cluster_Leader_Candidate { + std::string uuid; // empty = unknown (not electable) + std::string hostname; + uint16_t port = 0; + uint64_t weight = 0; + bool alive = false; +}; + +// Deterministic, ballot-free election over a locally-observed candidate set. +// Electable = alive && uuid non-empty. Highest weight wins; ties broken by +// lexicographically smallest uuid. Returns index into candidates, or -1. +int cluster_elect_leader(const std::vector& candidates); + +// Grace-window state machine: a computed leader (or leader loss, "") must be +// observed continuously for grace_ms before it becomes effective. +class Cluster_Leader_State { + public: + std::string current_leader_uuid; // empty = no leader + std::string pending_leader_uuid; + unsigned long long pending_since_ms = 0; + // Returns true when the effective leader changed. + bool update(const std::string& computed_uuid, unsigned long long now_ms, unsigned long long grace_ms); + void reset(); +}; + +#endif // __CLASS_PROXYSQL_CLUSTER_LEADER_H diff --git a/include/ProxySQL_Statistics.hpp b/include/ProxySQL_Statistics.hpp index ddd9e61821..4596315e50 100644 --- a/include/ProxySQL_Statistics.hpp +++ b/include/ProxySQL_Statistics.hpp @@ -8,6 +8,7 @@ #include #include #include +#include #define STATSDB_SQLITE_TABLE_MYSQL_CONNECTIONS_V1_4 "CREATE TABLE mysql_connections (timestamp INT NOT NULL, Client_Connections_aborted INT NOT NULL, Client_Connections_connected INT NOT NULL, Client_Connections_created INT NOT NULL, Server_Connections_aborted INT NOT NULL, Server_Connections_connected INT NOT NULL, Server_Connections_created INT NOT NULL, ConnPool_get_conn_failure INT NOT NULL, ConnPool_get_conn_immediate INT NOT NULL, ConnPool_get_conn_success INT NOT NULL, Questions INT NOT NULL, Slow_queries INT NOT NULL, PRIMARY KEY (timestamp))" @@ -110,6 +111,10 @@ // Backend health monitoring table #define STATSDB_SQLITE_TABLE_TSDB_BACKEND_HEALTH \ "CREATE TABLE tsdb_backend_health (timestamp INT NOT NULL, hostgroup INT NOT NULL, hostname TEXT NOT NULL, port INT NOT NULL, probe_up INT NOT NULL, connect_ms INT, PRIMARY KEY (timestamp, hostgroup, hostname, port)) WITHOUT ROWID" + +// Cluster-aggregated metrics table (leader-collected, per-node) +#define STATSDB_SQLITE_TABLE_TSDB_METRICS_CLUSTER \ +"CREATE TABLE tsdb_metrics_cluster (node VARCHAR NOT NULL , timestamp INTEGER NOT NULL , metric_name VARCHAR NOT NULL , labels VARCHAR NOT NULL DEFAULT '{}' , value REAL , PRIMARY KEY (node, timestamp, metric_name, labels)) WITHOUT ROWID" #endif class ProxySQL_Statistics { @@ -136,6 +141,29 @@ class ProxySQL_Statistics { unsigned long long next_timer_tsdb_monitor; unsigned long long next_timer_tsdb_retention; sqlite3_stmt *stmt_insert_tsdb_metric; + // Cluster aggregation (leader pulls peers' TSDB via pull+watermark) + unsigned long long next_timer_tsdb_cluster_check = 0; + pthread_t tsdb_agg_thread; + bool tsdb_agg_thread_started = false; // only touched by the admin main loop thread + std::atomic tsdb_agg_stop { false }; + // Set to true by the worker thread as its VERY LAST action before + // returning. tsdb_cluster_aggregation_check() must not pthread_join() + // the worker until this is observed true -- the worker can be blocked in + // peer I/O for up to ~11s (1s connect + 10s read/write timeouts), and a + // blocking join on the admin main loop thread would stall + // leader_election_tick(), called right after this check. + std::atomic tsdb_agg_thread_done { false }; + sqlite3_stmt *stmt_insert_tsdb_cluster_metric = NULL; + // Worker-thread-only bookkeeping (no locking needed: only the aggregation thread touches these) + std::map tsdb_agg_peer_last_wm; // last watermark observed per peer + std::map tsdb_agg_peer_stall_count; // consecutive unchanged cycles per peer + std::map tsdb_agg_peer_stall_logged; // once-flag: already logged the stall + std::map tsdb_agg_peer_cap_hit_count; // consecutive cap-hit cycles per peer + std::map tsdb_agg_peer_progress_stall_count; // consecutive no-progress-within-batch cycles per peer + void tsdb_cluster_aggregation_cycle(); + long tsdb_cluster_node_max_ts(const std::string& node); + void tsdb_cluster_replicate_self(const std::string& node, long watermark, int limit); + bool tsdb_cluster_replicate_peer(const std::string& host, int port, const std::string& node, long watermark, int limit, const std::string& user, const std::string& pass, long persisted_max_ts); #endif sqlite3_stmt *stmt_insert_backend_health; void MySQL_Threads_Handler_sets_v1(SQLite3_result *); @@ -162,6 +190,12 @@ class ProxySQL_Statistics { int tsdb_retention_days; int tsdb_monitor_enabled; int tsdb_monitor_interval; + int tsdb_cluster_aggregation; + int tsdb_cluster_interval; + int tsdb_cluster_backfill_hours; + int tsdb_cluster_retention_days; + int tsdb_cluster_batch_rows; + int tsdb_hourly_retention_days; #endif } variables; ProxySQL_Statistics(); @@ -239,7 +273,8 @@ class ProxySQL_Statistics { const std::map& label_filters, time_t from, time_t to, - const std::string& aggregation = ""); + const std::string& aggregation = "", + const std::string& node = ""); // Backend health queries SQLite3_result* get_backend_health_metrics(time_t from, time_t to, int hostgroup = -1); // Status @@ -259,6 +294,14 @@ class ProxySQL_Statistics { // Main loops void tsdb_sampler_loop(); void tsdb_monitor_loop(); + // Cluster aggregation (leader-only worker thread) + std::atomic tsdb_agg_active { false }; // thread running (read by REST) + std::atomic tsdb_agg_rows_total { 0 }; // rows replicated since start + std::atomic tsdb_agg_last_cycle_ts { 0 }; // unix ts of last completed cycle + std::atomic tsdb_agg_cap_hit_last_cycle { false }; + void tsdb_cluster_aggregation_check(unsigned long long curtime); + void tsdb_cluster_aggregation_thread_loop(); // thread body (public for the C trampoline) + SQLite3_result * get_tsdb_cluster_nodes(); // implemented in Task 4 #endif /** diff --git a/include/TSDB_Cluster_Aggregator.h b/include/TSDB_Cluster_Aggregator.h new file mode 100644 index 0000000000..00991205da --- /dev/null +++ b/include/TSDB_Cluster_Aggregator.h @@ -0,0 +1,23 @@ +#ifndef __CLASS_TSDB_CLUSTER_AGGREGATOR_H +#define __CLASS_TSDB_CLUSTER_AGGREGATOR_H + +// Pure planning logic for TSDB cluster aggregation (leader pulls peers' +// tsdb_metrics with a per-node watermark). Kept dependency-free so it is +// unit-testable in every tier. + +struct Tsdb_Agg_Fetch_Result { + long new_watermark = 0; + bool caught_up = false; +}; + +// Start-point for replicating a node: the max timestamp already replicated, +// clamped forward to the backfill horizon (now - backfill_hours). +// existing_max_ts <= 0 means "nothing replicated yet". +long tsdb_agg_effective_watermark(long existing_max_ts, long now, int backfill_hours); + +// Bookkeeping after one fetch of up to `limit` rows ordered by timestamp, +// where `last_row_ts` is the max timestamp among the fetched rows +// (ignored when rows_fetched == 0). +Tsdb_Agg_Fetch_Result tsdb_agg_apply_fetch(long prev_watermark, int rows_fetched, long last_row_ts, int limit); + +#endif // __CLASS_TSDB_CLUSTER_AGGREGATOR_H diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h index 9d7c68ea69..66851234f5 100644 --- a/include/proxysql_admin.h +++ b/include/proxysql_admin.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include "ProxySQL_RESTAPI_Server.hpp" @@ -34,6 +35,12 @@ enum SERVER_TYPE { SERVER_TYPE_PGSQL }; +enum admin_ro_mode_t { + ADMIN_RO_MODE_AUTO = 0, // read-only iff this node is a cluster follower (leader election) + ADMIN_RO_MODE_FORCED_RO = 1, // operator-forced read-only (PROXYSQL READONLY) + ADMIN_RO_MODE_FORCED_RW = 2, // operator-forced read-write (PROXYSQL READWRITE) +}; + class Scheduler_Row { public: unsigned int id; @@ -358,6 +365,9 @@ class ProxySQL_Admin { prometheus::SerialExposer serial_exposer; + std::atomic ro_mode { ADMIN_RO_MODE_AUTO }; + std::atomic cluster_follower { false }; + std::mutex proxysql_servers_mutex; void wrlock(); @@ -378,6 +388,9 @@ class ProxySQL_Admin { char * cluster_username; char * cluster_password; int cluster_check_interval_ms; + bool cluster_leader_election; + int cluster_leader_node_timeout_ms; + int cluster_leader_grace_ms; int cluster_check_status_frequency; int cluster_mysql_query_rules_diffs_before_sync; int cluster_mysql_servers_diffs_before_sync; @@ -649,8 +662,15 @@ class ProxySQL_Admin { * @details Modules ready when 'all_modules_started=true'. See 'all_modules_started'. */ void load_restapi_server(); - bool get_read_only() { return variables.admin_read_only; } - bool set_read_only(bool ro) { variables.admin_read_only=ro; return variables.admin_read_only; } + bool effective_read_only() { + int m = ro_mode.load(std::memory_order_relaxed); + if (m == ADMIN_RO_MODE_FORCED_RO) return true; + if (m == ADMIN_RO_MODE_FORCED_RW) return false; + return cluster_follower.load(std::memory_order_relaxed); + } + void set_ro_mode(admin_ro_mode_t m) { ro_mode.store((int)m, std::memory_order_relaxed); } + admin_ro_mode_t get_ro_mode() { return (admin_ro_mode_t)ro_mode.load(std::memory_order_relaxed); } + void set_cluster_follower(bool f) { cluster_follower.store(f, std::memory_order_relaxed); } bool has_variable(const char *name); void init_users(std::unique_ptr&& mysql_users_resultset = nullptr, const std::string& checksum = "", const time_t epoch = 0); void init_mysql_servers(); @@ -814,6 +834,7 @@ class ProxySQL_Admin { void stats___proxysql_servers_checksums(); void stats___proxysql_servers_metrics(); + void stats___proxysql_servers_status(); void stats___proxysql_message_metrics(bool reset); void stats___mysql_prepared_statements_info(); void stats___mysql_gtid_executed(); diff --git a/lib/Admin_Handler.cpp b/lib/Admin_Handler.cpp index 16f2e602a1..e2d0d06180 100644 --- a/lib/Admin_Handler.cpp +++ b/lib/Admin_Handler.cpp @@ -757,17 +757,25 @@ bool admin_handler_command_proxysql(char *query_no_space, unsigned int query_no_ } if (query_no_space_length==sizeof("PROXYSQL READONLY") - 1 && !strncasecmp("PROXYSQL READONLY",query_no_space, query_no_space_length)) { // this command enables admin_read_only , so the admin module is in read_only mode - proxy_info("Received PROXYSQL READONLY command\n"); + proxy_info("Received PROXYSQL READONLY command: forcing read-only mode (FORCED_RO)\n"); ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->set_read_only(true); + SPA->set_ro_mode(ADMIN_RO_MODE_FORCED_RO); SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); return false; } if (query_no_space_length==sizeof("PROXYSQL READWRITE") - 1 && !strncasecmp("PROXYSQL READWRITE",query_no_space, query_no_space_length)) { // this command disables admin_read_only , so the admin module won't be in read_only mode - proxy_info("Received PROXYSQL WRITE command\n"); + proxy_info("Received PROXYSQL WRITE command: forcing read-write mode (FORCED_RW)\n"); ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->set_read_only(false); + SPA->set_ro_mode(ADMIN_RO_MODE_FORCED_RW); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + if (query_no_space_length==strlen("PROXYSQL READONLY AUTO") && !strncasecmp("PROXYSQL READONLY AUTO",query_no_space, query_no_space_length)) { + // returns read-only control to the cluster leader election (AUTO mode) + proxy_info("Received PROXYSQL READONLY AUTO command\n"); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + SPA->set_ro_mode(ADMIN_RO_MODE_AUTO); SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); return false; } @@ -1528,6 +1536,68 @@ template bool admin_handler_command_load_or_save(char *query_no_space, unsigned int query_no_space_length, S* sess, ProxySQL_Admin *pa, char **q, unsigned int *ql) { proxy_debug(PROXY_DEBUG_ADMIN, 5, "Received command %s\n", query_no_space); + { + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + if (SPA->effective_read_only()) { + bool is_load = (!strncasecmp("LOAD ", query_no_space, 5)); + bool is_save = (!strncasecmp("SAVE ", query_no_space, 5)); + bool refuse = false; + if (query_no_space_length > 11 && !strncasecmp(" TO RUNTIME", query_no_space+query_no_space_length-11, 11)) { + refuse = true; // LOAD ... TO RUNTIME + } + if (query_no_space_length > 7 && !strncasecmp(" TO RUN", query_no_space+query_no_space_length-7, 7)) { + refuse = true; // abbreviation: LOAD ... TO RUN == LOAD ... TO RUNTIME + } + if (query_no_space_length > 8 && is_save && !strncasecmp(" TO DISK", query_no_space+query_no_space_length-8, 8)) { + refuse = true; // SAVE ... TO DISK + } + if (query_no_space_length > 12 && (is_load || is_save) && !strncasecmp(" FROM MEMORY", query_no_space+query_no_space_length-12, 12)) { + refuse = true; // aliases: LOAD x FROM MEMORY == LOAD x TO RUNTIME ; SAVE x FROM MEMORY == SAVE x TO DISK + } + if (query_no_space_length > 9 && (is_load || is_save) && !strncasecmp(" FROM MEM", query_no_space+query_no_space_length-9, 9)) { + refuse = true; // abbreviation: LOAD x FROM MEM == LOAD x FROM MEMORY ; SAVE x FROM MEM == SAVE x FROM MEMORY + } + // Direct SQL writes to the memory tier are blocked by PRAGMA + // query_only, but the forms below mutate the memory tier via C++ + // flush functions that bypass it entirely, so they must be + // refused here too. + if (query_no_space_length > 10 && is_load && !strncasecmp(" FROM DISK", query_no_space+query_no_space_length-10, 10)) { + refuse = true; // LOAD x FROM DISK (disk -> memory) + } + if (query_no_space_length > 12 && is_load && !strncasecmp(" FROM CONFIG", query_no_space+query_no_space_length-12, 12)) { + refuse = true; // LOAD x FROM CONFIG (config file -> memory) + } + if (query_no_space_length > 10 && (is_load || is_save) && !strncasecmp(" TO MEMORY", query_no_space+query_no_space_length-10, 10)) { + refuse = true; // LOAD x TO MEMORY (disk -> memory) ; SAVE x TO MEMORY (runtime -> memory) + } + if (query_no_space_length > 7 && (is_load || is_save) && !strncasecmp(" TO MEM", query_no_space+query_no_space_length-7, 7)) { + refuse = true; // abbreviation: x TO MEM == x TO MEMORY + } + if (query_no_space_length > 13 && is_save && !strncasecmp(" FROM RUNTIME", query_no_space+query_no_space_length-13, 13)) { + refuse = true; // SAVE x FROM RUNTIME (runtime -> memory) + } + if (query_no_space_length > 9 && is_save && !strncasecmp(" FROM RUN", query_no_space+query_no_space_length-9, 9)) { + refuse = true; // abbreviation: SAVE x FROM RUN == SAVE x FROM RUNTIME + } + if (refuse) { + std::string l_host; int l_port = 0; std::string l_uuid; + GloProxyCluster->get_leader_info(l_host, l_port, l_uuid); + char msg[512]; + if (l_host.length()) { + snprintf(msg, sizeof(msg), + "Admin is in read-only mode (cluster follower). Current leader is %s:%d (%s). Use PROXYSQL READWRITE to override.", + l_host.c_str(), l_port, l_uuid.c_str()); + } else { + snprintf(msg, sizeof(msg), + "Admin is in read-only mode. Use PROXYSQL READWRITE to override."); + } + proxy_warning("Refused '%s' : %s\n", query_no_space, msg); + SPA->send_error_msg_to_client(sess, msg); + return false; + } + } + } + #ifdef DEBUG if ((query_no_space_length>11) && ( (!strncasecmp("SAVE DEBUG ", query_no_space, 11)) || (!strncasecmp("LOAD DEBUG ", query_no_space, 11))) ) { if ( @@ -3816,7 +3886,7 @@ void admin_session_handler(S* sess, void *_pa, PtrSize_t *pkt) { query_length=strlen(q)+5; query=(char *)l_alloc(query_length); ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - bool ro=SPA->get_read_only(); + bool ro=SPA->effective_read_only(); //sprintf(query,q,( ro ? "ON" : "OFF")); PtrSize_t pkt_2; if (ro) { @@ -3841,7 +3911,7 @@ void admin_session_handler(S* sess, void *_pa, PtrSize_t *pkt) { query_length=strlen(q)+5; query=(char *)l_alloc(query_length); ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - bool ro=SPA->get_read_only(); + bool ro=SPA->effective_read_only(); //sprintf(query,q,( ro ? "ON" : "OFF")); PtrSize_t pkt_2; if (ro) { @@ -3916,6 +3986,31 @@ void admin_session_handler(S* sess, void *_pa, PtrSize_t *pkt) { goto __run_query; } + if ((query_no_space_length == strlen("SELECT GLOBAL_UUID()")) && (!strncasecmp("SELECT GLOBAL_UUID()", query_no_space, strlen("SELECT GLOBAL_UUID()")))) { + const char *uuid_val = (GloVars.uuid ? GloVars.uuid : ""); + uint16_t setStatus = 0; + auto *myds=sess->client_myds; + auto *myprot=&sess->client_myds->myprot; + myds->DSS=STATE_QUERY_SENT_DS; + int sid=1; + myprot->generate_pkt_column_count(true,NULL,NULL,sid,1); sid++; + myprot->generate_pkt_field(true,NULL,NULL,sid,(char *)"",(char *)"",(char *)"",(char *)"UUID",(char *)"",33,36,MYSQL_TYPE_VAR_STRING,0,0,false,0,NULL); sid++; + myds->DSS=STATE_COLUMN_DEFINITION; + myprot->generate_pkt_EOF(true,NULL,NULL,sid,0, setStatus); sid++; + char **p=(char **)malloc(sizeof(char*)*1); + unsigned long *l=(unsigned long *)malloc(sizeof(unsigned long)*1); + l[0]=strnlen(uuid_val, 64); + p[0]=(char *)uuid_val; + myprot->generate_pkt_row(true,NULL,NULL,sid,1,l,p); sid++; + myds->DSS=STATE_ROW; + myprot->generate_pkt_EOF(true,NULL,NULL,sid,0, setStatus); sid++; + myds->DSS=STATE_SLEEP; + run_query=false; + free(l); + free(p); + goto __run_query; + } + if ((query_no_space_length>8) && (!strncasecmp("PROXYSQL ", query_no_space, 8))) { proxy_debug(PROXY_DEBUG_ADMIN, 4, "Received PROXYSQL command\n"); @@ -5502,7 +5597,7 @@ void admin_session_handler(S* sess, void *_pa, PtrSize_t *pkt) { if (run_query) { ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; if (sess->session_type == PROXYSQL_SESSION_ADMIN) { // no stats - if (SPA->get_read_only()) { // disable writes if the admin interface is in read_only mode + if (SPA->effective_read_only()) { // disable writes if the admin interface is in read_only mode SPA->admindb->execute("PRAGMA query_only = ON"); SPA->admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset); SPA->admindb->execute("PRAGMA query_only = OFF"); diff --git a/lib/Makefile b/lib/Makefile index 63bcc297c4..30b3ee8180 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -93,7 +93,52 @@ MYCXXFLAGS := $(STDCPP) $(MYCFLAGS) $(PSQLCH) $(PSQL40) $(PSQL31) $(PSQLFFTO) $( default: libproxysql.a .PHONY: default -_OBJ_CXX := ProxySQL_GloVars.oo network.oo debug.oo configfile.oo Query_Cache.oo SpookyV2.oo MySQL_Authentication.oo MySQL_Passthrough_Auth_Cache.oo gen_utils.oo sqlite3db.oo mysql_connection.oo MySQL_HostGroups_Manager.oo mysql_data_stream.oo MySQL_Thread.oo MySQL_Session.oo MySQL_Protocol.oo mysql_backend.oo Query_Processor.oo MySQL_Query_Processor.oo PgSQL_Query_Processor.oo ProxySQL_Admin.oo ProxySQL_Config.oo ProxySQL_Restapi.oo MySQL_Monitor.oo MySQL_Logger.oo log_utils.oo thread.oo MySQL_PreparedStatement.oo ProxySQL_Cluster.oo ClickHouse_Authentication.oo ClickHouse_Server.oo ProxySQL_Statistics.oo Chart_bundle_js.oo ProxySQL_HTTP_Server.oo ProxySQL_RESTAPI_Server.oo font-awesome.min.css.oo main-bundle.min.css.oo MySQL_Variables.oo MySQL_User_Variables.oo c_tokenizer.oo proxysql_utils.oo proxysql_coredump.oo proxysql_sslkeylog.oo \ +_OBJ_CXX := \ + ProxySQL_GloVars.oo \ + network.oo \ + debug.oo \ + configfile.oo \ + Query_Cache.oo \ + SpookyV2.oo \ + MySQL_Authentication.oo \ + MySQL_Passthrough_Auth_Cache.oo \ + gen_utils.oo \ + sqlite3db.oo \ + mysql_connection.oo \ + MySQL_HostGroups_Manager.oo \ + mysql_data_stream.oo \ + MySQL_Thread.oo \ + MySQL_Session.oo \ + MySQL_Protocol.oo \ + mysql_backend.oo \ + Query_Processor.oo \ + MySQL_Query_Processor.oo \ + PgSQL_Query_Processor.oo \ + ProxySQL_Admin.oo \ + ProxySQL_Config.oo \ + ProxySQL_Restapi.oo \ + MySQL_Monitor.oo \ + MySQL_Logger.oo \ + log_utils.oo \ + thread.oo \ + MySQL_PreparedStatement.oo \ + ProxySQL_Cluster.oo \ + ClickHouse_Authentication.oo \ + ClickHouse_Server.oo \ + ProxySQL_Statistics.oo \ + Chart_bundle_js.oo \ + ProxySQL_HTTP_Server.oo \ + ProxySQL_RESTAPI_Server.oo \ + font-awesome.min.css.oo \ + main-bundle.min.css.oo \ + MySQL_Variables.oo \ + MySQL_User_Variables.oo \ + c_tokenizer.oo \ + proxysql_utils.oo \ + proxysql_coredump.oo \ + proxysql_sslkeylog.oo \ + ProxySQL_Cluster_Leader.oo \ + TSDB_Cluster_Aggregator.oo \ sha256crypt.oo \ ProxySQL_PluginManager.oo \ BaseSrvList.oo BaseHGC.oo Base_HostGroups_Manager.oo \ diff --git a/lib/MySQL_Logger.cpp b/lib/MySQL_Logger.cpp index 6698ab2032..21808277d0 100644 --- a/lib/MySQL_Logger.cpp +++ b/lib/MySQL_Logger.cpp @@ -2175,6 +2175,11 @@ void MySQL_Logger::insertMysqlEventsIntoDb(SQLite3DB * db, const std::string& ta char digest_hex_str[20]; // 2+sizeof(unsigned long long)*2+2 + // Serializes concurrent users of this SQLite3DB instance (e.g., admin main loop's + // periodic eventslog flush vs DUMP EVENTSLOG admin command). TSDB writer threads + // use a separate SQLite3DB connection to the same file, serialized at the SQLite + // file level. + db->wrlock(); db->execute("BEGIN"); int row_idx=0; @@ -2240,6 +2245,7 @@ void MySQL_Logger::insertMysqlEventsIntoDb(SQLite3DB * db, const std::string& ta row_idx++; } db->execute("COMMIT"); + db->wrunlock(); } diff --git a/lib/PgSQL_Logger.cpp b/lib/PgSQL_Logger.cpp index 7881c20737..0b9b5fa611 100644 --- a/lib/PgSQL_Logger.cpp +++ b/lib/PgSQL_Logger.cpp @@ -1558,6 +1558,11 @@ void PgSQL_Logger::insertPgSQLEventsIntoDb(SQLite3DB* db, const std::string& tab }; char digest_hex_str[20]; + // Serializes concurrent users of this SQLite3DB instance (e.g., admin main loop's + // periodic eventslog flush vs DUMP EVENTSLOG admin command). TSDB writer threads + // use a separate SQLite3DB connection to the same file, serialized at the SQLite + // file level. + db->wrlock(); db->execute("BEGIN"); int row_idx = 0; @@ -1603,6 +1608,7 @@ void PgSQL_Logger::insertPgSQLEventsIntoDb(SQLite3DB* db, const std::string& tab } db->execute("COMMIT"); + db->wrunlock(); } int PgSQL_Logger::processEvents(SQLite3DB* statsdb, SQLite3DB* statsdb_disk) { diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 48e127abcf..e2847c829c 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -436,6 +436,11 @@ static char * admin_variables_names[]= { (char *)"cluster_username", (char *)"cluster_password", (char *)"cluster_check_interval_ms", +#ifdef PROXYSQL31 + (char *)"cluster_leader_election", +#endif /* PROXYSQL31 */ + (char *)"cluster_leader_node_timeout_ms", + (char *)"cluster_leader_grace_ms", (char *)"cluster_check_status_frequency", (char *)"cluster_mysql_query_rules_diffs_before_sync", (char *)"cluster_mysql_servers_diffs_before_sync", @@ -1368,7 +1373,7 @@ bool ProxySQL_Admin::GenericRefreshStatistics(const char *query_no_space, unsign bool stats_proxysql_message_metrics = false; bool stats_proxysql_message_metrics_reset = false; - //bool stats_proxysql_servers_status = false; // temporary disabled because not implemented + bool stats_proxysql_servers_status = false; if (strcasestr(query_no_space, "pgsql processlist") || strcasestr(query_no_space, "pgsql activity") || @@ -1515,11 +1520,8 @@ bool ProxySQL_Admin::GenericRefreshStatistics(const char *query_no_space, unsign if (strstr(query_no_space,"stats_proxysql_message_metrics_reset")) { stats_proxysql_message_metrics_reset=true; refresh=true; } - // temporary disabled because not implemented -/* if (strstr(query_no_space,"stats_proxysql_servers_status")) { stats_proxysql_servers_status = true; refresh = true; } -*/ if (strstr(query_no_space,"stats_mysql_prepared_statements_info")) { stats_mysql_prepared_statements_info=true; refresh=true; } @@ -1747,10 +1749,9 @@ bool ProxySQL_Admin::GenericRefreshStatistics(const char *query_no_space, unsign } } - // temporary disabled because not implemented -// if (stats_proxysql_servers_status) { -// stats___proxysql_servers_status(); -// } + if (stats_proxysql_servers_status) { + stats___proxysql_servers_status(); + } if (stats_mysql_prepared_statements_info) { stats___mysql_prepared_statements_info(); } @@ -2657,7 +2658,11 @@ void * admin_main_loop(void *arg) { if (GloProxyStats->tsdb_retention_timetoget(curtime)) { GloProxyStats->tsdb_retention_cleanup(); } + GloProxyStats->tsdb_cluster_aggregation_check(curtime); #endif + if (GloProxyCluster) { + GloProxyCluster->leader_election_tick(curtime); + } } if (S_amll.get_version()!=version) { S_amll.wrlock(); @@ -2926,6 +2931,9 @@ ProxySQL_Admin::ProxySQL_Admin() : variables.cluster_username=strdup((char *)""); variables.cluster_password=strdup((char *)""); variables.cluster_check_interval_ms=1000; + variables.cluster_leader_election=false; + variables.cluster_leader_node_timeout_ms=3000; + variables.cluster_leader_grace_ms=3000; variables.cluster_check_status_frequency=10; variables.cluster_mysql_query_rules_diffs_before_sync = 3; variables.cluster_mysql_servers_diffs_before_sync = 3; @@ -3765,6 +3773,19 @@ char * ProxySQL_Admin::get_variable(char *name) { snprintf(intbuf, sizeof(intbuf),"%d",variables.cluster_check_interval_ms); return strdup(intbuf); } +#ifdef PROXYSQL31 + if (!strcasecmp(name,"cluster_leader_election")) { + return strdup((variables.cluster_leader_election ? "true" : "false")); + } +#endif /* PROXYSQL31 */ + if (!strcasecmp(name,"cluster_leader_node_timeout_ms")) { + snprintf(intbuf, sizeof(intbuf),"%d",variables.cluster_leader_node_timeout_ms); + return strdup(intbuf); + } + if (!strcasecmp(name,"cluster_leader_grace_ms")) { + snprintf(intbuf, sizeof(intbuf),"%d",variables.cluster_leader_grace_ms); + return strdup(intbuf); + } if (!strcasecmp(name,"cluster_check_status_frequency")) { snprintf(intbuf, sizeof(intbuf),"%d",variables.cluster_check_status_frequency); return strdup(intbuf); @@ -4276,6 +4297,56 @@ bool ProxySQL_Admin::set_variable(char *name, char *value, bool lock) { // this return false; } } +#ifdef PROXYSQL31 + if (!strcasecmp(name,"cluster_leader_election")) { + bool old_v = variables.cluster_leader_election; + if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { + variables.cluster_leader_election=true; + __sync_lock_test_and_set(&GloProxyCluster->cluster_leader_election, 1); + // Spec: with election enabled a node is effective-RO until the first + // election settles. Assume follower immediately; the next tick corrects + // it (the elected leader flips back to RW within tick+grace). + // Only flip on the false->true transition: this variable is + // re-applied on every LOAD ADMIN VARIABLES TO RUNTIME (including + // cluster syncs), and unconditionally forcing follower(true) here + // would kick an already-elected leader back to effective-RO on + // every reload. + if (old_v == false) { + set_cluster_follower(true); + } + return true; + } + if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { + variables.cluster_leader_election=false; + __sync_lock_test_and_set(&GloProxyCluster->cluster_leader_election, 0); + if (old_v == true) { + set_cluster_follower(false); // immediate, don't wait for the next tick + } + return true; + } + return false; + } +#endif /* PROXYSQL31 */ + if (!strcasecmp(name,"cluster_leader_node_timeout_ms")) { + int intv=atoi(value); + if (intv >= 1000 && intv <= 600000) { + variables.cluster_leader_node_timeout_ms=intv; + __sync_lock_test_and_set(&GloProxyCluster->cluster_leader_node_timeout_ms, intv); + return true; + } else { + return false; + } + } + if (!strcasecmp(name,"cluster_leader_grace_ms")) { + int intv=atoi(value); + if (intv >= 0 && intv <= 600000) { + variables.cluster_leader_grace_ms=intv; + __sync_lock_test_and_set(&GloProxyCluster->cluster_leader_grace_ms, intv); + return true; + } else { + return false; + } + } if (!strcasecmp(name,"cluster_check_status_frequency")) { int intv=atoi(value); if (intv >= 0 && intv <= 10000) { @@ -4847,12 +4918,19 @@ bool ProxySQL_Admin::set_variable(char *name, char *value, bool lock) { // this return false; } if (!strcasecmp(name,"read_only")) { + bool old_admin_read_only = variables.admin_read_only; if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { variables.admin_read_only=true; + if (old_admin_read_only != variables.admin_read_only) { + set_ro_mode(variables.admin_read_only ? ADMIN_RO_MODE_FORCED_RO : ADMIN_RO_MODE_AUTO); + } return true; } if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { variables.admin_read_only=false; + if (old_admin_read_only != variables.admin_read_only) { + set_ro_mode(variables.admin_read_only ? ADMIN_RO_MODE_FORCED_RO : ADMIN_RO_MODE_AUTO); + } return true; } return false; diff --git a/lib/ProxySQL_Admin_Stats.cpp b/lib/ProxySQL_Admin_Stats.cpp index d2738c6608..54667a7dfa 100644 --- a/lib/ProxySQL_Admin_Stats.cpp +++ b/lib/ProxySQL_Admin_Stats.cpp @@ -1583,6 +1583,44 @@ void ProxySQL_Admin::stats___proxysql_servers_checksums() { delete resultset; } +void ProxySQL_Admin::stats___proxysql_servers_status() { + // Same deadlock avoidance as stats___proxysql_servers_checksums: + // release sql_query_global_mutex while calling into the cluster nodes mutex. + pthread_mutex_unlock(&this->sql_query_global_mutex); + SQLite3_result* resultset = GloProxyCluster->get_stats_proxysql_servers_status(); + pthread_mutex_lock(&this->sql_query_global_mutex); + statsdb->execute("BEGIN"); + statsdb->execute("DELETE FROM stats_proxysql_servers_status"); + if (resultset) { + int rc; + sqlite3_stmt *statement1=NULL; + char *query1=NULL; + query1=(char *)"INSERT INTO stats_proxysql_servers_status VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)"; + auto [rc1, statement1_unique] = statsdb->prepare_v2(query1); + rc = rc1; + statement1 = statement1_unique.get(); + ASSERT_SQLITE_OK(rc, statsdb); + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r1=*it; + rc=(*proxy_sqlite3_bind_text)(statement1, 1, r1->fields[0], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 2, atoi(r1->fields[1])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 3, atoll(r1->fields[2])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement1, 4, r1->fields[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 5, atoll(r1->fields[4])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 6, atoll(r1->fields[5])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 7, atoll(r1->fields[6])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 8, atoll(r1->fields[7])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 9, atoll(r1->fields[8])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement1, 10, r1->fields[9], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + SAFE_SQLITE3_STEP2(statement1); + rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, statsdb); + } + } + statsdb->execute("COMMIT"); + delete resultset; +} + void ProxySQL_Admin::stats___proxysql_servers_metrics() { //SQLite3_result * resultset=GloProxyCluster->get_stats_proxysql_servers_metrics(); //if (resultset==NULL) return; diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index f82e571f45..58769e230f 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -163,6 +163,28 @@ extern MySQL_Authentication* GloMyAuth; extern PgSQL_Authentication *GloPgAuth; extern PgSQL_Query_Processor* GloPgQPro; +// Runs "SELECT GLOBAL_UUID()" on an already-connected peer connection and, on +// success, stores the result via Update_Node_UUID(). Shared by the +// post-handshake fetch and the per-poll retry (both query the same statement +// on the same connection and apply the same result validation). +// Returns true when a UUID was fetched and stored. +static bool cluster_fetch_peer_uuid(MYSQL *conn, char *hostname, uint16_t port) { + bool stored = false; + int rc_uuid = mysql_query(conn, (char *)"SELECT GLOBAL_UUID()"); + if (rc_uuid == 0) { + MYSQL_RES *uuid_res = mysql_store_result(conn); + if (uuid_res) { + MYSQL_ROW urow = mysql_fetch_row(uuid_res); + if (urow && urow[0] && strnlen(urow[0], 64) > 0) { + GloProxyCluster->Update_Node_UUID(hostname, port, urow[0]); + stored = true; + } + mysql_free_result(uuid_res); + } + } + return stored; +} + void * ProxySQL_Cluster_Monitor_thread(void *args) { pthread_attr_t thread_attr; size_t tmp_stack_size=0; @@ -187,6 +209,7 @@ void * ProxySQL_Cluster_Monitor_thread(void *args) { int query_error_counter = 0; char *query_error = NULL; int cluster_check_status_frequency_count = 0; + bool uuid_known = false; // set once GLOBAL_UUID() is successfully learned; avoids re-querying every poll iteration MYSQL *conn = mysql_init(NULL); if (conn==NULL) { @@ -242,6 +265,11 @@ void * ProxySQL_Cluster_Monitor_thread(void *args) { proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Sending CLUSTER_NODE_UUID %s to peer %s:%d\n", GloVars.uuid, node->hostname, node->port); proxy_info("Cluster: sending CLUSTER_NODE_UUID %s to peer %s:%d\n", GloVars.uuid, node->hostname, node->port); rc_query = mysql_query(conn, q.c_str()); + if (rc_query == 0) { + if (cluster_fetch_peer_uuid(conn, node->hostname, node->port)) { + uuid_known = true; + } + } } else { proxy_warning("Cluster: different ProxySQL version with peer %s:%d . Remote: %s . Self: %s\n", node->hostname, node->port, row[0], PROXYSQL_VERSION_); } @@ -259,6 +287,11 @@ void * ProxySQL_Cluster_Monitor_thread(void *args) { } rc_query = 1; } + } else { + // connect succeeded but the initial "SELECT @@version" failed: + // count it as a check failure for this cycle, same as a + // GLOBAL_CHECKSUM() failure further below. + GloProxyCluster->Update_Node_Failure(node->hostname, node->port); } while ( glovars.shutdown == 0 && rc_query == 0 && rc_bool == true) { unsigned long long start_time=monotonic_time(); @@ -275,6 +308,16 @@ void * ProxySQL_Cluster_Monitor_thread(void *args) { // FIXME: update metrics are not updated for now. We only check checksum //rc_bool = GloProxyCluster->Update_Node_Metrics(node->hostname, node->port, result, elapsed_time_us); + if (!uuid_known) { + // The initial UUID fetch (right after the version handshake) can + // fail transiently even though clustering is otherwise healthy. + // Retry it on every successful poll iteration until it succeeds, + // without re-querying once the UUID is known. + if (cluster_fetch_peer_uuid(conn, node->hostname, node->port)) { + uuid_known = true; + } + } + if (update_checksum) { unsigned long long before_query_time=monotonic_time(); rc_query = mysql_query(conn,query3); @@ -339,6 +382,7 @@ void * ProxySQL_Cluster_Monitor_thread(void *args) { ); } if (++query_error_counter == QUERY_ERROR_RATE) query_error_counter = 0; + GloProxyCluster->Update_Node_Failure(node->hostname, node->port); } unsigned long long end_time=monotonic_time(); if (rc_query == 0) { @@ -361,6 +405,7 @@ void * ProxySQL_Cluster_Monitor_thread(void *args) { } } else { proxy_warning("Cluster: unable to connect to peer %s:%d . Error: %s\n", node->hostname, node->port, mysql_error(conn)); + GloProxyCluster->Update_Node_Failure(node->hostname, node->port); node->resolve_hostname(); mysql_close(conn); conn = mysql_init(NULL); @@ -414,6 +459,11 @@ ProxySQL_Node_Entry::ProxySQL_Node_Entry(char* _hostname, uint16_t _port, uint64 global_checksum = 0; ip_addr = NULL; hostname = NULL; + uuid = NULL; + last_success_at_us = 0; + global_version = 0; + checks_ok = 0; + checks_err = 0; if (_hostname) { hostname = strdup(_hostname); } @@ -453,6 +503,10 @@ ProxySQL_Node_Entry::~ProxySQL_Node_Entry() { free(ip_addr); ip_addr = NULL; } + if (uuid) { + free(uuid); + uuid = NULL; + } for (int i = 0; i < PROXYSQL_NODE_METRICS_LEN ; i++) { delete metrics[i]; metrics[i] = NULL; @@ -497,6 +551,16 @@ void ProxySQL_Node_Entry::set_weight(uint64_t w) { weight = w; } +void ProxySQL_Node_Entry::set_uuid(const char* u) { + if (uuid) { + free(uuid); + uuid = NULL; + } + if (u) { + uuid = strdup(u); + } +} + void ProxySQL_Node_Entry::set_comment(char *s) { if (comment) { free(comment); @@ -3937,6 +4001,12 @@ cluster_nodes_metrics_map = std::make_tuple( "Number of frontend client connections currently open on the Cluster node.", metric_tags {} ), + std::make_tuple ( + p_cluster_nodes_dyn_gauge::proxysql_servers_alive, + "proxysql_servers_alive", + "1 when the peer answered the cluster liveness poll within admin-cluster_leader_node_timeout_ms, 0 otherwise.", + metric_tags {} + ), } ); @@ -4064,9 +4134,12 @@ bool ProxySQL_Cluster_Nodes::Update_Global_Checksum(char * _h, uint16_t _p, MYSQ } else { proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Global checksum for peer %s:%d is different from fetched one. Local checksum:[0x%lX] Fetched checksum:[0x%llX]\n", node->get_hostname(), node->get_port(), node->global_checksum, v); node->global_checksum = v; + node->global_version++; } } //pthread_mutex_unlock(&GloVars.checksum_mutex); + node->last_success_at_us = monotonic_time(); + node->checks_ok++; } pthread_mutex_unlock(&mutex); return ret; @@ -4152,6 +4225,26 @@ bool ProxySQL_Cluster_Nodes::Update_Node_Metrics(char * _h, uint16_t _p, MYSQL_R return ret; } +void ProxySQL_Cluster_Nodes::Update_Node_UUID(char * _hostname, uint16_t _port, const char * _uuid) { + uint64_t hash_ = generate_hash(_hostname, _port); + pthread_mutex_lock(&mutex); + auto ite = umap_proxy_nodes.find(hash_); + if (ite != umap_proxy_nodes.end()) { + ite->second->set_uuid(_uuid); + } + pthread_mutex_unlock(&mutex); +} + +void ProxySQL_Cluster_Nodes::Update_Node_Failure(char * _hostname, uint16_t _port) { + uint64_t hash_ = generate_hash(_hostname, _port); + pthread_mutex_lock(&mutex); + auto ite = umap_proxy_nodes.find(hash_); + if (ite != umap_proxy_nodes.end()) { + ite->second->checks_err++; + } + pthread_mutex_unlock(&mutex); +} + void ProxySQL_Cluster_Nodes::get_peer_to_sync_mysql_query_rules(char **host, uint16_t *port, char** ip_address) { get_peer_to_sync_variables_module("mysql_query_rules", host, port, ip_address, nullptr, nullptr); } @@ -4633,6 +4726,93 @@ SQLite3_result * ProxySQL_Cluster_Nodes::stats_proxysql_servers_metrics() { return result; } +SQLite3_result * ProxySQL_Cluster_Nodes::stats_proxysql_servers_status(const std::string& leader_uuid, unsigned long long alive_timeout_us) { + const int colnum=10; + SQLite3_result *result=new SQLite3_result(colnum); + result->add_column_definition(SQLITE_TEXT,"hostname"); + result->add_column_definition(SQLITE_TEXT,"port"); + result->add_column_definition(SQLITE_TEXT,"weight"); + result->add_column_definition(SQLITE_TEXT,"master"); + result->add_column_definition(SQLITE_TEXT,"global_version"); + result->add_column_definition(SQLITE_TEXT,"check_age_us"); + result->add_column_definition(SQLITE_TEXT,"ping_time_us"); + result->add_column_definition(SQLITE_TEXT,"checks_OK"); + result->add_column_definition(SQLITE_TEXT,"checks_ERR"); + result->add_column_definition(SQLITE_TEXT,"uuid"); + (void)alive_timeout_us; // liveness is derivable from check_age_us; kept for future use + + char buf[64]; + int k; + pthread_mutex_lock(&mutex); + unsigned long long now = monotonic_time(); + for( std::unordered_map::iterator it = umap_proxy_nodes.begin(); it != umap_proxy_nodes.end(); ) { + ProxySQL_Node_Entry * node = it->second; + char **pta=(char **)malloc(sizeof(char *)*colnum); + pta[0]=strdup(node->get_hostname()); + snprintf(buf, sizeof(buf), "%d", node->get_port()); + pta[1]=strdup(buf); + snprintf(buf, sizeof(buf), "%lu", node->get_weight()); + pta[2]=strdup(buf); + const char *nuuid = node->get_uuid(); + bool is_master = (nuuid != NULL && leader_uuid.empty() == false && leader_uuid == nuuid); + pta[3]=strdup(is_master ? "YES" : "NO"); + snprintf(buf, sizeof(buf), "%lu", (unsigned long)node->get_global_version()); + pta[4]=strdup(buf); + unsigned long long last = node->get_last_success_at_us(); + if (last == 0) { + pta[5]=strdup("-1"); + } else { + snprintf(buf, sizeof(buf), "%llu", now - last); + pta[5]=strdup(buf); + } + ProxySQL_Node_Metrics *curr = node->get_metrics_curr(); + snprintf(buf, sizeof(buf), "%llu", curr->response_time_us); + pta[6]=strdup(buf); + snprintf(buf, sizeof(buf), "%lu", (unsigned long)node->get_checks_ok()); + pta[7]=strdup(buf); + snprintf(buf, sizeof(buf), "%lu", (unsigned long)node->get_checks_err()); + pta[8]=strdup(buf); + pta[9]=strdup(nuuid ? nuuid : ""); + + result->add_row(pta); + for (k=0; k ProxySQL_Cluster_Nodes::get_leader_candidates(unsigned long long alive_timeout_us) { + std::vector candidates; + unsigned long long now = monotonic_time(); + pthread_mutex_lock(&mutex); + for (auto it = umap_proxy_nodes.begin(); it != umap_proxy_nodes.end(); it++) { + ProxySQL_Node_Entry * node = it->second; + Cluster_Leader_Candidate c; + c.uuid = (node->get_uuid() ? node->get_uuid() : ""); + c.hostname = node->get_hostname(); + c.port = node->get_port(); + c.weight = node->get_weight(); + bool is_self = (GloVars.uuid && node->get_uuid() && strcmp(node->get_uuid(), GloVars.uuid) == 0); + unsigned long long last = node->get_last_success_at_us(); + c.alive = is_self || (last != 0 && (now - last) < alive_timeout_us); + candidates.push_back(c); + } + pthread_mutex_unlock(&mutex); + return candidates; +} + SQLite3_result * ProxySQL_Cluster_Nodes::dump_table_proxysql_servers() { const int colnum=4; SQLite3_result *result=new SQLite3_result(colnum); @@ -4688,6 +4868,9 @@ void ProxySQL_Cluster_Nodes::update_prometheus_nodes_metrics() { vector cur_node_metrics {}; vector cur_node_checksums {}; + const unsigned long long alive_timeout_us = + (unsigned long long)__sync_fetch_and_add(&GloProxyCluster->cluster_leader_node_timeout_ms, 0) * 1000ULL; + // Update metrics for both 'servers_checksums' and 'servers_metrics' for (const auto& node_entry : umap_proxy_nodes) { const string hostname { node_entry.second->get_hostname() }; @@ -4747,11 +4930,14 @@ void ProxySQL_Cluster_Nodes::update_prometheus_nodes_metrics() { const double last_check_ms = (curtime - read_time_us) / 1000.0; const double response_time_ms = node_metrics->response_time_us / 1000.0; const double conns_connected = node_metrics->Client_Connections_connected; + const unsigned long long last_ok = node_entry.second->get_last_success_at_us(); + const double node_alive = (last_ok != 0 && (curtime - last_ok) < alive_timeout_us) ? 1.0 : 0.0; vector&, dyn_gauge::metric, double>> metric_gauges { std::make_tuple(std::ref(this->metrics.p_proxysql_servers_metrics_last_check_ms), dyn_gauge::proxysql_servers_metrics_last_check_ms, last_check_ms), std::make_tuple(std::ref(this->metrics.p_proxysql_servers_metrics_response_time_ms), dyn_gauge::proxysql_servers_metrics_response_time_ms, response_time_ms), std::make_tuple(std::ref(this->metrics.p_proxysql_servers_metrics_client_conns_connected), dyn_gauge::proxysql_servers_metrics_client_conns_connected, conns_connected), + std::make_tuple(std::ref(this->metrics.p_proxysql_servers_alive), dyn_gauge::proxysql_servers_alive, node_alive), }; for (const auto& metric_gauge : metric_gauges) { @@ -4796,6 +4982,7 @@ void ProxySQL_Cluster_Nodes::update_prometheus_nodes_metrics() { { metrics.p_proxysql_servers_metrics_response_time_ms, dyn_gauge::proxysql_servers_metrics_response_time_ms }, { metrics.p_proxysql_servers_metrics_last_check_ms, dyn_gauge::proxysql_servers_metrics_last_check_ms }, { metrics.p_proxysql_servers_metrics_client_conns_connected, dyn_gauge::proxysql_servers_metrics_client_conns_connected }, + { metrics.p_proxysql_servers_alive, dyn_gauge::proxysql_servers_alive }, { metrics.p_proxysql_servers_checksums_epoch, dyn_gauge::proxysql_servers_checksums_epoch }, { metrics.p_proxysql_servers_checksums_updated_at, dyn_gauge::proxysql_servers_checksums_updated_at }, @@ -5488,8 +5675,21 @@ cluster_metrics_map = std::make_tuple( { "reason", "version_one" } } ), + std::make_tuple ( + p_cluster_counter::cluster_leader_changes, + "proxysql_cluster_leader_changes_total", + "Number of times this node observed an effective cluster leader change.", + metric_tags {} + ), }, - cluster_gauge_vector {} + cluster_gauge_vector { + std::make_tuple ( + p_cluster_gauge::cluster_leader_status, + "proxysql_cluster_leader_status", + "1 when this node is the elected cluster leader, 0 otherwise.", + metric_tags {} + ), + } ); ProxySQL_Cluster::ProxySQL_Cluster() : proxysql_servers_to_monitor(NULL) { @@ -5505,6 +5705,13 @@ ProxySQL_Cluster::ProxySQL_Cluster() : proxysql_servers_to_monitor(NULL) { cluster_username = strdup((char *)""); cluster_password = strdup((char *)""); cluster_check_interval_ms = 1000; + cluster_leader_election = 0; + cluster_leader_node_timeout_ms = 3000; + cluster_leader_grace_ms = 3000; + leader_hostname = NULL; + leader_port = 0; + leader_next_check_at = 0; + pthread_mutex_init(&leader_mutex, NULL); cluster_check_status_frequency = 10; cluster_mysql_query_rules_diffs_before_sync = 3; cluster_mysql_servers_diffs_before_sync = 3; @@ -5539,12 +5746,80 @@ ProxySQL_Cluster::~ProxySQL_Cluster() { free(admin_mysql_ifaces); admin_mysql_ifaces = NULL; } + if (leader_hostname) { + free(leader_hostname); + leader_hostname = NULL; + } } void ProxySQL_Cluster::p_update_metrics() { this->nodes.update_prometheus_nodes_metrics(); + metrics.p_gauge_array[p_cluster_gauge::cluster_leader_status]->Set(is_leader() ? 1 : 0); }; +void ProxySQL_Cluster::leader_election_tick(unsigned long long curtime_us) { + if (curtime_us < leader_next_check_at) return; + leader_next_check_at = curtime_us + 500000; // evaluate at most every 500ms + int enabled = __sync_fetch_and_add(&cluster_leader_election, 0); + cluster_creds_t creds = get_credentials(); + bool clustering_active = (creds.user.empty() == false); + bool am_leader_or_standalone = true; + if (enabled == 0 || clustering_active == false) { + pthread_mutex_lock(&leader_mutex); + leader_state.reset(); + if (leader_hostname) { free(leader_hostname); leader_hostname = NULL; } + leader_port = 0; + pthread_mutex_unlock(&leader_mutex); + } else { + unsigned long long timeout_us = (unsigned long long)__sync_fetch_and_add(&cluster_leader_node_timeout_ms, 0) * 1000ULL; + unsigned long long grace_ms = (unsigned long long)__sync_fetch_and_add(&cluster_leader_grace_ms, 0); + std::vector candidates = nodes.get_leader_candidates(timeout_us); + if (candidates.empty()) { + // proxysql_servers is empty: standalone behavior + pthread_mutex_lock(&leader_mutex); + leader_state.reset(); + if (leader_hostname) { free(leader_hostname); leader_hostname = NULL; } + leader_port = 0; + pthread_mutex_unlock(&leader_mutex); + } else { + int idx = cluster_elect_leader(candidates); + std::string computed = (idx >= 0 ? candidates[idx].uuid : ""); + pthread_mutex_lock(&leader_mutex); + bool changed = leader_state.update(computed, curtime_us / 1000, grace_ms); + if (changed) { + if (leader_hostname) { free(leader_hostname); leader_hostname = NULL; } + leader_port = 0; + if (idx >= 0 && leader_state.current_leader_uuid == candidates[idx].uuid) { + leader_hostname = strdup(candidates[idx].hostname.c_str()); + leader_port = candidates[idx].port; + } + proxy_info("Cluster leader changed: new leader is %s (%s:%d)\n", + (leader_state.current_leader_uuid.empty() ? "NONE" : leader_state.current_leader_uuid.c_str()), + (leader_hostname ? leader_hostname : ""), leader_port); + metrics.p_counter_array[p_cluster_counter::cluster_leader_changes]->Increment(); + } + am_leader_or_standalone = (GloVars.uuid && leader_state.current_leader_uuid == GloVars.uuid); + pthread_mutex_unlock(&leader_mutex); + } + } + GloAdmin->set_cluster_follower(enabled != 0 && clustering_active && am_leader_or_standalone == false); +} + +bool ProxySQL_Cluster::is_leader() { + pthread_mutex_lock(&leader_mutex); + bool r = (GloVars.uuid && leader_state.current_leader_uuid.empty() == false && leader_state.current_leader_uuid == GloVars.uuid); + pthread_mutex_unlock(&leader_mutex); + return r; +} + +void ProxySQL_Cluster::get_leader_info(std::string& hostname, int& port, std::string& uuid) { + pthread_mutex_lock(&leader_mutex); + hostname = (leader_hostname ? leader_hostname : ""); + port = leader_port; + uuid = leader_state.current_leader_uuid; + pthread_mutex_unlock(&leader_mutex); +} + // this function returns credentials to the caller, used by monitoring threads cluster_creds_t ProxySQL_Cluster::get_credentials() { pthread_mutex_lock(&mutex); diff --git a/lib/ProxySQL_Cluster_Leader.cpp b/lib/ProxySQL_Cluster_Leader.cpp new file mode 100644 index 0000000000..06d6c4c02b --- /dev/null +++ b/lib/ProxySQL_Cluster_Leader.cpp @@ -0,0 +1,45 @@ +#include "ProxySQL_Cluster_Leader.h" + +int cluster_elect_leader(const std::vector& candidates) { + int best = -1; + for (size_t i = 0; i < candidates.size(); i++) { + const Cluster_Leader_Candidate& c = candidates[i]; + if (c.alive == false || c.uuid.empty()) { + continue; + } + if (best == -1) { + best = (int)i; + continue; + } + const Cluster_Leader_Candidate& b = candidates[best]; + if (c.weight > b.weight || (c.weight == b.weight && c.uuid < b.uuid)) { + best = (int)i; + } + } + return best; +} + +bool Cluster_Leader_State::update(const std::string& computed_uuid, unsigned long long now_ms, unsigned long long grace_ms) { + if (computed_uuid == current_leader_uuid) { + pending_leader_uuid.clear(); + pending_since_ms = 0; + return false; + } + if (pending_since_ms == 0 || pending_leader_uuid != computed_uuid) { + pending_leader_uuid = computed_uuid; + pending_since_ms = now_ms; + } + if (now_ms - pending_since_ms >= grace_ms) { + current_leader_uuid = pending_leader_uuid; + pending_leader_uuid.clear(); + pending_since_ms = 0; + return true; + } + return false; +} + +void Cluster_Leader_State::reset() { + current_leader_uuid.clear(); + pending_leader_uuid.clear(); + pending_since_ms = 0; +} diff --git a/lib/ProxySQL_RESTAPI_Server.cpp b/lib/ProxySQL_RESTAPI_Server.cpp index 1b361226cd..d9dd4420b4 100644 --- a/lib/ProxySQL_RESTAPI_Server.cpp +++ b/lib/ProxySQL_RESTAPI_Server.cpp @@ -407,16 +407,17 @@ class tsdb_resource : public http_resource { string s_to = req.get_arg("to"); if (!s_to.empty()) to = atol(s_to.c_str()); string agg = req.get_arg("agg"); + string node = req.get_arg("node"); std::map labels; auto all_args = req.get_args(); for (auto const& [key, val] : all_args) { - if (key != "metric" && key != "from" && key != "to" && key != "agg") { + if (key != "metric" && key != "from" && key != "to" && key != "agg" && key != "node") { labels[key] = val; } } - SQLite3_result *res = GloProxyStats->query_tsdb_metrics(metric, labels, from, to, agg); + SQLite3_result *res = GloProxyStats->query_tsdb_metrics(metric, labels, from, to, agg, node); if (!res) { j_resp = json::array(); } else { @@ -430,6 +431,11 @@ class tsdb_resource : public http_resource { row["labels"] = res->rows[i]->fields[2]; } row["value"] = atof(res->rows[i]->fields[3]); + if (res->columns >= 5) { + // Cluster-scoped query: node column identifies which + // node each row came from (relevant for node=*). + row["node"] = res->rows[i]->fields[4]; + } j_resp.push_back(row); } delete res; @@ -447,6 +453,34 @@ class tsdb_resource : public http_resource { j_resp["disk_size_bytes"] = status.disk_size_bytes; j_resp["oldest_datapoint"] = status.oldest_datapoint; j_resp["newest_datapoint"] = status.newest_datapoint; + j_resp["cluster_aggregation_active"] = GloProxyStats->tsdb_agg_active.load(); + j_resp["cluster_rows_replicated"] = GloProxyStats->tsdb_agg_rows_total.load(); + j_resp["cluster_last_cycle"] = GloProxyStats->tsdb_agg_last_cycle_ts.load(); + j_resp["cluster_cap_hit_last_cycle"] = GloProxyStats->tsdb_agg_cap_hit_last_cycle.load(); + } else if (req_path == "/api/tsdb/nodes") { + if (!GloProxyStats || !GloProxyStats->statsdb_disk) { + j_resp = json {{"error", "TSDB not initialized"}}; + auto response = std::shared_ptr(new string_response(j_resp.dump(), http::http_utils::http_internal_server_error)); + add_headers(response); + return response; + } + json nodes_arr = json::array(); + SQLite3_result *res = GloProxyStats->get_tsdb_cluster_nodes(); + time_t now = time(NULL); + if (res) { + for (std::vector::iterator it = res->rows.begin(); it != res->rows.end(); ++it) { + SQLite3_row *r = *it; + json jn; + jn["node"] = r->fields[0]; + long last_ts = atol(r->fields[1]); + jn["last_timestamp"] = last_ts; + jn["watermark_age_s"] = (long)now - last_ts; + jn["datapoints"] = atoll(r->fields[2]); + nodes_arr.push_back(jn); + } + delete res; + } + j_resp = nodes_arr; } else { return std::shared_ptr(new string_response("Not Found", http::http_utils::http_not_found)); } @@ -539,6 +573,7 @@ ProxySQL_RESTAPI_Server::ProxySQL_RESTAPI_Server( ws->register_resource("/api/tsdb/metrics", tsdb_endpoint.get(), true); ws->register_resource("/api/tsdb/query", tsdb_endpoint.get(), true); ws->register_resource("/api/tsdb/status", tsdb_endpoint.get(), true); + ws->register_resource("/api/tsdb/nodes", tsdb_endpoint.get(), true); /* Serve the dashboard HTML and its Chart.bundle.js asset on the * same port as the API, so the dashboard's relative fetch() calls diff --git a/lib/ProxySQL_Statistics.cpp b/lib/ProxySQL_Statistics.cpp index a954fbf907..da5a1f10f0 100644 --- a/lib/ProxySQL_Statistics.cpp +++ b/lib/ProxySQL_Statistics.cpp @@ -5,6 +5,10 @@ #include "ProxySQL_Statistics.hpp" #include "MySQL_HostGroups_Manager.h" #include "PgSQL_HostGroups_Manager.h" +#ifdef PROXYSQLTSDB +#include "TSDB_Cluster_Aggregator.h" +#include "ProxySQL_Cluster.hpp" +#endif #include "../deps/json/json.hpp" using json = nlohmann::json; @@ -147,9 +151,15 @@ ProxySQL_Statistics::ProxySQL_Statistics() { #ifdef PROXYSQLTSDB variables.tsdb_enabled = 0; variables.tsdb_sample_interval = 5; - variables.tsdb_retention_days = 7; + variables.tsdb_retention_days = 2; variables.tsdb_monitor_enabled = 0; variables.tsdb_monitor_interval = 10; + variables.tsdb_cluster_aggregation = 1; + variables.tsdb_cluster_interval = 10; + variables.tsdb_cluster_backfill_hours = 24; + variables.tsdb_cluster_retention_days = 1; + variables.tsdb_cluster_batch_rows = 10000; + variables.tsdb_hourly_retention_days = 365; #endif } @@ -164,6 +174,12 @@ static const struct { {"retention_days", 1, 3650}, {"monitor_enabled", 0, 1}, {"monitor_interval", 1, 3600}, + {"cluster_aggregation", 0, 1}, + {"cluster_interval", 5, 300}, + {"cluster_backfill_hours", 0, 168}, + {"cluster_retention_days", 1, 30}, + {"cluster_batch_rows", 1000, 100000}, + {"hourly_retention_days", 1, 3650}, {NULL, 0, 0} }; @@ -182,6 +198,12 @@ bool ProxySQL_Statistics::set_variable(const char *name, const char *value) { else if (i == 2) variables.tsdb_retention_days = (int)intv; else if (i == 3) variables.tsdb_monitor_enabled = (int)intv; else if (i == 4) variables.tsdb_monitor_interval = (int)intv; + else if (i == 5) variables.tsdb_cluster_aggregation = (int)intv; + else if (i == 6) variables.tsdb_cluster_interval = (int)intv; + else if (i == 7) variables.tsdb_cluster_backfill_hours = (int)intv; + else if (i == 8) variables.tsdb_cluster_retention_days = (int)intv; + else if (i == 9) variables.tsdb_cluster_batch_rows = (int)intv; + else if (i == 10) variables.tsdb_hourly_retention_days = (int)intv; return true; } return false; @@ -208,6 +230,24 @@ char *ProxySQL_Statistics::get_variable(const char *name) { } else if (!strcasecmp(name, "monitor_interval")) { snprintf(buf, sizeof(buf), "%d", variables.tsdb_monitor_interval); return strdup(buf); + } else if (!strcasecmp(name, "cluster_aggregation")) { + snprintf(buf, sizeof(buf), "%d", variables.tsdb_cluster_aggregation); + return strdup(buf); + } else if (!strcasecmp(name, "cluster_interval")) { + snprintf(buf, sizeof(buf), "%d", variables.tsdb_cluster_interval); + return strdup(buf); + } else if (!strcasecmp(name, "cluster_backfill_hours")) { + snprintf(buf, sizeof(buf), "%d", variables.tsdb_cluster_backfill_hours); + return strdup(buf); + } else if (!strcasecmp(name, "cluster_retention_days")) { + snprintf(buf, sizeof(buf), "%d", variables.tsdb_cluster_retention_days); + return strdup(buf); + } else if (!strcasecmp(name, "cluster_batch_rows")) { + snprintf(buf, sizeof(buf), "%d", variables.tsdb_cluster_batch_rows); + return strdup(buf); + } else if (!strcasecmp(name, "hourly_retention_days")) { + snprintf(buf, sizeof(buf), "%d", variables.tsdb_hourly_retention_days); + return strdup(buf); } return NULL; } @@ -234,9 +274,17 @@ bool ProxySQL_Statistics::has_variable(const char *name) { ProxySQL_Statistics::~ProxySQL_Statistics() { #ifdef PROXYSQLTSDB + if (tsdb_agg_thread_started) { + tsdb_agg_stop.store(true); + pthread_join(tsdb_agg_thread, NULL); + tsdb_agg_thread_started = false; + } if (stmt_insert_tsdb_metric) { (*proxy_sqlite3_finalize)(stmt_insert_tsdb_metric); } + if (stmt_insert_tsdb_cluster_metric) { + (*proxy_sqlite3_finalize)(stmt_insert_tsdb_cluster_metric); + } if (stmt_insert_backend_health) { (*proxy_sqlite3_finalize)(stmt_insert_backend_health); } @@ -295,6 +343,7 @@ void ProxySQL_Statistics::init() { insert_into_tables_defs(tables_defs_statsdb_disk,"tsdb_metrics", STATSDB_SQLITE_TABLE_TSDB_METRICS); insert_into_tables_defs(tables_defs_statsdb_disk,"tsdb_metrics_hour", STATSDB_SQLITE_TABLE_TSDB_METRICS_HOUR); insert_into_tables_defs(tables_defs_statsdb_disk,"tsdb_backend_health", STATSDB_SQLITE_TABLE_TSDB_BACKEND_HEALTH); + insert_into_tables_defs(tables_defs_statsdb_disk,"tsdb_metrics_cluster", STATSDB_SQLITE_TABLE_TSDB_METRICS_CLUSTER); #endif disk_upgrade_mysql_connections(); @@ -325,6 +374,7 @@ void ProxySQL_Statistics::init() { statsdb_disk->execute("CREATE INDEX IF NOT EXISTS idx_tsdb_metrics_hour_metric_bucket ON tsdb_metrics_hour(metric_name, bucket)"); statsdb_disk->execute("CREATE INDEX IF NOT EXISTS idx_tsdb_backend_health_time ON tsdb_backend_health(timestamp)"); statsdb_disk->execute("CREATE INDEX IF NOT EXISTS idx_tsdb_backend_health_host_time ON tsdb_backend_health(hostgroup, hostname, port, timestamp)"); + statsdb_disk->execute("CREATE INDEX IF NOT EXISTS idx_tsdb_metrics_cluster_node_metric_time ON tsdb_metrics_cluster (node, metric_name, timestamp)"); #endif } @@ -1583,7 +1633,12 @@ void ProxySQL_Statistics::tsdb_downsample_metrics() { last_hour > 0 ? last_hour + 3600 : 0, current_hour); + // Runs on the admin main loop and can otherwise execute inside the + // aggregation worker's open wrlocked transaction on the same shared + // statsdb_disk connection. + statsdb_disk->wrlock(); statsdb_disk->execute(buf); + statsdb_disk->wrunlock(); } } @@ -1595,16 +1650,22 @@ void ProxySQL_Statistics::tsdb_retention_cleanup() { const int retention_days = std::max(1, variables.tsdb_retention_days); char delete_buf[256]; + // Runs on the admin main loop and can otherwise execute inside the + // aggregation worker's open wrlocked transaction on the same shared + // statsdb_disk connection. Single lock span around all four DELETEs. + statsdb_disk->wrlock(); + // Retention: delete raw data older than configured days snprintf(delete_buf, sizeof(delete_buf), "DELETE FROM tsdb_metrics WHERE timestamp < %ld", ts - 86400 * retention_days); statsdb_disk->execute(delete_buf); - // Retention: delete hourly data older than 1 year + // Retention: delete hourly data older than configured days + const int hourly_retention_days = std::max(1, variables.tsdb_hourly_retention_days); snprintf(delete_buf, sizeof(delete_buf), "DELETE FROM tsdb_metrics_hour WHERE bucket < %ld", - ts - 86400 * 365); + ts - 86400L * hourly_retention_days); statsdb_disk->execute(delete_buf); // Retention: delete backend probe data older than configured days @@ -1612,6 +1673,15 @@ void ProxySQL_Statistics::tsdb_retention_cleanup() { "DELETE FROM tsdb_backend_health WHERE timestamp < %ld", ts - 86400 * retention_days); statsdb_disk->execute(delete_buf); + + // Retention: delete cluster-aggregated data older than configured days + const int cluster_retention_days = std::max(1, variables.tsdb_cluster_retention_days); + snprintf(delete_buf, sizeof(delete_buf), + "DELETE FROM tsdb_metrics_cluster WHERE timestamp < %ld", + ts - 86400L * cluster_retention_days); + statsdb_disk->execute(delete_buf); + + statsdb_disk->wrunlock(); } // TSDB Status @@ -1737,14 +1807,19 @@ SQLite3_result* ProxySQL_Statistics::query_tsdb_metrics( const std::map& label_filters, time_t from, time_t to, - const std::string& aggregation) { + const std::string& aggregation, + const std::string& node) { if (!statsdb_disk) return NULL; if (to < from) { std::swap(from, to); } - const bool use_hourly = (to - from > 86400); + bool use_hourly = (to - from > 86400); + if (node.length() > 0) { + // Cluster-scoped queries are always served from the raw cluster table. + use_hourly = false; + } const std::string agg = aggregation.empty() ? "raw" : aggregation; std::string query; @@ -1762,6 +1837,18 @@ SQLite3_result* ProxySQL_Statistics::query_tsdb_metrics( "FROM tsdb_metrics_hour " "WHERE metric_name='" + escape_sql_string_literal(metric_name) + "' " "AND bucket BETWEEN " + std::to_string(from) + " AND " + std::to_string(to); + } else if (node.length() > 0) { + // Include node as a 5th column: self-describing whether the query is + // node=* (multiple nodes -> otherwise unattributable rows) or a specific + // node (harmless, still correct). + query = + "SELECT timestamp AS ts, metric_name, labels, value, node " + "FROM tsdb_metrics_cluster " + "WHERE metric_name='" + escape_sql_string_literal(metric_name) + "' " + "AND timestamp BETWEEN " + std::to_string(from) + " AND " + std::to_string(to); + if (node != "*") { + query += " AND node='" + escape_sql_string_literal(node) + "'"; + } } else { query = "SELECT timestamp AS ts, metric_name, labels, value " @@ -1840,6 +1927,11 @@ void ProxySQL_Statistics::tsdb_sampler_loop() { update_modules_metrics(); auto metrics = GloVars.prometheus_registry->Collect(); time_t now = time(NULL); + // Shared statsdb_disk connection: also used by the TSDB cluster-aggregation + // worker thread (tsdb_cluster_replicate_self/peer). Take the write lock for + // the whole explicit transaction so the two threads' BEGIN..COMMIT blocks + // can't interleave on the same sqlite connection. + statsdb_disk->wrlock(); statsdb_disk->execute("BEGIN"); for (const auto& family : metrics) { for (const auto& metric : family.metric) { @@ -1890,6 +1982,7 @@ void ProxySQL_Statistics::tsdb_sampler_loop() { } } statsdb_disk->execute("COMMIT"); + statsdb_disk->wrunlock(); } } @@ -1995,6 +2088,10 @@ void ProxySQL_Statistics::tsdb_monitor_loop() { for (size_t j = i; j < batch_end; ++j) { batch_futures.push_back(std::async(std::launch::async, probe_backend, targets[j].hg, targets[j].host, targets[j].port, now)); } + // Shared statsdb_disk connection: see the comment in tsdb_sampler_loop() / + // tsdb_cluster_replicate_peer() — hold the write lock for the whole explicit + // transaction so it can't interleave with another thread's BEGIN..COMMIT. + statsdb_disk->wrlock(); statsdb_disk->execute("BEGIN"); for (auto& f : batch_futures) { try { @@ -2005,7 +2102,347 @@ void ProxySQL_Statistics::tsdb_monitor_loop() { } } statsdb_disk->execute("COMMIT"); + statsdb_disk->wrunlock(); + } +} + +// TSDB Cluster Aggregation +// The leader periodically pulls each cluster peer's own tsdb_metrics (pull + +// per-node watermark) into the local tsdb_metrics_cluster table, so that the +// cluster's TSDB dashboard can be served from a single node. See +// TSDB_Cluster_Aggregator.h for the pure watermark/fetch planning logic. + +extern ProxySQL_Cluster* GloProxyCluster; + +static void * tsdb_cluster_agg_thread_fn(void *arg) { + set_thread_name("TSDBClusterAgg"); + ((ProxySQL_Statistics *)arg)->tsdb_cluster_aggregation_thread_loop(); + return NULL; +} + +void ProxySQL_Statistics::tsdb_cluster_aggregation_check(unsigned long long curtime) { + if (curtime < next_timer_tsdb_cluster_check) return; + next_timer_tsdb_cluster_check = curtime + 1000000ULL; // evaluate at most every 1s + bool desired = false; + if (variables.tsdb_enabled && variables.tsdb_cluster_aggregation) { + if (GloProxyCluster && GloProxyCluster->is_leader()) { + desired = true; + } + } + + // A stop was previously requested and the worker may still be winding + // down (it can be blocked in peer I/O for up to ~11s). Never + // pthread_join() until the worker itself reports (via + // tsdb_agg_thread_done) that it has fully returned -- otherwise this call + // (on the admin main loop thread) would block, delaying + // leader_election_tick() called right after this check. + if (tsdb_agg_thread_started && tsdb_agg_stop.load()) { + if (tsdb_agg_thread_done.load() == false) { + // Still stopping: do nothing this tick, and in particular do NOT + // start a new worker even if leadership (and thus 'desired') was + // regained in the meantime -- the old worker must be fully + // reaped first. + return; + } + // Worker has returned: this join is effectively instantaneous. + pthread_join(tsdb_agg_thread, NULL); + tsdb_agg_thread_started = false; + tsdb_agg_stop.store(false); + tsdb_agg_thread_done.store(false); + proxy_info("TSDB cluster aggregation: stopped\n"); + } + + if (desired == true && tsdb_agg_thread_started == false) { + tsdb_agg_stop.store(false); + tsdb_agg_thread_done.store(false); + // Clear per-peer episode state before (re)starting the worker: these maps + // are worker-thread-only (no locking), so it's only safe to reset them + // here, before the new thread is created. The block above guarantees any + // previous worker has already been joined, so this is safe. Otherwise a + // restarted leadership inherits stale stall/cap-hit counters and + // watermarks from a previous leadership episode. + tsdb_agg_peer_last_wm.clear(); + tsdb_agg_peer_stall_count.clear(); + tsdb_agg_peer_stall_logged.clear(); + tsdb_agg_peer_cap_hit_count.clear(); + tsdb_agg_peer_progress_stall_count.clear(); + if (pthread_create(&tsdb_agg_thread, NULL, tsdb_cluster_agg_thread_fn, this) == 0) { + tsdb_agg_thread_started = true; + tsdb_agg_active.store(true); + proxy_info("TSDB cluster aggregation: started (this node is the cluster leader)\n"); + } else { + proxy_error("TSDB cluster aggregation: failed to create worker thread\n"); + } + } else if (desired == false && tsdb_agg_thread_started == true) { + // Request the stop but do NOT join here (see the reap block above -- + // reaping happens on a later tick once the worker sets + // tsdb_agg_thread_done). tsdb_agg_active goes false immediately since + // it is REST-visible and should reflect "no longer aggregating" as + // soon as the stop is requested, not only once the worker is reaped. + tsdb_agg_stop.store(true); + tsdb_agg_active.store(false); + proxy_info("TSDB cluster aggregation: stop requested (leadership lost)\n"); + } +} + +void ProxySQL_Statistics::tsdb_cluster_aggregation_thread_loop() { + while (tsdb_agg_stop.load() == false) { + tsdb_cluster_aggregation_cycle(); + tsdb_agg_last_cycle_ts.store((long long)time(NULL)); + int sleep_s = variables.tsdb_cluster_interval; + if (sleep_s < 5) sleep_s = 5; + for (int i = 0; i < sleep_s * 10 && tsdb_agg_stop.load() == false; i++) { + usleep(100000); + } + } + // VERY LAST action before this function (and the thread) returns: tells + // tsdb_cluster_aggregation_check() it is now safe to pthread_join() this + // thread without blocking the admin main loop. + tsdb_agg_thread_done.store(true); +} + +void ProxySQL_Statistics::tsdb_cluster_aggregation_cycle() { + if (GloProxyCluster == NULL) return; + if (GloProxyCluster->is_leader() == false) return; // deposed between checks + std::string self_host; int self_port = 0; std::string self_uuid; + GloProxyCluster->get_leader_info(self_host, self_port, self_uuid); + if (self_host.length() == 0) return; + std::string self_node = self_host + ":" + std::to_string(self_port); + cluster_creds_t creds = GloProxyCluster->get_credentials(); + SQLite3_result *servers = GloProxyCluster->dump_table_proxysql_servers(); + if (servers == NULL) return; + long now = (long)time(NULL); + int limit = variables.tsdb_cluster_batch_rows; + if (limit < 1000) limit = 1000; + bool cap_hit = false; + for (std::vector::iterator it = servers->rows.begin(); it != servers->rows.end(); ++it) { + if (tsdb_agg_stop.load()) break; + SQLite3_row *r = *it; + std::string node = std::string(r->fields[0]) + ":" + std::string(r->fields[1]); + // Persisted (raw) watermark: MAX(timestamp) actually replicated for this + // node so far, stable at 0 for a never-replicated peer. Fetched once here + // and reused below for stall detection, instead of letting the effective + // (backfill-adjusted) watermark — which moves every cycle for an empty + // peer — mask a permanently-stalled/never-replicated peer. + long persisted_max_ts = tsdb_cluster_node_max_ts(node); + long wm = tsdb_agg_effective_watermark(persisted_max_ts, now, variables.tsdb_cluster_backfill_hours); + if (node == self_node) { + tsdb_cluster_replicate_self(node, wm, limit); + } else { + if (creds.user.length() == 0) continue; // clustering unconfigured + bool hit = tsdb_cluster_replicate_peer(r->fields[0], atoi(r->fields[1]), node, wm, limit, creds.user, creds.pass, persisted_max_ts); + if (hit) cap_hit = true; + } + } + tsdb_agg_cap_hit_last_cycle.store(cap_hit); + delete servers; +} + +long ProxySQL_Statistics::tsdb_cluster_node_max_ts(const std::string& node) { + char *error = NULL; int cols = 0; int affected_rows = 0; + SQLite3_result *res = NULL; + std::string q = "SELECT COALESCE(MAX(timestamp),0) FROM tsdb_metrics_cluster WHERE node='" + escape_sql_string_literal(node) + "'"; + statsdb_disk->execute_statement(q.c_str(), &error, &cols, &affected_rows, &res); + long max_ts = 0; + if (error == NULL && res != NULL && res->rows_count > 0) { + max_ts = atol(res->rows[0]->fields[0]); + } + if (error) free(error); + if (res) delete res; + return max_ts; +} + +SQLite3_result * ProxySQL_Statistics::get_tsdb_cluster_nodes() { + char *error = NULL; int cols = 0; int affected_rows = 0; + SQLite3_result *res = NULL; + statsdb_disk->execute_statement( + "SELECT node, MAX(timestamp) AS last_timestamp, COUNT(*) AS datapoints FROM tsdb_metrics_cluster GROUP BY node ORDER BY node", + &error, &cols, &affected_rows, &res); + if (error) { + proxy_error("get_tsdb_cluster_nodes: %s\n", error); + free(error); + } + return res; // may be NULL on error; callers must handle +} + +void ProxySQL_Statistics::tsdb_cluster_replicate_self(const std::string& node, long watermark, int limit) { + std::string esc_node = escape_sql_string_literal(node); + // >= (not >): the sampler stamps every series in a tick with the same time_t, + // so rows arrive in same-timestamp groups. The watermark for the next cycle is + // re-derived as MAX(timestamp) already replicated (tsdb_cluster_node_max_ts), so + // re-fetching the boundary group is required whenever a previous cycle's LIMIT + // cut in the middle of it — otherwise the unreplicated remainder of that group + // is skipped forever. INSERT OR IGNORE + the (node, timestamp, metric_name, + // labels) PK make re-fetching the boundary group idempotent. + // + // No separate no-progress guard here (unlike tsdb_cluster_replicate_peer): + // this is a single INSERT..SELECT, and cheaply detecting "the whole LIMIT was + // consumed by one timestamp group" would require an extra read-only query to + // learn the max timestamp actually selected, which would complicate the + // single-statement path for a self-replication stall that shares the exact + // same root cause and the exact same operational remedy (raise + // tsdb-cluster_batch_rows) as the peer path. In any multi-node cluster the + // peer path's identical per-tick fan-out already surfaces the warning; a + // single-node cluster stalling here is a known limitation of this path. + std::string sql = + "INSERT OR IGNORE INTO tsdb_metrics_cluster (node, timestamp, metric_name, labels, value) " + "SELECT '" + esc_node + "', timestamp, metric_name, labels, value FROM tsdb_metrics " + "WHERE timestamp >= " + std::to_string(watermark) + " ORDER BY timestamp LIMIT " + std::to_string(limit); + // Even a single statement must take the write lock: on the shared statsdb_disk + // connection, an unlocked write here could execute inside another thread's + // still-open explicit transaction (tsdb_sampler_loop / tsdb_monitor_loop use + // the same connection from the admin thread). + statsdb_disk->wrlock(); + statsdb_disk->execute(sql.c_str()); + statsdb_disk->wrunlock(); +} + +bool ProxySQL_Statistics::tsdb_cluster_replicate_peer(const std::string& host, int port, const std::string& node, long watermark, int limit, const std::string& user, const std::string& pass, long persisted_max_ts) { + MYSQL *conn = mysql_init(NULL); + if (conn == NULL) return false; + // Same options as the cluster monitor threads (lib/ProxySQL_Cluster.cpp:207-217) + unsigned int timeout = 1; + mysql_options(conn, MYSQL_OPT_CONNECT_TIMEOUT, &timeout); + { + unsigned char val = 1; + mysql_options(conn, MYSQL_OPT_SSL_ENFORCE, &val); + mysql_options(conn, MARIADB_OPT_SSL_KEYLOG_CALLBACK, (void *)proxysql_keylog_write_line_callback); + } + // Unlike the cluster monitor threads (which run detached and can tolerate an + // unbounded stall), this worker is synchronously pthread_join()'d by the admin + // thread when leadership is lost. Bound read/write I/O so a stalled/unresponsive + // peer can't wedge the admin thread (and leader_election_tick with it) for the + // OS TCP timeout. + unsigned int rw_timeout = 10; + mysql_options(conn, MYSQL_OPT_READ_TIMEOUT, &rw_timeout); + mysql_options(conn, MYSQL_OPT_WRITE_TIMEOUT, &rw_timeout); + if (mysql_real_connect(conn, host.c_str(), user.c_str(), pass.c_str(), NULL, port, NULL, 0) == NULL) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "TSDB cluster aggregation: cannot connect to %s : %s\n", node.c_str(), mysql_error(conn)); + mysql_close(conn); + return false; } + + // Peer is reachable: track per-peer watermark progress for stall visibility. + // Worker-thread-only maps, no locking needed. + // Compared against the PERSISTED watermark (MAX(timestamp) actually + // replicated), not the effective/backfill-adjusted one: for a + // never-replicated (or permanently disabled) peer the persisted value is + // stable at 0 across cycles, whereas the effective watermark keeps moving + // with `now` every cycle and would mask a permanent stall as "progress". + std::map::iterator wm_it = tsdb_agg_peer_last_wm.find(node); + if (wm_it != tsdb_agg_peer_last_wm.end() && wm_it->second == persisted_max_ts) { + int stalled = ++tsdb_agg_peer_stall_count[node]; + if (stalled >= 10 && tsdb_agg_peer_stall_logged[node] == false) { + proxy_info("TSDB cluster aggregation: no new samples from %s — peer TSDB likely disabled\n", node.c_str()); + tsdb_agg_peer_stall_logged[node] = true; + } + } else { + tsdb_agg_peer_last_wm[node] = persisted_max_ts; + tsdb_agg_peer_stall_count[node] = 0; + tsdb_agg_peer_stall_logged[node] = false; + } + + char q[512]; + // >= (not >): see the comment in tsdb_cluster_replicate_self() — the sampler + // stamps every series in a tick with the same time_t, so a LIMIT can cut inside + // a same-timestamp group. The watermark is re-derived from MAX(timestamp) + // already replicated, so the boundary group must be re-fetched or its + // unreplicated remainder is skipped forever. INSERT OR IGNORE below makes + // re-fetching it idempotent. + snprintf(q, sizeof(q), + "SELECT timestamp, metric_name, labels, value FROM stats_history.tsdb_metrics " + "WHERE timestamp >= %ld ORDER BY timestamp LIMIT %d", + watermark, limit); + bool cap_hit = false; + bool no_progress = false; + if (mysql_query(conn, q) == 0) { + MYSQL_RES *res = mysql_store_result(conn); + if (res) { + int rc = 0; + if (stmt_insert_tsdb_cluster_metric == NULL) { + sqlite3 *mydb3 = statsdb_disk->get_db(); + const char *query = "INSERT OR IGNORE INTO tsdb_metrics_cluster (node, timestamp, metric_name, labels, value) VALUES (?1, ?2, ?3, ?4, ?5)"; + rc = (*proxy_sqlite3_prepare_v2)(mydb3, query, -1, &stmt_insert_tsdb_cluster_metric, 0); + if (rc != SQLITE_OK) { + proxy_error("Failed to prepare statement: %s\n", (*proxy_sqlite3_errmsg)(mydb3)); + mysql_free_result(res); + mysql_close(conn); + return false; + } + } + int rows = 0; + long last_row_ts = watermark; + // Same shared-connection concern as tsdb_cluster_replicate_self(): take the + // write lock for the whole explicit transaction so it can't interleave with + // tsdb_sampler_loop's / tsdb_monitor_loop's BEGIN..COMMIT on the admin thread. + // All rows were already buffered locally by mysql_store_result() above, so no + // network I/O happens while the lock is held. + statsdb_disk->wrlock(); + statsdb_disk->execute("BEGIN"); + sqlite3 *mydb = statsdb_disk->get_db(); + long long changes_before = (*proxy_sqlite3_total_changes64)(mydb); + MYSQL_ROW row; + while ((row = mysql_fetch_row(res))) { + rc = (*proxy_sqlite3_bind_text)(stmt_insert_tsdb_cluster_metric, 1, node.c_str(), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb_disk); + rc = (*proxy_sqlite3_bind_int64)(stmt_insert_tsdb_cluster_metric, 2, atoll(row[0])); ASSERT_SQLITE_OK(rc, statsdb_disk); + rc = (*proxy_sqlite3_bind_text)(stmt_insert_tsdb_cluster_metric, 3, row[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb_disk); + rc = (*proxy_sqlite3_bind_text)(stmt_insert_tsdb_cluster_metric, 4, (row[2] ? row[2] : "{}"), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb_disk); + rc = (*proxy_sqlite3_bind_double)(stmt_insert_tsdb_cluster_metric, 5, (row[3] ? atof(row[3]) : 0.0)); ASSERT_SQLITE_OK(rc, statsdb_disk); + SAFE_SQLITE3_STEP2(stmt_insert_tsdb_cluster_metric); + rc = (*proxy_sqlite3_clear_bindings)(stmt_insert_tsdb_cluster_metric); ASSERT_SQLITE_OK(rc, statsdb_disk); + rc = (*proxy_sqlite3_reset)(stmt_insert_tsdb_cluster_metric); ASSERT_SQLITE_OK(rc, statsdb_disk); + last_row_ts = atol(row[0]); + rows++; + } + statsdb_disk->execute("COMMIT"); + // Counter reflects net-new replicated rows; boundary re-fetches are ignored by + // the PK and not counted. Read the "after" counter before releasing the write + // lock, otherwise an unrelated writer that runs between wrunlock() and this + // read can inflate the delta. + long long inserted_rows = (*proxy_sqlite3_total_changes64)(mydb) - changes_before; + statsdb_disk->wrunlock(); + tsdb_agg_rows_total.fetch_add(inserted_rows); + Tsdb_Agg_Fetch_Result fr = tsdb_agg_apply_fetch(watermark, rows, last_row_ts, limit); + cap_hit = (fr.caught_up == false); + // fr.new_watermark is informational here: the watermark is re-derived + // from MAX(timestamp) in the table each cycle (restart-safe by design). + // No-progress: an entire full-limit fetch landed inside the single + // timestamp group at `watermark` (last_row_ts never moved past it). + // The watermark can never advance past this point until the batch + // size is raised — the next cycle will re-issue the identical query. + no_progress = (rows == limit && last_row_ts == watermark); + mysql_free_result(res); + } + } else { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "TSDB cluster aggregation: query failed on %s : %s\n", node.c_str(), mysql_error(conn)); + } + mysql_close(conn); + + // Falling-behind visibility: warn once per episode when the peer keeps + // hitting the per-cycle row cap for 3+ consecutive cycles. + if (cap_hit) { + int hits = ++tsdb_agg_peer_cap_hit_count[node]; + if (hits == 3) { + proxy_warning("TSDB cluster aggregation: peer %s is falling behind - hit the per-cycle row cap (%d rows) for %d consecutive cycles\n", node.c_str(), limit, hits); + } + } else { + tsdb_agg_peer_cap_hit_count[node] = 0; + } + + // Stuck visibility: a full-limit fetch that never moved past the timestamp + // group it started from means the watermark cannot advance at all — raising + // tsdb-cluster_batch_rows is the only way forward. Same episode-tracking + // style as the cap-hit warning above (log once per episode, at hits==3). + if (no_progress) { + int hits = ++tsdb_agg_peer_progress_stall_count[node]; + if (hits == 3) { + proxy_warning("TSDB cluster aggregation: tsdb-cluster_batch_rows smaller than per-timestamp series count for node %s; aggregation cannot progress - raise tsdb-cluster_batch_rows\n", node.c_str()); + } + } else { + tsdb_agg_peer_progress_stall_count[node] = 0; + } + + return cap_hit; } #endif diff --git a/lib/TSDB_Cluster_Aggregator.cpp b/lib/TSDB_Cluster_Aggregator.cpp new file mode 100644 index 0000000000..7826f2591e --- /dev/null +++ b/lib/TSDB_Cluster_Aggregator.cpp @@ -0,0 +1,21 @@ +#include "TSDB_Cluster_Aggregator.h" + +long tsdb_agg_effective_watermark(long existing_max_ts, long now, int backfill_hours) { + long horizon = now - (long)backfill_hours * 3600L; + if (existing_max_ts > horizon) { + return existing_max_ts; + } + return horizon; +} + +Tsdb_Agg_Fetch_Result tsdb_agg_apply_fetch(long prev_watermark, int rows_fetched, long last_row_ts, int limit) { + Tsdb_Agg_Fetch_Result r; + if (rows_fetched == 0) { + r.new_watermark = prev_watermark; + r.caught_up = true; + return r; + } + r.new_watermark = last_row_ts; + r.caught_up = (rows_fetched < limit); + return r; +} diff --git a/lib/TSDB_Dashboard_html.cpp b/lib/TSDB_Dashboard_html.cpp index c730d98423..deee8baa1f 100644 --- a/lib/TSDB_Dashboard_html.cpp +++ b/lib/TSDB_Dashboard_html.cpp @@ -26,6 +26,10 @@ const char * TSDB_Dashboard_html_c = R"HTML( +
+ + +