diff --git a/doc/AWS_Blue_Green/RDS_BGD_Simulator.md b/doc/AWS_Blue_Green/RDS_BGD_Simulator.md index 7c228a93bc..01777d12d8 100644 --- a/doc/AWS_Blue_Green/RDS_BGD_Simulator.md +++ b/doc/AWS_Blue_Green/RDS_BGD_Simulator.md @@ -78,7 +78,7 @@ Create the following tables in `SQLite3_Server::init()` and store them in the existing persistent `GloVars.sqlite3serverdb` database: ```sql -CREATE TABLE RDS_BGD_CONTROL ( +CREATE TABLE AWS_BGD_CONTROL ( backend_ip TEXT NOT NULL, backend_port INTEGER NOT NULL, topology_present INTEGER NOT NULL DEFAULT 0 CHECK (topology_present IN (0,1)), @@ -87,7 +87,7 @@ CREATE TABLE RDS_BGD_CONTROL ( PRIMARY KEY (backend_ip, backend_port) ); -CREATE TABLE RDS_BGD_TOPOLOGY ( +CREATE TABLE AWS_BGD_TOPOLOGY ( backend_ip TEXT NOT NULL, backend_port INTEGER NOT NULL, row_order INTEGER NOT NULL, @@ -99,7 +99,7 @@ CREATE TABLE RDS_BGD_TOPOLOGY ( PRIMARY KEY (backend_ip, backend_port, row_order) ); -CREATE TABLE RDS_BGD_PROBE_LOG ( +CREATE TABLE AWS_BGD_PROBE_LOG ( sequence_id INTEGER PRIMARY KEY AUTOINCREMENT, backend_ip TEXT NOT NULL, backend_port INTEGER NOT NULL, @@ -141,7 +141,7 @@ The TAP helper publishes control and topology changes atomically. Monitor probes read committed simulator state without holding a cross-query snapshot. The supported states are: -| `RDS_BGD_CONTROL` state | Topology rows | Meaning | +| `AWS_BGD_CONTROL` state | Topology rows | Meaning | |---|---|---| | No backend row | None | Backend is unconfigured; topology is absent. | | `topology_present=1`, `error_code=0` | One or more | Return the configured topology. | @@ -180,7 +180,7 @@ configured simulator errors may use `HY000` unless a test requires a specific mapping. Every handled topology-table check or metadata query appends one row to -`RDS_BGD_PROBE_LOG`, including empty and error responses. `sequence_id` +`AWS_BGD_PROBE_LOG`, including empty and error responses. `sequence_id` preserves order, `probe_kind` identifies the query, `backend_ip, backend_port` identify the destination, and `encrypted` records the accepted stream's TLS state. @@ -205,7 +205,7 @@ returns one `read_only` column. Table writes refresh the cache, and a missing entry returns the safe default `read_only=1`. BGD topology tasks send the production topology queries unchanged. Read-only -handling does not consult `RDS_BGD_CONTROL` or write `RDS_BGD_PROBE_LOG`. +handling does not consult `AWS_BGD_CONTROL` or write `AWS_BGD_PROBE_LOG`. ## TAP Helper API @@ -246,7 +246,7 @@ API; the helper closes it when destroyed. `read_only_update()` changes the ### Topology and Host Types ```cpp -struct RDS_BGD_Topology_Row { +struct BGD_Topology_Row { std::string id; std::string endpoint; int port; @@ -264,7 +264,7 @@ struct RDS_BGD_Host { }; ``` -`RDS_BGD_Topology_Row` represents one `mysql.rds_topology` row using +`BGD_Topology_Row` represents one `mysql.rds_topology` row using C++11-compatible field types. `RDS_BGD_Host` keeps the ProxySQL-facing hostname and simulator-facing IP together. @@ -282,7 +282,7 @@ public: std::vector get_blue_endpoints(); std::vector get_green_endpoints(); std::vector get_endpoints(); - std::vector get_topology(std::string status); + std::vector get_topology(std::string status); }; ``` @@ -300,7 +300,7 @@ reader mapping. ```cpp int topology_update( std::vector backends, - std::vector rows); + std::vector rows); int topology_delete(std::vector backends); @@ -330,26 +330,26 @@ not inherit simulator state from an earlier binary. ### Probe-Log Operations ```cpp -enum class RDS_BGD_Probe_Kind { +enum class BGD_Probe_Kind { table_check, metadata, }; -struct RDS_BGD_Probe_Log { +struct BGD_Probe_Log { uint64_t sequence_id; Endpoint backend; - RDS_BGD_Probe_Kind probe_kind; + BGD_Probe_Kind probe_kind; bool encrypted; }; rc_t probe_log_last_sequence(); -rc_t> probe_log_since(uint64_t sequence_id); +rc_t> probe_log_since(uint64_t sequence_id); -rc_t wait_for_probe_log( +rc_t wait_for_probe_log( uint64_t sequence_id, Endpoint backend, - RDS_BGD_Probe_Kind probe_kind, + BGD_Probe_Kind probe_kind, uint32_t timeout_ms, int encrypted = -1); ``` @@ -366,7 +366,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator simulator {}; + BGD_Simulator simulator {}; if (setup(cl, admin, simulator) != EXIT_SUCCESS) return exit_status(); diff --git a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md new file mode 100644 index 0000000000..5acaa5f887 --- /dev/null +++ b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md @@ -0,0 +1,484 @@ +# AWS Aurora Blue/Green Cluster Simulator and Testing Design + +**Date:** 2026-07-31 + +**Branch:** `spec/aws-aurora-bgd` + +**Status:** Design approved + +**Related designs:** + +- [Aurora BGD Configuration, Runtime Status, and Cluster Sync](2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md) +- [Aurora BGD Monitor Loop and FSM](2026-07-31-aurora-bgd-monitor-fsm-design.md) + +## 1. Scope + +This specification defines the simulator services, test controllers, endpoint +model, and executable coverage for AWS Aurora MySQL blue/green deployments. +The related designs remain authoritative for configuration, runtime status, +cluster synchronization, monitor scheduling, FSM transitions, and routing +behavior. + +The test environment drives the complete Aurora BGD lifecycle without live AWS +infrastructure and retains regression coverage for ordinary Aurora monitoring +and RDS Multi-AZ BGD. + +## 2. Test Architecture + +The `cluster_sim_aurora` group contains two complementary test styles: + +- `test_cluster_sim_aurora-t` runs JSON-defined ordinary Aurora scenarios + through the standalone `cluster_simulator` controller; +- interactive TAP binaries drive Aurora BGD phases through the + `BGD_Simulator` helper. + +RDS Multi-AZ BGD TAP binaries remain in `cluster_sim_rds_bgd`. A simulator +group is an execution and CI bucket; simulator capabilities come from the +ProxySQL build, SQLite-server handlers, control tables, and TAP helpers. +Only one test binary controls a given ProxySQL instance at a time, and every +binary owns setup and cleanup for its simulator state. + +The group runner enforces that ownership by starting one registered binary, +waiting for it to exit, and only then starting the next binary in sorted order. +Aurora BGD registration checks must retain that serial execution contract. If a +future runner executes binaries concurrently, each binary must receive an +isolated ProxySQL instance and simulator database. + +The build contract is: + +- `TEST_AURORA` enables ordinary Aurora and Aurora BGD simulation; +- `TEST_RDS_BGD` enables RDS Multi-AZ BGD simulation; +- AWS BGD topology services are available under either flag; +- Aurora replica services are available under `TEST_AURORA`; +- read-only simulation is available to Aurora BGD, RDS BGD, and focused + read-only simulator builds; +- no separate BGD simulator flag is used. + +Aurora monitor queries in simulator builds are the same production queries used +against AWS. The SQLite server identifies the simulated backend from the local +IP and port on which it accepted the monitor connection, then returns the state +assigned to that address. + +## 3. AWS BGD Topology Service + +Aurora BGD and RDS Multi-AZ BGD use the same simulated +`mysql.rds_topology` service. + +### 3.1 `AWS_BGD_CONTROL` + +One row controls topology behavior for one backend: + +```text +backend_ip TEXT NOT NULL +backend_port INTEGER NOT NULL +topology_present INTEGER NOT NULL DEFAULT 0 + CHECK (topology_present IN (0, 1)) +error_code INTEGER NOT NULL DEFAULT 0 +error_msg TEXT NOT NULL DEFAULT '' +PRIMARY KEY (backend_ip, backend_port) +``` + +`topology_present` is restricted to zero or one. A missing control row and a +row with `topology_present=0` both represent an absent topology table. Query +precedence is evaluated independently: + +| Query | Control state | Result | +|---|---|---| +| Table check | No row or `topology_present=0` | Successful empty result; `error_code` is ignored. | +| Table check | `topology_present=1` | Successful one-row result; `error_code` is ignored. | +| Metadata | No row | MySQL error 1146. | +| Metadata | `error_code!=0` | Configured MySQL error, including the defensive `topology_present=0` combination. | +| Metadata | `error_code=0`, `topology_present=0` | MySQL error 1146. | +| Metadata | `error_code=0`, `topology_present=1` | Ordered rows, including a successful empty result. | + +The helper normally publishes only consistent combinations: error 1146 marks +the table absent, and other errors retain a present table. The defensive matrix +keeps direct control-table writes deterministic. + +### 3.2 `AWS_BGD_TOPOLOGY` + +Successful topology results are stored as: + +```text +backend_ip TEXT NOT NULL +backend_port INTEGER NOT NULL +row_order INTEGER NOT NULL +id TEXT NOT NULL +endpoint TEXT NOT NULL +topology_port INTEGER NOT NULL +role TEXT NOT NULL +status TEXT NOT NULL +PRIMARY KEY (backend_ip, backend_port, row_order) +``` + +`row_order` makes the result deterministic. Aurora TARGET rows contain a +cluster endpoint; RDS Multi-AZ TARGET rows contain an instance endpoint. The +production monitor identifies the deployment type from the AWS metadata and +membership result rather than from simulator configuration. + +### 3.3 `AWS_BGD_PROBE_LOG` + +Every topology table check and metadata query records: + +```text +sequence_id INTEGER PRIMARY KEY AUTOINCREMENT +backend_ip TEXT NOT NULL +backend_port INTEGER NOT NULL +probe_kind TEXT NOT NULL # table_check or metadata +encrypted INTEGER NOT NULL + CHECK (encrypted IN (0, 1)) +``` + +`encrypted` is restricted to zero or one. The monotonic sequence supports +deterministic waits and ordering assertions. + +## 4. AWS Aurora Replica Service + +### 4.1 Membership sets + +`REPLICA_HOST_STATUS` stores each Aurora membership snapshot once: + +```text +REPLICA_SET_ID TEXT NOT NULL +SERVER_ID VARCHAR NOT NULL +SESSION_ID VARCHAR NOT NULL +CPU REAL NOT NULL +LAST_UPDATE_TIMESTAMP VARCHAR NOT NULL +REPLICA_LAG_IN_MILLISECONDS REAL NOT NULL +IS_CURRENT INTEGER NOT NULL DEFAULT 1 + CHECK (IS_CURRENT IN (0, 1)) +PRIMARY KEY (REPLICA_SET_ID, SERVER_ID) +``` + +`IS_CURRENT` is restricted to zero or one. It models the AWS column used by the +Aurora BGD membership query to exclude stale or decommissioned members. +Ordinary Aurora payloads default it to one. + +`REPLICA_SET_ID` is simulator-only and is never returned to ProxySQL. It groups +one membership snapshot independently of `DOMAIN_NAME`. The simulated table +does not contain `DOMAIN_NAME`; domain configuration remains in +`mysql_aws_aurora_hostgroups` and in the ordinary Aurora JSON input, where it +is used to construct member hostnames. + +Blue and green members share the same RDS domain suffix, so a BGD scenario uses +distinct replica-set identifiers for its blue and green snapshots. Multiple +backend addresses can map to the same set. A cluster with N members therefore +stores N membership rows plus lightweight backend mappings, rather than one +copy of the N rows per backend. + +### 4.2 `AWS_AURORA_REPLICA_CONTROL` + +One row maps a backend address to its replica set and controls the query +response: + +```text +backend_ip TEXT NOT NULL +backend_port INTEGER NOT NULL +replica_set_id TEXT NOT NULL +replica_table_present INTEGER NOT NULL DEFAULT 0 + CHECK (replica_table_present IN (0, 1)) +error_code INTEGER NOT NULL DEFAULT 0 +error_msg TEXT NOT NULL DEFAULT '' +PRIMARY KEY (backend_ip, backend_port) +``` + +`replica_table_present` is restricted to zero or one. Absence takes precedence +over a configured error. Response behavior is: + +- no control row or `replica_table_present=0`: MySQL error 1146, regardless of + `error_code`; +- present table and nonzero `error_code`: the configured MySQL error; +- present table, no error, and matching rows: return that set; +- present table, no error, and no matching rows: return a successful empty + result. + +Writer-only and intentionally incomplete snapshots are represented directly by +the rows stored for the selected set. No response-mode column is required. +Topology and replica errors remain independent because their controls are in +separate tables. + +### 4.3 `AWS_AURORA_REPLICA_PROBE_LOG` + +Every intercepted production `REPLICA_HOST_STATUS` query records: + +```text +sequence_id INTEGER PRIMARY KEY AUTOINCREMENT +backend_ip TEXT NOT NULL +backend_port INTEGER NOT NULL +probe_kind TEXT NOT NULL + CHECK (probe_kind IN ('ordinary', 'bgd_membership')) +replica_set_id TEXT NULL +encrypted INTEGER NOT NULL + CHECK (encrypted IN (0, 1)) +``` + +The accepted address is logged even when no control mapping exists or the query +returns an error. `probe_kind` distinguishes the ordinary production probe from +the Aurora BGD target-membership probe. `replica_set_id` is NULL when the +backend has no mapping. + +### 4.4 Query handling + +Ordinary Aurora and Aurora BGD issue their production-shaped +`REPLICA_HOST_STATUS` queries under `TEST_AURORA`. For a recognized monitor +query, the SQLite handler: + +1. reads the accepted backend IP and port; +2. records the probe; +3. reads the matching `AWS_AURORA_REPLICA_CONTROL` row; +4. returns the configured error or rewrites the query to select the mapped + replica set; +5. preserves the production query's result columns, filtering, and ordering. + +The ordinary Aurora query retains its timestamp, lag, and writer filtering. +The Aurora BGD query retains `IS_CURRENT` and `SERVER_ID` ordering. Other SQL +against the simulator tables executes normally so controllers can publish +state. + +## 5. State Publication + +### 5.1 Ordinary Aurora controller + +Ordinary Aurora JSON files retain their current schema. For each Aurora cluster, +the standalone `cluster_simulator` controller: + +1. uses the JSON `DOMAIN_NAME` value as the replica-set identifier; +2. constructs each member hostname from `SERVER_ID` and `DOMAIN_NAME`; +3. reads `CLUSTER_SIM_HOST_FILE` to resolve those fixed simulator hostnames to + loopback IPs; +4. writes the membership rows with a current timestamp and `IS_CURRENT=1`; +5. maps every backend that can serve the cluster membership to the same set in + `AWS_AURORA_REPLICA_CONTROL`; +6. commits the membership replacement and complete backend mapping atomically. + +The group environment already supplies `CLUSTER_SIM_HOST_FILE`; test +infrastructure uses it to populate the ProxySQL container's host map, and the +controller uses the same file as the authoritative hostname-to-IP mapping. A +configured or discovered member missing from that map is a test-setup failure. + +### 5.2 Aurora BGD controller + +Interactive Aurora BGD TAP tests publish state through `BGD_Simulator`. Each +membership publication supplies: + +- one stable replica-set identifier; +- one membership snapshot; +- every cluster endpoint or member backend that must return that snapshot. + +The helper replaces the set rows and complete backend mapping in one +transaction. Blue and green use separate identifiers. Tests do not store a +blue/green flag and do not infer environment identity from hostnames. + +The same helper publishes topology rows, topology errors, replica errors, +read-only state, and probe-log checkpoints. Non-1146 errors change only control +state, so clearing them exposes the retained data. An absent topology table +clears topology rows for the addressed backend, matching the topology service's +backend-scoped storage. An absent replica table preserves its shared membership +set because other backend mappings may still serve that set. + +### 5.3 Helper boundaries + +`BGD_Simulator` owns cross-service mechanics: SQL transactions, topology and +replica controls, read-only controls, probe-log reads and waits, endpoint +predicates, and cleanup. + +Deployment models remain engine-specific. `RDS_BGD_Cluster` and related RDS +types describe RDS instance topology. Aurora types describe cluster endpoints, +members, replica sets, rename identity, and expected runtime placement. + +### 5.4 Cleanup + +Set-scoped operations remove only the supplied replica set and backend +mappings. Full scenario cleanup clears: + +```text +READONLY_STATUS +AWS_BGD_CONTROL +AWS_BGD_TOPOLOGY +AWS_BGD_PROBE_LOG +AWS_AURORA_REPLICA_CONTROL +AWS_AURORA_REPLICA_PROBE_LOG +REPLICA_HOST_STATUS +``` + +Each TAP binary begins and ends with full cleanup. Sequential group execution +is expected, but correctness does not depend on a later test overwriting stale +state. + +## 6. Endpoint Model + +`cluster_sim_aurora/add-hosts` contains fixed aliases for: + +- canonical blue cluster members; +- the green target cluster endpoint returned by `mysql.rds_topology`; +- green target writer and reader members; +- canonical post-switchover member names; +- repeated deployments and concurrent test clusters. + +Independently controlled backends use distinct loopback IPs. Green and +canonical aliases may share an IP where the scenario represents AWS resource +renaming. Response selection always uses the accepted IP and port, not the +requested hostname. + +The fixed host map does not model mutable Route 53 propagation. DNS-related +tests assert cached-IP pins, accepted backend addresses, connection-pool +behavior, and explicit pin removal. + +## 7. Test Coverage + +### 7.1 Simulator contracts + +Focused tests cover: + +- table schemas, primary keys, defaults, and value constraints; +- topology and replica routing by accepted backend address; +- several backend addresses sharing one replica set; +- independent blue and green sets with overlapping `SERVER_ID` values; +- production-query result shape, filtering, and ordering; +- atomic membership and mapping replacement; +- successful complete, writer-only, incomplete, and empty results; +- missing tables and arbitrary MySQL errors; +- table-present/error precedence matrices for topology table-check, topology + metadata, and replica membership queries; +- independence of topology and replica errors; +- TLS state and ordered probe logging; +- `CLUSTER_SIM_HOST_FILE` parsing and missing-host failures; +- set-scoped deletion and full cleanup. + +### 7.2 Configuration, runtime, and cluster synchronization + +Tests owned by the configuration design cover: + +- configuration and runtime table schemas and column order; +- paired-NULL and hostgroup-conflict validation; +- config-file, memory, runtime, and disk round trips; +- runtime-only `bgd_status` initialization and preservation; +- exclusion of `bgd_status` from SAVE and cluster synchronization; +- synchronization of the configured green hostgroups, including NULL; +- independent node-local runtime status on cluster peers. + +### 7.3 Interactive Aurora BGD scenarios + +Interactive TAP tests cover: + +1. `AVAILABLE` discovery of the target writer and all current readers through + a complete target membership snapshot. +2. Green hostgroups configured and green hostgroups not configured, with all + Aurora hostgroup rows created by the test. +3. Independent topology, target-membership, and ordinary Aurora probes with the + cadence and on/off behavior defined by the monitor/FSM design, including + random blue-member topology probes and random green-member target probes. +4. `SWITCHOVER_INITIATED`, `SWITCHOVER_IN_PROGRESS`, and + `SWITCHOVER_IN_POST_PROCESSING` state retention and retry behavior. +5. Source-writer read-only observation and the corresponding writer-to-reader + placement. +6. Target member `SERVER_ID` rename with stable reader `SESSION_ID` identity + and retained cached IPs. +7. Idempotent per-member pinning, placement, and connection-retirement marking, + without treating asynchronous physical closure as an FSM prerequisite. +8. The first TARGET-only `SWITCHOVER_COMPLETED`, immediate effect-driven + cleanup, terminal latch, and runtime + `bgd_status='SWITCHOVER_COMPLETED'`. +9. Repeated completion observations, successful topology drain to `NONE`, and + rearming for a different deployment fingerprint. + +### 7.4 Resilience and lifecycle scenarios + +Coverage also includes: + +- zero-reader targets and membership row ordering; +- incomplete snapshots retaining the last complete target set; +- topology, replica, and read-only query failures in each relevant phase; +- rollback from initiated, in-progress, and post-processing states; +- late entry at initiated, in-progress, post-processing, and completed states; +- direct completion with an empty member map producing no production-member + routing or retirement actions; +- completion directly after in-progress restoring the writer without replaying + post-processing; +- completion after post-processing removing pins without reapplying connection + retirement or waiting for marked connections to close; +- configuration and hostgroup refresh during an active deployment; +- in-place worker configuration refresh with retained deployment state; +- deleting or deactivating an Aurora row during switchover; +- ONLINE, SHUNNED, OFFLINE_SOFT, and OFFLINE_HARD staging-pool members; +- TLS selection for topology and replica probes; +- repeated and concurrent Aurora deployments with isolated state. + +### 7.5 Regression suites + +The complete ordinary Aurora JSON suite runs through the production query path +and retains its role, failover, lag, autodiscovery, and autopurge expectations. + +The RDS Multi-AZ BGD TAP suite retains its instance-target, reader policy, +writer fallback, completion, and topology-drain expectations while using the +shared `BGD_Simulator` and `AWS_BGD_*` services. + +## 8. Determinism and Diagnostics + +Tests synchronize on observable state: probe sequence, runtime `bgd_status`, +server placement, pool counters, and committed simulator controls. Timeouts are +derived from configured monitor intervals with bounded slack. + +Fixed sleeps are permitted only for bounded negative assertions with no event +that can be awaited directly. A failed wait reports the latest runtime row, +relevant server and pool state, and probes observed since the scenario +checkpoint. + +Random probe-selection tests do not assert one exact random order. They assert +that every observed destination belongs to the scenario's eligible set and +that fallback reaches another eligible member within a bounded number of +attempts. Failure diagnostics include the complete probe sequence since the +scenario checkpoint, so production randomness cannot make the expected result +order-dependent. + +Scenarios use disjoint hostgroups, endpoints, replica-set identifiers, and +deployment fingerprints, or perform full cleanup before reuse. + +## 9. Build and CI Contract + +`make testaurora` produces the focused Aurora simulator build, including the +Aurora replica, shared BGD topology, and read-only services. `make test_rds_bgd` +produces the focused RDS BGD build. The combined `testall` build enables both +families. + +Ordinary Aurora and Aurora BGD binaries register in +`cluster_sim_aurora-g1`. RDS Multi-AZ BGD binaries register in +`cluster_sim_rds_bgd-g1`. Central simulator CI discovers groups and binaries +from `groups.json`, so no additional workflow, simulator group family, or CI +infrastructure is required. + +CI must continue to invoke the group through the serial group runner described +in Section 2. Registration coverage verifies that every Aurora BGD binary is in +that group and is not scheduled simultaneously against the same ProxySQL +instance. + +## 10. Non-Goals + +The simulator does not validate: + +- AWS control-plane APIs or real AWS timing; +- mutable Route 53 propagation; +- application-level latency or packet loss; +- cross-process persistence of transient FSM state; +- Aurora versions that violate the AWS-provided topology-status contract. + +The AWS-provided topology metadata semantics define the status-to-routing +contract. Live-AWS evidence remains the validation layer for observed status +ordering, member rename, DNS completion, and corroboration of the writability +transition. + +## 11. Acceptance Criteria + +The design is satisfied when: + +1. Ordinary Aurora and Aurora BGD use production monitor queries and the same + backend-address-to-replica-set service. +2. Blue and green membership snapshots are stored once and returned from every + mapped backend. +3. The complete Aurora BGD FSM and its failure paths run deterministically + without live AWS infrastructure. +4. Runtime status, routing, pools, cached IPs, probes, refresh, and cluster-sync + contracts have observable assertions. +5. Ordinary Aurora and RDS Multi-AZ BGD regressions retain their semantics. +6. The suite uses the existing Aurora and RDS simulator CI groups without a + separate feature flag or simulator family. diff --git a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md new file mode 100644 index 0000000000..fa1b7234bd --- /dev/null +++ b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md @@ -0,0 +1,444 @@ +# Aurora BGD Configuration, Runtime Status, and Cluster Sync Design + +**Date:** 2026-07-31 + +**Branch:** `spec/aws-aurora-bgd` + +**Status:** Design approved + +**Scope:** Configuration and runtime integration of Aurora MySQL blue/green +deployment handling with the existing `mysql_aws_aurora_hostgroups` subsystem. + +**Related designs:** + +- [Aurora BGD Monitor Loop and FSM](2026-07-31-aurora-bgd-monitor-fsm-design.md) +- [Aurora BGD Cluster Simulator and Testing](2026-07-31-aurora-bgd-cluster-simulator-testing-design.md) + +## Spec Boundary + +This specification owns the public configuration contract, the runtime-only +`bgd_status` contract, load/save and schema-upgrade behavior, and ProxySQL +Cluster synchronization. It treats the worker state machine as a consumer and +publisher through the interface defined here; membership discovery, routing +actions, and FSM internals belong to the monitor/FSM specification. + +## 1. Decision Summary + +Aurora blue/green handling is part of the existing Aurora monitor. It does not +require an entry in `mysql_aws_rds_bgd_hostgroups`. + +Two nullable columns are added to `mysql_aws_aurora_hostgroups` and +`runtime_mysql_aws_aurora_hostgroups`: + +```sql +green_writer_hostgroup INT DEFAULT NULL +green_reader_hostgroup INT DEFAULT NULL +``` + +The columns provide staging hostgroups when configured. When both are NULL, the +existing global variable `mysql-aws_blue_green_deployment_auto_discovery` +controls whether the worker may start BGD discovery for the row. + +No separate Aurora BGD enable column is added. + +The runtime table adds one runtime-only observability column: + +```sql +bgd_status VARCHAR NOT NULL DEFAULT 'NONE' +``` + +It reports the Aurora worker's local BGD FSM state. It is not configuration and +is excluded from persistence and ProxySQL Cluster synchronization. + +## 2. Table Schema + +The configuration table uses the following column order: + +```text +writer_hostgroup +reader_hostgroup +green_writer_hostgroup +green_reader_hostgroup +active +aurora_port +domain_name +max_lag_ms +check_interval_ms +check_timeout_ms +writer_is_also_reader +new_reader_weight +add_lag_ms +min_lag_ms +lag_num_checks +autopurge_missing_checks +comment +``` + +The runtime table uses the same order and appends: + +```text +bgd_status +``` + +The new column definitions are: + +```sql +green_writer_hostgroup INT DEFAULT NULL + CHECK (green_writer_hostgroup IS NULL OR green_writer_hostgroup >= 0), + +green_reader_hostgroup INT DEFAULT NULL + CHECK (green_reader_hostgroup IS NULL OR green_reader_hostgroup >= 0) +``` + +Both columns are nullable in the configuration and runtime tables. Their NULL +state is meaningful and must be preserved by every load, save, export, import, +and cluster-synchronization path. + +The runtime table mirrors these configured fields and adds only `bgd_status`. +Every Aurora row is user-created. There is no BGD mode or generated-row concept +in this table, so no `mode` or `auto_generated` column is present. + +## 3. Green Hostgroup Configuration and BGD Admission + +An active Aurora row behaves as follows: + +| Green hostgroup columns | `mysql-aws_blue_green_deployment_auto_discovery` | Behavior | +|---|---:|---| +| Both non-NULL | `0` or `1` | BGD discovery is admitted | +| Both NULL | `1` | BGD discovery is admitted | +| Both NULL | `0` | Aurora monitoring without BGD discovery | +| Exactly one NULL | `0` or `1` | Invalid configuration | +| Any values with `active=0` | `0` or `1` | Neither normal Aurora nor BGD monitoring is active | + +Changing `active` from `1` to `0` uses the worker-removal cleanup contract in +the monitor/FSM specification before the worker exits: remove every applied +pin, drain and purge affected production-hostname pools, restore safe writer +placement, release suspended monitoring, clear any terminal latch and retained +fingerprint, and publish `bgd_status=NONE`. The configuration and +`mysql_servers` rows remain user-owned and are not deleted by this cleanup. + +### 3.1 Green hostgroups configured + +When both green hostgroups are non-NULL: + +- BGD monitoring is enabled independently of the global auto-discovery value. +- The target cluster writer is associated with `green_writer_hostgroup`. +- All target cluster readers are associated with `green_reader_hostgroup`. +- Target membership still comes from the target cluster's + `REPLICA_HOST_STATUS`; configured green `mysql_servers` rows are not the + membership source of truth. +- The green hostgroups provide optional user-visible staging/routing pools in + addition to the internal BGD member map. + +### 3.2 Green hostgroups not configured + +When both green hostgroups are NULL and +`mysql-aws_blue_green_deployment_auto_discovery=true` at runtime: + +- The existing Aurora monitor detects BGD topology for the row. +- The monitor discovers the target writer and all target readers through + `REPLICA_HOST_STATUS`. +- The complete target member map and cached IPs remain internal. +- No green hostgroups or green `mysql_servers` rows are generated. +- Post-processing pinning covers the writer and every reader despite the green + hostgroup columns being NULL. + +This path does not use the writer-only fallback or reader shun/unshun policy +from the Multi-AZ instance implementation. + +### 3.3 BGD disabled for the row + +When both green hostgroups are NULL and +`mysql-aws_blue_green_deployment_auto_discovery=false`, the worker continues its +existing Aurora role and lag monitoring but does not start discovery of a new +BGD deployment. + +Disabling global auto-discovery after a switchover has started must not abandon +that switchover. The worker completes pin cleanup and enters the terminal latch +before disabling discovery for the row. The variable gates the start of a new +BGD state machine, not safe completion of one already in progress. + +## 4. Validation Rules + +For each `mysql_aws_aurora_hostgroups` row: + +1. `green_writer_hostgroup` and `green_reader_hostgroup` must either both be + NULL or both be non-NULL. +2. When non-NULL, all four hostgroups must be distinct: + + ```text + writer_hostgroup + reader_hostgroup + green_writer_hostgroup + green_reader_hostgroup + ``` + +3. A hostgroup assigned to one Aurora row must not conflict with any blue or + green role in another active Aurora row. +4. Existing writer-hostgroup primary-key and reader-hostgroup uniqueness rules + remain in effect. +5. Invalid rows use per-row isolation: each invalid row is excluded from runtime + loading, while the valid rows from the same LOAD remain eligible to publish. + The admin error identifies the rejected writer hostgroup, the conflicting + fields and values, and the other writer hostgroup involved in a cross-row + conflict. + +Validation of the complete candidate result set must finish before publishing +the filtered valid result set. Valid rows publish together only after that +validation succeeds and atomically replace the prior runtime configuration. A +rejected row contributes no new fields; if it previously owned a runtime +worker, its absence from the replacement triggers the worker-removal cleanup +contract rather than applying a mixture of old and invalid fields. + +## 5. Runtime Ownership + +`AWS_Aurora_Info`, keyed by `writer_hostgroup`, remains the runtime owner of the +cluster configuration. It is extended with optional integer fields: + +```text +green_writer_hostgroup = -1 when SQL NULL +green_reader_hostgroup = -1 when SQL NULL +``` + +The existing Aurora monitor worker remains one worker per active writer +hostgroup. The worker owns the BGD FSM and publishes each transition to the +runtime row's `bgd_status`. BGD admission and configured staging references are +determined from: + +```text +AWS_Aurora_Info.active +green_writer_hostgroup +green_reader_hostgroup +mysql_thread___aws_blue_green_deployment_auto_discovery +``` + +No Aurora configuration is copied into `mysql_aws_rds_bgd_hostgroups`, and no +second BGD worker is started for the same Aurora writer hostgroup. + +Following RDS BGD, the Hostgroups Manager's internal Aurora table is the source +used to materialize `runtime_mysql_aws_aurora_hostgroups`. It stores the +configured fields plus `bgd_status`. Its runtime dump includes all of those +fields; paths that write back to configuration explicitly project away +`bgd_status`. + +Configuration materialization and worker status publication use the same +Hostgroups Manager write-locked update path. Reload merges configured fields +into an existing writer-hostgroup row without writing its `bgd_status`; the +worker status API updates only `bgd_status` under that lock. A reload that began +from an older snapshot therefore cannot overwrite a transition published while +the reload is being applied. + +## 6. LOAD Behavior + +`LOAD MYSQL SERVERS TO RUNTIME` must: + +1. Read both new columns from `mysql_aws_aurora_hostgroups`. +2. Preserve SQL NULL as the internal unset value rather than converting it to + hostgroup `0`. +3. Validate paired NULL/non-NULL and hostgroup-conflict rules. +4. Update or create the `AWS_Aurora_Info` entry. +5. Include both fields in the Aurora monitor result set checksum. +6. Refresh the affected Aurora writer-hostgroup worker in place when either + green hostgroup changes. Apply only configuration-derived fields and staging + references; do not replace the worker-owned FSM object. +7. Preserve an active BGD FSM safely across an unrelated configuration refresh. +8. Initialize `bgd_status` to `NONE` for a newly published runtime row. +9. Preserve the existing `bgd_status` when merging an existing + writer-hostgroup runtime row, following the RDS BGD table-reload behavior. + The Aurora worker separately preserves the underlying FSM state required by + the monitor/FSM specification. +10. Remove `bgd_status` with a removed runtime row; active-operation teardown + follows the existing RDS BGD worker-removal cleanup pattern. + +Changing the green hostgroups during an active switchover must not lose cached +member identities, applied DNS pins, or completion-latch state. The worker must +apply the refreshed staging configuration without restarting the BGD operation +from `NONE`. + +`LOAD MYSQL VARIABLES TO RUNTIME` makes a change to +`mysql-aws_blue_green_deployment_auto_discovery` visible to Aurora monitor +workers. +The variable controls admission of new BGD operations for rows without green +hostgroups, as described in Section 3.3. + +## 7. Runtime Table Behavior + +`runtime_mysql_aws_aurora_hostgroups` exposes both new columns in the same +positions as the configuration table and appends the runtime-only +`bgd_status` column. + +`bgd_status` exposes the internal Aurora BGD FSM state, not merely the last raw +AWS status string. Its values are: + +```text +NONE +AVAILABLE +SWITCHOVER_INITIATED +SWITCHOVER_IN_PROGRESS +SWITCHOVER_IN_POST_PROCESSING +SWITCHOVER_COMPLETED +``` + +The worker updates this column as its FSM changes, following the existing RDS +BGD runtime-status mechanism wherever applicable. `SWITCHOVER_COMPLETED` is the +terminal rearm latch: all routing cleanup has already completed, and the status +remains visible until `mysql.rds_topology` drains. A successful empty/absent +topology result then changes it to `NONE`. Query or connection errors while +latched do not reset the status or repeat cleanup. + +An Aurora row that is not handling a BGD deployment reports `NONE`, including +ordinary Aurora monitoring when BGD discovery is not admitted. + +Examples: + +Green hostgroups configured: + +```sql +INSERT INTO mysql_aws_aurora_hostgroups ( + writer_hostgroup, + reader_hostgroup, + green_writer_hostgroup, + green_reader_hostgroup, + domain_name +) VALUES (10, 20, 11, 21, '.cluster-example.eu-north-1.rds.amazonaws.com'); +``` + +Green hostgroups not configured: + +```sql +INSERT INTO mysql_aws_aurora_hostgroups ( + writer_hostgroup, + reader_hostgroup, + green_writer_hostgroup, + green_reader_hostgroup, + domain_name +) VALUES (10, 20, NULL, NULL, '.cluster-example.eu-north-1.rds.amazonaws.com'); + +SET mysql-aws_blue_green_deployment_auto_discovery = 'true'; +``` + +The BGD FSM state is operational monitor state, not user configuration. +`bgd_status` provides its runtime observability without introducing a BGD mode +into the configuration contract. + +## 8. Persistence and Synchronization + +Both green hostgroup columns must be supported by: + +- admin-memory table creation; +- runtime table creation; +- disk database schema and in-place migration on ProxySQL startup; +- `LOAD MYSQL SERVERS TO RUNTIME`; +- `SAVE MYSQL SERVERS FROM RUNTIME`; +- `SAVE MYSQL SERVERS TO DISK`; +- ProxySQL configuration-file import and export; +- ProxySQL Cluster fetch, insert, checksum, and conflict handling; +- `dump_table_mysql("mysql_aws_aurora_hostgroups")`; +- test/bootstrap table definitions. + +ProxySQL Cluster synchronization must include the two configured values. NULL +must remain NULL on the receiving peer. + +Every configuration-bearing path must use an explicit configured-column +projection rather than `SELECT *`. In particular: + +- `SAVE MYSQL SERVERS FROM RUNTIME` copies the configured Aurora columns and + excludes `bgd_status`; +- the ProxySQL Cluster fetch/query, checksum, and insert paths include the two + green hostgroups and exclude `bgd_status`; +- disk saves and configuration-file export never persist `bgd_status`. + +BGD's `bgd_status`, transient member map, cached IPs, and pins are node-local +runtime state and are not cluster-synced as configuration. + +## 9. Disk Schema Migration + +A new Aurora hostgroups schema version must add the two nullable columns while +preserving every existing row and column value. + +Existing rows are migrated with: + +```text +green_writer_hostgroup = NULL +green_reader_hostgroup = NULL +``` + +After upgrade, existing active Aurora rows follow the global variable: + +- with auto-discovery enabled, they are eligible for BGD discovery; +- with auto-discovery disabled, they retain existing Aurora monitoring only. + +This is backward-compatible at the table-data level because no green +hostgroups are invented during migration. + +`bgd_status` requires no disk-schema migration because it exists only in the +runtime table. The runtime table is created with the new column, and every row +starts at `NONE` until its local Aurora worker publishes another state. + +## 10. Non-Goals of This Configuration Change + +This configuration design does not: + +- add an Aurora-specific configuration table; +- create Aurora rows in `mysql_aws_rds_bgd_hostgroups`; +- require green `mysql_servers` rows when green hostgroups are not configured; +- add a second per-cluster monitor worker; +- add a separate `bgd_enabled` column; +- add a `mode` or `auto_generated` column; +- persist transient BGD FSM state to disk; +- alter the existing RDS Multi-AZ BGD configuration contract. + +## 11. Required Configuration Tests + +1. Both green hostgroups NULL load successfully. +2. Both green hostgroups non-NULL load successfully. +3. Mixed NULL/non-NULL values are rejected. +4. Duplicate or overlapping blue/green hostgroups are rejected. +5. A LOAD containing valid and invalid rows excludes each invalid row, reports + its writer hostgroup and conflicting fields, and atomically publishes the + filtered valid result set. A previously active rejected row follows the safe + worker-removal cleanup path rather than receiving a partial update. +6. Configured green hostgroups admit BGD discovery with global auto-discovery + disabled. +7. A row without green hostgroups starts BGD discovery only when global + auto-discovery is enabled. +8. Disabling auto-discovery does not abort an active switchover. +9. Changing an active row to `active=0` removes pins, restores safe placement, + clears the latch and fingerprint, publishes `NONE`, and stops the worker + without deleting user configuration. +10. NULL values survive memory-to-runtime, runtime-to-memory, disk, config-file, + and cluster synchronization round trips. +11. In-place disk schema migration on ProxySQL startup preserves existing rows + and initializes both new fields to NULL. +12. Changing either green hostgroup refreshes only the affected Aurora worker + in place. +13. Unrelated LOAD operations preserve active BGD pins and terminal-latch state. +14. A LOAD concurrent with an FSM transition cannot overwrite the newer + `bgd_status`. +15. New runtime rows initialize `bgd_status` to `NONE`. +16. Runtime `bgd_status` follows every Aurora FSM transition. +17. `SWITCHOVER_COMPLETED` remains visible until topology drain and then changes + to `NONE`. +18. Reloading an existing Aurora row preserves its `bgd_status` and active FSM + state. +19. `SAVE MYSQL SERVERS FROM RUNTIME`, disk/config export, and ProxySQL Cluster + synchronization exclude `bgd_status`. +20. ProxySQL Cluster peers retain their own node-local `bgd_status` values. +21. Existing Multi-AZ BGD configuration and tests remain unchanged. + +## 12. Acceptance Criteria + +The configuration/runtime integration is complete when: + +1. Every Aurora configuration row is user-created, and its two green hostgroup + columns are either both configured or both NULL. +2. Configured green hostgroups admit BGD discovery independently of the global + variable; when both are NULL, the existing + `mysql-aws_blue_green_deployment_auto_discovery` variable controls admission. +3. All load, save, disk, config-file, upgrade, and cluster-sync paths preserve + the fields and their NULL values. +4. Exactly one Aurora monitor worker owns normal Aurora and BGD handling for a + writer hostgroup. +5. No Aurora row is generated in `mysql_aws_rds_bgd_hostgroups`. +6. `runtime_mysql_aws_aurora_hostgroups.bgd_status` exposes the local Aurora BGD + FSM state without being saved, exported, or cluster-synchronized. diff --git a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md new file mode 100644 index 0000000000..aa9354a4cb --- /dev/null +++ b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md @@ -0,0 +1,754 @@ +# Aurora BGD Monitor Loop and FSM Design + +**Date:** 2026-07-31 + +**Branch:** `spec/aws-aurora-bgd` + +**Status:** Design approved + +**Scope:** Aurora MySQL blue/green deployments exposed through +`mysql.rds_topology` and `INFORMATION_SCHEMA.REPLICA_HOST_STATUS`. + +**Evidence basis:** The AWS-team-provided *RDS Topology metadata – Overview* +document defines the routing semantics of the switchover statuses. A +timestamped live-Aurora switchover observation captured on 2026-07-30 +corroborates those semantics and supplies the observed Aurora-specific details. +Section 3 separates the supplied status contract, observed behavior, and design +policy. + +**Related designs:** + +- [Aurora BGD Configuration, Runtime Status, and Cluster Sync](2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md) +- [Aurora BGD Cluster Simulator and Testing](2026-07-31-aurora-bgd-cluster-simulator-testing-design.md) + +## Spec Boundary + +This specification owns the Aurora worker loop, probe ownership and cadence, +target-membership discovery, cached member identity and IP state, DNS and pool +actions, the BGD FSM, rollback, reload continuity, and completion-latch +behavior. Public table schemas, persistence, and cluster synchronization belong +to the configuration/runtime specification. Simulator mechanics and the +executable coverage matrix belong to the simulator/testing specification. + +## 1. Decision Summary + +Aurora BGD is part of the existing Aurora monitor. Each active, user-created +`mysql_aws_aurora_hostgroups` row continues to own one worker keyed by +`writer_hostgroup`. Every Aurora configuration row is user-created, and the +design starts no second BGD worker for it. + +The worker owns three logically separate probes: + +1. the existing production Aurora membership/lag probe; +2. an Aurora BGD `mysql.rds_topology` probe; +3. a target `REPLICA_HOST_STATUS` membership probe. + +The worker follows existing RDS BGD parsing, status publication, DNS pinning, +connection draining, writer-placement, rollback, reload, and cleanup patterns +where their semantics match. Aurora-specific behavior is retained where RDS +Multi-AZ instance behavior does not apply: + +- membership comes from the target Aurora cluster; +- the topology probe rotates across reachable target members instead of being + pinned to the target writer IP; +- normal Aurora probing is suspended during the active switchover phases; +- every current writer and reader must be mapped before traffic changes; +- readers are not shunned or unshunned; +- TARGET completion cleans up writer and reader routing immediately; +- topology drain is only the terminal-latch rearm signal. + +An Aurora row either has both green hostgroups configured or has neither. This +document describes the distinction only as green hostgroups configured or not +configured. The existing +`mysql-aws_blue_green_deployment_auto_discovery` variable remains only an +admission gate for starting BGD discovery when green hostgroups are not +configured, as defined by the configuration/runtime specification. + +## 2. Terminology + +- **Production cluster:** The source/current Aurora cluster before switchover + and the promoted target after completion. +- **Target cluster:** The Aurora cluster identified by the TARGET row before + promotion. +- **Canonical identifier:** The original production `SERVER_ID`, without the + temporary AWS `-green-` component. +- **Target identifier:** A pre-promotion target `SERVER_ID` containing the + temporary `-green-` component. +- **Production hostname:** The canonical instance hostname already configured + in the production writer or reader hostgroup. +- **Target hostname:** A hostname constructed from a target `SERVER_ID` and the + RDS domain suffix. +- **Traffic pin:** A permanent ProxySQL DNS-cache mapping from a production + hostname to the cached IP of its target counterpart. +- **Complete target snapshot:** Exactly one current target writer plus a unique + target counterpart for every current production member, with all target IPs + resolved. +- **Effect-driven cleanup:** The idempotent RDS BGD pattern that reconciles + placement and removes transient routing state from the worker's existing + member map. A member-scoped operation is a no-op when its map input is absent. +- **Deployment fingerprint:** TARGET topology identity retained after cleanup + to recognize repeated results. At minimum it contains TARGET `id`, endpoint, + and port. + +## 3. Evidence and AWS/Aurora Behavioral Contract + +### 3.1 AWS-provided status semantics + +The AWS-team-provided *RDS Topology metadata – Overview* document defines these +traffic-routing semantics for `mysql.rds_topology` during switchover: + +| Status | AWS-provided meaning | Write traffic | Read traffic | +|---|---|---|---| +| `SWITCHOVER_INITIATED` | Switchover was triggered, but no modifications have occurred and rollback remains possible. | Source | Source | +| `SWITCHOVER_IN_PROGRESS` | Source writes are disabled while target replication catches up; rollback remains possible. | Nowhere | Source | +| `SWITCHOVER_IN_POST_PROCESSING` | The target is promoted and can receive writes; rollback is no longer possible. | Target | Target | +| `SWITCHOVER_COMPLETED` | DNS propagation is complete and the original source endpoint points to the target. | Target | Target | + +The implementation treats these meanings as the AWS-supplied behavioral +contract for routing decisions. In particular, POST_PROCESSING definitively +means that the promoted target can accept write traffic; target readiness is +not inferred solely from one observed run. + +[AWS's Aurora switchover documentation](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/blue-green-deployments-switching.html) +also states that a DB cluster included in a switchover cannot be modified while +the switchover is running. The production and target member sets are therefore +fixed from INITIATED until the first COMPLETED observation. Membership can +change while the deployment is AVAILABLE, when the normal production probe +and target-membership probe remain active and refresh their snapshots. + +### 3.2 Directly observed behavior + +The 2026-07-30 run observed: + +1. `mysql.rds_topology` progressed through: + + ```text + AVAILABLE + -> SWITCHOVER_INITIATED + -> SWITCHOVER_IN_PROGRESS + -> SWITCHOVER_IN_POST_PROCESSING + -> TARGET-only SWITCHOVER_COMPLETED + ``` + +2. The target cluster endpoint returned exactly one writer and all current + readers through `REPLICA_HOST_STATUS`. +3. The writer row used `SESSION_ID='MASTER_SESSION_ID'`; reader rows used + distinct session identifiers. +4. During post-processing, target `SERVER_ID` values changed from temporary + green identifiers to canonical production identifiers. +5. Reader `SESSION_ID` values remained unchanged across that rename. +6. The production topology became `SWITCHOVER_IN_PROGRESS` at `T+9.864s`; the + source writer became read-only at `T+10.398s`. +7. The target writer became writable at `T+14.438s`; target post-processing was + observed at `T+15.795s`. +8. Every target member had a canonical `SERVER_ID` by `T+21.380s`. +9. TARGET completion was visible at `T+40.655s`; the topology table drained at + `T+65.730s`. + +The run corroborates the AWS-provided status semantics. Details learned only +from the observation, including timings, target-member rename behavior, and +table-drain timing, are implementation evidence rather than a formal AWS +compatibility guarantee. Tests must simulate the observed changes, and +unexpected or ambiguous metadata must fail closed. + +### 3.3 Design policy derived from the contract and observation + +- POST_PROCESSING is the routing barrier because the AWS-provided semantics say + that the promoted target can receive writes at that status; no additional + target-writability query gates traffic pinning. +- Normal monitoring refreshes membership while the deployment is AVAILABLE. + INITIATED freezes the last complete production snapshot for the duration of + the switchover, matching AWS's cluster-modification restriction. +- TARGET completion means Aurora writer and reader routing cleanup can occur + immediately. +- Table drain is not a reader-availability barrier for Aurora. +- A query error, incomplete membership result, or ambiguous identity does not + advance routing actions or cause rollback. + +## 4. Worker Ownership and Probe Model + +### 4.1 Single worker owner + +The existing Aurora worker for a `writer_hostgroup` is the sole owner of normal +Aurora monitoring and the BGD FSM. It owns all per-deployment state and +serializes the three probes and their actions. A separate Aurora BGD worker must +not be started for the same row because it would duplicate queries and race on +hostgroup state. + +### 4.2 Production Aurora probe + +This is the existing random-host `REPLICA_HOST_STATUS` probe used for Aurora +role, lag, and autopurge decisions. + +- It runs normally in `NONE` and `AVAILABLE`. +- It stops issuing queries in `SWITCHOVER_INITIATED`, + `SWITCHOVER_IN_PROGRESS`, and `SWITCHOVER_IN_POST_PROCESSING`. +- It resumes after completion cleanup enters `SWITCHOVER_COMPLETED`, or + immediately after rollback. + +Suspension means the query itself is skipped. Running the query while +discarding all role, lag, and autopurge actions provides no FSM input because +the target-membership probe owns target discovery. + +### 4.3 Topology probe + +In `NONE`, the topology probe uses the existing Aurora random-host selection: +choose a random reachable production member and fall back across the remaining +members when ping or connection setup fails. + +After the TARGET endpoint has bootstrapped a complete target member list, the +topology probe applies the same selection algorithm across the current target +members. It is not pinned to the target writer and does not use a dedicated +cached-IP probe pin. When `REPLICA_HOST_STATUS` reports renamed identifiers, +the target probe-host list is refreshed to the corresponding canonical +hostnames. + +After completion cleanup, topology probing returns to random reachable +canonical production members while the terminal latch waits for table drain. + +### 4.4 Target-membership probe + +The TARGET cluster endpoint from `mysql.rds_topology` bootstraps a separate +`REPLICA_HOST_STATUS` query. Its result is parsed only into the BGD target +snapshot. It must not be passed to normal Aurora evaluation before cutover, +because doing so could prematurely move or add target members in production +hostgroups. + +The target-membership probe continues through AVAILABLE, INITIATED, IN_PROGRESS, +and POST_PROCESSING. It stops after completion cleanup or rollback has removed +the need for target-side discovery. + +### 4.5 Cadence + +| FSM state | Topology probe | Target-membership probe | Production Aurora probe | +|---|---|---|---| +| `NONE` | Configured `check_interval_ms` when BGD discovery is admitted | Off | Configured interval | +| `AVAILABLE` | Configured `check_interval_ms` | Configured interval | Configured interval | +| `SWITCHOVER_INITIATED` | Fast, 100 ms | Fast, 100 ms | Suspended | +| `SWITCHOVER_IN_PROGRESS` | Fast, 100 ms | Fast, 100 ms | Suspended | +| `SWITCHOVER_IN_POST_PROCESSING` | Fast, 100 ms | Fast, 100 ms | Suspended | +| `SWITCHOVER_COMPLETED` | Configured interval | Off | Configured interval | + +The 100 ms active cadence follows the existing RDS BGD worker. Query duration +is accounted for so a slow query does not create an additional full-interval +sleep. + +## 5. Discovery Bootstrap and Admission + +For an eligible row in `NONE`: + +1. Probe `information_schema.TABLES` for `mysql.rds_topology`, following the + RDS BGD table-check state. +2. When the table exists, fetch and parse `SELECT * FROM mysql.rds_topology` so + AWS column-set differences remain tolerated by the shared parser. +3. Require a structurally valid BGD result with a TARGET row and non-empty + TARGET `id`, status, endpoint, and port. Construct the deployment fingerprint + from at least that validated `id`, endpoint, and port. +4. Map the observed TARGET status to an Aurora FSM state, publish it to + `bgd_status`, and leave `NONE` before starting membership discovery. +5. Use the TARGET cluster endpoint to bootstrap target membership after that + state transition. +6. After target membership is available, rotate topology probes across target + members. + +A membership failure does not undo a valid observed status. It prevents any +membership-dependent routing action and is retried at the cadence for the +current phase. Missing, empty, or malformed topology does not create a new +deployment. + +Raw topology statuses map to Aurora runtime states as follows: + +| Raw TARGET status | Aurora FSM and `bgd_status` value | +|---|---| +| `AVAILABLE` | `AVAILABLE` | +| `SWITCHOVER_INITIATED` | `SWITCHOVER_INITIATED` | +| `SWITCHOVER_IN_PROGRESS` | `SWITCHOVER_IN_PROGRESS` | +| `SWITCHOVER_IN_POST_PROCESSING` | `SWITCHOVER_IN_POST_PROCESSING` | +| `SWITCHOVER_COMPLETED` in a valid TARGET-only result | `SWITCHOVER_COMPLETED` terminal latch | + +`NONE` is an internal baseline and rearm state, not an arbitrary raw status. +An unsupported or unknown raw status is invalid metadata: a worker already in +an active or terminal state retains that state, and a worker in `NONE` remains +there. The value is never copied verbatim into `bgd_status`. Aurora does not use +the RDS Multi-AZ inferred `READER_SWITCHOVER_IN_PROGRESS` state; TARGET +completion maps directly to the Aurora terminal latch after cleanup. + +If green hostgroups are configured, BGD discovery is admitted for the +user-created Aurora row. If neither is configured, the existing global +auto-discovery variable gates admission of a new deployment. Changing that +variable after a deployment has started does not abandon the active FSM. + +## 6. Target Membership and Pair Mapping + +### 6.1 Query and current rows + +The target query must include at least: + +```sql +SELECT + SERVER_ID, + SESSION_ID, + LAST_UPDATE_TIMESTAMP, + IS_CURRENT +FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS +ORDER BY SERVER_ID; +``` + +The implementation may retain the existing lag and CPU columns. Only rows +identified as current are eligible. The writer is the sole current row whose +`SESSION_ID` is `MASTER_SESSION_ID`; all other eligible rows are readers. + +### 6.2 Complete-snapshot rule + +A new snapshot replaces the last complete snapshot only when: + +1. it contains exactly one current target writer; +2. each current production member has exactly one target counterpart; +3. no target member maps to more than one production member; +4. no production member maps to more than one target member; +5. every target hostname has a resolved IP. + +A target writer-only snapshot is valid only when the current production +snapshot also contains no readers. Query errors, empty results, multiple +writers, missing readers, duplicate identities, and unresolved IPs do not +replace the last complete snapshot. They also must not be reinterpreted as a +reader-less deployment. + +The production snapshot is refreshed by normal Aurora monitoring throughout +AVAILABLE. When the worker first accepts INITIATED, that last complete snapshot +becomes the fixed production membership for the switchover. The normal +production probe remains suspended while AWS prevents changes to either +included cluster; the frozen snapshot and continuously refreshed target +snapshot therefore require no separate member-set generation. + +POST_PROCESSING actions require a complete snapshot. If none exists, the +worker holds the current status, performs no partial traffic cutover, and +retries both active probes. + +### 6.3 Constructing member hostnames + +The TARGET cluster endpoint has the form: + +```text +.cluster- +``` + +A target member hostname is: + +```text +. +``` + +Each target member stores its role, current `SERVER_ID`, `SESSION_ID`, current +hostname, port, resolved IP, normalized canonical identifier, matching +production hostname, and per-action flags. + +### 6.4 Identity across rename + +Pair every target member to production by its normalized `SERVER_ID`: + +- before promotion, remove the AWS-added `-green-` component; +- after promotion, leave the already canonical identifier unchanged. + +For readers, `SESSION_ID` is an additional continuity check. A reader whose +`SERVER_ID` changes but whose session identifier matches the cached pair is the +same target reader. A session mismatch or a conflict between normalized name +and cached session identity makes the new snapshot ambiguous. + +For the writer, `MASTER_SESSION_ID` is only a role marker. It must not be used +as a unique instance identity. The sole writer is paired by normalized +`SERVER_ID`; no cached-IP writer identity rule is required. + +### 6.5 Cached IPs + +During AVAILABLE and every active switchover phase, resolve all target member +hostnames and retain the last complete set of IPs independently of DNS TTL +expiry. Membership discovery continues through post-processing so renamed +identifiers refresh the host list, but pre-rename cached IPs remain valid +traffic-pin values. + +Cached target IPs are used for traffic redirection only. They do not pin the +topology probe to a particular member. + +## 7. Worker-Owned State + +The runtime owner for a writer hostgroup must retain: + +```text +current FSM status +deployment fingerprint +last production membership snapshot +last complete target snapshot +normalized production/target member pairs +cached target IP per pair +traffic-pin-applied flag per pair +production-probe-suspended flag +configured green hostgroup identifiers, when present +``` + +Per-member flags make repeated observations idempotent. The fingerprint keeps a +repeated completed row from starting cleanup again after the active map has +been released. + +## 8. State Machine + +### 8.1 States + +```text +NONE +AVAILABLE +SWITCHOVER_INITIATED +SWITCHOVER_IN_PROGRESS +SWITCHOVER_IN_POST_PROCESSING +SWITCHOVER_COMPLETED +``` + +`runtime_mysql_aws_aurora_hostgroups.bgd_status` exposes these worker states. +The final state is a terminal rearm latch, not a separate reader-switchover +phase. + +### 8.2 Topology row-shape predicates + +The worker validates the complete topology result before publishing a new +status or running an FSM action: + +- An active deployment result contains exactly one SOURCE row and one TARGET + row. Both rows have non-empty `id`, endpoint, role, status, and a valid port; + both expose the same supported pre-completion status. +- A completion result contains exactly one TARGET row, has the validated + deployment-fingerprint fields, and reports `SWITCHOVER_COMPLETED`. +- A successful empty result or confirmed table absence is the explicit + cancellation or terminal-drain observation described below. +- SOURCE-only results, TARGET-only pre-completion results, duplicate SOURCE or + TARGET rows, extra or unknown roles, mismatched statuses, missing required + fields, and unsupported statuses are invalid or incomplete observations. + +Invalid or incomplete observations do not publish a new status, trigger +cleanup, release pins, or resume suspended monitoring. A worker in `NONE` +remains there; an active or latched worker retains its existing state and +retries. This validation is distinct from a successful empty or absent +topology, which has the explicit state-dependent meaning in the transition +table. + +### 8.3 Transition summary + +| Valid observation | Required transition and action | +|---|---| +| No deployment in `NONE` | Remain `NONE`; continue admitted discovery at configured cadence. | +| `AVAILABLE` | Enter/retain `AVAILABLE`; build target membership and cache IPs without changing production routing. | +| `SWITCHOVER_INITIATED` | Suspend production Aurora probing, capture rollback state, and switch BGD probes to fast cadence. | +| `SWITCHOVER_IN_PROGRESS` | Move the production writer to the reader hostgroup; do not route to target yet. | +| First `SWITCHOVER_IN_POST_PROCESSING` | With a complete snapshot, pin every production member, drain old connections, and restore the canonical writer to the writer hostgroup. | +| Repeated POST_PROCESSING | Retry only incomplete idempotent member actions. | +| First TARGET-only `SWITCHOVER_COMPLETED` | Run immediate effect-driven cleanup, resume normal Aurora monitoring, and enter the completed latch. | +| Repeated same completed TARGET | No-op while latched. | +| Successful empty/absent topology while latched | Release the fingerprint and return to `NONE`. | +| Earlier valid status before completion | Run rollback, then enter the earlier state. | +| Successful empty/absent topology before completion | Treat as cancellation and run rollback. | +| Query/connect error in any state | Retain state and retry; never infer rollback or table drain. | +| Different valid fingerprint while latched | Rearm and process the new deployment. | + +## 9. Phase Actions + +### 9.1 `AVAILABLE` + +On entry and repeated observations: + +1. Publish `bgd_status=AVAILABLE`. +2. Run normal production Aurora monitoring unchanged. +3. Build or refresh the last complete target snapshot. +4. Resolve and retain every target IP. +5. Keep all production DNS, pools, and hostgroup placement unchanged. + +Configured green hostgroups, when present, remain user configuration and are +not the target-membership source of truth. When they are absent, the complete +target map remains worker-local. No Aurora configuration row is generated in +either case. + +### 9.2 `SWITCHOVER_INITIATED` + +On entry: + +1. Publish `bgd_status=SWITCHOVER_INITIATED`. +2. Capture production writer placement and other state required for rollback. +3. Suspend the normal production Aurora probe. +4. Switch topology and target-membership probes to 100 ms. +5. Continue refreshing target membership and IPs. +6. Make no traffic-routing or hostgroup-placement change. + +### 9.3 `SWITCHOVER_IN_PROGRESS` + +On the first valid observation: + +1. Publish `bgd_status=SWITCHOVER_IN_PROGRESS`. +2. Retain fast BGD probes and the suspended production probe. +3. Move the current production writer to the reader hostgroup using the same + writer-demotion behavior as RDS BGD. +4. Retain the member map for writer-placement reconciliation during completion + or rollback. +5. Do not redirect any production hostname to target yet. + +The move is idempotent. A repeated observation does not repeat a completed +hostgroup action. + +### 9.4 `SWITCHOVER_IN_POST_PROCESSING` + +POST_PROCESSING is the only pre-completion traffic-redirection boundary. + +Before applying any action, require a complete target snapshot. Then: + +1. Publish `bgd_status=SWITCHOVER_IN_POST_PROCESSING`. +2. Do not issue an additional target-writability probe. +3. For every writer and reader pair, install: + + ```text + production hostname -> cached target IP + ``` + +4. Reuse the RDS BGD connection-drain behavior for each production hostname: + drop free backend connections immediately and mark used connections + unhealthy/non-reusable. +5. Purge the corresponding monitor connection-pool entries so new connections + resolve through the traffic pin. +6. After the writer pin is installed, restore the canonical writer to the + writer hostgroup. Its placement in the reader hostgroup follows the existing + `writer_is_also_reader` configuration. +7. Keep canonical readers eligible in the reader hostgroup. Do not shun or + unshun them. + +Every pair records that its pin-and-retirement action was applied after the DNS +pin, free-connection deletion, used-connection unhealthy/non-reusable marking, +and monitor-pool purge calls return. This flag does not mean that every used +connection has physically closed: destruction happens asynchronously when the +connection is released and is not a completion prerequisite. Repeated +POST_PROCESSING results retry only unapplied actions and do not reapply +retirement to an already transitioned member. + +### 9.5 `SWITCHOVER_COMPLETED` + +On the first TARGET-only completed result, run effect-driven cleanup +immediately, before publishing the terminal latch: + +1. If the existing member map contains a writer, reconcile that writer into + the writer hostgroup and apply its reader placement according to + `writer_is_also_reader`. This restores a writer moved during IN_PROGRESS when + POST_PROCESSING was not observed, and is a no-op when the writer is already + restored or the map is empty. +2. For every mapped pair, remove the production-hostname DNS entry and purge + the corresponding monitor-pool entries so normal DNS resolution resumes. + Removing an absent pin or pool entry is a no-op. +3. Do not reapply connection retirement, wait for marked connections to close, + wait for topology drain, or perform a separate DNS verification. +4. Drain only obsolete pools belonging to configured green hostgroups, subject + to the configured OFFLINE-status preservation policy. +5. Preserve every user-created configuration and `mysql_servers` row. +6. Clear any installed switchover guard and resume normal Aurora monitoring. +7. Move topology probing back to random reachable canonical production members + and the configured interval. +8. Release the active member map after retaining the deployment fingerprint. +9. Publish and enter the internal `SWITCHOVER_COMPLETED` latch. + +The cleanup path is the same regardless of the prior phase. It acts only on the +available member map and worker state, so absent inputs naturally produce +no-ops. Completion never replays a skipped phase or waits for earlier actions. + +While latched: + +- the same TARGET completed result is ignored; +- query/connect errors do not rollback, repin, or repeat cleanup; +- a successful empty result or confirmed table absence returns the worker to + `NONE`; +- a different valid deployment fingerprint rearms discovery. + +The topology drain is therefore only an FSM rearm signal. + +## 10. Rollback and Error Handling + +### 10.1 Rollback before completion + +A successful, structurally valid result for the same deployment with an +earlier status follows the existing RDS BGD backward-transition behavior. A +successful empty result or confirmed table absence before completion is treated +as cancellation. Either condition runs effect-driven cleanup in rollback mode: + +1. Remove every traffic pin that was applied. +2. Drain/purge affected production-hostname pools so subsequent connections use + restored canonical DNS. +3. Reconcile the mapped production writer into the writer hostgroup. This is a + no-op if the writer is already there or the map is empty. +4. Restore its reader placement according to `writer_is_also_reader`. +5. Resume normal production Aurora monitoring. +6. Return topology probing to production members and the appropriate cadence. +7. Preserve user-created green-hostgroup and server rows. +8. If AWS returned to AVAILABLE, rebuild target discovery in AVAILABLE; if the + topology disappeared, clear the deployment and return to `NONE`. + +Rollback does not drain configured green-hostgroup pools merely because an +attempt was cancelled. + +### 10.2 Errors and incomplete data + +- A topology or membership query/connect error retains the current state. +- An invalid or incomplete target snapshot retains the last complete snapshot. +- No error is interpreted as an empty table, reader-less cluster, cancellation, + or successful completion. +- No routing action runs without its state-specific prerequisites. +- Errors in the completed latch do not release or restart it. + +## 11. Late Entry, Reload, Removal, and Concurrency + +### 11.1 Late entry + +A worker that first observes INITIATED, IN_PROGRESS, or POST_PROCESSING rebuilds +all prerequisites before applying that phase's actions. In particular, +POST_PROCESSING cannot pin traffic until it has a complete target snapshot. + +A worker that first observes TARGET-only completion has an empty member map. +Its member-scoped completion actions are consequently no-ops. The worker +retains the fingerprint and enters the terminal latch without manufacturing or +replaying earlier phase actions. + +### 11.2 Configuration and variable refresh + +An unrelated `LOAD MYSQL SERVERS TO RUNTIME` or variable refresh must preserve +the FSM status, deployment fingerprint, complete snapshots, cached IPs, applied +pin flags, probe-suspension state, and terminal latch. + +Changing configured green hostgroups refreshes the staging/pool references but +does not restart an active deployment from `NONE`. Disabling the global +auto-discovery variable prevents admission of a new no-green-hostgroup +deployment; it does not abort one already active. + +### 11.3 Row removal and worker exit + +Removing or deactivating the owning Aurora row follows the existing RDS BGD +worker-removal cleanup pattern: remove applied pins, restore safe production +writer placement where possible, release suspended monitoring state, and then +terminate the worker. No user configuration row is deleted as a side effect. + +### 11.4 Concurrent deployments + +All state is keyed by production `writer_hostgroup`. Probes, fingerprints, +snapshots, pins, writer placement, reloads, rollback, and completion for one +Aurora row must not mutate another row's state. + +## 12. Separation from RDS Multi-AZ Instance Logic + +The following RDS Multi-AZ instance behaviors must not drive the Aurora branch: + +- pinning the topology probe to a single green writer IP; +- treating TARGET completion as writer-only completion; +- entering `READER_SWITCHOVER_IN_PROGRESS`; +- retaining reader pins until topology drain; +- shunning unmapped production readers; +- routing an empty reader hostgroup through the promoted writer as a missing-map + fallback. + +Aurora reuses the RDS machinery only where the behavior matches the contracts +in this specification. + +## 13. Current Implementation Gaps + +The existing RDS builder assumes the TARGET topology endpoint is an instance +writer and discovers readers from configured RDS green hostgroups. Aurora's +TARGET is a cluster endpoint and its complete membership comes from +`REPLICA_HOST_STATUS`. + +The Aurora implementation therefore requires: + +1. three logical probes in the existing per-writer-hostgroup worker; +2. target-cluster membership parsing separate from normal Aurora evaluation; +3. normalized `SERVER_ID` pairing plus reader `SESSION_ID` continuity; +4. complete-snapshot validation and cached per-member IPs; +5. normal-probe suspension during the active switchover; +6. RDS-style writer demotion, DNS pinning, pool draining, rollback, and status + publication; +7. Aurora-specific all-member POST_PROCESSING and immediate completion cleanup; +8. completion-fingerprint retention until topology drain. + +The existing RDS Multi-AZ builder and FSM remain unchanged. + +## 14. Required Tests + +### 14.1 Probe ownership and cadence + +- One Aurora worker owns all three logical probes. +- NONE and AVAILABLE use configured cadence and continue normal Aurora probes. +- INITIATED, IN_PROGRESS, and POST_PROCESSING use 100 ms BGD probes and issue no + normal Aurora query. +- Topology selection rotates across target members and falls back when a member + is unreachable. +- No target-writer IP probe pin is installed. +- COMPLETED resumes normal probing and production-member topology selection. + +### 14.2 Membership and mapping + +- Target cluster membership yields one writer and multiple reader pairs. +- No configured green hostgroups still produces a complete in-memory map. +- Green-suffixed `SERVER_ID`s normalize to production identifiers. +- Reader `SESSION_ID`s preserve identity after canonical rename. +- The writer is paired by normalized `SERVER_ID`, not `MASTER_SESSION_ID` or IP. +- A genuine no-reader cluster accepts a writer-only snapshot. +- Empty, failed, duplicate-writer, ambiguous, unresolved, and incomplete results + retain the last complete snapshot or defer actions. +- SOURCE-only, duplicate-role, mismatched-status, unknown-status, and + TARGET-only pre-completion topology results fail closed without publishing or + transitioning; only TARGET-only completion is accepted. + +### 14.3 FSM and routing + +- AVAILABLE discovers and resolves every member without routing changes. +- INITIATED suspends normal Aurora probing without moving servers. +- IN_PROGRESS moves the production writer to the reader hostgroup exactly once. +- POST_PROCESSING requires a complete snapshot. +- First POST_PROCESSING pins and drains every member and restores the writer. +- No target-writability query gates POST_PROCESSING. +- Readers remain eligible and are never shunned. +- Repeated POST_PROCESSING retries only unapplied work. +- The per-pair action flag records application of pinning and retirement + marking, not asynchronous physical connection closure. +- First TARGET completion runs immediate effect-driven cleanup using the + existing map and worker state. +- Direct completion with an empty map performs no production-member routing or + retirement action and enters the completed latch. +- Completion after IN_PROGRESS restores the writer even when POST_PROCESSING + was not observed. +- Completion after POST_PROCESSING removes applied pins without reapplying + connection retirement or waiting for marked connections to close. +- Completion performs no DNS verification and does not wait for table drain. +- Repeated completion is a no-op while latched. +- Empty/absent topology releases the latch and returns to `NONE`. + +### 14.4 Resilience and lifecycle + +- Backward status and pre-completion topology disappearance run rollback. +- Query errors never run rollback or release the completed latch. +- Rollback removes pins, restores writer placement, and resumes normal probes. +- Late entry at every active state reconstructs prerequisites before acting. +- Reload preserves active and terminal state. +- Changing or removing the owning row runs safe, isolated cleanup. +- Disabling auto-discovery does not abort an admitted deployment. +- Concurrent writer hostgroups remain isolated. +- Configured green pools are drained only on successful completion, with user + rows and statuses preserved. + +## 15. Acceptance Criteria + +The Aurora monitor/FSM design is satisfied when: + +1. One existing Aurora worker owns the three probes and all state for its + writer hostgroup. +2. Every production member is mapped from target `REPLICA_HOST_STATUS` before + traffic changes, regardless of whether green hostgroups are configured. +3. Normal Aurora queries stop during the three active switchover phases and + resume after completion or rollback. +4. IN_PROGRESS demotes the production writer using the RDS BGD behavior. +5. POST_PROCESSING redirects writer and reader traffic using cached target IPs, + drains old connections, and never shuns readers. +6. TARGET completion removes every traffic pin and finishes routing cleanup + without waiting for topology drain. +7. Repeated results, errors, reloads, late entry, rollback, worker removal, and + concurrent deployments behave idempotently and remain isolated. +8. The terminal completed state persists only until a successful topology drain + observation rearms the FSM. +9. RDS Multi-AZ instance behavior and tests remain unchanged. diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 1d88201a0b..bd81cffc35 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include "ev.h" #include "wqueue.h" @@ -52,6 +54,8 @@ #define MYHGM_MYSQL_GALERA_HOSTGROUPS "CREATE TABLE mysql_galera_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , backup_writer_hostgroup INT CHECK (backup_writer_hostgroup>=0 AND backup_writer_hostgroup<>writer_hostgroup) NOT NULL , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND backup_writer_hostgroup<>reader_hostgroup AND reader_hostgroup>0) , offline_hostgroup INT NOT NULL CHECK (offline_hostgroup<>writer_hostgroup AND offline_hostgroup<>reader_hostgroup AND backup_writer_hostgroup<>offline_hostgroup AND offline_hostgroup>=0) , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , max_writers INT NOT NULL CHECK (max_writers >= 0) DEFAULT 1 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1,2)) NOT NULL DEFAULT 0 , max_transactions_behind INT CHECK (max_transactions_behind>=0) NOT NULL DEFAULT 0 , comment VARCHAR , UNIQUE (reader_hostgroup) , UNIQUE (offline_hostgroup) , UNIQUE (backup_writer_hostgroup))" #define MYHGM_MYSQL_AWS_AURORA_HOSTGROUPS "CREATE TABLE mysql_aws_aurora_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , " \ + "green_writer_hostgroup INT DEFAULT NULL CHECK (green_writer_hostgroup IS NULL OR green_writer_hostgroup>=0) , " \ + "green_reader_hostgroup INT DEFAULT NULL CHECK (green_reader_hostgroup IS NULL OR green_reader_hostgroup>=0) , " \ "active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , aurora_port INT NOT NUlL DEFAULT 3306 , domain_name VARCHAR NOT NULL DEFAULT '' , " \ "max_lag_ms INT NOT NULL CHECK (max_lag_ms>= 10 AND max_lag_ms <= 600000) DEFAULT 600000 , " \ "check_interval_ms INT NOT NULL CHECK (check_interval_ms >= 100 AND check_interval_ms <= 600000) DEFAULT 1000 , " \ @@ -62,7 +66,7 @@ "min_lag_ms INT NOT NULL CHECK (min_lag_ms >= 0 AND min_lag_ms <= 600000) DEFAULT 30 , " \ "lag_num_checks INT NOT NULL CHECK (lag_num_checks >= 1 AND lag_num_checks <= 16) DEFAULT 1 , " \ "autopurge_missing_checks INT NOT NULL CHECK (autopurge_missing_checks >= 0 AND autopurge_missing_checks <= 100) DEFAULT 0 , " \ - "comment VARCHAR , UNIQUE (reader_hostgroup))" + "comment VARCHAR , bgd_status VARCHAR NOT NULL DEFAULT 'NONE' , UNIQUE (reader_hostgroup))" #define MYHGM_MYSQL_AWS_RDS_BGD_HOSTGROUPS "CREATE TABLE mysql_aws_rds_bgd_hostgroups ("\ "writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , "\ @@ -356,6 +360,8 @@ class AWS_Aurora_Info { public: int writer_hostgroup; int reader_hostgroup; + int green_writer_hostgroup; + int green_reader_hostgroup; int aurora_port; int max_lag_ms; int add_lag_ms; @@ -372,13 +378,25 @@ class AWS_Aurora_Info { char * comment; bool active; bool active_; - AWS_Aurora_Info(int w, int r, int _port, char *_end_addr, int maxl, int al, int minl, int lnc, int ci, int ct, bool _a, int wiar, int nrw, int amc, char *c); - bool update(int r, int _port, char *_end_addr, int maxl, int al, int minl, int lnc, int ci, int ct, bool _a, int wiar, int nrw, int amc, char *c); + AWS_Aurora_Info(int w, int r, int gw, int gr, int _port, char *_end_addr, int maxl, int al, int minl, int lnc, int ci, int ct, bool _a, int wiar, int nrw, int amc, char *c); + bool update(int r, int gw, int gr, int _port, char *_end_addr, int maxl, int al, int minl, int lnc, int ci, int ct, bool _a, int wiar, int nrw, int amc, char *c); ~AWS_Aurora_Info(); AWS_Aurora_Info(const AWS_Aurora_Info&) = delete; AWS_Aurora_Info& operator=(const AWS_Aurora_Info&) = delete; }; +/** + * @brief Validate an Aurora hostgroup candidate and return its canonical configured projection. + * + * The candidate must exactly match the configured Aurora Admin table projection. + * The returned result contains only valid rows in that canonical order. The + * caller owns the returned result. + */ +SQLite3_result* validate_and_filter_aws_aurora_hostgroups( + const SQLite3_result* candidate, + std::vector& errors +); + struct p_hg_counter { enum metric { servers_table_version = 0, @@ -770,6 +788,7 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { public: // Friend declaration for WebUI monitoring metrics collector friend class ProxySQL::Monitoring::MetricsCollector; + friend class TestAuroraBGDRuntime; std::mutex galera_set_writer_mutex; /** @@ -979,6 +998,7 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { * @brief Creates a resultset with the current full content of the target table. * @param string The target table. Valid values are: * - "mysql_aws_aurora_hostgroups" + * - "runtime_mysql_aws_aurora_hostgroups" * - "mysql_galera_hostgroups" * - "mysql_group_replication_hostgroups" * - "mysql_replication_hostgroups" @@ -1104,6 +1124,14 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { * @param status AWS_RDS_BGD_Status underlying value. */ void aws_rds_bgd_set_runtime_status(unsigned int writer_hg, int status); + /** + * @brief Publish the node-local Aurora BGD state for one runtime row. + * + * The caller owns validation of the state vocabulary. Writer hostgroups not + * present at runtime are ignored. Configuration reloads do not write this + * column. + */ + void update_aws_aurora_bgd_status(int writer_hostgroup, const std::string& bgd_status); /** * @brief Aligns the runtime 'mysql_servers' table + checksums with the server state in MyHGM. * @@ -1199,6 +1227,10 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { //void update_aws_aurora_set_reader(int _whid, int _rhid, char *_hostname, int _port); bool aws_aurora_replication_lag_action(int _whid, int _rhid, char *server_id, float current_replication_lag_ms, bool enable, bool is_writer, bool verbose=true); void update_aws_aurora_set_writer(int _whid, int _rhid, char *server_id, bool verbose=true); + void update_aws_aurora_set_writer( + int _whid, int _rhid, char *server_id, bool verbose, + const char *domain_name, int aurora_port, int writer_is_also_reader, + int new_reader_weight); void update_aws_aurora_set_reader(int _whid, int _rhid, char *server_id); /** * @brief Updates the resultset and corresponding checksum used by Monitor for AWS Aurora. diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index ad599493a5..4a2637f61a 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -62,6 +62,26 @@ struct cmp_str { #define AWS_ENDPOINT_SUFFIX_STRING "rds.amazonaws.com" #define QUERY_AWS_RDS_TOPOLOGY_DISCOVERY "SELECT * FROM mysql.rds_topology" #define QUERY_AWS_RDS_TOPOLOGY_TABLE_CHECK "SELECT 1 FROM information_schema.TABLES WHERE TABLE_SCHEMA='mysql' AND TABLE_NAME='rds_topology'" +#define QUERY_AWS_AURORA_REPLICA_HOST_STATUS \ + "SELECT SERVER_ID," \ + "IF(" \ + "SESSION_ID = 'MASTER_SESSION_ID' AND " \ + "SERVER_ID <> (SELECT SERVER_ID FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE SESSION_ID = 'MASTER_SESSION_ID' ORDER BY LAST_UPDATE_TIMESTAMP DESC LIMIT 1), " \ + "'probably_former_MASTER_SESSION_ID', SESSION_ID" \ + ") SESSION_ID, " \ + "LAST_UPDATE_TIMESTAMP, " \ + "IF(SESSION_ID = 'MASTER_SESSION_ID', 0, REPLICA_LAG_IN_MILLISECONDS) AS REPLICA_LAG_IN_MILLISECONDS, " \ + "CPU " \ + "FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE" \ + " ( " \ + "(REPLICA_LAG_IN_MILLISECONDS >= 0 AND REPLICA_LAG_IN_MILLISECONDS <= 600000)" \ + " OR SESSION_ID = 'MASTER_SESSION_ID'" \ + " ) " \ + "AND LAST_UPDATE_TIMESTAMP > NOW() - INTERVAL 180 SECOND" \ + " ORDER BY SERVER_ID" +#define QUERY_AWS_AURORA_BGD_REPLICA_HOST_STATUS \ + "SELECT SERVER_ID, SESSION_ID, LAST_UPDATE_TIMESTAMP, IS_CURRENT " \ + "FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS ORDER BY SERVER_ID" /* @@ -77,14 +97,6 @@ A single AWS_Aurora_monitor_node will have a AWS_Aurora_status_entry per check. */ -#ifdef TEST_AURORA - -#define TEST_AURORA_MONITOR_BASE_QUERY \ - "SELECT SERVER_ID, SESSION_ID, LAST_UPDATE_TIMESTAMP, REPLICA_LAG_IN_MILLISECONDS, CPU"\ - " FROM REPLICA_HOST_STATUS ORDER BY SERVER_ID " - -#endif - class AWS_Aurora_replica_host_status_entry { public: char * server_id = nullptr; @@ -552,6 +564,126 @@ struct AWS_RDS_BGD_Probe_Host { int use_ssl = 0; }; +/** + * @brief Aurora blue/green deployment phase published by the Aurora worker. + */ +enum class AWS_Aurora_BGD_Status { + NONE = 0, + AVAILABLE, + SWITCHOVER_INITIATED, + SWITCHOVER_IN_PROGRESS, + SWITCHOVER_IN_POST_PROCESSING, + SWITCHOVER_COMPLETED, +}; + +/** + * @brief Column positions in `AWS_Aurora_Hosts_resultset`. + */ +enum AWS_Aurora_Hosts_Column { + AWS_AURORA_WRITER_HOSTGROUP = 0, + AWS_AURORA_READER_HOSTGROUP, + AWS_AURORA_HOSTNAME, + AWS_AURORA_PORT, + AWS_AURORA_USE_SSL, + AWS_AURORA_MAX_LAG_MS, + AWS_AURORA_CHECK_INTERVAL_MS, + AWS_AURORA_CHECK_TIMEOUT_MS, + AWS_AURORA_ADD_LAG_MS, + AWS_AURORA_MIN_LAG_MS, + AWS_AURORA_LAG_NUM_CHECKS, + AWS_AURORA_AUTOPURGE_MISSING_CHECKS, + AWS_AURORA_DOMAIN_NAME, + AWS_AURORA_GREEN_WRITER_HOSTGROUP, + AWS_AURORA_GREEN_READER_HOSTGROUP, + AWS_AURORA_WRITER_IS_ALSO_READER, + AWS_AURORA_NEW_READER_WEIGHT, + AWS_AURORA_HOSTS_COLUMNS, +}; + +/** + * @brief Monitor worker owned by one Aurora writer hostgroup. + */ +struct AWS_Aurora_BGD_Worker { + int writer_hg = 0; + pthread_t thread {}; + std::atomic_bool worker_stop {false}; + std::atomic current_checksum {0}; +}; + +/** + * @brief Stable identity of one Aurora blue/green deployment. + */ +struct AWS_Aurora_BGD_Fingerprint { + std::string target_id; + std::string target_endpoint; + int target_port = 0; + + bool empty() const { + return target_id.empty() || target_endpoint.empty() || target_port <= 0; + } + + bool operator==(const AWS_Aurora_BGD_Fingerprint& rhs) const { + return target_id == rhs.target_id + && target_endpoint == rhs.target_endpoint + && target_port == rhs.target_port; + } +}; + +/** + * @brief One production or target Aurora member retained by the BGD worker. + */ +struct AWS_Aurora_BGD_Member { + std::string server_id; + std::string normalized_server_id; + std::string session_id; + std::string hostname; + std::string production_hostname; + std::string target_ip; + int port = 0; + int use_ssl = 0; + bool is_writer = false; + bool traffic_pin_applied = false; +}; + +/** + * @brief State carried by one existing per-writer Aurora monitor worker. + */ +struct AWS_Aurora_BGD_State { + unsigned int writer_hg = 0; + unsigned int reader_hg = 0; + int green_writer_hg = -1; + int green_reader_hg = -1; + unsigned int max_lag_ms = 0; + unsigned int check_interval_ms = 0; + unsigned int check_timeout_ms = 0; + unsigned int add_lag_ms = 0; + unsigned int min_lag_ms = 0; + unsigned int lag_num_checks = 1; + unsigned int autopurge_missing_checks = 0; + int writer_is_also_reader = 0; + int new_reader_weight = 1; + int target_use_ssl = 0; + std::string domain_name; + + AWS_Aurora_BGD_Status status = AWS_Aurora_BGD_Status::NONE; + RDS_BGD_Topology_Monitor_State topology_state = TOPOLOGY_TABLE_CHECK; + AWS_Aurora_BGD_Fingerprint fingerprint; + std::vector production_probe_hosts; + std::vector production_members; + std::vector target_members; + + bool production_snapshot_frozen = false; + bool production_probe_suspended = false; + bool target_snapshot_complete = false; + + bool has_complete_target_snapshot() const { + return target_snapshot_complete; + } +}; + +// Maps an Aurora switchover status enum to its runtime string. +const char* aws_aurora_bgd_status_str(AWS_Aurora_BGD_Status status); + /** * @brief Switchover state carried by RDS BGD worker thread. * @@ -711,6 +843,68 @@ class MySQL_Monitor { void * monitor_group_replication_2(); void * monitor_galera(); void * monitor_aws_aurora(); + /** + * @brief Load the configuration rows for one Aurora worker. + * + * @return true when the rows still match the coordinator-provided checksum. + */ + bool aws_aurora_bgd_load_worker_config( + int writer_hg, uint64_t current_checksum, AWS_Aurora_BGD_State& candidate); + /** + * @brief Refresh only configuration-derived fields of a running Aurora worker. + */ + bool aws_aurora_bgd_refresh_worker_config( + AWS_Aurora_BGD_State& st, uint64_t current_checksum, + unsigned long long& next_loop_at); + /** + * @brief Refresh the Aurora BGD worker's last complete production snapshot. + * + * @details Invalid or incomplete observations retain the previous snapshot. + */ + void aws_aurora_bgd_refresh_production_snapshot( + AWS_Aurora_BGD_State& st, const AWS_Aurora_status_entry& result); + /** + * @brief Apply the routing actions for an accepted active Aurora BGD state. + * + * @details INITIATED only suspends production probing. IN_PROGRESS demotes + * the snapshotted writer on entry. POST_PROCESSING applies each complete + * member mapping once and restores canonical writer placement. + */ + void aws_aurora_bgd_apply_active_actions( + AWS_Aurora_BGD_State& st, bool status_changed); + /** + * @brief Run immediate effect-driven cleanup for TARGET completion. + * + * @details Reconciles writer placement, removes mapped traffic pins, drains + * eligible configured green pools, resumes production probing, and enters + * the completed latch while retaining the deployment fingerprint. + */ + void aws_aurora_bgd_apply_completion( + AWS_Aurora_BGD_State& st, + const AWS_Aurora_BGD_Fingerprint& completed_fingerprint); + /** + * @brief Reverse applied pre-completion effects and enter a safe earlier state. + * + * @details Removes only applied traffic pins, drains affected production + * pools, restores canonical writer placement, resumes ordinary probing, + * and optionally clears the deployment identity. + */ + void aws_aurora_bgd_apply_rollback( + AWS_Aurora_BGD_State& st, AWS_Aurora_BGD_Status next_status, + bool clear_deployment); + /** + * @brief Release the completed latch after a successful topology drain. + */ + void aws_aurora_bgd_release_completed_latch(AWS_Aurora_BGD_State& st); + /** + * @brief Run the topology and target-membership probes owned by an Aurora worker. + * + * @details Discovery is serialized with the ordinary Aurora probe. This method + * validates topology before publishing status and replaces target membership + * only with a complete, unambiguous, fully resolved snapshot. + */ + void aws_aurora_bgd_run_discovery_cycle( + AWS_Aurora_BGD_State& st, std::atomic_bool& worker_stop); /** * @brief AWS RDS BGD monitor thread entry point. * diff --git a/include/ProxySQL_Admin_Tables_Definitions.h b/include/ProxySQL_Admin_Tables_Definitions.h index 4514853bd0..164ec01856 100644 --- a/include/ProxySQL_Admin_Tables_Definitions.h +++ b/include/ProxySQL_Admin_Tables_Definitions.h @@ -235,9 +235,47 @@ // V2_0_10: Adds autopurge_missing_checks column #define ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS_V2_0_10 "CREATE TABLE mysql_aws_aurora_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , aurora_port INT NOT NUlL DEFAULT 3306 , domain_name VARCHAR NOT NULL CHECK (SUBSTR(domain_name,1,1) = '.') , max_lag_ms INT NOT NULL CHECK (max_lag_ms>= 10 AND max_lag_ms <= 600000) DEFAULT 600000 , check_interval_ms INT NOT NULL CHECK (check_interval_ms >= 100 AND check_interval_ms <= 600000) DEFAULT 1000 , check_timeout_ms INT NOT NULL CHECK (check_timeout_ms >= 80 AND check_timeout_ms <= 3000) DEFAULT 800 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , new_reader_weight INT CHECK (new_reader_weight >= 0 AND new_reader_weight <=10000000) NOT NULL DEFAULT 1 , add_lag_ms INT NOT NULL CHECK (add_lag_ms >= 0 AND add_lag_ms <= 600000) DEFAULT 30 , min_lag_ms INT NOT NULL CHECK (min_lag_ms >= 0 AND min_lag_ms <= 600000) DEFAULT 30 , lag_num_checks INT NOT NULL CHECK (lag_num_checks >= 1 AND lag_num_checks <= 16) DEFAULT 1 , autopurge_missing_checks INT NOT NULL CHECK (autopurge_missing_checks >= 0 AND autopurge_missing_checks <= 100) DEFAULT 0 , comment VARCHAR , UNIQUE (reader_hostgroup))" -#define ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS_V2_0_10 - -#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_AURORA_HOSTGROUPS "CREATE TABLE runtime_mysql_aws_aurora_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , aurora_port INT NOT NUlL DEFAULT 3306 , domain_name VARCHAR NOT NULL CHECK (SUBSTR(domain_name,1,1) = '.') , max_lag_ms INT NOT NULL CHECK (max_lag_ms>= 10 AND max_lag_ms <= 600000) DEFAULT 600000 , check_interval_ms INT NOT NULL CHECK (check_interval_ms >= 100 AND check_interval_ms <= 600000) DEFAULT 1000 , check_timeout_ms INT NOT NULL CHECK (check_timeout_ms >= 80 AND check_timeout_ms <= 3000) DEFAULT 800 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , new_reader_weight INT CHECK (new_reader_weight >= 0 AND new_reader_weight <=10000000) NOT NULL DEFAULT 1 , add_lag_ms INT NOT NULL CHECK (add_lag_ms >= 0 AND add_lag_ms <= 600000) DEFAULT 30 , min_lag_ms INT NOT NULL CHECK (min_lag_ms >= 0 AND min_lag_ms <= 600000) DEFAULT 30 , lag_num_checks INT NOT NULL CHECK (lag_num_checks >= 1 AND lag_num_checks <= 16) DEFAULT 1 , autopurge_missing_checks INT NOT NULL CHECK (autopurge_missing_checks >= 0 AND autopurge_missing_checks <= 100) DEFAULT 0 , comment VARCHAR , UNIQUE (reader_hostgroup))" +// V3_0_12: Adds nullable Aurora blue/green staging hostgroups. +#define ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS_V3_0_12 "CREATE TABLE mysql_aws_aurora_hostgroups ("\ + "writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , "\ + "reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , "\ + "green_writer_hostgroup INT DEFAULT NULL CHECK (green_writer_hostgroup IS NULL OR green_writer_hostgroup>=0) , "\ + "green_reader_hostgroup INT DEFAULT NULL CHECK (green_reader_hostgroup IS NULL OR green_reader_hostgroup>=0) , "\ + "active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , "\ + "aurora_port INT NOT NUlL DEFAULT 3306 , "\ + "domain_name VARCHAR NOT NULL CHECK (SUBSTR(domain_name,1,1) = '.') , "\ + "max_lag_ms INT NOT NULL CHECK (max_lag_ms>= 10 AND max_lag_ms <= 600000) DEFAULT 600000 , "\ + "check_interval_ms INT NOT NULL CHECK (check_interval_ms >= 100 AND check_interval_ms <= 600000) DEFAULT 1000 , "\ + "check_timeout_ms INT NOT NULL CHECK (check_timeout_ms >= 80 AND check_timeout_ms <= 3000) DEFAULT 800 , "\ + "writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , "\ + "new_reader_weight INT CHECK (new_reader_weight >= 0 AND new_reader_weight <=10000000) NOT NULL DEFAULT 1 , "\ + "add_lag_ms INT NOT NULL CHECK (add_lag_ms >= 0 AND add_lag_ms <= 600000) DEFAULT 30 , "\ + "min_lag_ms INT NOT NULL CHECK (min_lag_ms >= 0 AND min_lag_ms <= 600000) DEFAULT 30 , "\ + "lag_num_checks INT NOT NULL CHECK (lag_num_checks >= 1 AND lag_num_checks <= 16) DEFAULT 1 , "\ + "autopurge_missing_checks INT NOT NULL CHECK (autopurge_missing_checks >= 0 AND autopurge_missing_checks <= 100) DEFAULT 0 , "\ + "comment VARCHAR , UNIQUE (reader_hostgroup))" + +#define ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS_V3_0_12 + +#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_AURORA_HOSTGROUPS "CREATE TABLE runtime_mysql_aws_aurora_hostgroups ("\ + "writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , "\ + "reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , "\ + "green_writer_hostgroup INT DEFAULT NULL CHECK (green_writer_hostgroup IS NULL OR green_writer_hostgroup>=0) , "\ + "green_reader_hostgroup INT DEFAULT NULL CHECK (green_reader_hostgroup IS NULL OR green_reader_hostgroup>=0) , "\ + "active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , "\ + "aurora_port INT NOT NUlL DEFAULT 3306 , "\ + "domain_name VARCHAR NOT NULL CHECK (SUBSTR(domain_name,1,1) = '.') , "\ + "max_lag_ms INT NOT NULL CHECK (max_lag_ms>= 10 AND max_lag_ms <= 600000) DEFAULT 600000 , "\ + "check_interval_ms INT NOT NULL CHECK (check_interval_ms >= 100 AND check_interval_ms <= 600000) DEFAULT 1000 , "\ + "check_timeout_ms INT NOT NULL CHECK (check_timeout_ms >= 80 AND check_timeout_ms <= 3000) DEFAULT 800 , "\ + "writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , "\ + "new_reader_weight INT CHECK (new_reader_weight >= 0 AND new_reader_weight <=10000000) NOT NULL DEFAULT 1 , "\ + "add_lag_ms INT NOT NULL CHECK (add_lag_ms >= 0 AND add_lag_ms <= 600000) DEFAULT 30 , "\ + "min_lag_ms INT NOT NULL CHECK (min_lag_ms >= 0 AND min_lag_ms <= 600000) DEFAULT 30 , "\ + "lag_num_checks INT NOT NULL CHECK (lag_num_checks >= 1 AND lag_num_checks <= 16) DEFAULT 1 , "\ + "autopurge_missing_checks INT NOT NULL CHECK (autopurge_missing_checks >= 0 AND autopurge_missing_checks <= 100) DEFAULT 0 , "\ + "comment VARCHAR , "\ + "bgd_status VARCHAR NOT NULL DEFAULT 'NONE' , UNIQUE (reader_hostgroup))" // AWS RDS hostgroups; adds blue/green (green_*_hostgroup) over aurora. // The runtime table carries one extra runtime-only column: auto_generated. diff --git a/include/ProxySQL_Cluster.hpp b/include/ProxySQL_Cluster.hpp index eaccc4bbb6..6f1e07fb6c 100644 --- a/include/ProxySQL_Cluster.hpp +++ b/include/ProxySQL_Cluster.hpp @@ -70,7 +70,7 @@ #define CLUSTER_QUERY_PGSQL_SERVERS_SSL_PARAMS "PROXY_SELECT hostname, port, username, ssl_ca, ssl_cert, ssl_key, ssl_crl, ssl_crlpath, ssl_protocol_version_range, comment FROM runtime_pgsql_servers_ssl_params ORDER BY hostname, port, username" /* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_aws_aurora_hostgroups'. See top comment for details. */ -#define CLUSTER_QUERY_MYSQL_AWS_AURORA "PROXY_SELECT writer_hostgroup, reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, autopurge_missing_checks, comment FROM runtime_mysql_aws_aurora_hostgroups ORDER BY writer_hostgroup" +#define CLUSTER_QUERY_MYSQL_AWS_AURORA "PROXY_SELECT writer_hostgroup, reader_hostgroup, green_writer_hostgroup, green_reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, autopurge_missing_checks, comment FROM runtime_mysql_aws_aurora_hostgroups ORDER BY writer_hostgroup" /* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_aws_rds_bgd_hostgroups'. See top comment for details. */ #define CLUSTER_QUERY_MYSQL_AWS_RDS_BGD "PROXY_SELECT writer_hostgroup, reader_hostgroup, green_writer_hostgroup, green_reader_hostgroup, active, writer_is_also_reader, check_interval_ms, check_timeout_ms, comment, auto_generated, status FROM runtime_mysql_aws_rds_bgd_hostgroups WHERE auto_generated=0 ORDER BY writer_hostgroup" diff --git a/include/SQLite3_Server.h b/include/SQLite3_Server.h index a2ceaabe2a..eaf3cd017a 100644 --- a/include/SQLite3_Server.h +++ b/include/SQLite3_Server.h @@ -51,13 +51,13 @@ class SQLite3_Server { std::unordered_map grouprep_map; std::vector *tables_defs_grouprep; #endif // TEST_GROUPREP -#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) +#if defined(TEST_AURORA) || defined(TEST_READONLY) || defined(TEST_RDS_BGD) std::vector *tables_defs_readonly; std::unordered_map readonly_map; -#endif // TEST_READONLY || TEST_RDS_BGD -#ifdef TEST_RDS_BGD - std::vector *tables_defs_rds_bgd; -#endif // TEST_RDS_BGD +#endif // TEST_AURORA || TEST_READONLY || TEST_RDS_BGD +#if defined(TEST_AURORA) || defined(TEST_RDS_BGD) + std::vector *tables_defs_aws_bgd; +#endif // TEST_AURORA || TEST_RDS_BGD #ifdef TEST_REPLICATIONLAG std::unordered_map> replicationlag_map; std::vector* tables_defs_replicationlag; @@ -70,20 +70,9 @@ class SQLite3_Server { public: SQLite3DB *sessdb; #ifdef TEST_AURORA - unsigned int cur_aurora_writer[3]; - unsigned int num_aurora_servers[3]; - unsigned int max_num_aurora_servers; - pthread_mutex_t aurora_mutex; - /** - * @brief Handles queries to table 'REPLICA_HOST_STATUS'. - * @details This function needs to be called with lock on mutex aurora_mutex already acquired. - * @param sess The session which request is to be handled. - */ - void populate_aws_aurora_table(MySQL_Session *sess, uint32_t whg); void init_aurora_ifaces_string(std::string& s); #endif // TEST_AURORA #ifdef TEST_GALERA - //unsigned int cur_aurora_writer[3]; unsigned int num_galera_servers[3]; unsigned int max_num_galera_servers; pthread_mutex_t galera_mutex; @@ -97,14 +86,14 @@ class SQLite3_Server { void init_grouprep_ifaces_string(std::string& s); group_rep_status grouprep_test_value(const std::string& srv_addr); #endif // TEST_GROUPREP -#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) +#if defined(TEST_AURORA) || defined(TEST_READONLY) || defined(TEST_RDS_BGD) pthread_mutex_t test_readonly_mutex; void load_readonly_table(MySQL_Session *sess); int readonly_test_value(char *p); int readonly_map_size() { return readonly_map.size(); } -#endif // TEST_READONLY || TEST_RDS_BGD +#endif // TEST_AURORA || TEST_READONLY || TEST_RDS_BGD #ifdef TEST_REPLICATIONLAG pthread_mutex_t test_replicationlag_mutex; void load_replicationlag_table(MySQL_Session* sess); diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h index d642b2756e..4907c3245e 100644 --- a/include/proxysql_admin.h +++ b/include/proxysql_admin.h @@ -320,6 +320,25 @@ struct FlushVariableStats { int unknown = 0; }; +/** + * @brief Replace an Admin Aurora hostgroup table with an HGM runtime snapshot. + * + * @param db Destination Admin database. + * @param resultset Aurora rows returned by MySQL_HostGroups_Manager. + * @param runtime Selects runtime_mysql_aws_aurora_hostgroups when true and + * mysql_aws_aurora_hostgroups when false. + * @return true when the destination table was replaced. + */ +bool materialize_mysql_aws_aurora_hostgroups( + SQLite3DB* db, + const SQLite3_result* resultset, + bool runtime +); + +/** Copy the configured Aurora hostgroup projection between Admin and disk. */ +bool copy_mysql_aws_aurora_hostgroups_from_disk(SQLite3DB* db); +bool copy_mysql_aws_aurora_hostgroups_to_disk(SQLite3DB* db); + class ProxySQL_Admin { friend class TestDiskUpgrade; private: diff --git a/lib/Admin_Handler.cpp b/lib/Admin_Handler.cpp index 16f2e602a1..f17a43ec04 100644 --- a/lib/Admin_Handler.cpp +++ b/lib/Admin_Handler.cpp @@ -4605,7 +4605,11 @@ void admin_session_handler(S* sess, void *_pa, PtrSize_t *pkt) { (query_no_space_length==sizeof("CHECKSUM MEM MYSQL AURORA HOSTGROUPS") - 1 && !strncasecmp("CHECKSUM MEM MYSQL AURORA HOSTGROUPS", query_no_space, query_no_space_length)) || (query_no_space_length==sizeof("CHECKSUM MYSQL AURORA HOSTGROUPS") - 1 && !strncasecmp("CHECKSUM MYSQL AURORA HOSTGROUPS", query_no_space, query_no_space_length))){ - char *q=(char *)"SELECT * FROM mysql_aws_aurora_hostgroups ORDER BY writer_hostgroup"; + char *q=(char *) + "SELECT writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active," + "aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader," + "new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment " + "FROM mysql_aws_aurora_hostgroups ORDER BY writer_hostgroup"; tablename=(char *)"MYSQL AURORA HOSTGROUPS"; SPA->admindb->execute_statement(q, &error, &cols, &affected_rows, &resultset); } diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index f7a9824841..fd572ef66c 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -9,6 +9,7 @@ using json = nlohmann::json; #include "MySQL_PreparedStatement.h" #include "MySQL_Data_Stream.h" +#include #include #include #include @@ -30,6 +31,7 @@ using json = nlohmann::json; #include #include +#include #include using std::function; @@ -58,6 +60,185 @@ class MyHGC; const int MYSQL_ERRORS_STATS_FIELD_NUM = 11; +namespace { + +constexpr int AWS_AURORA_CONFIG_COLUMN_COUNT = 17; + +const char* const AWS_AURORA_CONFIG_COLUMNS[AWS_AURORA_CONFIG_COLUMN_COUNT] = { + "writer_hostgroup", + "reader_hostgroup", + "green_writer_hostgroup", + "green_reader_hostgroup", + "active", + "aurora_port", + "domain_name", + "max_lag_ms", + "check_interval_ms", + "check_timeout_ms", + "writer_is_also_reader", + "new_reader_weight", + "add_lag_ms", + "min_lag_ms", + "lag_num_checks", + "autopurge_missing_checks", + "comment" +}; + +struct Aurora_Hostgroup_Role { + const char* field; + int value; +}; + +struct Aurora_Config_Row { + std::vector fields; + std::vector roles; + std::vector errors; + int writer_hostgroup; + bool active; + bool locally_valid; +}; + +std::string aws_aurora_nullable_value(const char* value) { + return value ? value : "NULL"; +} + +} // namespace + +SQLite3_result* validate_and_filter_aws_aurora_hostgroups( + const SQLite3_result* candidate, + std::vector& errors +) { + SQLite3_result* filtered = new SQLite3_result(AWS_AURORA_CONFIG_COLUMN_COUNT); + for (const char* column : AWS_AURORA_CONFIG_COLUMNS) { + filtered->add_column_definition(SQLITE_TEXT, column); + } + + if (candidate == nullptr) { + return filtered; + } + + if (candidate->columns != static_cast(candidate->column_definition.size())) { + errors.emplace_back( + "mysql_aws_aurora_hostgroups rejected: candidate projection metadata is inconsistent" + ); + return filtered; + } + + const int comparable_columns = std::min(candidate->columns, AWS_AURORA_CONFIG_COLUMN_COUNT); + for (int i = 0; i < comparable_columns; ++i) { + const char* actual_name = candidate->column_definition[i]->name; + if (strcasecmp(actual_name, AWS_AURORA_CONFIG_COLUMNS[i]) != 0) { + std::ostringstream message; + message << "mysql_aws_aurora_hostgroups rejected: candidate projection column " << i + << " must be " << AWS_AURORA_CONFIG_COLUMNS[i] + << ", got " << actual_name; + errors.emplace_back(message.str()); + return filtered; + } + } + + if (candidate->columns != AWS_AURORA_CONFIG_COLUMN_COUNT) { + errors.emplace_back( + "mysql_aws_aurora_hostgroups rejected: candidate projection must contain exactly 17 columns" + ); + return filtered; + } + + std::vector rows; + rows.reserve(candidate->rows.size()); + + for (const SQLite3_row* source_row : candidate->rows) { + Aurora_Config_Row row; + row.fields.resize(AWS_AURORA_CONFIG_COLUMN_COUNT, nullptr); + for (int i = 0; i < AWS_AURORA_CONFIG_COLUMN_COUNT; ++i) { + row.fields[i] = source_row->fields[i]; + } + + row.writer_hostgroup = row.fields[0] ? atoi(row.fields[0]) : -1; + row.active = row.fields[4] && atoi(row.fields[4]) != 0; + row.locally_valid = true; + + const bool has_green_writer = row.fields[2] != nullptr; + const bool has_green_reader = row.fields[3] != nullptr; + if (has_green_writer != has_green_reader) { + std::ostringstream message; + message << "green_writer_hostgroup=" << aws_aurora_nullable_value(row.fields[2]) + << " and green_reader_hostgroup=" << aws_aurora_nullable_value(row.fields[3]) + << " must both be NULL or both be non-NULL"; + row.errors.emplace_back(message.str()); + row.locally_valid = false; + } + + row.roles.push_back({"writer_hostgroup", row.writer_hostgroup}); + row.roles.push_back({"reader_hostgroup", row.fields[1] ? atoi(row.fields[1]) : -1}); + if (has_green_writer && has_green_reader) { + row.roles.push_back({"green_writer_hostgroup", atoi(row.fields[2])}); + row.roles.push_back({"green_reader_hostgroup", atoi(row.fields[3])}); + } + + if (row.locally_valid) { + for (size_t left = 0; left < row.roles.size(); ++left) { + for (size_t right = left + 1; right < row.roles.size(); ++right) { + if (row.roles[left].value == row.roles[right].value) { + std::ostringstream message; + message << "conflicting fields " << row.roles[left].field << '=' << row.roles[left].value + << " and " << row.roles[right].field << '=' << row.roles[right].value; + row.errors.emplace_back(message.str()); + row.locally_valid = false; + } + } + } + } + + rows.emplace_back(std::move(row)); + } + + // A locally invalid row is already absent from the replacement and must not + // make an otherwise valid row fail. Cross-row conflicts are directional: + // every row is checked against the roles owned by every other active row. + for (size_t row_index = 0; row_index < rows.size(); ++row_index) { + Aurora_Config_Row& row = rows[row_index]; + if (!row.locally_valid) { + continue; + } + + for (size_t other_index = 0; other_index < rows.size(); ++other_index) { + const Aurora_Config_Row& other = rows[other_index]; + if (row_index == other_index || !other.locally_valid || !other.active) { + continue; + } + + for (const Aurora_Hostgroup_Role& role : row.roles) { + for (const Aurora_Hostgroup_Role& other_role : other.roles) { + if (role.value == other_role.value) { + std::ostringstream message; + message << "conflicting field " << role.field << '=' << role.value + << " overlaps " << other_role.field << '=' << other_role.value + << " in writer_hostgroup=" << other.writer_hostgroup; + row.errors.emplace_back(message.str()); + } + } + } + } + } + + for (Aurora_Config_Row& row : rows) { + if (row.errors.empty()) { + filtered->add_row(row.fields.data()); + continue; + } + + for (const std::string& reason : row.errors) { + std::ostringstream message; + message << "mysql_aws_aurora_hostgroups writer_hostgroup=" << row.writer_hostgroup + << " rejected: " << reason; + errors.emplace_back(message.str()); + } + } + + return filtered; +} + struct ev_io * new_connect_watcher(char *address, uint16_t gtid_port, uint16_t mysql_port); void * GTID_syncer_run(); @@ -957,7 +1138,16 @@ void MySQL_HostGroups_Manager::CUCFT1( int cols=0; int affected_rows=0; SQLite3_result *resultset=NULL; - string query = "SELECT * FROM " + TableName + " ORDER BY " + ColumnName; + string query; + if (TableName == "mysql_aws_aurora_hostgroups") { + query = + "SELECT writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup," + "active,aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms," + "writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks," + "autopurge_missing_checks,comment FROM mysql_aws_aurora_hostgroups ORDER BY " + ColumnName; + } else { + query = "SELECT * FROM " + TableName + " ORDER BY " + ColumnName; + } mydb->execute_statement(query.c_str(), &error , &cols , &affected_rows , &resultset); if (resultset) { if (resultset->rows_count) { @@ -1544,8 +1734,6 @@ bool MySQL_HostGroups_Manager::commit( // AWS Aurora if (incoming_aws_aurora_hostgroups) { - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_aws_aurora_hostgroups\n"); - mydb->execute("DELETE FROM mysql_aws_aurora_hostgroups"); generate_mysql_aws_aurora_hostgroups_table(); } @@ -2285,8 +2473,11 @@ void MySQL_HostGroups_Manager::update_table_mysql_servers_for_monitor(bool lock) SQLite3_result * MySQL_HostGroups_Manager::dump_table_mysql(const string& name) { char * query = (char *)""; if (name == "mysql_aws_aurora_hostgroups") { - query=(char *)"SELECT writer_hostgroup,reader_hostgroup,active,aurora_port,domain_name,max_lag_ms," + query=(char *)"SELECT writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,aurora_port,domain_name,max_lag_ms," "check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment FROM mysql_aws_aurora_hostgroups"; + } else if (name == "runtime_mysql_aws_aurora_hostgroups") { + query=(char *)"SELECT writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,aurora_port,domain_name,max_lag_ms," + "check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment,bgd_status FROM mysql_aws_aurora_hostgroups"; } else if (name == "mysql_aws_rds_bgd_hostgroups") { query=(char *)"SELECT writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,writer_is_also_reader," "check_interval_ms,check_timeout_ms,comment,auto_generated,status FROM mysql_aws_rds_bgd_hostgroups"; @@ -4014,6 +4205,20 @@ void MySQL_HostGroups_Manager::aws_rds_bgd_set_runtime_status(unsigned int write wrunlock(); } +void MySQL_HostGroups_Manager::update_aws_aurora_bgd_status(int writer_hostgroup, const std::string& bgd_status) { + wrlock(); + const char* query = "UPDATE mysql_aws_aurora_hostgroups SET bgd_status=?1 WHERE writer_hostgroup=?2"; + auto [prepare_rc, statement_unique] = mydb->prepare_v2(query); + ASSERT_SQLITE_OK(prepare_rc, mydb); + sqlite3_stmt* statement = statement_unique.get(); + int rc = (*proxy_sqlite3_bind_text)(statement, 1, bgd_status.c_str(), -1, SQLITE_TRANSIENT); + ASSERT_SQLITE_OK(rc, mydb); + rc = (*proxy_sqlite3_bind_int64)(statement, 2, writer_hostgroup); + ASSERT_SQLITE_OK(rc, mydb); + SAFE_SQLITE3_STEP2(statement); + wrunlock(); +} + /** * @brief Aligns the runtime 'mysql_servers' table + checksums with the server state in MyHGM. * @@ -6057,13 +6262,15 @@ SQLite3_result * MySQL_HostGroups_Manager::get_mysql_errors(bool reset) { return result; } -AWS_Aurora_Info::AWS_Aurora_Info(int w, int r, int _port, char *_end_addr, int maxl, int al, int minl, int lnc, int ci, int ct, bool _a, int wiar, int nrw, int amc, char *c) { +AWS_Aurora_Info::AWS_Aurora_Info(int w, int r, int gw, int gr, int _port, char *_end_addr, int maxl, int al, int minl, int lnc, int ci, int ct, bool _a, int wiar, int nrw, int amc, char *c) { comment=NULL; if (c) { comment=strdup(c); } writer_hostgroup=w; reader_hostgroup=r; + green_writer_hostgroup=gw; + green_reader_hostgroup=gr; max_lag_ms=maxl; add_lag_ms=al; min_lag_ms=minl; @@ -6091,13 +6298,21 @@ AWS_Aurora_Info::~AWS_Aurora_Info() { } } -bool AWS_Aurora_Info::update(int r, int _port, char *_end_addr, int maxl, int al, int minl, int lnc, int ci, int ct, bool _a, int wiar, int nrw, int amc, char *c) { +bool AWS_Aurora_Info::update(int r, int gw, int gr, int _port, char *_end_addr, int maxl, int al, int minl, int lnc, int ci, int ct, bool _a, int wiar, int nrw, int amc, char *c) { bool ret=false; active_=true; if (reader_hostgroup!=r) { reader_hostgroup=r; ret=true; } + if (green_writer_hostgroup!=gw) { + green_writer_hostgroup=gw; + ret=true; + } + if (green_reader_hostgroup!=gr) { + green_reader_hostgroup=gr; + ret=true; + } if (max_lag_ms!=maxl) { max_lag_ms=maxl; ret=true; @@ -6447,14 +6662,81 @@ void MySQL_HostGroups_Manager::generate_mysql_aws_aurora_hostgroups_table() { if (incoming_aws_aurora_hostgroups==NULL) { return; } + + struct RuntimeRow { + int reader_hostgroup; + std::string bgd_status; + }; + + std::map runtime_rows; + std::map incoming_reader_hostgroups; + for (SQLite3_row* row : incoming_aws_aurora_hostgroups->rows) { + incoming_reader_hostgroups.emplace(atoi(row->fields[0]), atoi(row->fields[1])); + } + + char* error = NULL; + int cols = 0; + int affected_rows = 0; + SQLite3_result* resultset = NULL; + const char* select_query = + "SELECT writer_hostgroup, reader_hostgroup, bgd_status FROM mysql_aws_aurora_hostgroups"; + mydb->execute_statement(select_query, &error, &cols, &affected_rows, &resultset); + if (error) { + proxy_error("Error on %s : %s\n", select_query, error); + free(error); + assert(0); + } + if (resultset) { + for (SQLite3_row* row : resultset->rows) { + runtime_rows.emplace( + atoi(row->fields[0]), + RuntimeRow {atoi(row->fields[1]), row->fields[2] ? row->fields[2] : "NONE"} + ); + } + delete resultset; + } + int rc; - //sqlite3 *mydb3=mydb->get_db(); - char *query=(char *)"INSERT INTO mysql_aws_aurora_hostgroups(writer_hostgroup,reader_hostgroup,active,aurora_port,domain_name,max_lag_ms,check_interval_ms," - "check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment) VALUES " - "(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15)"; - auto [rc1, statement_unique] = mydb->prepare_v2(query); - ASSERT_SQLITE_OK(rc1, mydb); - sqlite3_stmt *statement = statement_unique.get(); + const char* delete_query = "DELETE FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=?1"; + auto [delete_rc, delete_statement_unique] = mydb->prepare_v2(delete_query); + ASSERT_SQLITE_OK(delete_rc, mydb); + sqlite3_stmt* delete_statement = delete_statement_unique.get(); + + // Delete missing deployments and release a changed reader hostgroup before inserting its replacement. + for (const auto& [writer_hostgroup, runtime_row] : runtime_rows) { + auto incoming_it = incoming_reader_hostgroups.find(writer_hostgroup); + bool removed = incoming_it == incoming_reader_hostgroups.end(); + bool reader_changed = !removed && incoming_it->second != runtime_row.reader_hostgroup; + if (!removed && !reader_changed) { + continue; + } + rc=(*proxy_sqlite3_bind_int64)(delete_statement, 1, writer_hostgroup); ASSERT_SQLITE_OK(rc, mydb); + SAFE_SQLITE3_STEP2(delete_statement); + rc=(*proxy_sqlite3_clear_bindings)(delete_statement); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_reset)(delete_statement); ASSERT_SQLITE_OK(rc, mydb); + } + + const char* update_query = + "UPDATE mysql_aws_aurora_hostgroups SET " + "reader_hostgroup=?1, green_writer_hostgroup=?2, green_reader_hostgroup=?3, active=?4, " + "aurora_port=?5, domain_name=?6, max_lag_ms=?7, check_interval_ms=?8, check_timeout_ms=?9, " + "writer_is_also_reader=?10, new_reader_weight=?11, add_lag_ms=?12, min_lag_ms=?13, " + "lag_num_checks=?14, autopurge_missing_checks=?15, comment=?16 " + "WHERE writer_hostgroup=?17"; + auto [update_rc, update_statement_unique] = mydb->prepare_v2(update_query); + ASSERT_SQLITE_OK(update_rc, mydb); + sqlite3_stmt* update_statement = update_statement_unique.get(); + + const char* insert_query = + "INSERT INTO mysql_aws_aurora_hostgroups(" + "writer_hostgroup, reader_hostgroup, green_writer_hostgroup, green_reader_hostgroup, active, " + "aurora_port, domain_name, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, " + "new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, autopurge_missing_checks, comment, bgd_status" + ") VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18)"; + auto [insert_rc, insert_statement_unique] = mydb->prepare_v2(insert_query); + ASSERT_SQLITE_OK(insert_rc, mydb); + sqlite3_stmt* insert_statement = insert_statement_unique.get(); + proxy_info("New mysql_aws_aurora_hostgroups table\n"); pthread_mutex_lock(&AWS_Aurora_Info_mutex); for (std::map::iterator it1 = AWS_Aurora_Info_Map.begin() ; it1 != AWS_Aurora_Info_Map.end(); ++it1) { @@ -6466,34 +6748,69 @@ void MySQL_HostGroups_Manager::generate_mysql_aws_aurora_hostgroups_table() { SQLite3_row *r=*it; int writer_hostgroup=atoi(r->fields[0]); int reader_hostgroup=atoi(r->fields[1]); - int active=atoi(r->fields[2]); - int aurora_port = atoi(r->fields[3]); - int max_lag_ms = atoi(r->fields[5]); - int check_interval_ms = atoi(r->fields[6]); - int check_timeout_ms = atoi(r->fields[7]); - int writer_is_also_reader = atoi(r->fields[8]); - int new_reader_weight = atoi(r->fields[9]); - int add_lag_ms = atoi(r->fields[10]); - int min_lag_ms = atoi(r->fields[11]); - int lag_num_checks = atoi(r->fields[12]); - int autopurge_missing_checks = atoi(r->fields[13]); - proxy_info("Loading AWS Aurora info for (%d,%d,%s,%d,\"%s\",%d,%d,%d,%d,%d,%d,%d,\"%s\")\n", writer_hostgroup,reader_hostgroup,(active ? "on" : "off"),aurora_port, - r->fields[4],max_lag_ms,add_lag_ms,min_lag_ms,lag_num_checks,check_interval_ms,check_timeout_ms,autopurge_missing_checks,r->fields[14]); - rc=(*proxy_sqlite3_bind_int64)(statement, 1, writer_hostgroup); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 2, reader_hostgroup); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 3, active); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 4, aurora_port); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_text)(statement, 5, r->fields[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 6, max_lag_ms); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 7, check_interval_ms); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 8, check_timeout_ms); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 9, writer_is_also_reader); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 10, new_reader_weight); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 11, add_lag_ms); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 12, min_lag_ms); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 13, lag_num_checks); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 14, autopurge_missing_checks); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_text)(statement, 15, r->fields[14], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + const char* green_writer_value = r->fields[2]; + const char* green_reader_value = r->fields[3]; + int green_writer_hostgroup = green_writer_value ? atoi(green_writer_value) : -1; + int green_reader_hostgroup = green_reader_value ? atoi(green_reader_value) : -1; + int active=atoi(r->fields[4]); + int aurora_port = atoi(r->fields[5]); + int max_lag_ms = atoi(r->fields[7]); + int check_interval_ms = atoi(r->fields[8]); + int check_timeout_ms = atoi(r->fields[9]); + int writer_is_also_reader = atoi(r->fields[10]); + int new_reader_weight = atoi(r->fields[11]); + int add_lag_ms = atoi(r->fields[12]); + int min_lag_ms = atoi(r->fields[13]); + int lag_num_checks = atoi(r->fields[14]); + int autopurge_missing_checks = atoi(r->fields[15]); + proxy_info("Loading AWS Aurora info for (%d,%d,%d,%d,%s,%d,\"%s\",%d,%d,%d,%d,%d,%d,%d,\"%s\")\n", writer_hostgroup,reader_hostgroup, + green_writer_hostgroup, green_reader_hostgroup, + (active ? "on" : "off"),aurora_port,r->fields[6],max_lag_ms,add_lag_ms,min_lag_ms,lag_num_checks, + check_interval_ms,check_timeout_ms,autopurge_missing_checks,r->fields[16]); + + auto runtime_it = runtime_rows.find(writer_hostgroup); + bool update_existing = + runtime_it != runtime_rows.end() && + runtime_it->second.reader_hostgroup == reader_hostgroup; + sqlite3_stmt* statement = update_existing ? update_statement : insert_statement; + int field_offset = update_existing ? 0 : 1; + if (!update_existing) { + rc=(*proxy_sqlite3_bind_int64)(statement, 1, writer_hostgroup); ASSERT_SQLITE_OK(rc, mydb); + } + rc=(*proxy_sqlite3_bind_int64)(statement, 1 + field_offset, reader_hostgroup); ASSERT_SQLITE_OK(rc, mydb); + if (green_writer_hostgroup >= 0) { + rc=(*proxy_sqlite3_bind_int64)(statement, 2 + field_offset, green_writer_hostgroup); + } else { + rc=(*proxy_sqlite3_bind_null)(statement, 2 + field_offset); + } + ASSERT_SQLITE_OK(rc, mydb); + if (green_reader_hostgroup >= 0) { + rc=(*proxy_sqlite3_bind_int64)(statement, 3 + field_offset, green_reader_hostgroup); + } else { + rc=(*proxy_sqlite3_bind_null)(statement, 3 + field_offset); + } + ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 4 + field_offset, active); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 5 + field_offset, aurora_port); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement, 6 + field_offset, r->fields[6], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 7 + field_offset, max_lag_ms); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 8 + field_offset, check_interval_ms); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 9 + field_offset, check_timeout_ms); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 10 + field_offset, writer_is_also_reader); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 11 + field_offset, new_reader_weight); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 12 + field_offset, add_lag_ms); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 13 + field_offset, min_lag_ms); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 14 + field_offset, lag_num_checks); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 15 + field_offset, autopurge_missing_checks); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement, 16 + field_offset, r->fields[16], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + if (update_existing) { + rc=(*proxy_sqlite3_bind_int64)(statement, 17, writer_hostgroup); ASSERT_SQLITE_OK(rc, mydb); + } else { + const std::string bgd_status = runtime_it == runtime_rows.end() + ? std::string("NONE") + : runtime_it->second.bgd_status; + rc=(*proxy_sqlite3_bind_text)(statement, 18, bgd_status.c_str(), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + } SAFE_SQLITE3_STEP2(statement); rc=(*proxy_sqlite3_clear_bindings)(statement); ASSERT_SQLITE_OK(rc, mydb); @@ -6504,12 +6821,12 @@ void MySQL_HostGroups_Manager::generate_mysql_aws_aurora_hostgroups_table() { if (it2!=AWS_Aurora_Info_Map.end()) { info=it2->second; bool changed=false; - changed=info->update(reader_hostgroup, aurora_port, r->fields[4], max_lag_ms, add_lag_ms, min_lag_ms, lag_num_checks, check_interval_ms, check_timeout_ms, (bool)active, writer_is_also_reader, new_reader_weight, autopurge_missing_checks, r->fields[14]); + changed=info->update(reader_hostgroup, green_writer_hostgroup, green_reader_hostgroup, aurora_port, r->fields[6], max_lag_ms, add_lag_ms, min_lag_ms, lag_num_checks, check_interval_ms, check_timeout_ms, (bool)active, writer_is_also_reader, new_reader_weight, autopurge_missing_checks, r->fields[16]); if (changed) { //info->need_converge=true; } } else { - info=new AWS_Aurora_Info(writer_hostgroup, reader_hostgroup, aurora_port, r->fields[4], max_lag_ms, add_lag_ms, min_lag_ms, lag_num_checks, check_interval_ms, check_timeout_ms, (bool)active, writer_is_also_reader, new_reader_weight, autopurge_missing_checks, r->fields[14]); + info=new AWS_Aurora_Info(writer_hostgroup, reader_hostgroup, green_writer_hostgroup, green_reader_hostgroup, aurora_port, r->fields[6], max_lag_ms, add_lag_ms, min_lag_ms, lag_num_checks, check_interval_ms, check_timeout_ms, (bool)active, writer_is_also_reader, new_reader_weight, autopurge_missing_checks, r->fields[16]); //info->need_converge=true; AWS_Aurora_Info_Map.insert(AWS_Aurora_Info_Map.begin(), std::pair(writer_hostgroup,info)); } @@ -6879,6 +7196,15 @@ int MySQL_HostGroups_Manager::remove_server_in_hg(uint32_t hid, const string& ad // FIXME: complete this!! void MySQL_HostGroups_Manager::update_aws_aurora_set_writer(int _whid, int _rhid, char *_server_id, bool verbose) { + update_aws_aurora_set_writer( + _whid, _rhid, _server_id, verbose, nullptr, -1, -1, -1); +} + +void MySQL_HostGroups_Manager::update_aws_aurora_set_writer( + int _whid, int _rhid, char *_server_id, bool verbose, + const char *domain_name_override, int aurora_port_override, + int writer_is_also_reader_override, int new_reader_weight_override +) { int cols=0; int affected_rows=0; SQLite3_result *resultset=NULL; @@ -6886,7 +7212,7 @@ void MySQL_HostGroups_Manager::update_aws_aurora_set_writer(int _whid, int _rhid char *q=NULL; char *error=NULL; //q=(char *)"SELECT hostgroup_id FROM mysql_servers JOIN mysql_galera_hostgroups ON hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup OR hostgroup_id=backup_writer_hostgroup OR hostgroup_id=offline_hostgroup WHERE hostname='%s' AND port=%d AND status<>3"; - q=(char *)"SELECT hostgroup_id FROM mysql_servers JOIN mysql_aws_aurora_hostgroups ON hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup WHERE hostname='%s%s' AND port=%d AND status<>3 AND hostgroup_id IN (%d, %d)"; + q=(char *)"SELECT hostgroup_id FROM mysql_servers WHERE hostname='%s%s' AND port=%d AND status<>3 AND hostgroup_id IN (%d, %d)"; int writer_is_also_reader=0; int new_reader_weight = 1; @@ -6896,7 +7222,14 @@ void MySQL_HostGroups_Manager::update_aws_aurora_set_writer(int _whid, int _rhid int aurora_port = 3306; char *domain_name = strdup((char *)""); int read_HG=-1; - { + if (domain_name_override != nullptr) { + free(domain_name); + domain_name = strdup(domain_name_override); + aurora_port = aurora_port_override; + writer_is_also_reader = writer_is_also_reader_override; + new_reader_weight = new_reader_weight_override; + read_HG = _rhid; + } else { pthread_mutex_lock(&AWS_Aurora_Info_mutex); std::map::iterator it2; it2 = AWS_Aurora_Info_Map.find(_writer_hostgroup); @@ -7000,10 +7333,10 @@ void MySQL_HostGroups_Manager::update_aws_aurora_set_writer(int _whid, int _rhid char *q1 = NULL; char *q2 = NULL; char *error=NULL; - q1 = (char *)"SELECT DISTINCT hostgroup_id, hostname, port, gtid_port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, mysql_servers.comment FROM mysql_servers JOIN mysql_aws_aurora_hostgroups ON hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup WHERE writer_hostgroup=%d ORDER BY hostgroup_id, hostname, port"; - q2 = (char *)"SELECT DISTINCT hostgroup_id, hostname, port, gtid_port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, mysql_servers_incoming.comment FROM mysql_servers_incoming JOIN mysql_aws_aurora_hostgroups ON hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup WHERE writer_hostgroup=%d ORDER BY hostgroup_id, hostname, port"; + q1 = (char *)"SELECT DISTINCT hostgroup_id, hostname, port, gtid_port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers WHERE hostgroup_id IN (%d,%d) ORDER BY hostgroup_id, hostname, port"; + q2 = (char *)"SELECT DISTINCT hostgroup_id, hostname, port, gtid_port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers_incoming WHERE hostgroup_id IN (%d,%d) ORDER BY hostgroup_id, hostname, port"; query = (char *)malloc(strlen(q2)+128); - sprintf(query,q1,_writer_hostgroup); + sprintf(query,q1,_writer_hostgroup,_rhid); mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset_servers); if (error == NULL) { if (resultset_servers) { @@ -7014,7 +7347,7 @@ void MySQL_HostGroups_Manager::update_aws_aurora_set_writer(int _whid, int _rhid delete resultset_servers; resultset_servers = NULL; } - sprintf(query,q2,_writer_hostgroup); + sprintf(query,q2,_writer_hostgroup,_rhid); mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset_servers); if (error == NULL) { if (resultset_servers) { @@ -7232,7 +7565,8 @@ void MySQL_HostGroups_Manager::update_aws_aurora_set_reader(int _whid, int _rhid const char SELECT_AWS_AURORA_SERVERS_FOR_MONITOR[] { "SELECT writer_hostgroup, reader_hostgroup, hostname, port, MAX(use_ssl) use_ssl, max_lag_ms, check_interval_ms," - " check_timeout_ms, add_lag_ms, min_lag_ms, lag_num_checks, autopurge_missing_checks, domain_name FROM mysql_servers" + " check_timeout_ms, add_lag_ms, min_lag_ms, lag_num_checks, autopurge_missing_checks, domain_name," + " green_writer_hostgroup, green_reader_hostgroup, writer_is_also_reader, new_reader_weight FROM mysql_servers" " JOIN mysql_aws_aurora_hostgroups ON" " hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup WHERE active=1 AND status NOT IN (2,3)" " GROUP BY writer_hostgroup, hostname, port" diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index ff3c31389b..d9adc334b5 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -6034,43 +6034,851 @@ bool AWS_Aurora_monitor_node::add_entry(AWS_Aurora_status_entry *ase) { } -typedef struct _host_def_t { - char *host; - int port; - int use_ssl; -} host_def_t; +const char* aws_aurora_bgd_status_str(AWS_Aurora_BGD_Status status) { + switch (status) { + case AWS_Aurora_BGD_Status::NONE: + return "NONE"; + case AWS_Aurora_BGD_Status::AVAILABLE: + return BGD_STATUS_AVAILABLE; + case AWS_Aurora_BGD_Status::SWITCHOVER_INITIATED: + return BGD_STATUS_INITIATED; + case AWS_Aurora_BGD_Status::SWITCHOVER_IN_PROGRESS: + return BGD_STATUS_IN_PROGRESS; + case AWS_Aurora_BGD_Status::SWITCHOVER_IN_POST_PROCESSING: + return BGD_STATUS_POST_PROC; + case AWS_Aurora_BGD_Status::SWITCHOVER_COMPLETED: + return BGD_STATUS_COMPLETED; + } + return "NONE"; +} + +namespace { + +struct AWS_Aurora_BGD_Query_Result { + unique_ptr mmsd; + int rc = 1; + unsigned int mysql_error = 0; +}; + +struct AWS_Aurora_BGD_Topology_Observation { + bool valid = false; + bool completed = false; + AWS_Aurora_BGD_Status status = AWS_Aurora_BGD_Status::NONE; + AWS_Aurora_BGD_Fingerprint fingerprint; +}; + +static std::string aws_aurora_bgd_normalize_server_id(const std::string& server_id) { + const size_t green_pos = server_id.find("-green-"); + return green_pos == std::string::npos ? server_id : server_id.substr(0, green_pos); +} + +static std::string aws_aurora_bgd_member_hostname( + const std::string& server_id, const std::string& domain_name +) { + if (domain_name.empty()) { + return server_id; + } + return domain_name.front() == '.' + ? server_id + domain_name + : server_id + "." + domain_name; +} + +static bool aws_aurora_bgd_status_from_raw( + const std::string& raw_status, AWS_Aurora_BGD_Status& status +) { + if (strcasecmp(raw_status.c_str(), BGD_STATUS_AVAILABLE) == 0) { + status = AWS_Aurora_BGD_Status::AVAILABLE; + } else if (strcasecmp(raw_status.c_str(), BGD_STATUS_INITIATED) == 0) { + status = AWS_Aurora_BGD_Status::SWITCHOVER_INITIATED; + } else if (strcasecmp(raw_status.c_str(), BGD_STATUS_IN_PROGRESS) == 0) { + status = AWS_Aurora_BGD_Status::SWITCHOVER_IN_PROGRESS; + } else if (strcasecmp(raw_status.c_str(), BGD_STATUS_POST_PROC) == 0) { + status = AWS_Aurora_BGD_Status::SWITCHOVER_IN_POST_PROCESSING; + } else if (strcasecmp(raw_status.c_str(), BGD_STATUS_COMPLETED) == 0) { + status = AWS_Aurora_BGD_Status::SWITCHOVER_COMPLETED; + } else { + return false; + } + return true; +} + +static bool aws_aurora_bgd_required_topology_fields(const AWS_RDS_Topology_Node& node) { + return !node.id.empty() && !node.endpoint.empty() && node.port > 0 + && !node.role.empty() && !node.status.empty(); +} + +static AWS_Aurora_BGD_Topology_Observation aws_aurora_bgd_validate_topology( + const AWS_RDS_Topology_Result& topology +) { + AWS_Aurora_BGD_Topology_Observation observation; + if (!topology.blue_green || topology.nodes.empty()) { + return observation; + } + + const AWS_RDS_Topology_Node* source = nullptr; + const AWS_RDS_Topology_Node* target = nullptr; + for (const AWS_RDS_Topology_Node& node : topology.nodes) { + if (!aws_aurora_bgd_required_topology_fields(node)) { + return observation; + } + if (strcasecmp(node.role.c_str(), BGD_ROLE_SOURCE) == 0) { + if (source != nullptr) { + return observation; + } + source = &node; + } else if (strcasecmp(node.role.c_str(), BGD_ROLE_TARGET) == 0) { + if (target != nullptr) { + return observation; + } + target = &node; + } else { + return observation; + } + } + + if (target == nullptr || !aws_aurora_bgd_status_from_raw(target->status, observation.status)) { + return observation; + } + + if (observation.status == AWS_Aurora_BGD_Status::SWITCHOVER_COMPLETED) { + if (topology.nodes.size() != 1 || source != nullptr) { + return AWS_Aurora_BGD_Topology_Observation {}; + } + observation.completed = true; + } else { + if (topology.nodes.size() != 2 || source == nullptr + || strcasecmp(source->status.c_str(), target->status.c_str()) != 0) { + return AWS_Aurora_BGD_Topology_Observation {}; + } + } + + observation.fingerprint = { + target->id, + target->endpoint, + target->port, + }; + observation.valid = !observation.fingerprint.empty(); + return observation; +} + +static bool aws_aurora_bgd_select_reachable_host( + const std::vector& hosts, + unsigned int writer_hg, + AWS_RDS_BGD_Probe_Host& selected +) { + if (hosts.empty()) { + return false; + } + + const size_t first = static_cast(rand()) % hosts.size(); + for (size_t offset = 0; offset < hosts.size(); ++offset) { + const AWS_RDS_BGD_Probe_Host& host = hosts[(first + offset) % hosts.size()]; + if (GloMyMon->server_responds_to_ping( + const_cast(host.hostname.c_str()), host.port)) { + selected = host; + return true; + } + MyHGM->p_update_mysql_error_counter( + p_mysql_error_type::proxysql, writer_hg, + const_cast(host.hostname.c_str()), host.port, + ER_PROXYSQL_AWS_NO_PINGABLE_SRV + ); + } + return false; +} + +static AWS_Aurora_BGD_Query_Result aws_aurora_bgd_query( + const AWS_RDS_BGD_Probe_Host& host, + unsigned int writer_hg, + unsigned int timeout_ms, + MySQL_Monitor_State_Data_Task_Type task_type, + const char* query, + std::atomic_bool& worker_stop +) { + AWS_Aurora_BGD_Query_Result out; + if (worker_stop.load()) { + out.rc = 2; + return out; + } + out.mmsd.reset(new MySQL_Monitor_State_Data( + task_type, const_cast(host.hostname.c_str()), host.port, host.use_ssl, writer_hg)); + MySQL_Monitor_State_Data* mmsd = out.mmsd.get(); + mmsd->writer_hostgroup = writer_hg; + mmsd->aws_aurora_check_timeout_ms = timeout_ms; + mmsd->mysql = GloMyMon->My_Conn_Pool->get_connection(mmsd->hostname, mmsd->port, mmsd); + + bool new_connection = false; + if (mmsd->mysql == nullptr) { + new_connection = true; + if (!mmsd->create_new_connection()) { + out.mysql_error = mmsd->mysql ? mysql_errno(mmsd->mysql) : 0; + return out; + } + GloMyMon->My_Conn_Pool->conn_register(mmsd); + } + + out.rc = GloMyMon->aws_rds_bgd_async_query(mmsd, query, worker_stop); + out.mysql_error = mmsd->mysql ? mysql_errno(mmsd->mysql) : 0; + + if (out.rc != 0) { + GloMyMon->My_Conn_Pool->destroy_mysql_connection(mmsd); + } else if (new_connection) { + if (mmsd->set_wait_timeout()) { + GloMyMon->My_Conn_Pool->put_connection(mmsd->hostname, mmsd); + } else { + GloMyMon->My_Conn_Pool->destroy_mysql_connection(mmsd); + } + } else { + GloMyMon->My_Conn_Pool->put_connection(mmsd->hostname, mmsd); + } + + return out; +} + +static void aws_aurora_bgd_set_status( + AWS_Aurora_BGD_State& st, AWS_Aurora_BGD_Status status +) { + if (st.status == status) { + return; + } + proxy_info( + "AWS Aurora BGD [wHG=%u rHG=%u]: switchover status '%s' -> '%s'\n", + st.writer_hg, st.reader_hg, + aws_aurora_bgd_status_str(st.status), aws_aurora_bgd_status_str(status)); + st.status = status; + MyHGM->update_aws_aurora_bgd_status(st.writer_hg, aws_aurora_bgd_status_str(status)); +} + +static bool aws_aurora_bgd_same_production_snapshot( + const std::vector& lhs, + const std::vector& rhs +) { + if (lhs.size() != rhs.size()) { + return false; + } + for (const AWS_Aurora_BGD_Member& member : lhs) { + auto found = std::find_if(rhs.begin(), rhs.end(), [&](const AWS_Aurora_BGD_Member& candidate) { + return candidate.normalized_server_id == member.normalized_server_id + && candidate.is_writer == member.is_writer; + }); + if (found == rhs.end()) { + return false; + } + } + return true; +} + +static bool aws_aurora_bgd_parse_target_membership( + AWS_Aurora_BGD_State& st, MYSQL_RES* result, + std::vector& snapshot +) { + if (result == nullptr || st.production_members.empty()) { + return false; + } + + const unsigned int num_fields = mysql_num_fields(result); + MYSQL_FIELD* fields = mysql_fetch_fields(result); + int server_id_idx = -1; + int session_id_idx = -1; + int is_current_idx = -1; + for (unsigned int i = 0; i < num_fields; ++i) { + if (fields[i].name == nullptr) { + continue; + } + if (strcasecmp(fields[i].name, "SERVER_ID") == 0) { + server_id_idx = static_cast(i); + } else if (strcasecmp(fields[i].name, "SESSION_ID") == 0) { + session_id_idx = static_cast(i); + } else if (strcasecmp(fields[i].name, "IS_CURRENT") == 0) { + is_current_idx = static_cast(i); + } + } + if (server_id_idx < 0 || session_id_idx < 0 || is_current_idx < 0) { + return false; + } + + std::unordered_map production_by_id; + for (const AWS_Aurora_BGD_Member& member : st.production_members) { + if (!production_by_id.emplace(member.normalized_server_id, &member).second) { + return false; + } + } + + std::unordered_map previous_reader_session_by_id; + std::unordered_map previous_reader_id_by_session; + std::unordered_map previous_ip_by_id; + std::unordered_map previous_action_by_id; + for (const AWS_Aurora_BGD_Member& member : st.target_members) { + previous_ip_by_id[member.normalized_server_id] = member.target_ip; + previous_action_by_id[member.normalized_server_id] = member.traffic_pin_applied; + if (!member.is_writer) { + previous_reader_session_by_id[member.normalized_server_id] = member.session_id; + previous_reader_id_by_session[member.session_id] = member.normalized_server_id; + } + } + + std::unordered_set target_ids; + std::unordered_set reader_sessions; + unsigned int writers = 0; + MYSQL_ROW row; + while ((row = mysql_fetch_row(result))) { + if (row[is_current_idx] == nullptr) { + continue; + } + const bool is_current = atoi(row[is_current_idx]) != 0 + || strcasecmp(row[is_current_idx], "true") == 0; + if (!is_current) { + continue; + } + if (row[server_id_idx] == nullptr || row[server_id_idx][0] == '\0' + || row[session_id_idx] == nullptr || row[session_id_idx][0] == '\0') { + return false; + } + + AWS_Aurora_BGD_Member member; + member.server_id = row[server_id_idx]; + member.normalized_server_id = aws_aurora_bgd_normalize_server_id(member.server_id); + member.session_id = row[session_id_idx]; + member.is_writer = strcasecmp(member.session_id.c_str(), "MASTER_SESSION_ID") == 0; + if (member.normalized_server_id.empty() || !target_ids.insert(member.normalized_server_id).second) { + return false; + } + + auto production = production_by_id.find(member.normalized_server_id); + if (production == production_by_id.end() + || production->second->is_writer != member.is_writer) { + return false; + } + member.production_hostname = production->second->hostname; + member.hostname = aws_aurora_bgd_member_hostname(member.server_id, st.domain_name); + member.port = st.fingerprint.target_port; + member.use_ssl = st.target_use_ssl; + + if (member.is_writer) { + writers++; + } else { + if (!reader_sessions.insert(member.session_id).second) { + return false; + } + auto old_session = previous_reader_session_by_id.find(member.normalized_server_id); + if (old_session != previous_reader_session_by_id.end() + && old_session->second != member.session_id) { + return false; + } + auto old_id = previous_reader_id_by_session.find(member.session_id); + if (old_id != previous_reader_id_by_session.end() + && old_id->second != member.normalized_server_id) { + return false; + } + } + + size_t ip_count = 0; + std::string resolved_ip = MySQL_Monitor::dns_lookup(member.hostname, false, &ip_count); + if (resolved_ip.empty()) { + const int ai_family = mysql_resolution_family_to_ai_family(mysql_thread___resolution_family); + std::vector ips = dns_resolve(member.hostname, ai_family); + if (!ips.empty()) { + resolved_ip = ips.front(); + } + } + if (resolved_ip.empty()) { + return false; + } + auto previous_ip = previous_ip_by_id.find(member.normalized_server_id); + member.target_ip = previous_ip != previous_ip_by_id.end() && !previous_ip->second.empty() + ? previous_ip->second : resolved_ip; + auto previous_action = previous_action_by_id.find(member.normalized_server_id); + member.traffic_pin_applied = previous_action != previous_action_by_id.end() + && previous_action->second; + snapshot.push_back(std::move(member)); + } + + return writers == 1 && snapshot.size() == st.production_members.size(); +} + +static std::vector aws_aurora_bgd_target_probe_hosts( + const AWS_Aurora_BGD_State& st +) { + std::vector hosts; + if (st.has_complete_target_snapshot()) { + hosts.reserve(st.target_members.size()); + for (const AWS_Aurora_BGD_Member& member : st.target_members) { + hosts.push_back({member.hostname, member.port, member.use_ssl}); + } + } else if (!st.fingerprint.empty()) { + hosts.push_back({ + st.fingerprint.target_endpoint, + st.fingerprint.target_port, + st.target_use_ssl, + }); + } + return hosts; +} + +} // namespace + +void MySQL_Monitor::aws_aurora_bgd_refresh_production_snapshot( + AWS_Aurora_BGD_State& st, const AWS_Aurora_status_entry& result +) { + if (st.production_snapshot_frozen || result.error != nullptr + || result.host_statuses == nullptr || result.host_statuses->empty()) { + return; + } + + std::vector snapshot; + std::unordered_set member_ids; + unsigned int writers = 0; + for (const AWS_Aurora_replica_host_status_entry* row : *result.host_statuses) { + if (row == nullptr || row->server_id == nullptr || row->server_id[0] == '\0' + || row->session_id == nullptr) { + return; + } + AWS_Aurora_BGD_Member member; + member.server_id = row->server_id; + member.normalized_server_id = aws_aurora_bgd_normalize_server_id(member.server_id); + member.session_id = row->session_id; + member.hostname = aws_aurora_bgd_member_hostname(member.server_id, st.domain_name); + member.production_hostname = member.hostname; + member.port = st.production_probe_hosts.empty() ? 0 : st.production_probe_hosts.front().port; + member.use_ssl = st.production_probe_hosts.empty() ? 0 : st.production_probe_hosts.front().use_ssl; + member.is_writer = strcasecmp(member.session_id.c_str(), "MASTER_SESSION_ID") == 0; + if (member.normalized_server_id.empty() || !member_ids.insert(member.normalized_server_id).second) { + return; + } + writers += member.is_writer ? 1 : 0; + snapshot.push_back(std::move(member)); + } + if (writers != 1) { + return; + } + + if (!aws_aurora_bgd_same_production_snapshot(st.production_members, snapshot)) { + st.target_snapshot_complete = false; + } + st.production_members = std::move(snapshot); +} + +void MySQL_Monitor::aws_aurora_bgd_apply_active_actions( + AWS_Aurora_BGD_State& st, bool status_changed +) { + if (st.status < AWS_Aurora_BGD_Status::SWITCHOVER_INITIATED + || st.status > AWS_Aurora_BGD_Status::SWITCHOVER_IN_POST_PROCESSING) { + return; + } + + st.production_snapshot_frozen = true; + st.production_probe_suspended = true; + + if (st.status == AWS_Aurora_BGD_Status::SWITCHOVER_IN_PROGRESS + && status_changed) { + auto writer = std::find_if( + st.production_members.begin(), st.production_members.end(), + [](const AWS_Aurora_BGD_Member& member) { return member.is_writer; }); + if (writer == st.production_members.end()) { + proxy_error( + "AWS Aurora BGD [wHG=%u rHG=%u]: cannot demote writer without a complete production snapshot\n", + st.writer_hg, st.reader_hg); + return; + } + MyHGM->update_aws_aurora_set_reader( + st.writer_hg, st.reader_hg, const_cast(writer->server_id.c_str())); + return; + } + + if (st.status != AWS_Aurora_BGD_Status::SWITCHOVER_IN_POST_PROCESSING + || !st.has_complete_target_snapshot()) { + return; + } + + bool applied_member_action = false; + bool writer_pin_applied = false; + for (AWS_Aurora_BGD_Member& member : st.target_members) { + if (!member.traffic_pin_applied) { + dns_cache->pin(member.production_hostname, member.target_ip); + MyHGM->wrlock(); + MyHGM->drain_server_connections(member.production_hostname.c_str(), member.port); + MyHGM->wrunlock(); + My_Conn_Pool->purge_connections(member.production_hostname.c_str(), member.port); + member.traffic_pin_applied = true; + applied_member_action = true; + + proxy_info( + "AWS Aurora BGD [wHG=%u rHG=%u]: repointed production '%s' to target IP %s\n", + st.writer_hg, st.reader_hg, member.production_hostname.c_str(), + member.target_ip.c_str()); + } + writer_pin_applied |= member.is_writer && member.traffic_pin_applied; + } + + if (!writer_pin_applied || (!status_changed && !applied_member_action)) { + return; + } + + auto writer = std::find_if( + st.production_members.begin(), st.production_members.end(), + [](const AWS_Aurora_BGD_Member& member) { return member.is_writer; }); + if (writer != st.production_members.end()) { + MyHGM->update_aws_aurora_set_writer( + st.writer_hg, st.reader_hg, const_cast(writer->server_id.c_str()), true, + st.domain_name.c_str(), writer->port, st.writer_is_also_reader, + st.new_reader_weight); + } +} + +void MySQL_Monitor::aws_aurora_bgd_apply_completion( + AWS_Aurora_BGD_State& st, + const AWS_Aurora_BGD_Fingerprint& completed_fingerprint +) { + if (st.status == AWS_Aurora_BGD_Status::SWITCHOVER_COMPLETED + && st.fingerprint == completed_fingerprint) { + return; + } + + auto writer = std::find_if( + st.production_members.begin(), st.production_members.end(), + [](const AWS_Aurora_BGD_Member& member) { return member.is_writer; }); + if (writer != st.production_members.end()) { + MyHGM->update_aws_aurora_set_writer( + st.writer_hg, st.reader_hg, const_cast(writer->server_id.c_str()), true, + st.domain_name.c_str(), writer->port, st.writer_is_also_reader, + st.new_reader_weight); + } + + for (const AWS_Aurora_BGD_Member& member : st.target_members) { + dns_cache->remove(member.production_hostname); + My_Conn_Pool->purge_connections(member.production_hostname.c_str(), member.port); + } + + struct green_server_t { + int hostgroup; + std::string hostname; + int port; + }; + std::vector green_servers; + MyHGM->wrlock(); + for (int hostgroup : {st.green_writer_hg, st.green_reader_hg}) { + if (hostgroup < 0) { + continue; + } + MyHGC* hgc = MyHGM->MyHGC_find(hostgroup); + if (hgc == nullptr || hgc->mysrvs == nullptr) { + continue; + } + for (unsigned int i = 0; i < hgc->mysrvs->cnt(); ++i) { + MySrvC* server = hgc->mysrvs->idx(i); + if (server->get_status() == MYSQL_SERVER_STATUS_OFFLINE_SOFT + || server->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD) { + continue; + } + green_servers.push_back({hostgroup, server->address, server->port}); + MyHGM->drain_server_connections(server->address, server->port); + } + } + MyHGM->wrunlock(); + + for (const green_server_t& server : green_servers) { + dns_cache->remove(server.hostname); + My_Conn_Pool->purge_connections(server.hostname.c_str(), server.port); + proxy_info( + "AWS Aurora BGD [wHG=%u rHG=%u]: connections drained from green HG %d server '%s:%d'\n", + st.writer_hg, st.reader_hg, server.hostgroup, + server.hostname.c_str(), server.port); + } + + st.fingerprint = completed_fingerprint; + st.production_snapshot_frozen = false; + st.production_probe_suspended = false; + st.target_snapshot_complete = false; + st.target_members.clear(); + aws_aurora_bgd_set_status(st, AWS_Aurora_BGD_Status::SWITCHOVER_COMPLETED); + + proxy_info( + "AWS Aurora BGD [wHG=%u rHG=%u]: switchover cleanup complete; completion latched\n", + st.writer_hg, st.reader_hg); +} + +void MySQL_Monitor::aws_aurora_bgd_apply_rollback( + AWS_Aurora_BGD_State& st, AWS_Aurora_BGD_Status next_status, + bool clear_deployment +) { + for (AWS_Aurora_BGD_Member& member : st.target_members) { + if (!member.traffic_pin_applied) { + continue; + } + dns_cache->remove(member.production_hostname); + MyHGM->wrlock(); + MyHGM->drain_server_connections(member.production_hostname.c_str(), member.port); + MyHGM->wrunlock(); + My_Conn_Pool->purge_connections(member.production_hostname.c_str(), member.port); + member.traffic_pin_applied = false; + } + + auto writer = std::find_if( + st.production_members.begin(), st.production_members.end(), + [](const AWS_Aurora_BGD_Member& member) { return member.is_writer; }); + if (writer != st.production_members.end()) { + MyHGM->update_aws_aurora_set_writer( + st.writer_hg, st.reader_hg, const_cast(writer->server_id.c_str()), true, + st.domain_name.c_str(), writer->port, st.writer_is_also_reader, + st.new_reader_weight); + } + + st.production_snapshot_frozen = false; + st.production_probe_suspended = false; + st.target_snapshot_complete = false; + st.target_members.clear(); + if (clear_deployment) { + st.fingerprint = AWS_Aurora_BGD_Fingerprint {}; + } + + aws_aurora_bgd_set_status(st, next_status); + if (next_status >= AWS_Aurora_BGD_Status::SWITCHOVER_INITIATED + && next_status <= AWS_Aurora_BGD_Status::SWITCHOVER_IN_POST_PROCESSING) { + aws_aurora_bgd_apply_active_actions(st, true); + } + + proxy_info( + "AWS Aurora BGD [wHG=%u rHG=%u]: rollback cleanup complete; status '%s'\n", + st.writer_hg, st.reader_hg, aws_aurora_bgd_status_str(st.status)); +} + +void MySQL_Monitor::aws_aurora_bgd_release_completed_latch(AWS_Aurora_BGD_State& st) { + if (st.status != AWS_Aurora_BGD_Status::SWITCHOVER_COMPLETED) { + return; + } + + st.fingerprint = AWS_Aurora_BGD_Fingerprint {}; + st.target_members.clear(); + st.target_snapshot_complete = false; + aws_aurora_bgd_set_status(st, AWS_Aurora_BGD_Status::NONE); + + proxy_info( + "AWS Aurora BGD [wHG=%u rHG=%u]: topology drained; completion latch released\n", + st.writer_hg, st.reader_hg); +} + +void MySQL_Monitor::aws_aurora_bgd_run_discovery_cycle( + AWS_Aurora_BGD_State& st, std::atomic_bool& worker_stop +) { + if (worker_stop.load()) { + return; + } + const bool configured_green_hgs = st.green_writer_hg >= 0 && st.green_reader_hg >= 0; + const bool discovery_admitted = configured_green_hgs + || st.status != AWS_Aurora_BGD_Status::NONE + || mysql_thread___aws_blue_green_deployment_auto_discovery != 0; + if (!discovery_admitted) { + return; + } -static void shuffle_hosts(host_def_t *array, size_t n) { - char tmp[sizeof(host_def_t)]; - char *arr = (char *)array; - size_t stride = sizeof(host_def_t) * sizeof(char); + std::vector topology_hosts = st.has_complete_target_snapshot() + ? aws_aurora_bgd_target_probe_hosts(st) + : st.production_probe_hosts; + AWS_RDS_BGD_Probe_Host topology_host; + if (!aws_aurora_bgd_select_reachable_host(topology_hosts, st.writer_hg, topology_host)) { + proxy_error("No node is pingable for AWS Aurora BGD topology checks with writer HG %u\n", st.writer_hg); + return; + } - if (n > 1) { - size_t i; - for (i = 0; i < n - 1 ; ++i) { - size_t rnd = (size_t) fastrand(); - size_t j = i + rnd / (0x7FFF / (n - i) + 1); - memcpy(tmp, arr + j * stride, sizeof(host_def_t)); - memcpy(arr + j * stride, arr + i * stride, sizeof(host_def_t)); - memcpy(arr + i * stride, tmp, sizeof(host_def_t)); + if (st.topology_state == TOPOLOGY_TABLE_CHECK) { + AWS_Aurora_BGD_Query_Result query = aws_aurora_bgd_query( + topology_host, st.writer_hg, st.check_timeout_ms, + MON_AWS_RDS_BGD, QUERY_AWS_RDS_TOPOLOGY_TABLE_CHECK, worker_stop); + if (worker_stop.load()) { + return; + } + if (query.rc == 0 && query.mmsd->result) { + if (mysql_num_rows(query.mmsd->result) > 0) { + st.topology_state = TOPOLOGY_METADATA_FETCH; + } else if (st.status != AWS_Aurora_BGD_Status::NONE + && st.status != AWS_Aurora_BGD_Status::SWITCHOVER_COMPLETED) { + aws_aurora_bgd_apply_rollback( + st, AWS_Aurora_BGD_Status::NONE, true); + } else { + aws_aurora_bgd_release_completed_latch(st); + } + } + } else { + AWS_Aurora_BGD_Query_Result query = aws_aurora_bgd_query( + topology_host, st.writer_hg, st.check_timeout_ms, + MON_AWS_RDS_BGD, QUERY_AWS_RDS_TOPOLOGY_DISCOVERY, worker_stop); + if (worker_stop.load()) { + return; + } + if (query.rc != 0) { + if (query.mysql_error == 1146) { + st.topology_state = TOPOLOGY_TABLE_CHECK; + } + } else if (query.mmsd->result) { + if (mysql_num_rows(query.mmsd->result) == 0) { + if (st.status != AWS_Aurora_BGD_Status::NONE + && st.status != AWS_Aurora_BGD_Status::SWITCHOVER_COMPLETED) { + aws_aurora_bgd_apply_rollback( + st, AWS_Aurora_BGD_Status::NONE, true); + } else { + aws_aurora_bgd_release_completed_latch(st); + } + } else { + AWS_RDS_Topology_Result topology = parse_aws_rds_topology(query.mmsd->result); + AWS_Aurora_BGD_Topology_Observation observation = + aws_aurora_bgd_validate_topology(topology); + if (observation.valid && observation.completed) { + const bool same_active_deployment = st.fingerprint.empty() + || st.fingerprint == observation.fingerprint + || st.status == AWS_Aurora_BGD_Status::SWITCHOVER_COMPLETED; + if (same_active_deployment) { + aws_aurora_bgd_apply_completion(st, observation.fingerprint); + } + } else if (observation.valid) { + const bool same_deployment = st.fingerprint.empty() + || st.fingerprint == observation.fingerprint; + const bool rollback_transition = + static_cast(observation.status) < static_cast(st.status); + if (same_deployment && rollback_transition) { + aws_aurora_bgd_apply_rollback( + st, observation.status, false); + } else if (same_deployment) { + const bool status_changed = st.status != observation.status; + st.fingerprint = observation.fingerprint; + st.target_use_ssl = topology_host.use_ssl; + aws_aurora_bgd_set_status(st, observation.status); + aws_aurora_bgd_apply_active_actions(st, status_changed); + } + } + } } } + + if (st.status == AWS_Aurora_BGD_Status::NONE + || st.status == AWS_Aurora_BGD_Status::SWITCHOVER_COMPLETED + || st.fingerprint.empty() || worker_stop.load()) { + return; + } + + std::vector membership_hosts = aws_aurora_bgd_target_probe_hosts(st); + AWS_RDS_BGD_Probe_Host membership_host; + if (!aws_aurora_bgd_select_reachable_host(membership_hosts, st.writer_hg, membership_host)) { + proxy_error("No node is pingable for AWS Aurora BGD membership checks with writer HG %u\n", st.writer_hg); + return; + } + + AWS_Aurora_BGD_Query_Result membership = aws_aurora_bgd_query( + membership_host, st.writer_hg, st.check_timeout_ms, + MON_AWS_AURORA, QUERY_AWS_AURORA_BGD_REPLICA_HOST_STATUS, worker_stop); + if (membership.rc != 0) { + return; + } + + std::vector snapshot; + if (aws_aurora_bgd_parse_target_membership(st, membership.mmsd->result, snapshot)) { + st.target_members = std::move(snapshot); + st.target_snapshot_complete = true; + proxy_debug(PROXY_DEBUG_MONITOR, 7, + "AWS Aurora BGD [wHG=%u rHG=%u]: retained complete target membership with %zu members\n", + st.writer_hg, st.reader_hg, st.target_members.size()); + aws_aurora_bgd_apply_active_actions(st, false); + } +} + +bool MySQL_Monitor::aws_aurora_bgd_load_worker_config( + int writer_hg, uint64_t current_checksum, AWS_Aurora_BGD_State& candidate +) { + SQLite3_result cluster_result(AWS_AURORA_HOSTS_COLUMNS); + pthread_mutex_lock(&aws_aurora_mutex); + if (AWS_Aurora_Hosts_resultset) { + for (SQLite3_row* row : AWS_Aurora_Hosts_resultset->rows) { + if (atoi(row->fields[AWS_AURORA_WRITER_HOSTGROUP]) == writer_hg) { + cluster_result.add_row(row); + } + } + } + pthread_mutex_unlock(&aws_aurora_mutex); + + if (cluster_result.rows.empty() || cluster_result.raw_checksum() != current_checksum) { + return false; + } + + candidate.writer_hg = writer_hg; + bool first_row = true; + for (SQLite3_row* row : cluster_result.rows) { + if (first_row) { + candidate.reader_hg = atoi(row->fields[AWS_AURORA_READER_HOSTGROUP]); + candidate.green_writer_hg = row->fields[AWS_AURORA_GREEN_WRITER_HOSTGROUP] + ? atoi(row->fields[AWS_AURORA_GREEN_WRITER_HOSTGROUP]) : -1; + candidate.green_reader_hg = row->fields[AWS_AURORA_GREEN_READER_HOSTGROUP] + ? atoi(row->fields[AWS_AURORA_GREEN_READER_HOSTGROUP]) : -1; + candidate.max_lag_ms = atoi(row->fields[AWS_AURORA_MAX_LAG_MS]); + candidate.check_interval_ms = atoi(row->fields[AWS_AURORA_CHECK_INTERVAL_MS]); + candidate.check_timeout_ms = atoi(row->fields[AWS_AURORA_CHECK_TIMEOUT_MS]); + candidate.add_lag_ms = atoi(row->fields[AWS_AURORA_ADD_LAG_MS]); + candidate.min_lag_ms = atoi(row->fields[AWS_AURORA_MIN_LAG_MS]); + candidate.lag_num_checks = atoi(row->fields[AWS_AURORA_LAG_NUM_CHECKS]); + candidate.autopurge_missing_checks = + atoi(row->fields[AWS_AURORA_AUTOPURGE_MISSING_CHECKS]); + candidate.domain_name = row->fields[AWS_AURORA_DOMAIN_NAME] + ? row->fields[AWS_AURORA_DOMAIN_NAME] : ""; + candidate.writer_is_also_reader = + atoi(row->fields[AWS_AURORA_WRITER_IS_ALSO_READER]); + candidate.new_reader_weight = atoi(row->fields[AWS_AURORA_NEW_READER_WEIGHT]); + first_row = false; + } + + candidate.production_probe_hosts.push_back(AWS_RDS_BGD_Probe_Host { + row->fields[AWS_AURORA_HOSTNAME], + atoi(row->fields[AWS_AURORA_PORT]), + atoi(row->fields[AWS_AURORA_USE_SSL]), + }); + } + + if (candidate.production_probe_hosts.empty()) { + return false; + } + candidate.target_use_ssl = candidate.production_probe_hosts.front().use_ssl; + return true; +} + +bool MySQL_Monitor::aws_aurora_bgd_refresh_worker_config( + AWS_Aurora_BGD_State& st, uint64_t current_checksum, + unsigned long long& next_loop_at +) { + AWS_Aurora_BGD_State candidate; + if (!aws_aurora_bgd_load_worker_config(st.writer_hg, current_checksum, candidate)) { + return false; + } + + st.reader_hg = candidate.reader_hg; + st.green_writer_hg = candidate.green_writer_hg; + st.green_reader_hg = candidate.green_reader_hg; + st.max_lag_ms = candidate.max_lag_ms; + st.check_interval_ms = candidate.check_interval_ms; + st.check_timeout_ms = candidate.check_timeout_ms; + st.add_lag_ms = candidate.add_lag_ms; + st.min_lag_ms = candidate.min_lag_ms; + st.lag_num_checks = candidate.lag_num_checks; + st.autopurge_missing_checks = candidate.autopurge_missing_checks; + st.writer_is_also_reader = candidate.writer_is_also_reader; + st.new_reader_weight = candidate.new_reader_weight; + st.domain_name = candidate.domain_name; + st.production_probe_hosts = std::move(candidate.production_probe_hosts); + if (st.status == AWS_Aurora_BGD_Status::NONE && !st.target_snapshot_complete) { + st.target_use_ssl = candidate.target_use_ssl; + } + + next_loop_at = 0; + proxy_info( + "AWS Aurora BGD [wHG=%u rHG=%u]: applied checksum %llu with in-place refresh at %s\n", + st.writer_hg, st.reader_hg, (unsigned long long)current_checksum, + aws_aurora_bgd_status_str(st.status)); + return true; } void * monitor_AWS_Aurora_thread_HG(void *arg) { - unsigned int wHG = *(unsigned int *)arg; - unsigned int rHG = 0; - unsigned int num_hosts = 0; + AWS_Aurora_BGD_Worker* worker = static_cast(arg); + unsigned int wHG = worker->writer_hg; unsigned int cur_host_idx = 0; - unsigned int max_lag_ms = 0; - unsigned int check_interval_ms = 0; - unsigned int check_timeout_ms = 0; - unsigned int add_lag_ms = 0; - unsigned int min_lag_ms = 0; - unsigned int lag_num_checks = 1; - unsigned int autopurge_missing_checks = 0; - std::string domain_name; std::map autopurge_counter; + AWS_Aurora_BGD_State bgd_state; + bgd_state.writer_hg = wHG; set_thread_name("MonitorAuroraHG", GloVars.set_thread_name); proxy_info("Started Monitor thread for AWS Aurora writer HG %u\n", wHG); @@ -6082,8 +6890,6 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { MySQL_Monitor__thread_MySQL_Thread_Variables_version=GloMTH->get_global_version(); mysql_thr->refresh_variables(); - uint64_t initial_raw_checksum = 0; - // this is a static array of the latest reads unsigned int ase_idx = 0; AWS_Aurora_status_entry *lasts_ase[N_L_ASE]; @@ -6091,60 +6897,13 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { lasts_ase[i] = NULL; } - // initial data load - pthread_mutex_lock(&GloMyMon->aws_aurora_mutex); - initial_raw_checksum = GloMyMon->AWS_Aurora_Hosts_resultset_checksum; - // count the number of hosts - for (std::vector::iterator it = GloMyMon->AWS_Aurora_Hosts_resultset->rows.begin() ; it != GloMyMon->AWS_Aurora_Hosts_resultset->rows.end(); ++it) { - SQLite3_row *r=*it; - if (atoi(r->fields[0]) == (int)wHG) { - num_hosts++; - if (max_lag_ms == 0) { - max_lag_ms = atoi(r->fields[5]); - } - if (check_interval_ms == 0) { - check_interval_ms = atoi(r->fields[6]); - } - if (check_timeout_ms == 0) { - check_timeout_ms = atoi(r->fields[7]); - } - if (rHG == 0) { - rHG = atoi(r->fields[1]); - } - add_lag_ms = atoi(r->fields[8]); - min_lag_ms = atoi(r->fields[9]); - lag_num_checks = atoi(r->fields[10]); - autopurge_missing_checks = atoi(r->fields[11]); - if (domain_name.empty() && r->fields[12]) { - domain_name = r->fields[12]; - } - } - } - host_def_t *hpa = (host_def_t *)malloc(sizeof(host_def_t)*num_hosts); - for (std::vector::iterator it = GloMyMon->AWS_Aurora_Hosts_resultset->rows.begin() ; it != GloMyMon->AWS_Aurora_Hosts_resultset->rows.end(); ++it) { - SQLite3_row *r=*it; - if (atoi(r->fields[0]) == (int)wHG) { - hpa[cur_host_idx].host = strdup(r->fields[2]); - hpa[cur_host_idx].port = atoi(r->fields[3]); - hpa[cur_host_idx].use_ssl = atoi(r->fields[4]); - cur_host_idx++; - } - } - // NOTE: 'cur_host_idx' should never be higher than 'num_hosts' otherwise later an invalid memory access - // can table place later when accessing 'hpa[cur_host_idx]'. - if (cur_host_idx >= num_hosts) { - cur_host_idx = num_hosts - 1; - } - pthread_mutex_unlock(&GloMyMon->aws_aurora_mutex); - - bool exit_now = false; unsigned long long t1 = 0; //unsigned long long t2 = 0; unsigned long long next_loop_at = 0; bool crc = false; - uint64_t current_raw_checksum = 0; + uint64_t last_checksum = 0; size_t rnd; bool found_pingable_host = false; bool rc_ping = false; @@ -6153,7 +6912,8 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { t1 = monotonic_time(); unsigned long long start_time=t1; - while (GloMyMon->shutdown==false && mysql_thread___monitor_enabled==true && exit_now==false) { + while (GloMyMon->shutdown==false && mysql_thread___monitor_enabled==true + && worker->worker_stop.load()==false) { unsigned int glover; @@ -6175,14 +6935,17 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { next_loop_at=0; } - pthread_mutex_lock(&GloMyMon->aws_aurora_mutex); - current_raw_checksum = GloMyMon->AWS_Aurora_Hosts_resultset_checksum; - pthread_mutex_unlock(&GloMyMon->aws_aurora_mutex); - - if (current_raw_checksum != initial_raw_checksum) { - // the content of AWS_Aurora_Hosts_resultset has changed. Exit - exit_now=true; - break; + uint64_t current_checksum = worker->current_checksum.load(); + if (current_checksum != last_checksum) { + if (!GloMyMon->aws_aurora_bgd_refresh_worker_config( + bgd_state, current_checksum, next_loop_at)) { + usleep(50000); + continue; + } + last_checksum = current_checksum; + if (cur_host_idx >= bgd_state.production_probe_hosts.size()) { + cur_host_idx = 0; + } } //fprintf(stderr,"%u : %llu %llu\n", wHG, t1, next_loop_at); if (t1 < next_loop_at) { @@ -6195,39 +6958,43 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { //proxy_info("Looping Monitor thread for AWS Aurora writer HG %u\n", wHG); continue; } + if (bgd_state.production_probe_suspended) { + if (mmsd) { + delete mmsd; + mmsd = NULL; + } + GloMyMon->aws_aurora_bgd_run_discovery_cycle( + bgd_state, worker->worker_stop); + const unsigned int interval_ms = bgd_state.production_probe_suspended + ? 100 : bgd_state.check_interval_ms; + next_loop_at = t1 + static_cast(interval_ms) * 1000; + continue; + } //proxy_info("Running check AWS Aurora writer HG %u\n", wHG); found_pingable_host = false; - rc_ping = false; - // pick a random host - rnd = (size_t) rand(); - rnd %= num_hosts; - rc_ping = GloMyMon->server_responds_to_ping(hpa[rnd].host, hpa[rnd].port); - //proxy_info("Looping Monitor thread for AWS Aurora writer HG %u\n", wHG); + const size_t num_hosts = bgd_state.production_probe_hosts.size(); + if (num_hosts != 0) { + rnd = static_cast(rand()) % num_hosts; + for (size_t offset = 0; !found_pingable_host && offset < num_hosts; ++offset) { + const size_t host_idx = (rnd + offset) % num_hosts; + const AWS_RDS_BGD_Probe_Host& host = + bgd_state.production_probe_hosts[host_idx]; + rc_ping = GloMyMon->server_responds_to_ping( + const_cast(host.hostname.c_str()), host.port); #ifdef TEST_AURORA_RANDOM - if (rand() % 100 < 30) { - // we randomly fail 30% of the requests - rc_ping = false; - } + if (offset == 0 && rand() % 100 < 30) { + rc_ping = false; + } #endif // TEST_AURORA_RANDOM - if (rc_ping) { - found_pingable_host = true; - cur_host_idx = rnd; - } else { - MyHGM->p_update_mysql_error_counter( - p_mysql_error_type::proxysql, wHG, hpa[rnd].host, hpa[rnd].port, ER_PROXYSQL_AWS_NO_PINGABLE_SRV - ); - // the randomly picked host didn't work work - shuffle_hosts(hpa,num_hosts); - for (unsigned int i=0; (found_pingable_host == false && iserver_responds_to_ping(hpa[i].host, hpa[i].port); if (rc_ping) { found_pingable_host = true; - cur_host_idx = i; + cur_host_idx = host_idx; } else { MyHGM->p_update_mysql_error_counter( - p_mysql_error_type::proxysql, wHG, hpa[i].host, hpa[i].port, ER_PROXYSQL_AWS_NO_PINGABLE_SRV - ); + p_mysql_error_type::proxysql, wHG, + const_cast(host.hostname.c_str()), host.port, + ER_PROXYSQL_AWS_NO_PINGABLE_SRV); } } } @@ -6241,22 +7008,31 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { if (found_pingable_host == false) { proxy_error("No node is pingable for AWS Aurora cluster with writer HG %u\n", wHG); - next_loop_at = t1 + check_interval_ms * 1000; + GloMyMon->aws_aurora_bgd_run_discovery_cycle( + bgd_state, worker->worker_stop); + const unsigned int interval_ms = bgd_state.production_probe_suspended + ? 100 : bgd_state.check_interval_ms; + next_loop_at = t1 + static_cast(interval_ms) * 1000; continue; } #ifdef TEST_AURORA if (rand() % 1000 == 0) { // suppress 99.9% of the output, too verbose - proxy_info("Running check for AWS Aurora writer HG %u on %s:%d\n", wHG , hpa[cur_host_idx].host, hpa[cur_host_idx].port); + const AWS_RDS_BGD_Probe_Host& host = bgd_state.production_probe_hosts[cur_host_idx]; + proxy_info("Running check for AWS Aurora writer HG %u on %s:%d\n", wHG, + host.hostname.c_str(), host.port); } #endif // TEST_AURORA if (mmsd) { delete mmsd; mmsd = NULL; } - //mmsd = NULL; - mmsd = new MySQL_Monitor_State_Data(MON_AWS_AURORA, hpa[cur_host_idx].host, hpa[cur_host_idx].port, hpa[cur_host_idx].use_ssl); + const AWS_RDS_BGD_Probe_Host& probe_host = + bgd_state.production_probe_hosts[cur_host_idx]; + mmsd = new MySQL_Monitor_State_Data( + MON_AWS_AURORA, const_cast(probe_host.hostname.c_str()), + probe_host.port, probe_host.use_ssl); mmsd->writer_hostgroup = wHG; - mmsd->aws_aurora_check_timeout_ms = check_timeout_ms; + mmsd->aws_aurora_check_timeout_ms = bgd_state.check_timeout_ms; mmsd->mysql=GloMyMon->My_Conn_Pool->get_connection(mmsd->hostname, mmsd->port, mmsd); //unsigned long long start_time=mysql_thr->curtime; start_time=t1; @@ -6290,12 +7066,6 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { mmsd->t1=monotonic_time(); mmsd->interr=0; // reset the value -#ifdef TEST_AURORA - { - string query { TEST_AURORA_MONITOR_BASE_QUERY + std::to_string(wHG) }; - mmsd->async_exit_status = mysql_query_start(&mmsd->interr, mmsd->mysql, query.c_str()); - } -#else // for reference we list the old queries. // original implementation: // mmsd->async_exit_status = mysql_query_start(&mmsd->interr, mmsd->mysql, "SELECT SERVER_ID, SESSION_ID, LAST_UPDATE_TIMESTAMP, IF(SESSION_ID = 'MASTER_SESSION_ID', 0, REPLICA_LAG_IN_MILLISECONDS) AS REPLICA_LAG_IN_MILLISECONDS, CPU FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE (REPLICA_LAG_IN_MILLISECONDS > 0 AND REPLICA_LAG_IN_MILLISECONDS <= 600000) OR SESSION_ID = 'MASTER_SESSION_ID' ORDER BY SERVER_ID"); @@ -6306,27 +7076,8 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { // with: // "REPLICA_LAG_IN_MILLISECONDS >= 0" // mmsd->async_exit_status = mysql_query_start(&mmsd->interr, mmsd->mysql, "SELECT SERVER_ID, SESSION_ID, LAST_UPDATE_TIMESTAMP, IF(SESSION_ID = 'MASTER_SESSION_ID', 0, REPLICA_LAG_IN_MILLISECONDS) AS REPLICA_LAG_IN_MILLISECONDS, CPU FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE (REPLICA_LAG_IN_MILLISECONDS >= 0 AND REPLICA_LAG_IN_MILLISECONDS <= 600000) OR SESSION_ID = 'MASTER_SESSION_ID' ORDER BY SERVER_ID"); - { - const char * query = - "SELECT SERVER_ID," - "IF(" - "SESSION_ID = 'MASTER_SESSION_ID' AND " - "SERVER_ID <> (SELECT SERVER_ID FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE SESSION_ID = 'MASTER_SESSION_ID' ORDER BY LAST_UPDATE_TIMESTAMP DESC LIMIT 1), " - "'probably_former_MASTER_SESSION_ID', SESSION_ID" - ") SESSION_ID, " // it seems that during a failover, the old writer can keep MASTER_SESSION_ID because not updated - "LAST_UPDATE_TIMESTAMP, " - "IF(SESSION_ID = 'MASTER_SESSION_ID', 0, REPLICA_LAG_IN_MILLISECONDS) AS REPLICA_LAG_IN_MILLISECONDS, " - "CPU " - "FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE" - " ( " - "(REPLICA_LAG_IN_MILLISECONDS >= 0 AND REPLICA_LAG_IN_MILLISECONDS <= 600000)" // lag between 0 and 10 minutes - " OR SESSION_ID = 'MASTER_SESSION_ID'" // or server with MASTER_SESSION_ID - " ) " - "AND LAST_UPDATE_TIMESTAMP > NOW() - INTERVAL 180 SECOND" // ignore decommissioned or renamed nodes, see https://github.com/sysown/proxysql/issues/3484 - " ORDER BY SERVER_ID"; - mmsd->async_exit_status = mysql_query_start(&mmsd->interr, mmsd->mysql, query); - } -#endif // TEST_AURORA + mmsd->async_exit_status = mysql_query_start( + &mmsd->interr, mmsd->mysql, QUERY_AWS_AURORA_REPLICA_HOST_STATUS); while (mmsd->async_exit_status) { mmsd->async_exit_status=wait_for_mysql(mmsd->mysql, mmsd->async_exit_status); #ifdef DEBUG @@ -6340,7 +7091,7 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, ER_PROXYSQL_AWS_HEALTH_CHECK_TIMEOUT); goto __exit_monitor_aws_aurora_HG_thread; } - if (GloMyMon->shutdown==true) { + if (GloMyMon->shutdown==true || worker->worker_stop.load()) { goto __fast_exit_monitor_aws_aurora_HG_thread; // exit immediately } if ((mmsd->async_exit_status & MYSQL_WAIT_TIMEOUT) == 0) { @@ -6361,7 +7112,7 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, ER_PROXYSQL_AWS_HEALTH_CHECK_TIMEOUT); goto __exit_monitor_aws_aurora_HG_thread; } - if (GloMyMon->shutdown==true) { + if (GloMyMon->shutdown==true || worker->worker_stop.load()) { goto __fast_exit_monitor_aws_aurora_HG_thread; // exit immediately } if ((mmsd->async_exit_status & MYSQL_WAIT_TIMEOUT) == 0) { @@ -6375,7 +7126,7 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { __exit_monitor_aws_aurora_HG_thread: mmsd->t2=monotonic_time(); - next_loop_at = t1 + (check_interval_ms * 1000); + next_loop_at = t1 + (bgd_state.check_interval_ms * 1000); if (mmsd->t2 > t1) { next_loop_at -= (mmsd->t2 - t1); } @@ -6413,18 +7164,26 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { mysql_free_result(mmsd->result); mmsd->result=NULL; } + GloMyMon->aws_aurora_bgd_refresh_production_snapshot(bgd_state, *ase); if (lasts_ase[ase_idx]) { AWS_Aurora_status_entry * l_ase = lasts_ase[ase_idx]; delete l_ase; } lasts_ase[ase_idx] = ase_l; - GloMyMon->evaluate_aws_aurora_results(wHG, rHG, &lasts_ase[0], ase_idx, max_lag_ms, add_lag_ms, min_lag_ms, lag_num_checks); + GloMyMon->evaluate_aws_aurora_results( + wHG, bgd_state.reader_hg, &lasts_ase[0], ase_idx, + bgd_state.max_lag_ms, bgd_state.add_lag_ms, + bgd_state.min_lag_ms, bgd_state.lag_num_checks); // Auto-purge servers that disappear from REPLICA_HOST_STATUS // Only process if autopurge is enabled and query was successful with results - if (autopurge_missing_checks > 0 && mmsd->interr == 0 && ase->host_statuses->size() > 0) { - GloMyMon->aws_aurora_autopurge_servers(wHG, rHG, ase, autopurge_missing_checks, autopurge_counter, domain_name); + if (bgd_state.autopurge_missing_checks > 0 && mmsd->interr == 0 + && ase->host_statuses->size() > 0) { + GloMyMon->aws_aurora_autopurge_servers( + wHG, bgd_state.reader_hg, ase, + bgd_state.autopurge_missing_checks, autopurge_counter, + bgd_state.domain_name); } for (auto h : *(ase_l->host_statuses)) { @@ -6502,8 +7261,19 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { } } } + if (GloMyMon->shutdown == false && worker->worker_stop.load() == false) { + GloMyMon->aws_aurora_bgd_run_discovery_cycle( + bgd_state, worker->worker_stop); + const unsigned int interval_ms = bgd_state.production_probe_suspended + ? 100 : bgd_state.check_interval_ms; + next_loop_at = t1 + static_cast(interval_ms) * 1000; + } } __exit_monitor_AWS_Aurora_thread_HG_now: + if (bgd_state.status != AWS_Aurora_BGD_Status::NONE) { + GloMyMon->aws_aurora_bgd_apply_rollback( + bgd_state, AWS_Aurora_BGD_Status::NONE, true); + } if (mmsd) { delete (mmsd); mmsd = NULL; @@ -6515,7 +7285,6 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { } } - free(hpa); if (mysql_thr) { delete mysql_thr; mysql_thr=NULL; @@ -6532,9 +7301,7 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { void * MySQL_Monitor::monitor_aws_aurora() { - // initialize the MySQL Thread (note: this is not a real thread, just the structures associated with it) - // Wait for GloMTH to be initialized - if (!wait_for_glo_mth()) return NULL; // quick exit during shutdown/restart + if (!wait_for_glo_mth()) return NULL; unsigned int MySQL_Monitor__thread_MySQL_Thread_Variables_version; MySQL_Thread * mysql_thr = new MySQL_Thread(); mysql_thr->curtime=monotonic_time(); @@ -6542,20 +7309,11 @@ void * MySQL_Monitor::monitor_aws_aurora() { mysql_thr->refresh_variables(); uint64_t last_raw_checksum = 0; - - // ADD here an unordered map , Writer HG => next time at - // when empty, a new map is populated - // when next_loop_at = 0 , the tables is emptied so to be populated again - - unsigned int *hgs_array = NULL; - pthread_t *pthreads_array = NULL; - unsigned int hgs_num = 0; + std::unordered_map> workers; while (GloMyMon->shutdown==false && mysql_thread___monitor_enabled==true) { - unsigned int glover; - - if (!GloMTH) return NULL; // quick exit during shutdown/restart + if (!GloMTH) break; // if variables has changed, triggers new checks glover=GloMTH->get_global_version(); @@ -6564,66 +7322,96 @@ void * MySQL_Monitor::monitor_aws_aurora() { mysql_thr->refresh_variables(); } - // if list of servers or HG or options has changed, triggers new checks + uint64_t new_raw_checksum = 0; + std::unordered_map cluster_checksums; pthread_mutex_lock(&aws_aurora_mutex); - uint64_t new_raw_checksum = AWS_Aurora_Hosts_resultset->raw_checksum(); + if (AWS_Aurora_Hosts_resultset) { + new_raw_checksum = AWS_Aurora_Hosts_resultset->raw_checksum(); + if (new_raw_checksum != last_raw_checksum) { + std::unordered_map> cluster_results; + for (SQLite3_row* row : AWS_Aurora_Hosts_resultset->rows) { + const int writer_hg = atoi(row->fields[AWS_AURORA_WRITER_HOSTGROUP]); + auto cluster_it = cluster_results.find(writer_hg); + if (cluster_it == cluster_results.end()) { + cluster_it = cluster_results.emplace( + writer_hg, std::unique_ptr( + new SQLite3_result(AWS_AURORA_HOSTS_COLUMNS))).first; + } + cluster_it->second->add_row(row); + } + for (const auto& [writer_hg, result] : cluster_results) { + cluster_checksums[writer_hg] = result->raw_checksum(); + } + } + } pthread_mutex_unlock(&aws_aurora_mutex); + if (new_raw_checksum != last_raw_checksum) { proxy_info("Detected new/changed definition for AWS Aurora monitoring\n"); last_raw_checksum = new_raw_checksum; - if (pthreads_array) { - // wait all threads to terminate - for (unsigned int i=0; i < hgs_num; i++) { - pthread_join(pthreads_array[i], NULL); - proxy_info("Stopped Monitor thread for AWS Aurora writer HG %u\n", hgs_array[i]); - } - free(pthreads_array); - free(hgs_array); - pthreads_array = NULL; - hgs_array = NULL; - } - hgs_num = 0; - pthread_mutex_lock(&aws_aurora_mutex); - // scan all the writer HGs - unsigned int num_rows = AWS_Aurora_Hosts_resultset->rows_count; - if (num_rows) { - unsigned int *tmp_hgs_array = (unsigned int *)malloc(sizeof(unsigned int)*num_rows); - for (std::vector::iterator it = AWS_Aurora_Hosts_resultset->rows.begin() ; it != AWS_Aurora_Hosts_resultset->rows.end(); ++it) { - SQLite3_row *r=*it; - int wHG = atoi(r->fields[0]); - bool found = false; - // very simple search. Far from optimal, but assuming very few HGs it is fast enough - for (unsigned int i=0; i < hgs_num; i++) { - if (tmp_hgs_array[i] == (unsigned int)wHG) { - found = true; - } - } - if (found == false) { - // new wHG found - tmp_hgs_array[hgs_num]=wHG; - hgs_num++; - } + std::vector stopped_workers; + + for (auto& [writer_hg, worker] : workers) { + auto cluster_it = cluster_checksums.find(writer_hg); + if (cluster_it == cluster_checksums.end()) { + worker->worker_stop.store(true); + stopped_workers.push_back(writer_hg); + proxy_info( + "AWS Aurora BGD [wHG=%d]: stopping worker; owning row is inactive, removed, or has no monitored server\n", + writer_hg); + continue; } - proxy_info("Activating Monitoring of %u AWS Aurora clusters\n", hgs_num); - hgs_array = (unsigned int *)malloc(sizeof(unsigned int)*hgs_num); - pthreads_array = (pthread_t *)malloc(sizeof(pthread_t)*hgs_num); - for (unsigned int i=0; i < hgs_num; i++) { - hgs_array[i] = tmp_hgs_array[i]; - proxy_info("Starting Monitor thread for AWS Aurora writer HG %u\n", hgs_array[i]); - if (pthread_create(&pthreads_array[i], NULL, monitor_AWS_Aurora_thread_HG, &hgs_array[i]) != 0) { - // LCOV_EXCL_START - proxy_error("Thread creation\n"); - assert(0); - // LCOV_EXCL_STOP - } + + uint64_t old_checksum = worker->current_checksum.load(); + if (old_checksum != cluster_it->second) { + worker->current_checksum.store(cluster_it->second); + proxy_info( + "AWS Aurora BGD [wHG=%d]: signaling config refresh, checksum %llu -> %llu\n", + writer_hg, (unsigned long long)old_checksum, + (unsigned long long)cluster_it->second); } - free(tmp_hgs_array); } - pthread_mutex_unlock(&aws_aurora_mutex); + + for (const auto& [writer_hg, checksum] : cluster_checksums) { + if (workers.find(writer_hg) != workers.end()) { + continue; + } + std::unique_ptr worker(new AWS_Aurora_BGD_Worker); + worker->writer_hg = writer_hg; + worker->current_checksum.store(checksum); + AWS_Aurora_BGD_Worker* worker_arg = worker.get(); + workers.emplace(writer_hg, std::move(worker)); + proxy_info("Starting Monitor thread for AWS Aurora writer HG %d\n", writer_hg); + if (pthread_create( + &worker_arg->thread, NULL, monitor_AWS_Aurora_thread_HG, worker_arg) != 0) { + // LCOV_EXCL_START + proxy_error("Thread creation\n"); + assert(0); + // LCOV_EXCL_STOP + } + } + + for (int writer_hg : stopped_workers) { + auto worker_it = workers.find(writer_hg); + if (worker_it == workers.end()) { + continue; + } + pthread_join(worker_it->second->thread, NULL); + proxy_info("Stopped Monitor thread for AWS Aurora writer HG %d\n", writer_hg); + workers.erase(worker_it); + } } usleep(10000); } + for (auto& [writer_hg, worker] : workers) { + worker->worker_stop.store(true); + } + for (auto& [writer_hg, worker] : workers) { + pthread_join(worker->thread, NULL); + proxy_info("Stopped Monitor thread for AWS Aurora writer HG %d\n", writer_hg); + } + workers.clear(); if (mysql_thr) { delete mysql_thr; mysql_thr=NULL; diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index b7a9ce5562..0a8dcdae95 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -143,7 +143,6 @@ static const vector mysql_servers_tablenames = { "mysql_replication_hostgroups", "mysql_group_replication_hostgroups", "mysql_galera_hostgroups", - "mysql_aws_aurora_hostgroups", "mysql_aws_rds_bgd_hostgroups", "mysql_hostgroup_attributes", "mysql_servers_ssl_params", @@ -189,6 +188,11 @@ static unordered_map&> module_tablenames = { { "pgsql_query_rules", pgsql_query_rules_tablenames }, }; +static const string mysql_aws_aurora_config_columns = + "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active," + "aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader," + "new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment"; + static void BQE1(SQLite3DB *db, const vector& tbs, const string& p1, const string& p2, const string& p3) { string query; for (auto it = tbs.begin(); it != tbs.end(); it++) { @@ -203,6 +207,27 @@ static void BQE1(SQLite3DB *db, const vector& tbs, const string& p1, con } } +static bool copy_mysql_aws_aurora_hostgroups( + SQLite3DB* db, + const string& destination_schema, + const string& source_schema +) { + const string table_name = "mysql_aws_aurora_hostgroups"; + const string query = + "INSERT OR REPLACE INTO " + destination_schema + "." + table_name + + " (" + mysql_aws_aurora_config_columns + ") SELECT " + + mysql_aws_aurora_config_columns + " FROM " + source_schema + "." + table_name; + return db->execute(query.c_str()); +} + +bool copy_mysql_aws_aurora_hostgroups_from_disk(SQLite3DB* db) { + return copy_mysql_aws_aurora_hostgroups(db, "main", "disk"); +} + +bool copy_mysql_aws_aurora_hostgroups_to_disk(SQLite3DB* db) { + return copy_mysql_aws_aurora_hostgroups(db, "disk", "main"); +} + static int round_intv_to_time_interval(const char* name, int _intv) { int intv = _intv; @@ -5760,6 +5785,7 @@ int ProxySQL_Admin::flush_debug_levels_database_to_runtime(SQLite3DB *db) { void ProxySQL_Admin::__insert_or_replace_maintable_select_disktable() { admindb->execute("PRAGMA foreign_keys = OFF"); BQE1(admindb, mysql_servers_tablenames, "", "INSERT OR REPLACE INTO main.", " SELECT * FROM disk."); + copy_mysql_aws_aurora_hostgroups_from_disk(admindb); BQE1(admindb, mysql_query_rules_tablenames, "", "INSERT OR REPLACE INTO main.", " SELECT * FROM disk."); admindb->execute("INSERT OR REPLACE INTO main.mysql_users SELECT * FROM disk.mysql_users"); BQE1(admindb, mysql_firewall_tablenames, "", "INSERT OR REPLACE INTO main.", " SELECT * FROM disk."); @@ -5844,6 +5870,7 @@ void ProxySQL_Admin::__insert_or_replace_maintable_select_disktable() { void ProxySQL_Admin::__insert_or_replace_disktable_select_maintable() { BQE1(admindb, mysql_servers_tablenames, "", "INSERT OR REPLACE INTO disk.", " SELECT * FROM main."); + copy_mysql_aws_aurora_hostgroups_to_disk(admindb); BQE1(admindb, mysql_query_rules_tablenames, "", "INSERT OR REPLACE INTO disk.", " SELECT * FROM main."); admindb->execute("INSERT OR REPLACE INTO disk.mysql_users SELECT * FROM main.mysql_users"); BQE1(admindb, mysql_firewall_tablenames, "", "INSERT OR REPLACE INTO disk.", " SELECT * FROM main."); @@ -5952,8 +5979,16 @@ void ProxySQL_Admin::flush_GENERIC__from_to(const string& name, const string& di assert(it != module_tablenames.end()); if (direction == "disk_to_memory") { BQE1(admindb, it->second, "DELETE FROM main.", "INSERT INTO main.", " SELECT * FROM disk."); + if (name == "mysql_servers") { + admindb->execute("DELETE FROM main.mysql_aws_aurora_hostgroups"); + copy_mysql_aws_aurora_hostgroups_from_disk(admindb); + } } else if (direction == "memory_to_disk") { BQE1(admindb, it->second, "DELETE FROM disk.", "INSERT INTO disk.", " SELECT * FROM main."); + if (name == "mysql_servers") { + admindb->execute("DELETE FROM disk.mysql_aws_aurora_hostgroups"); + copy_mysql_aws_aurora_hostgroups_to_disk(admindb); + } } else { assert(0); } @@ -7311,6 +7346,83 @@ void ProxySQL_Admin::save_scheduler_runtime_to_database(bool _runtime) { free(args); } +bool materialize_mysql_aws_aurora_hostgroups( + SQLite3DB* db, + const SQLite3_result* resultset, + bool runtime +) { + if (db == nullptr) { + return false; + } + + const int expected_columns = runtime ? 18 : 17; + if (resultset != nullptr && resultset->columns < expected_columns) { + proxy_error( + "Cannot materialize mysql_aws_aurora_hostgroups: expected at least %d columns, got %d\n", + expected_columns, + resultset->columns + ); + return false; + } + + const char* delete_query = runtime + ? "DELETE FROM main.runtime_mysql_aws_aurora_hostgroups" + : "DELETE FROM main.mysql_aws_aurora_hostgroups"; + proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", delete_query); + if (!db->execute(delete_query)) { + return false; + } + if (resultset == nullptr) { + return true; + } + + const char* insert_query = runtime + ? "INSERT INTO runtime_mysql_aws_aurora_hostgroups(writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment,bgd_status) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18)" + : "INSERT INTO mysql_aws_aurora_hostgroups(writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17)"; + auto [prepare_rc, statement_unique] = db->prepare_v2(insert_query); + ASSERT_SQLITE_OK(prepare_rc, db); + sqlite3_stmt* statement = statement_unique.get(); + + for (const SQLite3_row* row : resultset->rows) { + int rc = (*proxy_sqlite3_bind_int64)(statement, 1, atoi(row->fields[0])); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_int64)(statement, 2, atoi(row->fields[1])); ASSERT_SQLITE_OK(rc, db); + if (row->fields[2]) { + rc = (*proxy_sqlite3_bind_int64)(statement, 3, atoi(row->fields[2])); + } else { + rc = (*proxy_sqlite3_bind_null)(statement, 3); + } + ASSERT_SQLITE_OK(rc, db); + if (row->fields[3]) { + rc = (*proxy_sqlite3_bind_int64)(statement, 4, atoi(row->fields[3])); + } else { + rc = (*proxy_sqlite3_bind_null)(statement, 4); + } + ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_int64)(statement, 5, atoi(row->fields[4])); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_int64)(statement, 6, atoi(row->fields[5])); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_text)(statement, 7, row->fields[6], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_int64)(statement, 8, atoi(row->fields[7])); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_int64)(statement, 9, atoi(row->fields[8])); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_int64)(statement, 10, atoi(row->fields[9])); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_int64)(statement, 11, atoi(row->fields[10])); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_int64)(statement, 12, atoi(row->fields[11])); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_int64)(statement, 13, atoi(row->fields[12])); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_int64)(statement, 14, atoi(row->fields[13])); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_int64)(statement, 15, atoi(row->fields[14])); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_int64)(statement, 16, atoi(row->fields[15])); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_text)(statement, 17, row->fields[16], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, db); + if (runtime) { + rc = (*proxy_sqlite3_bind_text)(statement, 18, row->fields[17], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, db); + } + + SAFE_SQLITE3_STEP2(statement); + rc = (*proxy_sqlite3_clear_bindings)(statement); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_reset)(statement); ASSERT_SQLITE_OK(rc, db); + } + + return true; +} + void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) { // make sure that the caller has called mysql_servers_wrlock() char *query=NULL; @@ -7524,54 +7636,10 @@ void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) { resultset = NULL; // dump mysql_aws_aurora_hostgroups - - if (_runtime) { - query=(char *)"DELETE FROM main.runtime_mysql_aws_aurora_hostgroups"; - } else { - query=(char *)"DELETE FROM main.mysql_aws_aurora_hostgroups"; - } - proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); - admindb->execute(query); - resultset=MyHGM->dump_table_mysql("mysql_aws_aurora_hostgroups"); - if (resultset) { - int rc; - sqlite3_stmt *statement=NULL; - - char *query=NULL; - if (_runtime) { - query=(char *)"INSERT INTO runtime_mysql_aws_aurora_hostgroups(writer_hostgroup,reader_hostgroup,active,aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15)"; - } else { - query=(char *)"INSERT INTO mysql_aws_aurora_hostgroups(writer_hostgroup,reader_hostgroup,active,aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15)"; - } - - auto [rc1, statement_unique] = admindb->prepare_v2(query); - rc = rc1; - statement = statement_unique.get(); - ASSERT_SQLITE_OK(rc, admindb); - - for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { - SQLite3_row *r=*it; - rc=(*proxy_sqlite3_bind_int64)(statement, 1, atoi(r->fields[0])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 2, atoi(r->fields[1])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 3, atoi(r->fields[2])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 4, atoi(r->fields[3])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_text)(statement, 5, r->fields[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 6, atoi(r->fields[5])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 7, atoi(r->fields[6])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 8, atoi(r->fields[7])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 9, atoi(r->fields[8])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 10, atoi(r->fields[9])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 11, atoi(r->fields[10])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 12, atoi(r->fields[11])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 13, atoi(r->fields[12])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 14, atoi(r->fields[13])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_text)(statement, 15, r->fields[14], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); - - SAFE_SQLITE3_STEP2(statement); - rc=(*proxy_sqlite3_clear_bindings)(statement); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_reset)(statement); ASSERT_SQLITE_OK(rc, admindb); - } - } + resultset=MyHGM->dump_table_mysql( + _runtime ? "runtime_mysql_aws_aurora_hostgroups" : "mysql_aws_aurora_hostgroups" + ); + materialize_mysql_aws_aurora_hostgroups(admindb, resultset, _runtime); if(resultset) delete resultset; resultset=NULL; @@ -8110,27 +8178,11 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime(const incoming_servers_t& inc } // support for AWS Aurora, table mysql_aws_aurora_hostgroups - - // look for invalid combinations - query=(char *)"SELECT a.* FROM mysql_aws_aurora_hostgroups a JOIN mysql_aws_aurora_hostgroups b ON a.writer_hostgroup=b.reader_hostgroup WHERE b.reader_hostgroup"; - proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); - admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset); - if (error) { - proxy_error("Error on %s : %s\n", query, error); - } else { - for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { - SQLite3_row *r=*it; - proxy_error("Incompatible entry in mysql_aws_aurora_hostgroups will be ignored : ( %s , %s , %s , %s )\n", r->fields[0], r->fields[1], r->fields[2], r->fields[3]); - } - } - if (resultset) delete resultset; - resultset=NULL; - -//#ifdef TEST_AURORA // temporary enabled only for testing purpose - query=(char *)"SELECT a.* FROM mysql_aws_aurora_hostgroups a LEFT JOIN mysql_aws_aurora_hostgroups b ON (a.writer_hostgroup=b.reader_hostgroup) WHERE b.reader_hostgroup IS NULL ORDER BY writer_hostgroup"; -//#else -// query=(char *)"SELECT a.* FROM mysql_aws_aurora_hostgroups a WHERE 1=0"; -//#endif + query=(char *) + "SELECT writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active," + "aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader," + "new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment " + "FROM mysql_aws_aurora_hostgroups ORDER BY writer_hostgroup"; proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); if (incoming_aurora_hostgroups == nullptr) { admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset_aws_aurora); @@ -8140,8 +8192,17 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime(const incoming_servers_t& inc if (error) { proxy_error("Error on %s : %s\n", query, error); } else { - // Pass the resultset to MyHGM - MyHGM->save_incoming_mysql_table(resultset_aws_aurora,"mysql_aws_aurora_hostgroups"); + std::vector validation_errors; + SQLite3_result* filtered_aws_aurora = validate_and_filter_aws_aurora_hostgroups( + resultset_aws_aurora, + validation_errors + ); + for (const std::string& validation_error : validation_errors) { + proxy_error("%s\n", validation_error.c_str()); + } + delete resultset_aws_aurora; + resultset_aws_aurora = nullptr; + MyHGM->save_incoming_mysql_table(filtered_aws_aurora,"mysql_aws_aurora_hostgroups"); } // support for AWS RDS, table mysql_aws_rds_bgd_hostgroups @@ -9164,7 +9225,7 @@ void ProxySQL_Admin::enable_galera_testing() { void ProxySQL_Admin::enable_aurora_testing_populate_mysql_servers() { sqlite3_stmt *statement=NULL; - unsigned int num_aurora_servers = GloSQLite3Server->num_aurora_servers[0]; + constexpr unsigned int num_aurora_servers = 4; admindb->execute("DELETE FROM mysql_servers WHERE hostgroup_id BETWEEN 1271 AND 1276"); char *query=(char *)"INSERT INTO mysql_servers (hostgroup_id,hostname,use_ssl,comment) VALUES (?1, ?2, ?3, ?4)"; auto [rc1, statement_unique] = admindb->prepare_v2(query); diff --git a/lib/ProxySQL_Admin_Disk_Upgrade.cpp b/lib/ProxySQL_Admin_Disk_Upgrade.cpp index fbf3f2b638..b8cb964210 100644 --- a/lib/ProxySQL_Admin_Disk_Upgrade.cpp +++ b/lib/ProxySQL_Admin_Disk_Upgrade.cpp @@ -504,6 +504,28 @@ void ProxySQL_Admin::disk_upgrade_mysql_servers() { "add_lag_ms, min_lag_ms, lag_num_checks, comment FROM mysql_aws_aurora_hostgroups_v209"); } + // Upgrade mysql_aws_aurora_hostgroups from V2_0_10 to V3_0_12 + // (adds nullable green_writer_hostgroup and green_reader_hostgroup columns). + rci=configdb->check_table_structure("mysql_aws_aurora_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS_V2_0_10); + if (rci) { + proxy_warning("Detected version pre-v3.0.12 of mysql_aws_aurora_hostgroups\n"); + proxy_warning("ONLINE UPGRADE of table mysql_aws_aurora_hostgroups in progress\n"); + configdb->execute("DROP TABLE IF EXISTS mysql_aws_aurora_hostgroups_v2010"); + configdb->execute("ALTER TABLE mysql_aws_aurora_hostgroups RENAME TO mysql_aws_aurora_hostgroups_v2010"); + configdb->build_table("mysql_aws_aurora_hostgroups",ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS,false); + configdb->execute( + "INSERT INTO mysql_aws_aurora_hostgroups (" + "writer_hostgroup, reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, " + "check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, " + "add_lag_ms, min_lag_ms, lag_num_checks, autopurge_missing_checks, comment" + ") SELECT " + "writer_hostgroup, reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, " + "check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, " + "add_lag_ms, min_lag_ms, lag_num_checks, autopurge_missing_checks, comment " + "FROM mysql_aws_aurora_hostgroups_v2010" + ); + } + // upgrade mysql_hostgroup_attributes rci=configdb->check_table_structure("mysql_hostgroup_attributes",ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES_V2_5_0); if (rci) { diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index f82e571f45..bc223ec646 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -2389,41 +2389,62 @@ void ProxySQL_Cluster::pull_mysql_servers_v2_from_peer(const mysql_servers_v2_ch proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Writing mysql_aws_aurora_hostgroups table\n"); proxy_info("Cluster: Writing mysql_aws_aurora_hostgroups table\n"); GloAdmin->admindb->execute(SQLQueries::DELETE_MYSQL_AWS_AURORA_HOSTGROUPS); - q = (char*)"INSERT INTO mysql_aws_aurora_hostgroups ( " - "writer_hostgroup, reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, check_interval_ms, " - "check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, autopurge_missing_checks, comment) "; - while ((row = mysql_fetch_row(results[4]))) { - int l = 0; - for (int i = 0; i < 14; i++) { - l += strlen(row[i]); - } - char* o = nullptr; - char* query = nullptr; - std::string fqs = q; - - if (row[14] != nullptr) { - fqs += "VALUES (%s, %s, %s, %s, '%s', %s, %s, %s, %s, %s, %s, %s, %s, %s, '%s')"; - o = escape_string_single_quotes(row[14], false); - query = (char*)malloc(strlen(fqs.c_str()) + l + strlen(o) + 64); - sprintf(query, fqs.c_str(), row[0], row[1], row[2], row[3], row[4], row[5], row[6], row[7], row[8], row[9], row[10], row[11], row[12], row[13], o); - // free in case of 'o' being a copy - if (o != row[14]) { - free(o); + { + const char* insert_aurora = + "INSERT INTO mysql_aws_aurora_hostgroups (" + "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active," + "aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader," + "new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment" + ") VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17)"; + auto [prepare_rc, statement_unique] = GloAdmin->admindb->prepare_v2(insert_aurora); + ASSERT_SQLITE_OK(prepare_rc, GloAdmin->admindb); + sqlite3_stmt* statement = statement_unique.get(); + + while ((row = mysql_fetch_row(results[4]))) { + int rc=(*proxy_sqlite3_bind_int64)(statement, 1, atol(row[0])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 2, atol(row[1])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + if (row[2]) { + rc=(*proxy_sqlite3_bind_int64)(statement, 3, atol(row[2])); + } else { + rc=(*proxy_sqlite3_bind_null)(statement, 3); } - } else { - // In case of comment being null, placeholder must not have '' - fqs += "VALUES (%s, %s, %s, %s, '%s', %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"; - o = const_cast("NULL"); - query = (char*)malloc(strlen(fqs.c_str()) + l + strlen("NULL") + 64); - sprintf(query, fqs.c_str(), row[0], row[1], row[2], row[3], row[4], row[5], row[6], row[7], row[8], row[9], row[10], row[11], row[12], row[13], o); + ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + if (row[3]) { + rc=(*proxy_sqlite3_bind_int64)(statement, 4, atol(row[3])); + } else { + rc=(*proxy_sqlite3_bind_null)(statement, 4); + } + ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 5, atol(row[4])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 6, atol(row[5])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_bind_text)(statement, 7, row[6], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 8, atol(row[7])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 9, atol(row[8])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 10, atol(row[9])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 11, atol(row[10])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 12, atol(row[11])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 13, atol(row[12])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 14, atol(row[13])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 15, atol(row[14])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 16, atol(row[15])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + if (row[16]) { + rc=(*proxy_sqlite3_bind_text)(statement, 17, row[16], -1, SQLITE_TRANSIENT); + } else { + rc=(*proxy_sqlite3_bind_null)(statement, 17); + } + ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + SAFE_SQLITE3_STEP2(statement); + rc=(*proxy_sqlite3_clear_bindings)(statement); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_reset)(statement); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); } - - GloAdmin->admindb->execute(query); - free(query); } proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Dumping fetched 'mysql_aws_aurora_hostgroups'\n"); proxy_info("Dumping fetched 'mysql_aws_aurora_hostgroups'\n"); - GloAdmin->admindb->execute_statement((char*)"SELECT * FROM mysql_aws_aurora_hostgroups", &error, &cols, &affected_rows, &resultset); + GloAdmin->admindb->execute_statement((char*) + "SELECT writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active," + "aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader," + "new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment " + "FROM mysql_aws_aurora_hostgroups", &error, &cols, &affected_rows, &resultset); resultset->dump_to_stderr(); delete resultset; diff --git a/lib/ProxySQL_Config.cpp b/lib/ProxySQL_Config.cpp index 560e70c4cf..b4c7e910fb 100644 --- a/lib/ProxySQL_Config.cpp +++ b/lib/ProxySQL_Config.cpp @@ -1282,7 +1282,10 @@ int ProxySQL_Config::Write_MySQL_Servers_to_configfile(std::string& data) { sqlite_resultset = NULL; } - query=(char *)"SELECT * FROM mysql_aws_aurora_hostgroups"; + query=(char *)"SELECT writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active," + "aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader," + "new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment " + "FROM mysql_aws_aurora_hostgroups"; admindb->execute_statement(query, &error, &cols, &affected_rows, &sqlite_resultset); if (error) { // tolerate missing table (e.g. partial schemas in unit tests or old DBs) @@ -1298,18 +1301,21 @@ int ProxySQL_Config::Write_MySQL_Servers_to_configfile(std::string& data) { data += "\t{\n"; addField(data, "writer_hostgroup", r->fields[0], ""); addField(data, "reader_hostgroup", r->fields[1], ""); - addField(data, "active", r->fields[2], ""); - addField(data, "aurora_port", r->fields[3], ""); - addField(data, "domain_name", r->fields[4]); - addField(data, "max_lag_ms", r->fields[5], ""); - addField(data, "check_interval_ms", r->fields[6], ""); - addField(data, "check_timeout_ms", r->fields[7], ""); - addField(data, "writer_is_also_reader", r->fields[8], ""); - addField(data, "new_reader_weight", r->fields[9], ""); - addField(data, "add_lag_ms", r->fields[10], ""); - addField(data, "min_lag_ms", r->fields[11], ""); - addField(data, "lag_num_checks", r->fields[12], ""); - addField(data, "comment", r->fields[13]); + addField(data, "green_writer_hostgroup", r->fields[2], ""); + addField(data, "green_reader_hostgroup", r->fields[3], ""); + addField(data, "active", r->fields[4], ""); + addField(data, "aurora_port", r->fields[5], ""); + addField(data, "domain_name", r->fields[6]); + addField(data, "max_lag_ms", r->fields[7], ""); + addField(data, "check_interval_ms", r->fields[8], ""); + addField(data, "check_timeout_ms", r->fields[9], ""); + addField(data, "writer_is_also_reader", r->fields[10], ""); + addField(data, "new_reader_weight", r->fields[11], ""); + addField(data, "add_lag_ms", r->fields[12], ""); + addField(data, "min_lag_ms", r->fields[13], ""); + addField(data, "lag_num_checks", r->fields[14], ""); + addField(data, "autopurge_missing_checks", r->fields[15], ""); + addField(data, "comment", r->fields[16]); data += "\t}"; isNext = true; @@ -1727,11 +1733,14 @@ int ProxySQL_Config::Read_MySQL_Servers_from_configfile(std::string& error) { if (root.exists("mysql_aws_aurora_hostgroups")==true) { const Setting &mysql_aws_aurora_hostgroups = root["mysql_aws_aurora_hostgroups"]; int count = mysql_aws_aurora_hostgroups.getLength(); - char *q=(char *)"INSERT OR REPLACE INTO mysql_aws_aurora_hostgroups (writer_hostgroup, reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, autopurge_missing_checks, comment ) VALUES (%d, %d, %d, %d, '%s', %d, %d, %d, %d, %d, %d, %d, %d, %d, '%s')"; + // Green hostgroups are nullable -> passed as %s ("NULL" or an integer). + char *q=(char *)"INSERT OR REPLACE INTO mysql_aws_aurora_hostgroups (writer_hostgroup, reader_hostgroup, green_writer_hostgroup, green_reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, autopurge_missing_checks, comment ) VALUES (%d, %d, %s, %s, %d, %d, '%s', %d, %d, %d, %d, %d, %d, %d, %d, %d, '%s')"; for (i=0; i< count; i++) { const Setting &line = mysql_aws_aurora_hostgroups[i]; int writer_hostgroup; int reader_hostgroup; + int green_writer_hostgroup; + int green_reader_hostgroup; int active=1; // default int aurora_port; int max_lag_ms; @@ -1753,6 +1762,19 @@ int ProxySQL_Config::Read_MySQL_Servers_from_configfile(std::string& error) { proxy_error("Admin: detected a mysql_aws_aurora_hostgroups in config file without a mandatory reader_hostgroup\n"); continue; } + std::string green_writer_str; + std::string green_reader_str; + if (line.lookupValue("green_writer_hostgroup", green_writer_hostgroup)==false) { + green_writer_str = "NULL"; + } else { + green_writer_str = std::to_string(green_writer_hostgroup); + } + if (line.lookupValue("green_reader_hostgroup", green_reader_hostgroup)==false) { + green_reader_str = "NULL"; + } else { + green_reader_str = std::to_string(green_reader_hostgroup); + } + if (line.lookupValue("active", active)==false) active=1; if (line.lookupValue("aurora_port", aurora_port)==false) aurora_port=3306; if (line.lookupValue("max_lag_ms", max_lag_ms)==false) max_lag_ms=600000; if (line.lookupValue("check_interval_ms", check_interval_ms)==false) check_interval_ms=1000; @@ -1776,7 +1798,7 @@ int ProxySQL_Config::Read_MySQL_Servers_from_configfile(std::string& error) { const size_t safe_domain_len = safe_strlen(safe_domain); const size_t query_len = query_base_len + safe_comment_len + safe_domain_len + 256; // 128 vs sizeof(int)*8 char *query=(char *)l_alloc(query_len); - format_query(query, query_len, q, writer_hostgroup, reader_hostgroup, active, aurora_port, safe_domain, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, autopurge_missing_checks, safe_comment); + format_query(query, query_len, q, writer_hostgroup, reader_hostgroup, green_writer_str.c_str(), green_reader_str.c_str(), active, aurora_port, safe_domain, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, autopurge_missing_checks, safe_comment); //fprintf(stderr, "%s\n", query); admindb->execute(query); if (o!=o1) free(o); diff --git a/src/SQLite3_Server.cpp b/src/SQLite3_Server.cpp index f29a055df8..c5898dead0 100644 --- a/src/SQLite3_Server.cpp +++ b/src/SQLite3_Server.cpp @@ -11,7 +11,7 @@ #include "proxysql_utils.h" #include "MySQL_Query_Processor.h" #include "SQLite3_Server.h" -#ifdef TEST_RDS_BGD +#if defined(TEST_AURORA) || defined(TEST_RDS_BGD) #include "MySQL_Monitor.hpp" #endif @@ -47,6 +47,20 @@ static int random_replication_lag_seconds() { } #endif +#ifdef TEST_AURORA +static std::string simulator_sql_quote(const std::string& value) { + std::string quoted { "'" }; + for (char c : value) { + quoted += c; + if (c == '\'') { + quoted += '\''; + } + } + quoted += '\''; + return quoted; +} +#endif + #ifdef TEST_REPLICATIONLAG static void ensure_replicationlag_table_loaded(SQLite3_Server* server, MySQL_Session* sess) { if (server->replicationlag_map_size() == 0) { @@ -285,7 +299,7 @@ class sqlite3server_main_loop_listeners { static sqlite3server_main_loop_listeners S_amll; -#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) +#if defined(TEST_AURORA) || defined(TEST_READONLY) || defined(TEST_RDS_BGD) static void ensure_readonly_table(SQLite3_Server *server, MySQL_Session *sess) { if (server->readonly_map_size() == 0) { server->load_readonly_table(sess); @@ -413,6 +427,9 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p int cols; int affected_rows; bool run_query=true; +#ifdef TEST_AURORA + bool aws_aurora_replica_transaction=false; +#endif SQLite3_result *resultset=NULL; char *strA=NULL; char *strB=NULL; @@ -897,17 +914,17 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p if (run_query) { #if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) || defined(TEST_READONLY) || defined(TEST_REPLICATIONLAG) || defined(TEST_RDS_BGD) if (strncasecmp("SELECT",query_no_space,6)==0) { -#ifdef TEST_RDS_BGD - const bool rds_bgd_table_check = +#if defined(TEST_AURORA) || defined(TEST_RDS_BGD) + const bool aws_bgd_table_check = strcasecmp(query_no_space, QUERY_AWS_RDS_TOPOLOGY_TABLE_CHECK) == 0; - const bool rds_bgd_metadata = + const bool aws_bgd_metadata = strcasecmp(query_no_space, QUERY_AWS_RDS_TOPOLOGY_DISCOVERY) == 0; - if (rds_bgd_table_check || rds_bgd_metadata) { + if (aws_bgd_table_check || aws_bgd_metadata) { if (sess->client_myds->proxy_addr.addr == NULL || sess->client_myds->proxy_addr.port <= 0) { GloSQLite3Server->send_MySQL_ERR( &sess->client_myds->myprot, 1105, - "RDS BGD simulator could not identify the accepted backend address"); + "AWS BGD simulator could not identify the accepted backend address"); run_query=false; } else { SQLite3_Session *sqlite_sess = (SQLite3_Session *)sess->thread->gen_args; @@ -918,16 +935,16 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p std::to_string(backend_port) }; const std::string log_query { - "INSERT INTO RDS_BGD_PROBE_LOG" + "INSERT INTO AWS_BGD_PROBE_LOG" "(backend_ip,backend_port,probe_kind,encrypted) VALUES ('" + backend_ip + "'," + std::to_string(backend_port) + ",'" + - (rds_bgd_table_check ? "table_check" : "metadata") + "'," + + (aws_bgd_table_check ? "table_check" : "metadata") + "'," + (sess->client_myds->encrypted ? "1" : "0") + ")" }; if (!sqlite_sess->sessdb->execute(log_query.c_str())) { GloSQLite3Server->send_MySQL_ERR( &sess->client_myds->myprot, 1105, - "RDS BGD simulator failed to record the topology probe"); + "AWS BGD simulator failed to record the topology probe"); run_query=false; } else { char *control_error=NULL; @@ -935,7 +952,7 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p int control_affected_rows=0; SQLite3_result *control_result=NULL; const std::string control_query { - "SELECT topology_present,error_code,error_msg FROM RDS_BGD_CONTROL WHERE " + + "SELECT topology_present,error_code,error_msg FROM AWS_BGD_CONTROL WHERE " + predicate }; sqlite_sess->sessdb->execute_statement( @@ -960,7 +977,7 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p } delete control_result; - if (run_query && rds_bgd_table_check) { + if (run_query && aws_bgd_table_check) { const char* topology_sql = topology_present ? "SELECT 1" : "SELECT 1 WHERE 0"; static constexpr size_t topology_sql_len = sizeof("SELECT 1") - 1; @@ -983,7 +1000,7 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p } else if (run_query) { const std::string topology_query { "SELECT id,endpoint,topology_port AS port,role,status " - "FROM RDS_BGD_TOPOLOGY WHERE " + predicate + + "FROM AWS_BGD_TOPOLOGY WHERE " + predicate + " ORDER BY row_order" }; l_free(query_length,query); @@ -994,40 +1011,134 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p } } -#endif // TEST_RDS_BGD - #ifdef TEST_AURORA - if (strstr(query_no_space,(char *)"REPLICA_HOST_STATUS")) { - pthread_mutex_lock(&GloSQLite3Server->aurora_mutex); +#endif // TEST_AURORA || TEST_RDS_BGD +#ifdef TEST_AURORA + const bool aws_aurora_replica_query = + strcasecmp(query_no_space, QUERY_AWS_AURORA_REPLICA_HOST_STATUS) == 0; + const bool aws_aurora_bgd_replica_query = + strcasecmp(query_no_space, QUERY_AWS_AURORA_BGD_REPLICA_HOST_STATUS) == 0; + if (aws_aurora_replica_query || aws_aurora_bgd_replica_query) { + if (sess->client_myds->proxy_addr.addr == NULL || + sess->client_myds->proxy_addr.port <= 0) { + GloSQLite3Server->send_MySQL_ERR( + &sess->client_myds->myprot, 1105, + "AWS Aurora simulator could not identify the accepted backend address"); + run_query=false; + } else { + SQLite3_Session *sqlite_sess = (SQLite3_Session *)sess->thread->gen_args; + sqlite3 *db = sqlite_sess->sessdb->get_db(); + if ((*proxy_sqlite3_get_autocommit)(db)==1) { + if (!sqlite_sess->sessdb->execute("BEGIN IMMEDIATE")) { + GloSQLite3Server->send_MySQL_ERR( + &sess->client_myds->myprot, 1105, + "AWS Aurora simulator failed to start the replica probe transaction"); + run_query=false; + } else { + aws_aurora_replica_transaction=true; + } + } - if (strcasestr(query_no_space, TEST_AURORA_MONITOR_BASE_QUERY)) { - string s_whg { query_no_space + (sizeof(TEST_AURORA_MONITOR_BASE_QUERY) - 1) }; - uint32_t whg = atoi(s_whg.c_str()); + const std::string backend_ip { sess->client_myds->proxy_addr.addr }; + const int backend_port = sess->client_myds->proxy_addr.port; + const std::string predicate { + "backend_ip=" + simulator_sql_quote(backend_ip) + + " AND backend_port=" + std::to_string(backend_port) + }; + char *control_error=nullptr; + int control_cols=0; + int control_affected_rows=0; + SQLite3_result *control_result=nullptr; + const std::string control_query { + "SELECT replica_set_id,replica_table_present,error_code,error_msg " + "FROM AWS_AURORA_REPLICA_CONTROL WHERE " + predicate + }; + if (run_query) { + sqlite_sess->sessdb->execute_statement( + control_query.c_str(), &control_error, &control_cols, + &control_affected_rows, &control_result); + } - GloSQLite3Server->populate_aws_aurora_table(sess, whg); - vector hgs_info { get_hgs_info(GloAdmin->admindb) }; + std::string replica_set_id {}; + bool replica_table_present=false; + unsigned int configured_error=0; + std::string configured_error_msg {}; + if (run_query && control_error == nullptr && control_result && + control_result->rows_count == 1) { + SQLite3_row *row=control_result->rows.front(); + replica_set_id=row->fields[0] ? row->fields[0] : ""; + replica_table_present=atoi(row->fields[1]) != 0; + configured_error=static_cast(atoi(row->fields[2])); + configured_error_msg=row->fields[3] ? row->fields[3] : ""; + } + delete control_result; - const auto match_writer = [&whg](const aurora_hg_info_t& hg_info) { - return std::get(hg_info) == whg; - }; - const auto hg_info_it = std::find_if(hgs_info.begin(), hgs_info.end(), match_writer); - string select_query { - "SELECT SERVER_ID,SESSION_ID,LAST_UPDATE_TIMESTAMP,REPLICA_LAG_IN_MILLISECONDS,CPU" - " FROM REPLICA_HOST_STATUS " - }; + const std::string log_query { + "INSERT INTO AWS_AURORA_REPLICA_PROBE_LOG" + "(backend_ip,backend_port,probe_kind,replica_set_id,encrypted) VALUES (" + + simulator_sql_quote(backend_ip) + "," + + std::to_string(backend_port) + "," + + simulator_sql_quote( + aws_aurora_replica_query ? "ordinary" : "bgd_membership") + "," + + (replica_set_id.empty() ? "NULL" : simulator_sql_quote(replica_set_id)) + + "," + (sess->client_myds->encrypted ? "1" : "0") + ")" + }; - if (hg_info_it == hgs_info.end()) { - select_query += " LIMIT 0"; + if (!run_query) { + // The transaction-start error was already sent above. + } else if (control_error != nullptr) { + GloSQLite3Server->send_MySQL_ERR( + &sess->client_myds->myprot, 1105, control_error); + free(control_error); + run_query=false; + } else if (!sqlite_sess->sessdb->execute(log_query.c_str())) { + GloSQLite3Server->send_MySQL_ERR( + &sess->client_myds->myprot, 1105, + "AWS Aurora simulator failed to record the replica probe"); + run_query=false; + } else if (!replica_table_present) { + GloSQLite3Server->send_MySQL_ERR( + &sess->client_myds->myprot, 1146, + "Table 'information_schema.REPLICA_HOST_STATUS' doesn't exist"); + run_query=false; + } else if (configured_error != 0) { + GloSQLite3Server->send_MySQL_ERR( + &sess->client_myds->myprot, + static_cast(configured_error), + configured_error_msg.c_str()); + run_query=false; + } else { + const std::string set_literal { simulator_sql_quote(replica_set_id) }; + std::string select_query {}; + if (aws_aurora_replica_query) { + select_query = + "SELECT SERVER_ID," + "CASE WHEN SESSION_ID='MASTER_SESSION_ID' AND SERVER_ID<>(" + "SELECT SERVER_ID FROM REPLICA_HOST_STATUS WHERE REPLICA_SET_ID=" + + set_literal + " AND SESSION_ID='MASTER_SESSION_ID' " + "ORDER BY LAST_UPDATE_TIMESTAMP DESC LIMIT 1) " + "THEN 'probably_former_MASTER_SESSION_ID' ELSE SESSION_ID END AS SESSION_ID," + "LAST_UPDATE_TIMESTAMP," + "CASE WHEN SESSION_ID='MASTER_SESSION_ID' THEN 0 " + "ELSE REPLICA_LAG_IN_MILLISECONDS END AS REPLICA_LAG_IN_MILLISECONDS," + "CPU FROM REPLICA_HOST_STATUS WHERE REPLICA_SET_ID=" + set_literal + + " AND ((REPLICA_LAG_IN_MILLISECONDS>=0 AND " + "REPLICA_LAG_IN_MILLISECONDS<=600000) OR " + "SESSION_ID='MASTER_SESSION_ID')" + " AND LAST_UPDATE_TIMESTAMP>datetime('now','-180 seconds')" + " ORDER BY SERVER_ID"; } else { - const string& domain_name { std::get(*hg_info_it) }; - select_query += " WHERE DOMAIN_NAME='" + domain_name + "' ORDER BY SERVER_ID"; + select_query = + "SELECT SERVER_ID,SESSION_ID,LAST_UPDATE_TIMESTAMP,IS_CURRENT " + "FROM REPLICA_HOST_STATUS WHERE REPLICA_SET_ID=" + set_literal + + " ORDER BY SERVER_ID"; } - free(query); - query = static_cast(malloc(select_query.length() + 1)); - memcpy(query, select_query.c_str(), select_query.length()); - query[select_query.length()] = '\0'; + l_free(query_length,query); + query=l_strdup(select_query.c_str()); + query_length=select_query.length()+1; } } + } #endif // TEST_AURORA #ifdef TEST_GALERA if (strstr(query_no_space,(char *)"HOST_STATUS_GALERA")) { @@ -1077,7 +1188,7 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p } } #endif // TEST_GROUPREP -#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) +#if defined(TEST_AURORA) || defined(TEST_READONLY) || defined(TEST_RDS_BGD) if (strncasecmp("SELECT @@global.read_only read_only ",query_no_space, k_select_read_only_len)==0 && query_no_space_length > k_select_read_only_len+5) { pthread_mutex_lock(&GloSQLite3Server->test_readonly_mutex); @@ -1091,7 +1202,7 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p query_length = formatted_query.size() + 1; pthread_mutex_unlock(&GloSQLite3Server->test_readonly_mutex); } -#endif // TEST_READONLY || TEST_RDS_BGD +#endif // TEST_AURORA || TEST_READONLY || TEST_RDS_BGD #ifdef TEST_REPLICATIONLAG const bool replica_status = strncasecmp("SELECT REPLICA STATUS ", query_no_space, k_select_replica_status_len) == 0; const uint64_t addr_offset { @@ -1126,6 +1237,12 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p } #endif // TEST_AURORA || TEST_GALERA || TEST_GROUPREP || TEST_READONLY || TEST_REPLICATIONLAG || TEST_RDS_BGD if (!run_query) { +#ifdef TEST_AURORA + if (aws_aurora_replica_transaction) { + SQLite3_Session *sqlite_sess = (SQLite3_Session *)sess->thread->gen_args; + sqlite_sess->sessdb->execute("COMMIT"); + } +#endif l_free(pkt->size-sizeof(mysql_hdr),query_no_space); l_free(query_length,query); return; @@ -1139,19 +1256,13 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p } } sqlite_sess->sessdb->execute_statement(query, &error , &cols , &affected_rows , &resultset); -#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) - if (strncasecmp("SELECT",query_no_space,6)==0) { #ifdef TEST_AURORA - if (strstr(query_no_space,(char *)"REPLICA_HOST_STATUS")) { - pthread_mutex_unlock(&GloSQLite3Server->aurora_mutex); -#ifdef TEST_AURORA_RANDOM - if (rand() % 100 == 0) { - // randomly add some latency on 1% of the traffic - sleep(2); - } + if (aws_aurora_replica_transaction) { + sqlite_sess->sessdb->execute("COMMIT"); + } #endif - } -#endif // TEST_AURORA +#if defined(TEST_GALERA) || defined(TEST_GROUPREP) + if (strncasecmp("SELECT",query_no_space,6)==0) { #ifdef TEST_GALERA if (strstr(query_no_space,(char *)"HOST_STATUS_GALERA")) { pthread_mutex_unlock(&GloSQLite3Server->galera_mutex); @@ -1184,7 +1295,7 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p } } } -#endif // TEST_AURORA || TEST_GALERA || TEST_GROUPREP +#endif // TEST_GALERA || TEST_GROUPREP sqlite3 *db = sqlite_sess->sessdb->get_db(); bool in_trans = false; if ((*proxy_sqlite3_get_autocommit)(db)==0) { @@ -1193,7 +1304,7 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p bool deprecate_eof = sess->client_myds->myconn->options.client_flag & CLIENT_DEPRECATE_EOF; sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot, in_trans, deprecate_eof); delete resultset; -#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) +#if defined(TEST_AURORA) || defined(TEST_READONLY) || defined(TEST_RDS_BGD) if (strncasecmp("SELECT",query_no_space,6)) { if (strstr(query_no_space,(char *)"READONLY_STATUS")) { // the table is writable @@ -1202,7 +1313,7 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p pthread_mutex_unlock(&GloSQLite3Server->test_readonly_mutex); } } -#endif // TEST_READONLY || TEST_RDS_BGD +#endif // TEST_AURORA || TEST_READONLY || TEST_RDS_BGD #ifdef TEST_REPLICATIONLAG if (strncasecmp("SELECT", query_no_space, 6)) { if (strstr(query_no_space, (char*)"REPLICATIONLAG_HOST_STATUS")) { @@ -1512,29 +1623,23 @@ SQLite3_Server::~SQLite3_Server() { delete tables_defs_grouprep; #endif // TEST_GROUPREP -#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) +#if defined(TEST_AURORA) || defined(TEST_READONLY) || defined(TEST_RDS_BGD) drop_tables_defs(tables_defs_readonly); delete tables_defs_readonly; #endif -#ifdef TEST_RDS_BGD - drop_tables_defs(tables_defs_rds_bgd); - delete tables_defs_rds_bgd; -#endif // TEST_RDS_BGD +#if defined(TEST_AURORA) || defined(TEST_RDS_BGD) + drop_tables_defs(tables_defs_aws_bgd); + delete tables_defs_aws_bgd; +#endif // TEST_AURORA || TEST_RDS_BGD }; #ifdef TEST_AURORA void SQLite3_Server::init_aurora_ifaces_string(std::string& s) { if(!s.empty()) s += ";"; - pthread_mutex_init(&aurora_mutex,NULL); - unsigned int nas = time(NULL); - nas = nas % 3; // range - nas += 4; // min - max_num_aurora_servers = 10; // hypothetical maximum number of nodes + constexpr unsigned int max_num_aurora_servers = 10; for (unsigned int j=1; j<4; j++) { - cur_aurora_writer[j-1] = 0; - num_aurora_servers[j-1] = nas; for (unsigned int i=11; i(cpu_i) / 100; - - return cpu; -} - -string get_curtime_str() { - time_t __timer; - char lut[30]; - struct tm __tm_info; - time(&__timer); - localtime_r(&__timer, &__tm_info); - strftime(lut, 25, "%Y-%m-%d %H:%M:%S", &__tm_info); - string s = string(lut); - return s; -} - -void bind_query_params( - SQLite3DB* db, - sqlite3_stmt* stmt, - const string& server_id, - const string& domain, - const string& session_id, - float cpu, - const string& lut, - int32_t lag_ms -) { - int rc = 0; - - rc=(*proxy_sqlite3_bind_text)(stmt, 1, server_id.c_str(), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, db); - rc=(*proxy_sqlite3_bind_text)(stmt, 2, domain.c_str(), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, db); - rc=(*proxy_sqlite3_bind_text)(stmt, 3, session_id.c_str(), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, db); - rc=(*proxy_sqlite3_bind_double)(stmt, 4, cpu); ASSERT_SQLITE_OK(rc, db); - rc=(*proxy_sqlite3_bind_text)(stmt, 5, lut.c_str(), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, db); - rc=(*proxy_sqlite3_bind_double)(stmt, 6, lag_ms); ASSERT_SQLITE_OK(rc, db); - SAFE_SQLITE3_STEP2(stmt); - rc=(*proxy_sqlite3_clear_bindings)(stmt); ASSERT_SQLITE_OK(rc, db); - rc=(*proxy_sqlite3_reset)(stmt); ASSERT_SQLITE_OK(rc, db); -} - -/** - * @brief Extracts SERVER_ID from the supplied hostname using DOMAIN_NAME. - * @param hostname The server hostname (SERVER_ID + DOMAIN_NAME)). - * @param domain_name The server DOMAIN_NAME as in 'mysql_aws_aurora_hostgroups' - * @return Either the SERVER_ID in the supplied hostname or empty if DOMAIN_NAME failed to match. - */ -string get_server_id(const string& hostname, const string& domain_name) { - string::size_type pos = hostname.find(domain_name); - - if (pos == string::npos) { - return {}; - } else { - return hostname.substr(0, pos); - } -} - -void SQLite3_Server::populate_aws_aurora_table(MySQL_Session *sess, uint32_t whg) { - int rc = 0; - sqlite3_stmt* stmt = NULL; - const char query[] { "INSERT INTO REPLICA_HOST_STATUS VALUES (?1, ?2, ?3, ?4, ?5, ?6)" }; - - auto [stmt_rc, stmt_unique] = sessdb->prepare_v2(query); - rc = stmt_rc; - stmt = stmt_unique.get(); - ASSERT_SQLITE_OK(rc, sessdb); - -#ifndef TEST_AURORA_RANDOM - SQLite3_result* host_status = NULL; - - { - char* error = NULL; - int cols = 0; - int affected_rows = 0; - - string query { - "SELECT SERVER_ID,DOMAIN_NAME,SESSION_ID,LAST_UPDATE_TIMESTAMP,REPLICA_LAG_IN_MILLISECONDS" - " FROM REPLICA_HOST_STATUS" - }; - sessdb->execute_statement(query.c_str(), &error, &cols, &affected_rows, &host_status); - } - - // If empty, we fill the map with sensible defaults for performing manual testing. - if (host_status->rows.empty()) { - vector hgs_info { get_hgs_info(GloAdmin->admindb) }; - SQLite3_result* resultset = nullptr; - - { - char* error = nullptr; - int cols = 0; - int affected_rows = 0; - - GloAdmin->admindb->execute_statement( - "SELECT hostname, hostgroup_id FROM mysql_servers WHERE hostgroup_id BETWEEN 1270 AND 1300" - " GROUP BY HOSTNAME", - &error, &cols, &affected_rows, &resultset - ); - } - - sessdb->execute("DELETE FROM REPLICA_HOST_STATUS"); - vector proc_srvs {}; - - for (const aurora_hg_info_t& hg_info : hgs_info) { - const auto match_writer = [&hg_info](const SQLite3_row* row) { - return atoi(row->fields[1]) == std::get(hg_info); - }; - const auto mysrv_it = std::find_if(resultset->rows.begin(), resultset->rows.end(), match_writer); - bool writer_set = false; - - for (const SQLite3_row* r : resultset->rows) { - const string srv_hostname { r->fields[0] }; - const uint32_t srv_hg_id = atoi(r->fields[1]); - const string& aurora_domain { std::get(hg_info) }; - - if ( - srv_hostname.find(aurora_domain) == string::npos || - std::find(proc_srvs.begin(), proc_srvs.end(), srv_hostname) != proc_srvs.end() - ) { - continue; - } - - const string server_id { - get_server_id(srv_hostname, std::get(hg_info)) - }; - - string session_id {}; - - if ( - (mysrv_it == resultset->rows.end() && writer_set == false) || - (srv_hg_id == std::get(hg_info) && writer_set == false) - ) { - session_id = "MASTER_SESSION_ID"; - writer_set = true; - } else { - session_id = "TESTID-" + server_id + aurora_domain + "-R"; - } - - const float cpu = get_rand_cpu(); - const string lut { get_curtime_str() }; - const int lag_ms = 0; - - bind_query_params(sessdb, stmt, server_id, aurora_domain, session_id, cpu, lut, lag_ms); - proc_srvs.push_back(srv_hostname); - } - } - - delete resultset; - } else { - // We just re-generate deterministic 'SESSION_IDS', preserving 'MASTER_SESSION_ID' values: - // 'SESSION_IDS' are preserved, 'MASTER_SESSION_ID' or others. - for (SQLite3_row* row : host_status->rows) { - const char* server_id = row->fields[0]; - const char* domain_name = row->fields[1]; - - const char update_query_t[] { - "UPDATE REPLICA_HOST_STATUS SET SESSION_ID='%s',CPU=%f,LAST_UPDATE_TIMESTAMP='%s'" - " WHERE SERVER_ID='%s' AND DOMAIN_NAME='%s' AND SESSION_ID!='MASTER_SESSION_ID'" - }; - - const string session_id { "TESTID-" + string { server_id } + domain_name + "-R" }; - const float cpu = get_rand_cpu(); - const string lut { get_curtime_str() }; - - const string update_query { - cstr_format(update_query_t, session_id.c_str(), cpu, lut.c_str(), server_id, domain_name).str - }; - - sessdb->execute(update_query.c_str()); - } - } - - delete host_status; -#else - sessdb->execute("DELETE FROM REPLICA_HOST_STATUS"); - - string lut { get_curtime_str() }; - string myip = string(sess->client_myds->proxy_addr.addr); - string clu_id_s = myip.substr(6,1); - unsigned int cluster_id = atoi(clu_id_s.c_str()); - cluster_id--; - - if (rand() % 20000 == 0) { - // simulate a failover - cur_aurora_writer[cluster_id] = rand() % num_aurora_servers[cluster_id]; - proxy_info("Simulating a failover for AWS Aurora cluster %d , HGs (%d:%d)\n", cluster_id, 1270 + cluster_id*2+1 , 1270 + cluster_id*2+2); - } - if (rand() % 1000 == 0) { - if (num_aurora_servers[cluster_id] < max_num_aurora_servers) { - num_aurora_servers[cluster_id]++; - proxy_info("Simulating the add of a new server for AWS Aurora Cluster %d , HGs (%d:%d). Now adding server num %d\n", cluster_id, 1270 + cluster_id*2+1 , 1270 + cluster_id*2+2, num_aurora_servers[cluster_id]); - } - } - if (rand() % 1000 == 0) { - if (num_aurora_servers[cluster_id] > 1) { - if (cur_aurora_writer[cluster_id] != (num_aurora_servers[cluster_id] - 1) ) { - num_aurora_servers[cluster_id]--; - proxy_info("Simulating the deletion of a server from AWS Aurora Cluster %d , HGs (%d:%d). Removing server num %d\n", cluster_id, 1270 + cluster_id*2+1 , 1270 + cluster_id*2+2, num_aurora_servers[cluster_id]+1); - } - } - } - for (unsigned int i=0; ienable_grouprep_testing(); #endif // TEST_GALERA -#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) +#if defined(TEST_AURORA) || defined(TEST_READONLY) || defined(TEST_RDS_BGD) tables_defs_readonly = new std::vector; insert_into_tables_defs(tables_defs_readonly, (const char *)"READONLY_STATUS", @@ -2093,32 +1993,32 @@ bool SQLite3_Server::init() { #ifdef TEST_READONLY GloAdmin->enable_readonly_testing(); #endif // TEST_READONLY -#endif // TEST_READONLY || TEST_RDS_BGD -#ifdef TEST_RDS_BGD - tables_defs_rds_bgd = new std::vector; - insert_into_tables_defs(tables_defs_rds_bgd, - (const char *)"RDS_BGD_CONTROL", - (const char *)"CREATE TABLE RDS_BGD_CONTROL (" +#endif // TEST_AURORA || TEST_READONLY || TEST_RDS_BGD +#if defined(TEST_AURORA) || defined(TEST_RDS_BGD) + tables_defs_aws_bgd = new std::vector; + insert_into_tables_defs(tables_defs_aws_bgd, + (const char *)"AWS_BGD_CONTROL", + (const char *)"CREATE TABLE AWS_BGD_CONTROL (" "backend_ip TEXT NOT NULL, backend_port INTEGER NOT NULL, " "topology_present INTEGER NOT NULL DEFAULT 0 CHECK (topology_present IN (0,1)), " "error_code INTEGER NOT NULL DEFAULT 0, error_msg TEXT NOT NULL DEFAULT '', " "PRIMARY KEY (backend_ip, backend_port))"); - insert_into_tables_defs(tables_defs_rds_bgd, - (const char *)"RDS_BGD_TOPOLOGY", - (const char *)"CREATE TABLE RDS_BGD_TOPOLOGY (" + insert_into_tables_defs(tables_defs_aws_bgd, + (const char *)"AWS_BGD_TOPOLOGY", + (const char *)"CREATE TABLE AWS_BGD_TOPOLOGY (" "backend_ip TEXT NOT NULL, backend_port INTEGER NOT NULL, row_order INTEGER NOT NULL, " "id TEXT NOT NULL, endpoint TEXT NOT NULL, topology_port INTEGER NOT NULL, " "role TEXT NOT NULL, status TEXT NOT NULL, " "PRIMARY KEY (backend_ip, backend_port, row_order))"); - insert_into_tables_defs(tables_defs_rds_bgd, - (const char *)"RDS_BGD_PROBE_LOG", - (const char *)"CREATE TABLE RDS_BGD_PROBE_LOG (" + insert_into_tables_defs(tables_defs_aws_bgd, + (const char *)"AWS_BGD_PROBE_LOG", + (const char *)"CREATE TABLE AWS_BGD_PROBE_LOG (" "sequence_id INTEGER PRIMARY KEY AUTOINCREMENT, backend_ip TEXT NOT NULL, " "backend_port INTEGER NOT NULL, probe_kind TEXT NOT NULL " "CHECK (probe_kind IN ('table_check','metadata')), encrypted INTEGER NOT NULL " "CHECK (encrypted IN (0,1)))"); - check_and_build_standard_tables(sessdb, tables_defs_rds_bgd); -#endif // TEST_RDS_BGD + check_and_build_standard_tables(sessdb, tables_defs_aws_bgd); +#endif // TEST_AURORA || TEST_RDS_BGD #ifdef TEST_REPLICATIONLAG tables_defs_replicationlag = new std::vector; insert_into_tables_defs(tables_defs_replicationlag, @@ -2248,7 +2148,7 @@ void SQLite3_Server::send_MySQL_ERR(MySQL_Protocol *myprot, uint16_t error_code, myds->DSS=STATE_SLEEP; } -#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) +#if defined(TEST_AURORA) || defined(TEST_READONLY) || defined(TEST_RDS_BGD) void SQLite3_Server::load_readonly_table(MySQL_Session *sess) { // this function needs to be called with lock on mutex readonly_mutex already acquired GloAdmin->mysql_servers_wrlock(); @@ -2289,7 +2189,7 @@ int SQLite3_Server::readonly_test_value(char *p) { } return rc; } -#endif // TEST_READONLY || TEST_RDS_BGD +#endif // TEST_AURORA || TEST_READONLY || TEST_RDS_BGD #ifdef TEST_REPLICATIONLAG void SQLite3_Server::load_replicationlag_table(MySQL_Session* sess) { diff --git a/test/deps/cluster_simulator/cluster_simulator.cpp b/test/deps/cluster_simulator/cluster_simulator.cpp index 2ee974de1a..dc23952a0a 100644 --- a/test/deps/cluster_simulator/cluster_simulator.cpp +++ b/test/deps/cluster_simulator/cluster_simulator.cpp @@ -1719,7 +1719,9 @@ std::pair simulate_aws_aurora_cluster_state( // change the values for the cluster state as a last step prep_aurora_init_state_err = prepare_aurora_cluster_state( - proxysql_sqlite, aurora_init_servers_state, hostgroup_attributes.empty() ? 1 : 2 + proxysql_sqlite, + aurora_init_servers_state, + aurora_publication_mode::reset_scenario ); if (prep_aurora_init_state_err.first) { @@ -1794,16 +1796,13 @@ std::pair simulate_aws_aurora_cluster_state( aurora_servers_state_diff(aurora_init_servers_state, aurora_new_servers_state) }; - // set the new servers state - const auto& aurora_new_state_to_set { - aurora_update_cluster_state(aurora_init_servers_state, aurora_new_servers_state) - }; - // cleanup=1 deletes REPLICA_HOST_STATUS rows whose SERVER_ID is not in the new - // state, so payloads can simulate a server being removed from the topology - // (e.g. for autopurge_missing_checks coverage). Backwards-compatible: existing - // payloads keep the full server list in new_state, so nothing extra is purged. - prep_aurora_final_state_err = - prepare_aurora_cluster_state(proxysql_sqlite, aurora_new_servers_state, 1); + // Replace the published replica sets and their complete backend mappings so + // payloads can simulate members being removed (for example, autopurge coverage). + prep_aurora_final_state_err = prepare_aurora_cluster_state( + proxysql_sqlite, + aurora_new_servers_state, + aurora_publication_mode::replace_snapshot_retaining_backends + ); if (prep_aurora_final_state_err.first) { result = internal_error(prep_aurora_final_state_err.second, __FILE__, __LINE__); diff --git a/test/deps/cluster_simulator/lib/aurora_utils.cpp b/test/deps/cluster_simulator/lib/aurora_utils.cpp index 920acd30d9..fb7b1ef9ec 100644 --- a/test/deps/cluster_simulator/lib/aurora_utils.cpp +++ b/test/deps/cluster_simulator/lib/aurora_utils.cpp @@ -1,10 +1,15 @@ #include "aurora_utils.h" #include +#include +#include +#include #include +#include #include #include #include +#include // NOTE: Only needed during testing #include @@ -354,89 +359,260 @@ std::pair prepare_mysql_aurora_hostgroups( return { EXIT_SUCCESS, "" }; } -const char t_aurora_server_state_insert[] { - "INSERT OR REPLACE INTO REPLICA_HOST_STATUS(" - " SERVER_ID," - " DOMAIN_NAME," - " SESSION_ID," - " CPU," - " LAST_UPDATE_TIMESTAMP," - " REPLICA_LAG_IN_MILLISECONDS" - ") VALUES (" - "'%s', '%s', '%s', %d, '%s', %d" - ")" -}; +namespace { + +string aurora_sql_quote(const string& value) { + string quoted { "'" }; + for (char c : value) { + quoted += c; + if (c == '\'') { + quoted += '\''; + } + } + quoted += '\''; + return quoted; +} + +string aurora_utc_timestamp() { + time_t now = time(nullptr); + struct tm utc_time {}; + gmtime_r(&now, &utc_time); + char timestamp[20] {}; + strftime(timestamp, sizeof(timestamp), "%Y-%m-%d %H:%M:%S", &utc_time); + return timestamp; +} + +std::pair aurora_scalar_uint64(MYSQL* connection, const string& query) { + if (mysql_query(connection, query.c_str()) != 0) { + return { EXIT_FAILURE, 0 }; + } + MYSQL_RES* result = mysql_store_result(connection); + if (result == nullptr) { + return { EXIT_FAILURE, 0 }; + } + MYSQL_ROW row = mysql_fetch_row(result); + const bool valid = row != nullptr && row[0] != nullptr; + const uint64_t value = valid ? strtoull(row[0], nullptr, 10) : 0; + mysql_free_result(result); + return { valid ? EXIT_SUCCESS : EXIT_FAILURE, value }; +} + +std::pair load_aurora_backend_addresses( + std::unordered_map& addresses) +{ + const char* host_file_path = getenv("CLUSTER_SIM_HOST_FILE"); + if (host_file_path == nullptr || *host_file_path == '\0') { + return { EXIT_FAILURE, "CLUSTER_SIM_HOST_FILE is not configured" }; + } + + std::ifstream host_file { host_file_path }; + if (!host_file.is_open()) { + return { + EXIT_FAILURE, + "Unable to open CLUSTER_SIM_HOST_FILE '" + string { host_file_path } + "'" + }; + } + + string line {}; + uint64_t line_number = 0; + while (std::getline(host_file, line)) { + ++line_number; + std::istringstream fields { line }; + string hostname {}; + string ip {}; + if (!(fields >> hostname) || hostname.front() == '#') { + continue; + } + if (!(fields >> ip)) { + return { + EXIT_FAILURE, + "Missing IP in CLUSTER_SIM_HOST_FILE at line " + std::to_string(line_number) + }; + } + + auto existing = addresses.find(hostname); + if (existing != addresses.end() && existing->second != ip) { + return { + EXIT_FAILURE, + "Conflicting CLUSTER_SIM_HOST_FILE mappings for '" + hostname + "'" + }; + } + addresses[hostname] = ip; + } + + return { EXIT_SUCCESS, "" }; +} + +} // namespace std::pair prepare_aurora_cluster_state( MYSQL* proxysql_sqlite, const vector& servers, - uint32_t cleanup + aurora_publication_mode mode ) { - int query_error = 0; + std::unordered_map backend_addresses {}; + auto [host_file_rc, host_file_error] = + load_aurora_backend_addresses(backend_addresses); + if (host_file_rc != EXIT_SUCCESS) { + return { EXIT_FAILURE, host_file_error }; + } - if (cleanup) { - string srv_ids {}; - string domain_names {}; + std::map> replica_sets {}; + for (const aurora_server_state_t& server : servers) { + replica_sets[std::get(server)].push_back(server); + } - for (const auto& server : servers) { - srv_ids += "'" + std::get(server) + "'"; - domain_names += "'" + std::get(server) + "'"; + int query_error = mysql_query(proxysql_sqlite, "BEGIN IMMEDIATE"); + if (query_error) { + return create_query_error(proxysql_sqlite, "BEGIN IMMEDIATE", __FILE__, __LINE__); + } - if (&server != &servers.back()) { - srv_ids += ","; - domain_names += ","; - } + const auto execute_or_rollback = [proxysql_sqlite](const string& query) { + if (mysql_query(proxysql_sqlite, query.c_str()) == 0) { + return std::pair { EXIT_SUCCESS, "" }; } + auto error = create_query_error(proxysql_sqlite, query, __FILE__, __LINE__); + (void)mysql_query(proxysql_sqlite, "ROLLBACK"); + return error; + }; - string cleanup_query {}; + string replica_set_list {}; + for (const auto& replica_set : replica_sets) { + if (!replica_set_list.empty()) replica_set_list += ","; + replica_set_list += aurora_sql_quote(replica_set.first); + } - if (cleanup == 1) { - cleanup_query = "DELETE FROM REPLICA_HOST_STATUS WHERE SERVER_ID NOT IN (" + - srv_ids + ") OR DOMAIN_NAME NOT IN (" + domain_names + ")"; - } else { - cleanup_query = "DELETE FROM REPLICA_HOST_STATUS"; + uint64_t probe_checkpoint = 0; + if (mode == aurora_publication_mode::replace_snapshot_retaining_backends) { + auto [checkpoint_rc, checkpoint] = aurora_scalar_uint64( + proxysql_sqlite, + "SELECT COALESCE(MAX(sequence_id),0) FROM AWS_AURORA_REPLICA_PROBE_LOG"); + if (checkpoint_rc != EXIT_SUCCESS) { + (void)mysql_query(proxysql_sqlite, "ROLLBACK"); + return { EXIT_FAILURE, "Unable to read the Aurora replica probe checkpoint" }; } + probe_checkpoint = checkpoint; + } - query_error = mysql_query(proxysql_sqlite, cleanup_query.c_str()); - if (query_error) { - return create_query_error(proxysql_sqlite, cleanup_query, __FILE__, __LINE__); + if (mode == aurora_publication_mode::reset_scenario) { + auto [control_rc, control_error] = + execute_or_rollback("DELETE FROM AWS_AURORA_REPLICA_CONTROL"); + if (control_rc != EXIT_SUCCESS) return { control_rc, control_error }; + auto [rows_rc, rows_error] = execute_or_rollback("DELETE FROM REPLICA_HOST_STATUS"); + if (rows_rc != EXIT_SUCCESS) return { rows_rc, rows_error }; + } else if (mode == aurora_publication_mode::replace_snapshot_retaining_backends) { + const string delete_controls { + replica_set_list.empty() + ? "DELETE FROM AWS_AURORA_REPLICA_CONTROL" + : "DELETE FROM AWS_AURORA_REPLICA_CONTROL WHERE replica_set_id NOT IN (" + + replica_set_list + ")" + }; + auto [control_rc, control_error] = execute_or_rollback(delete_controls); + if (control_rc != EXIT_SUCCESS) return { control_rc, control_error }; + if (!replica_set_list.empty()) { + auto [reset_rc, reset_error] = execute_or_rollback( + "UPDATE AWS_AURORA_REPLICA_CONTROL SET replica_table_present=1," + "error_code=0,error_msg='' WHERE replica_set_id IN (" + + replica_set_list + ")"); + if (reset_rc != EXIT_SUCCESS) return { reset_rc, reset_error }; + } + auto [rows_rc, rows_error] = execute_or_rollback("DELETE FROM REPLICA_HOST_STATUS"); + if (rows_rc != EXIT_SUCCESS) return { rows_rc, rows_error }; + } else { + for (const auto& replica_set : replica_sets) { + const string set_literal { aurora_sql_quote(replica_set.first) }; + auto [control_rc, control_error] = execute_or_rollback( + "DELETE FROM AWS_AURORA_REPLICA_CONTROL WHERE replica_set_id=" + set_literal); + if (control_rc != EXIT_SUCCESS) return { control_rc, control_error }; + auto [rows_rc, rows_error] = execute_or_rollback( + "DELETE FROM REPLICA_HOST_STATUS WHERE REPLICA_SET_ID=" + set_literal); + if (rows_rc != EXIT_SUCCESS) return { rows_rc, rows_error }; } } - usleep(1000 * 1000); - - // NOTE: We adquire a 'write lock' so there are no dirty reads on ProxySQL side - // while we write the new values. - query_error = mysql_query(proxysql_sqlite, "BEGIN IMMEDIATE"); - if (query_error) { - return create_query_error(proxysql_sqlite, "BEGIN IMMEDIATE", __FILE__, __LINE__); - } + const string timestamp { aurora_utc_timestamp() }; + for (const auto& replica_set : replica_sets) { + const string& replica_set_id = replica_set.first; + std::map, bool> mapped_backends {}; + for (const aurora_server_state_t& server : replica_set.second) { + const string& server_id = std::get(server); + string session_id = std::get(server); + if (session_id.empty()) { + session_id = "TESTID-" + server_id + replica_set_id + "-R"; + } + const string hostname { server_id + replica_set_id }; + auto address = backend_addresses.find(hostname); + if (address == backend_addresses.end()) { + (void)mysql_query(proxysql_sqlite, "ROLLBACK"); + return { + EXIT_FAILURE, + "Missing CLUSTER_SIM_HOST_FILE mapping for Aurora member '" + + hostname + "'" + }; + } - for (const auto& server : servers) { - string server_insert_query {}; + const string row_query { + "INSERT INTO REPLICA_HOST_STATUS" + "(REPLICA_SET_ID,SERVER_ID,SESSION_ID,CPU,LAST_UPDATE_TIMESTAMP," + "REPLICA_LAG_IN_MILLISECONDS,IS_CURRENT) VALUES (" + + aurora_sql_quote(replica_set_id) + "," + aurora_sql_quote(server_id) + + "," + aurora_sql_quote(session_id) + ",0," + + aurora_sql_quote(timestamp) + "," + + std::to_string(std::get(server)) + + ",1)" + }; + auto [row_rc, row_error] = execute_or_rollback(row_query); + if (row_rc != EXIT_SUCCESS) return { row_rc, row_error }; - string_format( - t_aurora_server_state_insert, - server_insert_query, - std::get(server).c_str(), - std::get(server).c_str(), - std::get(server).c_str(), - 0, - "", - std::get(server) - ); + mapped_backends[{ address->second, 3306 }] = true; + } - query_error = mysql_query(proxysql_sqlite, server_insert_query.c_str()); - if (query_error) { - return create_query_error(proxysql_sqlite, server_insert_query, __FILE__, __LINE__); + for (const auto& backend : mapped_backends) { + const string control_query { + "INSERT OR REPLACE INTO AWS_AURORA_REPLICA_CONTROL" + "(backend_ip,backend_port,replica_set_id,replica_table_present,error_code,error_msg) " + "VALUES (" + aurora_sql_quote(backend.first.first) + "," + + std::to_string(backend.first.second) + "," + aurora_sql_quote(replica_set_id) + + ",1,0,'')" + }; + auto [control_rc, control_error] = execute_or_rollback(control_query); + if (control_rc != EXIT_SUCCESS) return { control_rc, control_error }; } } query_error = mysql_query(proxysql_sqlite, "COMMIT"); if (query_error) { + (void)mysql_query(proxysql_sqlite, "ROLLBACK"); return create_query_error(proxysql_sqlite, "COMMIT", __FILE__, __LINE__); } + if (mode == aurora_publication_mode::replace_snapshot_retaining_backends && + !replica_sets.empty()) { + const uint64_t deadline = monotonic_time() + 10000000; + const string observed_sets_query { + "SELECT COUNT(DISTINCT replica_set_id) FROM AWS_AURORA_REPLICA_PROBE_LOG " + "WHERE sequence_id>" + std::to_string(probe_checkpoint) + + " AND probe_kind='ordinary' AND replica_set_id IN (" + + replica_set_list + ")" + }; + do { + auto [observed_rc, observed_sets] = + aurora_scalar_uint64(proxysql_sqlite, observed_sets_query); + if (observed_rc != EXIT_SUCCESS) { + return { EXIT_FAILURE, "Unable to read the Aurora replica probe log" }; + } + if (observed_sets == replica_sets.size()) { + return { EXIT_SUCCESS, "" }; + } + usleep(50000); + } while (monotonic_time() < deadline); + + return { + EXIT_FAILURE, + "Timed out waiting for every Aurora replica set to be probed" + }; + } + return { EXIT_SUCCESS, "" }; } @@ -548,67 +724,3 @@ cluster_state_changes aurora_servers_state_diff( return result; } - -aurora_server_state_t aurora_update_state( - const aurora_server_state_t& st1, - const aurora_server_state_t& st2 -) { - aurora_server_state_t result {}; - - // SERVER_ID and DOMAIN_NAME **can't** be changed, because the are part of the server 'id'. Only the other - // fields are allowed to change, otherwise, the verification step should have failed. - - const string st1_session_id { std::get(st1) }; - const string st2_session_id { std::get(st2) }; - - int32_t st1_read_only { std::get(st1) }; - int32_t st2_read_only { std::get(st2) }; - - // Since empty 'SESSION_IDs' have no meaning, we ignore them for updated states - if (st2_session_id != "" && st1_session_id != st2_session_id) { - std::get<2>(result) = st2_session_id; - } - - if (st2_read_only != -1 && st1_read_only != st2_read_only) { - std::get<3>(result) = st2_read_only; - } - - return result; -} - -vector aurora_update_cluster_state( - const vector& servers_state_p, - const vector& servers_state_n -) { - vector result {}; - - vector s_servers_state_p { sort_aurora_server_state(servers_state_p) }; - vector s_servers_state_n { sort_aurora_server_state(servers_state_n) }; - - // find the differences - for (const auto& server_state_n : s_servers_state_n) { - for (const auto& server_state_p : s_servers_state_p) { - const string n_server_state_id { - std::get(server_state_n) + ":" + - std::get(server_state_n) - }; - const string p_server_state_id { - std::get(server_state_p) + ":" + - std::get(server_state_p) - }; - - bool diff_server_status = - ( n_server_state_id == p_server_state_id ) && - ( server_state_n != server_state_p ); - - if (diff_server_status) { - const aurora_server_state_t server_state_update { - aurora_update_state(server_state_p, server_state_n) - }; - result.push_back(server_state_update); - } - } - } - - return result; -} diff --git a/test/deps/cluster_simulator/lib/aurora_utils.h b/test/deps/cluster_simulator/lib/aurora_utils.h index 1c75495abf..5006d50ccd 100644 --- a/test/deps/cluster_simulator/lib/aurora_utils.h +++ b/test/deps/cluster_simulator/lib/aurora_utils.h @@ -50,6 +50,12 @@ enum class aurora_state_id { new_state = 1 }; +enum class aurora_publication_mode { + replace_sets, + replace_snapshot_retaining_backends, + reset_scenario +}; + std::pair extract_aurora_servers_state( const aurora_state_id& state_id, const json& aurora_test_def, @@ -61,10 +67,15 @@ std::pair prepare_mysql_aurora_hostgroups( const std::vector& hostgroups_configs ); +/** + * Publishes ordinary Aurora JSON state through backend-address replica sets. + * DOMAIN_NAME identifies the set, and CLUSTER_SIM_HOST_FILE resolves every + * SERVER_ID + DOMAIN_NAME member hostname to its simulated backend address. + */ std::pair prepare_aurora_cluster_state( MYSQL* proxysql_sqlite, const std::vector& servers, - uint32_t cleanup = 0 + aurora_publication_mode mode = aurora_publication_mode::replace_sets ); /** @@ -88,10 +99,4 @@ cluster_state_changes aurora_servers_state_diff( const std::vector& servers_state_n ); -std::vector aurora_update_cluster_state( - const std::vector& servers_state_p, - const std::vector& servers_state_n -); - #endif - diff --git a/test/tap/groups/cluster_sim_aurora/add-hosts b/test/tap/groups/cluster_sim_aurora/add-hosts index 77d02bd058..b8853c07d1 100644 --- a/test/tap/groups/cluster_sim_aurora/add-hosts +++ b/test/tap/groups/cluster_sim_aurora/add-hosts @@ -2,7 +2,7 @@ # Format: " " per line; '#' comments allowed. # These are injected into the ProxySQL container's /etc/hosts via Docker # --add-host by test/infra/control/start-proxysql-isolated.bash when -# CLUSTER_SIM_HOST_FILE points at this file (see test/tap/groups/aurora-sim/env.sh). +# CLUSTER_SIM_HOST_FILE points at this file (see test/tap/groups/cluster_sim_aurora/env.sh). # Cluster 1 host.1.11.aws-test.com 127.0.1.11 @@ -28,6 +28,11 @@ host.2.18.cluster2.aws.test 127.0.2.18 host.2.19.cluster2.aws.test 127.0.2.19 host.2.20.cluster2.aws.test 127.0.2.20 +# Cluster 2 members reused with the first test domain. +host.2.11.aws-test.com 127.0.2.11 +host.2.12.aws-test.com 127.0.2.12 +host.2.13.aws-test.com 127.0.2.13 + # Cluster 3 host.3.11.aws.3.test.com 127.0.3.11 host.3.12.aws.3.test.com 127.0.3.12 @@ -39,3 +44,36 @@ host.3.17.aws.3.test.com 127.0.3.17 host.3.18.aws.3.test.com 127.0.3.18 host.3.19.aws.3.test.com 127.0.3.19 host.3.20.aws.3.test.com 127.0.3.20 + +# Aurora blue/green deployment A: production members and target resources. +aurora-a-writer.a1.us-east-1.rds.amazonaws.com 127.0.11.11 +aurora-a-reader-1.a1.us-east-1.rds.amazonaws.com 127.0.11.12 +aurora-a-reader-2.a1.us-east-1.rds.amazonaws.com 127.0.11.13 +aurora-a-green.cluster-a1.us-east-1.rds.amazonaws.com 127.0.11.20 +aurora-a-green-writer.a1.us-east-1.rds.amazonaws.com 127.0.11.21 +aurora-a-green-reader-1.a1.us-east-1.rds.amazonaws.com 127.0.11.22 +aurora-a-green-reader-2.a1.us-east-1.rds.amazonaws.com 127.0.11.23 +aurora-a-writer-green-k7m2.a1.us-east-1.rds.amazonaws.com 127.0.11.21 +aurora-a-reader-1-green-p4q8.a1.us-east-1.rds.amazonaws.com 127.0.11.22 +aurora-a-reader-2-green-v9n3.a1.us-east-1.rds.amazonaws.com 127.0.11.23 + +# Canonical names observed after target-member rename retain target addresses. +aurora-a-canonical-writer.a1.us-east-1.rds.amazonaws.com 127.0.11.21 +aurora-a-canonical-reader-1.a1.us-east-1.rds.amazonaws.com 127.0.11.22 +aurora-a-canonical-reader-2.a1.us-east-1.rds.amazonaws.com 127.0.11.23 + +# Aurora blue/green deployment B supports repeated and concurrent scenarios. +aurora-b-writer.b1.us-east-1.rds.amazonaws.com 127.0.12.11 +aurora-b-reader-1.b1.us-east-1.rds.amazonaws.com 127.0.12.12 +aurora-b-green.cluster-b1.us-east-1.rds.amazonaws.com 127.0.12.20 +aurora-b-green-writer.b1.us-east-1.rds.amazonaws.com 127.0.12.21 +aurora-b-green-reader-1.b1.us-east-1.rds.amazonaws.com 127.0.12.22 +aurora-b-writer-green-h2s6.b1.us-east-1.rds.amazonaws.com 127.0.12.21 +aurora-b-reader-1-green-r8t4.b1.us-east-1.rds.amazonaws.com 127.0.12.22 +aurora-b-canonical-writer.b1.us-east-1.rds.amazonaws.com 127.0.12.21 +aurora-b-canonical-reader-1.b1.us-east-1.rds.amazonaws.com 127.0.12.22 + +# Aurora blue/green deployment C supports three-cluster isolation scenarios. +aurora-c-writer.c1.us-east-1.rds.amazonaws.com 127.0.13.11 +aurora-c-green.cluster-c1.us-east-1.rds.amazonaws.com 127.0.13.20 +aurora-c-writer-green-m5n9.c1.us-east-1.rds.amazonaws.com 127.0.13.21 diff --git a/test/tap/groups/groups.json b/test/tap/groups/groups.json index 361903314e..c4de080fa4 100644 --- a/test/tap/groups/groups.json +++ b/test/tap/groups/groups.json @@ -11,6 +11,7 @@ "ai_error_handling_edge_cases-t" : [ "ai-g1","@proxysql_min_version:4.0" ], "ai_llm_retry_scenarios-t" : [ "ai-g1","@proxysql_min_version:4.0" ], "ai_validation-t" : [ "ai-g1","@proxysql_min_version:4.0" ], + "aurora_bgd_config_unit-t" : [ "unit-tests-g1" ], "auth_unit-t" : [ "unit-tests-g1" ], "backend_sync_unit-t" : [ "unit-tests-g1" ], "basic-t" : [ "legacy-g1","mariadb10-galera-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1","mysql84-g1","mysql84-gr-g1","mysql90-g1","mysql90-gr-g1","mysql93-g1","mysql93-gr-g1","mysql95-g1","mysql95-gr-g1" ], @@ -362,6 +363,12 @@ "test_client_limit_error-t" : [ "todo-g1" ], "test_cluster1-t" : [ "legacy-g5","mariadb10-galera-g5","mysql-auto_increment_delay_multiplex=0-g3","mysql-multiplexing=false-g3","mysql-query_digests=0-g3","mysql-query_digests_keep_comment=1-g3","mysql84-g5","mysql84-gr-g5","mysql90-g5","mysql95-g5" ], "test_cluster_sim_aurora-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_discovery-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_active-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_completion-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_cluster_sync-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_resilience-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_lifecycle-t" : [ "cluster_sim_aurora-g1" ], "test_cluster_sim_galera-t" : [ "cluster_sim_galera-g1" ], "test_cluster_sim_group_repl-t" : [ "cluster_sim_group_repl-g1" ], "test_cluster_sim_read_only-t" : [ "cluster_sim_read_only-g1" ], diff --git a/test/tap/tap/Makefile b/test/tap/tap/Makefile index ead866e075..568de7a136 100644 --- a/test/tap/tap/Makefile +++ b/test/tap/tap/Makefile @@ -83,11 +83,17 @@ mcp_client.o: mcp_client.cpp mcp_client.h libcurl$(SHLIB_EXT) cluster_simulator.o: cluster_simulator.cpp cluster_simulator.h $(CXX) -fPIC -c cluster_simulator.cpp $(IDIRS) -I$(MARIADB_IDIR) $(OPT) -rds_bgd_simulator.o: rds_bgd_simulator.cpp rds_bgd_simulator.h cluster_simulator.h utils.h +bgd_simulator.o: bgd_simulator.cpp bgd_simulator.h cluster_simulator.h utils.h tap.h command_line.h + $(CXX) -fPIC -c bgd_simulator.cpp $(IDIRS) -I$(MARIADB_IDIR) $(OPT) + +aurora_bgd_simulator.o: aurora_bgd_simulator.cpp aurora_bgd_simulator.h bgd_simulator.h cluster_simulator.h utils.h command_line.h + $(CXX) -fPIC -c aurora_bgd_simulator.cpp $(IDIRS) -I$(MARIADB_IDIR) $(OPT) + +rds_bgd_simulator.o: rds_bgd_simulator.cpp rds_bgd_simulator.h bgd_simulator.h cluster_simulator.h utils.h command_line.h $(CXX) -fPIC -c rds_bgd_simulator.cpp $(IDIRS) -I$(MARIADB_IDIR) $(OPT) -libtap_mariadb.a: tap.o command_line.o utils_mariadb.o noise_utils_mariadb.o mcp_client.o cluster_simulator.o rds_bgd_simulator.o cpp-dotenv/static/cpp-dotenv/libcpp_dotenv.a - $(AR) rcs libtap_mariadb.a tap.o command_line.o utils_mariadb.o noise_utils_mariadb.o mcp_client.o cluster_simulator.o rds_bgd_simulator.o $(SQLITE3_LDIR)/sqlite3.o $(PROXYSQL_LDIR)/obj/sha256crypt.oo +libtap_mariadb.a: tap.o command_line.o utils_mariadb.o noise_utils_mariadb.o mcp_client.o cluster_simulator.o bgd_simulator.o aurora_bgd_simulator.o rds_bgd_simulator.o cpp-dotenv/static/cpp-dotenv/libcpp_dotenv.a + $(AR) rcs libtap_mariadb.a tap.o command_line.o utils_mariadb.o noise_utils_mariadb.o mcp_client.o cluster_simulator.o bgd_simulator.o aurora_bgd_simulator.o rds_bgd_simulator.o $(SQLITE3_LDIR)/sqlite3.o $(PROXYSQL_LDIR)/obj/sha256crypt.oo libtap_mysql57.a: tap.o command_line.o utils_mysql57.o noise_utils_mysql57.o mcp_client.o cpp-dotenv/static/cpp-dotenv/libcpp_dotenv.a $(AR) rcs libtap_mysql57.a tap.o command_line.o utils_mysql57.o noise_utils_mysql57.o mcp_client.o $(SQLITE3_LDIR)/sqlite3.o $(PROXYSQL_LDIR)/obj/sha256crypt.oo diff --git a/test/tap/tap/aurora_bgd_simulator.cpp b/test/tap/tap/aurora_bgd_simulator.cpp new file mode 100644 index 0000000000..1fb37f78c6 --- /dev/null +++ b/test/tap/tap/aurora_bgd_simulator.cpp @@ -0,0 +1,40 @@ +#include "aurora_bgd_simulator.h" + +using namespace std; + +Endpoint Aurora_BGD_Endpoint::backend() { + return { ip, port }; +} + +Endpoint Aurora_BGD_Endpoint::host_endpoint() { + return { hostname, port }; +} + +Aurora_Replica_Row Aurora_BGD_Member::replica_row() { + return { + server_id, + session_id, + cpu, + last_update_timestamp, + replica_lag_in_milliseconds, + is_current, + }; +} + +vector Aurora_BGD_Membership_Set::replica_rows() { + vector rows {}; + rows.reserve(members.size()); + for (Aurora_BGD_Member& member : members) { + rows.push_back(member.replica_row()); + } + return rows; +} + +vector Aurora_BGD_Membership_Set::backends() { + vector endpoints {}; + endpoints.reserve(serving_endpoints.size()); + for (Aurora_BGD_Endpoint& endpoint : serving_endpoints) { + endpoints.push_back(endpoint.backend()); + } + return endpoints; +} diff --git a/test/tap/tap/aurora_bgd_simulator.h b/test/tap/tap/aurora_bgd_simulator.h new file mode 100644 index 0000000000..6df8f01db2 --- /dev/null +++ b/test/tap/tap/aurora_bgd_simulator.h @@ -0,0 +1,52 @@ +#ifndef TAP_AURORA_BGD_SIMULATOR_H +#define TAP_AURORA_BGD_SIMULATOR_H + +#include +#include + +#include "bgd_simulator.h" + +using namespace std; + +/** An AWS-style Aurora hostname and its fixed simulator backend address. */ +struct Aurora_BGD_Endpoint { + string hostname; + string ip; + int port; + + Endpoint backend(); + Endpoint host_endpoint(); +}; + +/** One Aurora cluster member published through REPLICA_HOST_STATUS. */ +struct Aurora_BGD_Member { + string server_id; + string session_id; + Aurora_BGD_Endpoint endpoint; + double cpu { 0 }; + string last_update_timestamp; + double replica_lag_in_milliseconds { 0 }; + bool is_current { true }; + + Aurora_Replica_Row replica_row(); +}; + +/** A membership snapshot and every backend allowed to return it. */ +struct Aurora_BGD_Membership_Set { + string replica_set_id; + vector members; + vector serving_endpoints; + + vector replica_rows(); + vector backends(); +}; + +/** Stable identity used to describe an AWS member rename during switchover. */ +struct Aurora_BGD_Rename { + string production_server_id; + string target_server_id; + string session_id; + string cached_target_ip; +}; + +#endif // TAP_AURORA_BGD_SIMULATOR_H diff --git a/test/tap/tap/aurora_bgd_tap.h b/test/tap/tap/aurora_bgd_tap.h new file mode 100644 index 0000000000..c71d8100c1 --- /dev/null +++ b/test/tap/tap/aurora_bgd_tap.h @@ -0,0 +1,354 @@ +#ifndef TAP_TESTS_AURORA_BGD_TAP_H +#define TAP_TESTS_AURORA_BGD_TAP_H + +#include +#include +#include +#include +#include +#include + +#include "aurora_bgd_simulator.h" +#include "tap.h" + +using namespace std; + +struct Aurora_BGD_Test_Deployment { + string name; + string domain_name; + string blue_replica_set; + string target_replica_set; + Aurora_BGD_Endpoint target_cluster_endpoint; + Aurora_BGD_Membership_Set production; + Aurora_BGD_Membership_Set target; + string source_topology_id; + string target_topology_id; +}; + +inline Aurora_BGD_Member aurora_bgd_member( + string server_id, string session_id, Aurora_BGD_Endpoint endpoint, bool current = true +) { + Aurora_BGD_Member member; + member.server_id = move(server_id); + member.session_id = move(session_id); + member.endpoint = move(endpoint); + member.last_update_timestamp = "2099-01-01 00:00:00"; + member.is_current = current; + return member; +} + +inline Aurora_BGD_Test_Deployment aurora_bgd_deployment_a() { + Aurora_BGD_Test_Deployment deployment; + deployment.name = "Aurora BGD deployment A"; + deployment.domain_name = ".a1.us-east-1.rds.amazonaws.com"; + deployment.blue_replica_set = "aurora-bgd-blue-a"; + deployment.target_replica_set = "aurora-bgd-target-a"; + deployment.source_topology_id = "aurora-bgd-source-a"; + deployment.target_topology_id = "aurora-bgd-target-a"; + deployment.target_cluster_endpoint = { + "aurora-a-green.cluster-a1.us-east-1.rds.amazonaws.com", "127.0.11.20", 3306 + }; + deployment.production = { + deployment.blue_replica_set, + { + aurora_bgd_member("aurora-a-writer", "MASTER_SESSION_ID", + {"aurora-a-writer.a1.us-east-1.rds.amazonaws.com", "127.0.11.11", 3306}), + aurora_bgd_member("aurora-a-reader-1", "reader-a-1", + {"aurora-a-reader-1.a1.us-east-1.rds.amazonaws.com", "127.0.11.12", 3306}), + aurora_bgd_member("aurora-a-reader-2", "reader-a-2", + {"aurora-a-reader-2.a1.us-east-1.rds.amazonaws.com", "127.0.11.13", 3306}), + }, + {} + }; + for (Aurora_BGD_Member& member : deployment.production.members) { + deployment.production.serving_endpoints.push_back(member.endpoint); + } + deployment.target = { + deployment.target_replica_set, + { + aurora_bgd_member("aurora-a-writer-green-k7m2", "MASTER_SESSION_ID", + {"aurora-a-writer-green-k7m2.a1.us-east-1.rds.amazonaws.com", "127.0.11.21", 3306}), + aurora_bgd_member("aurora-a-reader-1-green-p4q8", "reader-a-1", + {"aurora-a-reader-1-green-p4q8.a1.us-east-1.rds.amazonaws.com", "127.0.11.22", 3306}), + aurora_bgd_member("aurora-a-reader-2-green-v9n3", "reader-a-2", + {"aurora-a-reader-2-green-v9n3.a1.us-east-1.rds.amazonaws.com", "127.0.11.23", 3306}), + }, + {deployment.target_cluster_endpoint} + }; + for (Aurora_BGD_Member& member : deployment.target.members) { + deployment.target.serving_endpoints.push_back(member.endpoint); + } + return deployment; +} + +inline Aurora_BGD_Test_Deployment aurora_bgd_deployment_b_writer_only() { + Aurora_BGD_Test_Deployment deployment; + deployment.name = "Aurora BGD deployment B writer-only"; + deployment.domain_name = ".b1.us-east-1.rds.amazonaws.com"; + deployment.blue_replica_set = "aurora-bgd-blue-b"; + deployment.target_replica_set = "aurora-bgd-target-b"; + deployment.source_topology_id = "aurora-bgd-source-b"; + deployment.target_topology_id = "aurora-bgd-target-b"; + deployment.target_cluster_endpoint = { + "aurora-b-green.cluster-b1.us-east-1.rds.amazonaws.com", "127.0.12.20", 3306 + }; + deployment.production = { + deployment.blue_replica_set, + { + aurora_bgd_member("aurora-b-writer", "MASTER_SESSION_ID", + {"aurora-b-writer.b1.us-east-1.rds.amazonaws.com", "127.0.12.11", 3306}), + }, + {} + }; + deployment.production.serving_endpoints.push_back(deployment.production.members.front().endpoint); + deployment.target = { + deployment.target_replica_set, + { + aurora_bgd_member("aurora-b-writer-green-h2s6", "MASTER_SESSION_ID", + {"aurora-b-writer-green-h2s6.b1.us-east-1.rds.amazonaws.com", "127.0.12.21", 3306}), + }, + {deployment.target_cluster_endpoint} + }; + deployment.target.serving_endpoints.push_back(deployment.target.members.front().endpoint); + return deployment; +} + +inline Aurora_BGD_Test_Deployment aurora_bgd_deployment_c_writer_only() { + Aurora_BGD_Test_Deployment deployment; + deployment.name = "Aurora BGD deployment C writer-only"; + deployment.domain_name = ".c1.us-east-1.rds.amazonaws.com"; + deployment.blue_replica_set = "aurora-bgd-blue-c"; + deployment.target_replica_set = "aurora-bgd-target-c"; + deployment.source_topology_id = "aurora-bgd-source-c"; + deployment.target_topology_id = "aurora-bgd-target-c"; + deployment.target_cluster_endpoint = { + "aurora-c-green.cluster-c1.us-east-1.rds.amazonaws.com", "127.0.13.20", 3306 + }; + deployment.production = { + deployment.blue_replica_set, + { + aurora_bgd_member("aurora-c-writer", "MASTER_SESSION_ID", + {"aurora-c-writer.c1.us-east-1.rds.amazonaws.com", "127.0.13.11", 3306}), + }, + {} + }; + deployment.production.serving_endpoints.push_back(deployment.production.members.front().endpoint); + deployment.target = { + deployment.target_replica_set, + { + aurora_bgd_member("aurora-c-writer-green-m5n9", "MASTER_SESSION_ID", + {"aurora-c-writer-green-m5n9.c1.us-east-1.rds.amazonaws.com", "127.0.13.21", 3306}), + }, + {deployment.target_cluster_endpoint} + }; + deployment.target.serving_endpoints.push_back(deployment.target.members.front().endpoint); + return deployment; +} + +inline string aurora_bgd_sql_quote(const string& value) { + string quoted {"'"}; + for (char c : value) { + quoted += c; + if (c == '\'') { + quoted += '\''; + } + } + quoted += '\''; + return quoted; +} + +inline int aurora_bgd_execute_all(MYSQL* admin, const vector& queries) { + for (const string& query : queries) { + if (mysql_query(admin, query.c_str()) != 0) { + diag("Aurora BGD admin query failed: %s; query: %s", mysql_error(admin), query.c_str()); + return EXIT_FAILURE; + } + } + return EXIT_SUCCESS; +} + +inline int aurora_bgd_admin_cleanup(MYSQL* admin) { + return aurora_bgd_execute_all(admin, { + "SET mysql-aws_blue_green_deployment_auto_discovery='false'", + "LOAD MYSQL VARIABLES TO RUNTIME", + "DELETE FROM mysql_aws_aurora_hostgroups", + "DELETE FROM mysql_servers", + "UPDATE mysql_users SET default_hostgroup=0 WHERE username='testuser'", + "LOAD MYSQL SERVERS TO RUNTIME", + "LOAD MYSQL USERS TO RUNTIME", + }); +} + +inline int aurora_bgd_admin_setup( + MYSQL* admin, Aurora_BGD_Test_Deployment& deployment, + int writer_hg, int reader_hg, int green_writer_hg, int green_reader_hg, + bool auto_discovery, int check_interval_ms = 100, + bool writer_is_also_reader = false +) { + vector queries { + "SET mysql-monitor_username='aurora1'", + "SET mysql-monitor_password='pass1'", + "SET mysql-monitor_enabled='true'", + "SET mysql-aws_blue_green_deployment_auto_discovery='" + + string(auto_discovery ? "true" : "false") + "'", + "UPDATE mysql_users SET default_hostgroup=" + to_string(writer_hg) + + " WHERE username='testuser'", + }; + + string green_columns = green_writer_hg >= 0 + ? to_string(green_writer_hg) + "," + to_string(green_reader_hg) + : "NULL,NULL"; + queries.push_back( + "INSERT INTO mysql_aws_aurora_hostgroups(" + "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup," + "active,aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms," + "writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks," + "autopurge_missing_checks,comment) VALUES (" + + to_string(writer_hg) + "," + to_string(reader_hg) + "," + green_columns + + ",1,3306," + aurora_bgd_sql_quote(deployment.domain_name) + + ",200," + to_string(check_interval_ms) + ",800," + + to_string(writer_is_also_reader ? 1 : 0) + + ",1,30,30,1,0," + aurora_bgd_sql_quote(deployment.name) + ")"); + + for (Aurora_BGD_Member& member : deployment.production.members) { + const int hostgroup = member.session_id == "MASTER_SESSION_ID" ? writer_hg : reader_hg; + queries.push_back( + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,use_ssl,comment) VALUES (" + + to_string(hostgroup) + "," + aurora_bgd_sql_quote(member.endpoint.hostname) + + ",3306,'ONLINE',0,'Aurora BGD production member')"); + } + queries.push_back("LOAD MYSQL VARIABLES TO RUNTIME"); + queries.push_back("LOAD MYSQL SERVERS TO RUNTIME"); + queries.push_back("LOAD MYSQL USERS TO RUNTIME"); + return aurora_bgd_execute_all(admin, queries); +} + +inline vector aurora_bgd_available_topology( + Aurora_BGD_Test_Deployment& deployment +) { + return { + { + deployment.source_topology_id, + deployment.production.members.front().endpoint.hostname, + 3306, + "BLUE_GREEN_DEPLOYMENT_SOURCE", + "AVAILABLE", + }, + { + deployment.target_topology_id, + deployment.target_cluster_endpoint.hostname, + 3306, + "BLUE_GREEN_DEPLOYMENT_TARGET", + "AVAILABLE", + }, + }; +} + +inline vector aurora_bgd_topology( + Aurora_BGD_Test_Deployment& deployment, const string& status +) { + vector rows = aurora_bgd_available_topology(deployment); + for (BGD_Topology_Row& row : rows) { + row.status = status; + } + return rows; +} + +inline vector aurora_bgd_completed_topology( + Aurora_BGD_Test_Deployment& deployment +) { + return { + { + deployment.target_topology_id, + deployment.target_cluster_endpoint.hostname, + deployment.target_cluster_endpoint.port, + "BLUE_GREEN_DEPLOYMENT_TARGET", + "SWITCHOVER_COMPLETED", + }, + }; +} + +inline vector aurora_bgd_topology_backends(Aurora_BGD_Test_Deployment& deployment) { + vector backends = deployment.production.backends(); + vector target_backends = deployment.target.backends(); + backends.insert(backends.end(), target_backends.begin(), target_backends.end()); + return backends; +} + +inline int aurora_bgd_publish( + BGD_Simulator& sim, Aurora_BGD_Test_Deployment& deployment +) { + if (sim.replica_update( + deployment.production.replica_set_id, + deployment.production.replica_rows(), + deployment.production.backends()) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + if (sim.replica_update( + deployment.target.replica_set_id, + deployment.target.replica_rows(), + deployment.target.backends()) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return sim.topology_update( + aurora_bgd_topology_backends(deployment), + aurora_bgd_available_topology(deployment)); +} + +inline int aurora_bgd_wait_for_status( + MYSQL* admin, int writer_hg, const string& status, uint32_t timeout_seconds +) { + string query = + "SELECT COUNT(*)=1 FROM runtime_mysql_aws_aurora_hostgroups WHERE writer_hostgroup=" + + to_string(writer_hg) + " AND bgd_status=" + aurora_bgd_sql_quote(status); + return wait_for_cond(admin, query, timeout_seconds); +} + +inline rc_t aurora_bgd_wait_for_topology_probe( + BGD_Simulator& sim, uint64_t sequence, const vector& backends, + BGD_Probe_Kind kind, uint32_t timeout_ms +) { + const uint64_t deadline = monotonic_time() + static_cast(timeout_ms) * 1000; + do { + auto [rc, logs] = sim.probe_log_since(sequence); + if (rc != EXIT_SUCCESS) { + return {EXIT_FAILURE, {}}; + } + for (const BGD_Probe_Log& log : logs) { + for (const Endpoint& backend : backends) { + if (log.backend.host == backend.host && log.backend.port == backend.port + && log.probe_kind == kind) { + return {EXIT_SUCCESS, log}; + } + } + } + usleep(50000); + } while (monotonic_time() < deadline); + return {ETIMEDOUT, {}}; +} + +inline rc_t aurora_bgd_wait_for_replica_probe( + BGD_Simulator& sim, uint64_t sequence, const vector& backends, + Aurora_Replica_Probe_Kind kind, uint32_t timeout_ms, const string& replica_set = "" +) { + const uint64_t deadline = monotonic_time() + static_cast(timeout_ms) * 1000; + do { + auto [rc, logs] = sim.replica_probe_log_since(sequence); + if (rc != EXIT_SUCCESS) { + return {EXIT_FAILURE, {}}; + } + for (const Aurora_Replica_Probe_Log& log : logs) { + for (const Endpoint& backend : backends) { + if (log.backend.host == backend.host && log.backend.port == backend.port + && log.probe_kind == kind + && (replica_set.empty() || log.replica_set_id == replica_set)) { + return {EXIT_SUCCESS, log}; + } + } + } + usleep(50000); + } while (monotonic_time() < deadline); + return {ETIMEDOUT, {}}; +} + +#endif // TAP_TESTS_AURORA_BGD_TAP_H diff --git a/test/tap/tap/bgd_simulator.cpp b/test/tap/tap/bgd_simulator.cpp new file mode 100644 index 0000000000..8c0641fa24 --- /dev/null +++ b/test/tap/tap/bgd_simulator.cpp @@ -0,0 +1,429 @@ +#include "bgd_simulator.h" + +#include +#include +#include +#include + +#include "tap.h" + +using namespace std; + +namespace { + +const char* probe_kind_string(BGD_Probe_Kind kind) { + return kind == BGD_Probe_Kind::table_check ? "table_check" : "metadata"; +} + +rc_t parse_probe_kind(string value) { + if (value == "table_check") { + return { EXIT_SUCCESS, BGD_Probe_Kind::table_check }; + } + if (value == "metadata") { + return { EXIT_SUCCESS, BGD_Probe_Kind::metadata }; + } + return { EXIT_FAILURE, BGD_Probe_Kind::table_check }; +} + +const char* replica_probe_kind_string(Aurora_Replica_Probe_Kind kind) { + return kind == Aurora_Replica_Probe_Kind::ordinary ? "ordinary" : "bgd_membership"; +} + +rc_t parse_replica_probe_kind(string value) { + if (value == "ordinary") { + return { EXIT_SUCCESS, Aurora_Replica_Probe_Kind::ordinary }; + } + if (value == "bgd_membership") { + return { EXIT_SUCCESS, Aurora_Replica_Probe_Kind::bgd_membership }; + } + return { EXIT_FAILURE, Aurora_Replica_Probe_Kind::ordinary }; +} + +} // namespace + +int BGD_Simulator::topology_update(vector backends, vector rows) { + if (backends.empty()) { + return EXIT_FAILURE; + } + + vector statements {}; + for (Endpoint& backend : backends) { + string predicate { backend_predicate(backend) }; + statements.push_back("DELETE FROM AWS_BGD_TOPOLOGY WHERE " + predicate); + statements.push_back( + "INSERT OR REPLACE INTO AWS_BGD_CONTROL" + "(backend_ip,backend_port,topology_present,error_code,error_msg) VALUES (" + + sql_quote(backend.host) + "," + to_string(backend.port) + ",1,0,'')"); + + for (size_t row_order = 0; row_order < rows.size(); ++row_order) { + BGD_Topology_Row& row = rows[row_order]; + statements.push_back( + "INSERT INTO AWS_BGD_TOPOLOGY" + "(backend_ip,backend_port,row_order,id,endpoint,topology_port,role,status) VALUES (" + + sql_quote(backend.host) + "," + to_string(backend.port) + "," + + to_string(row_order) + "," + sql_quote(row.id) + "," + + sql_quote(row.endpoint) + "," + to_string(row.port) + "," + + sql_quote(row.role) + "," + sql_quote(row.status) + ")"); + } + } + + return execute_transaction(statements); +} + +int BGD_Simulator::topology_delete(vector backends) { + if (backends.empty()) { + return EXIT_FAILURE; + } + + vector statements {}; + for (Endpoint& backend : backends) { + statements.push_back( + "DELETE FROM AWS_BGD_TOPOLOGY WHERE " + backend_predicate(backend)); + statements.push_back( + "INSERT OR REPLACE INTO AWS_BGD_CONTROL" + "(backend_ip,backend_port,topology_present,error_code,error_msg) VALUES (" + + sql_quote(backend.host) + "," + to_string(backend.port) + ",1,0,'')"); + } + return execute_transaction(statements); +} + +int BGD_Simulator::topology_drop(vector backends) { + return topology_error(backends, 1146, "Table 'mysql.rds_topology' doesn't exist"); +} + +int BGD_Simulator::topology_error(vector backends, int error_code, string error_msg) { + if (backends.empty() || error_code == 0) { + return EXIT_FAILURE; + } + + bool topology_present = error_code != 1146; + vector statements {}; + for (Endpoint& backend : backends) { + if (!topology_present) { + statements.push_back( + "DELETE FROM AWS_BGD_TOPOLOGY WHERE " + backend_predicate(backend)); + } + statements.push_back( + "INSERT OR REPLACE INTO AWS_BGD_CONTROL" + "(backend_ip,backend_port,topology_present,error_code,error_msg) VALUES (" + + sql_quote(backend.host) + "," + to_string(backend.port) + "," + + (topology_present ? "1" : "0") + "," + to_string(error_code) + "," + + sql_quote(error_msg) + ")"); + } + return execute_transaction(statements); +} + +int BGD_Simulator::replica_update( + string replica_set_id, + vector rows, + vector backends) +{ + if (replica_set_id.empty() || backends.empty()) { + return EXIT_FAILURE; + } + + vector statements { + "DELETE FROM REPLICA_HOST_STATUS WHERE REPLICA_SET_ID=" + + sql_quote(replica_set_id), + "DELETE FROM AWS_AURORA_REPLICA_CONTROL WHERE replica_set_id=" + + sql_quote(replica_set_id), + }; + for (Aurora_Replica_Row& row : rows) { + statements.push_back( + "INSERT INTO REPLICA_HOST_STATUS" + "(REPLICA_SET_ID,SERVER_ID,SESSION_ID,CPU,LAST_UPDATE_TIMESTAMP," + "REPLICA_LAG_IN_MILLISECONDS,IS_CURRENT) VALUES (" + + sql_quote(replica_set_id) + "," + sql_quote(row.server_id) + "," + + sql_quote(row.session_id) + "," + to_string(row.cpu) + "," + + sql_quote(row.last_update_timestamp) + "," + + to_string(row.replica_lag_in_milliseconds) + "," + + (row.is_current ? "1" : "0") + ")"); + } + for (Endpoint& backend : backends) { + statements.push_back( + "INSERT OR REPLACE INTO AWS_AURORA_REPLICA_CONTROL" + "(backend_ip,backend_port,replica_set_id,replica_table_present,error_code,error_msg) " + "VALUES (" + sql_quote(backend.host) + "," + to_string(backend.port) + "," + + sql_quote(replica_set_id) + ",1,0,'')"); + } + + return execute_transaction(statements); +} + +int BGD_Simulator::replica_delete(string replica_set_id) { + if (replica_set_id.empty()) { + return EXIT_FAILURE; + } + vector statements { + "DELETE FROM AWS_AURORA_REPLICA_CONTROL WHERE replica_set_id=" + + sql_quote(replica_set_id), + "DELETE FROM REPLICA_HOST_STATUS WHERE REPLICA_SET_ID=" + + sql_quote(replica_set_id), + }; + return execute_transaction(statements); +} + +int BGD_Simulator::replica_drop(vector backends) { + return replica_error( + backends, 1146, + "Table 'information_schema.REPLICA_HOST_STATUS' doesn't exist"); +} + +int BGD_Simulator::replica_error( + vector backends, int error_code, string error_msg) +{ + if (backends.empty() || error_code == 0) { + return EXIT_FAILURE; + } + + const bool table_present = error_code != 1146; + vector statements {}; + for (Endpoint& backend : backends) { + const string predicate { backend_predicate(backend) }; + statements.push_back( + "INSERT OR REPLACE INTO AWS_AURORA_REPLICA_CONTROL" + "(backend_ip,backend_port,replica_set_id,replica_table_present,error_code,error_msg) " + "VALUES (" + sql_quote(backend.host) + "," + to_string(backend.port) + "," + + "COALESCE((SELECT replica_set_id FROM AWS_AURORA_REPLICA_CONTROL WHERE " + + predicate + "),'')," + (table_present ? "1" : "0") + "," + + to_string(error_code) + "," + sql_quote(error_msg) + ")"); + } + return execute_transaction(statements); +} + +int BGD_Simulator::cleanup() { + vector statements { + "DELETE FROM READONLY_STATUS", + "DELETE FROM AWS_BGD_TOPOLOGY", + "DELETE FROM AWS_BGD_CONTROL", + "DELETE FROM AWS_BGD_PROBE_LOG", + }; + + if (connection() == nullptr) { + return EXIT_FAILURE; + } + auto [rc, rows] = mysql_query_ext_rows( + connection(), + "SELECT name FROM sqlite_master WHERE type='table' AND name IN (" + "'AWS_AURORA_REPLICA_CONTROL','AWS_AURORA_REPLICA_PROBE_LOG'," + "'REPLICA_HOST_STATUS')"); + if (rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + for (mysql_res_row& row : rows) { + if (row.size() != 1) { + return EXIT_FAILURE; + } + statements.push_back("DELETE FROM " + row.front()); + } + return execute_transaction(statements); +} + +rc_t BGD_Simulator::probe_log_last_sequence() { + if (connection() == nullptr) { + return { EXIT_FAILURE, 0 }; + } + + auto [rc, rows] = mysql_query_ext_rows( + connection(), "SELECT COALESCE(MAX(sequence_id),0) FROM AWS_BGD_PROBE_LOG"); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows.front().size() != 1) { + return { EXIT_FAILURE, 0 }; + } + + return { + EXIT_SUCCESS, + static_cast(strtoull(rows.front().front().c_str(), nullptr, 10)) + }; +} + +rc_t> BGD_Simulator::probe_log_since(uint64_t sequence_id) { + if (connection() == nullptr) { + return { EXIT_FAILURE, {} }; + } + + string query { + "SELECT sequence_id,backend_ip,backend_port,probe_kind,encrypted " + "FROM AWS_BGD_PROBE_LOG WHERE sequence_id>" + to_string(sequence_id) + + " ORDER BY sequence_id" + }; + auto [rc, rows] = mysql_query_ext_rows(connection(), query); + if (rc != EXIT_SUCCESS) { + return { EXIT_FAILURE, {} }; + } + + vector logs {}; + for (mysql_res_row& row : rows) { + if (row.size() != 5) { + return { EXIT_FAILURE, {} }; + } + auto [kind_rc, kind] = parse_probe_kind(row[3]); + if (kind_rc != EXIT_SUCCESS) { + return { EXIT_FAILURE, {} }; + } + logs.push_back({ + static_cast(strtoull(row[0].c_str(), nullptr, 10)), + { row[1], atoi(row[2].c_str()) }, + kind, + atoi(row[4].c_str()) != 0, + }); + } + + return { EXIT_SUCCESS, move(logs) }; +} + +rc_t BGD_Simulator::wait_for_probe_log( + uint64_t sequence_id, + Endpoint backend, + BGD_Probe_Kind probe_kind, + uint32_t timeout_ms, + int encrypted) +{ + uint64_t deadline = monotonic_time() + static_cast(timeout_ms) * 1000; + do { + auto [rc, logs] = probe_log_since(sequence_id); + if (rc != EXIT_SUCCESS) { + return { EXIT_FAILURE, {} }; + } + for (BGD_Probe_Log& log : logs) { + if (log.backend.host == backend.host && log.backend.port == backend.port && + log.probe_kind == probe_kind && + (encrypted < 0 || log.encrypted == (encrypted != 0))) { + return { EXIT_SUCCESS, log }; + } + } + usleep(50000); + } while (monotonic_time() < deadline); + + auto [rc, logs] = probe_log_since(sequence_id); + if (rc == EXIT_SUCCESS) { + for (BGD_Probe_Log& log : logs) { + diag( + "Observed BGD probe sequence=%llu backend=%s:%d kind=%s encrypted=%d", + static_cast(log.sequence_id), + log.backend.host.c_str(), log.backend.port, + probe_kind_string(log.probe_kind), log.encrypted ? 1 : 0); + } + } + diag( + "Timed out waiting for BGD probe backend=%s:%d kind=%s encrypted=%d", + backend.host.c_str(), backend.port, probe_kind_string(probe_kind), encrypted); + return { ETIMEDOUT, {} }; +} + +rc_t BGD_Simulator::replica_probe_log_last_sequence() { + if (connection() == nullptr) { + return { EXIT_FAILURE, 0 }; + } + auto [rc, rows] = mysql_query_ext_rows( + connection(), + "SELECT COALESCE(MAX(sequence_id),0) FROM AWS_AURORA_REPLICA_PROBE_LOG"); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows.front().size() != 1) { + return { EXIT_FAILURE, 0 }; + } + return { + EXIT_SUCCESS, + static_cast(strtoull(rows.front().front().c_str(), nullptr, 10)) + }; +} + +rc_t> BGD_Simulator::replica_probe_log_since( + uint64_t sequence_id) +{ + if (connection() == nullptr) { + return { EXIT_FAILURE, {} }; + } + string query { + "SELECT sequence_id,backend_ip,backend_port,probe_kind," + "COALESCE(replica_set_id,''),encrypted " + "FROM AWS_AURORA_REPLICA_PROBE_LOG WHERE sequence_id>" + + to_string(sequence_id) + " ORDER BY sequence_id" + }; + auto [rc, rows] = mysql_query_ext_rows(connection(), query); + if (rc != EXIT_SUCCESS) { + return { EXIT_FAILURE, {} }; + } + + vector logs {}; + for (mysql_res_row& row : rows) { + if (row.size() != 6) { + return { EXIT_FAILURE, {} }; + } + auto [kind_rc, probe_kind] = parse_replica_probe_kind(row[3]); + if (kind_rc != EXIT_SUCCESS) { + return { EXIT_FAILURE, {} }; + } + logs.push_back({ + static_cast(strtoull(row[0].c_str(), nullptr, 10)), + { row[1], atoi(row[2].c_str()) }, + probe_kind, + row[4], + atoi(row[5].c_str()) != 0, + }); + } + return { EXIT_SUCCESS, move(logs) }; +} + +rc_t BGD_Simulator::wait_for_replica_probe_log( + uint64_t sequence_id, + Endpoint backend, + Aurora_Replica_Probe_Kind probe_kind, + uint32_t timeout_ms, + int encrypted, + string replica_set_id) +{ + uint64_t deadline = monotonic_time() + static_cast(timeout_ms) * 1000; + do { + auto [rc, logs] = replica_probe_log_since(sequence_id); + if (rc != EXIT_SUCCESS) { + return { EXIT_FAILURE, {} }; + } + for (Aurora_Replica_Probe_Log& log : logs) { + if (log.backend.host == backend.host && log.backend.port == backend.port && + log.probe_kind == probe_kind && + (replica_set_id.empty() || log.replica_set_id == replica_set_id) && + (encrypted < 0 || log.encrypted == (encrypted != 0))) { + return { EXIT_SUCCESS, log }; + } + } + usleep(50000); + } while (monotonic_time() < deadline); + + auto [rc, logs] = replica_probe_log_since(sequence_id); + if (rc == EXIT_SUCCESS) { + for (Aurora_Replica_Probe_Log& log : logs) { + diag( + "Observed Aurora replica probe sequence=%llu backend=%s:%d kind=%s set=%s encrypted=%d", + static_cast(log.sequence_id), + log.backend.host.c_str(), log.backend.port, + replica_probe_kind_string(log.probe_kind), + log.replica_set_id.c_str(), log.encrypted ? 1 : 0); + } + } + diag( + "Timed out waiting for Aurora replica probe backend=%s:%d kind=%s set=%s encrypted=%d", + backend.host.c_str(), backend.port, replica_probe_kind_string(probe_kind), + replica_set_id.c_str(), encrypted); + return { ETIMEDOUT, {} }; +} + +int BGD_Simulator::execute_transaction(vector& statements) { + if (execute("START TRANSACTION") != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + for (string& statement : statements) { + if (execute(statement) != EXIT_SUCCESS) { + (void)execute("ROLLBACK"); + return EXIT_FAILURE; + } + } + if (execute("COMMIT") != EXIT_SUCCESS) { + (void)execute("ROLLBACK"); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +string BGD_Simulator::backend_predicate(Endpoint backend) { + return "backend_ip=" + sql_quote(backend.host) + + " AND backend_port=" + to_string(backend.port); +} diff --git a/test/tap/tap/bgd_simulator.h b/test/tap/tap/bgd_simulator.h new file mode 100644 index 0000000000..30650da764 --- /dev/null +++ b/test/tap/tap/bgd_simulator.h @@ -0,0 +1,103 @@ +#ifndef TAP_BGD_SIMULATOR_H +#define TAP_BGD_SIMULATOR_H + +#include +#include +#include + +#include "cluster_simulator.h" +#include "utils.h" + +using namespace std; + +/** One row returned by the simulated mysql.rds_topology table. */ +struct BGD_Topology_Row { + string id; + string endpoint; + int port; + string role; + string status; +}; + +/** Identifies a topology query recorded by the shared AWS BGD simulator. */ +enum class BGD_Probe_Kind { + table_check, + metadata, +}; + +/** One topology query observed by the shared AWS BGD simulator. */ +struct BGD_Probe_Log { + uint64_t sequence_id; + Endpoint backend; + BGD_Probe_Kind probe_kind; + bool encrypted; +}; + +/** One row returned by the simulated Aurora replica-status service. */ +struct Aurora_Replica_Row { + string server_id; + string session_id; + double cpu; + string last_update_timestamp; + double replica_lag_in_milliseconds; + bool is_current; +}; + +/** Identifies which Aurora replica-status query reached the simulator. */ +enum class Aurora_Replica_Probe_Kind { + ordinary, + bgd_membership, +}; + +/** One Aurora replica-status query observed by the shared AWS simulator. */ +struct Aurora_Replica_Probe_Log { + uint64_t sequence_id; + Endpoint backend; + Aurora_Replica_Probe_Kind probe_kind; + string replica_set_id; + bool encrypted; +}; + +/** Controls shared AWS BGD topology and Aurora replica simulator responses. */ +class BGD_Simulator : public Cluster_Simulator { +public: + int topology_update(vector backends, vector rows); + int topology_delete(vector backends); + int topology_drop(vector backends); + int topology_error(vector backends, int error_code, string error_msg); + int replica_update( + string replica_set_id, + vector rows, + vector backends + ); + int replica_delete(string replica_set_id); + int replica_drop(vector backends); + int replica_error(vector backends, int error_code, string error_msg); + int cleanup(); + + rc_t probe_log_last_sequence(); + rc_t> probe_log_since(uint64_t sequence_id); + rc_t wait_for_probe_log( + uint64_t sequence_id, + Endpoint backend, + BGD_Probe_Kind probe_kind, + uint32_t timeout_ms, + int encrypted = -1 + ); + rc_t replica_probe_log_last_sequence(); + rc_t> replica_probe_log_since(uint64_t sequence_id); + rc_t wait_for_replica_probe_log( + uint64_t sequence_id, + Endpoint backend, + Aurora_Replica_Probe_Kind probe_kind, + uint32_t timeout_ms, + int encrypted = -1, + string replica_set_id = "" + ); + +protected: + static string backend_predicate(Endpoint backend); + int execute_transaction(vector& statements); +}; + +#endif // TAP_BGD_SIMULATOR_H diff --git a/test/tap/tap/rds_bgd_simulator.cpp b/test/tap/tap/rds_bgd_simulator.cpp index 5b53f0aad4..673aa50790 100644 --- a/test/tap/tap/rds_bgd_simulator.cpp +++ b/test/tap/tap/rds_bgd_simulator.cpp @@ -1,28 +1,7 @@ #include "rds_bgd_simulator.h" -#include -#include -#include -#include - -#include "tap.h" - using namespace std; -const char* probe_kind_string(RDS_BGD_Probe_Kind kind) { - return kind == RDS_BGD_Probe_Kind::table_check ? "table_check" : "metadata"; -} - -rc_t parse_probe_kind(string value) { - if (value == "table_check") { - return { EXIT_SUCCESS, RDS_BGD_Probe_Kind::table_check }; - } - if (value == "metadata") { - return { EXIT_SUCCESS, RDS_BGD_Probe_Kind::metadata }; - } - return { EXIT_FAILURE, RDS_BGD_Probe_Kind::table_check }; -} - Endpoint RDS_BGD_Host::endpoint() { return { ip, port }; } @@ -54,7 +33,7 @@ vector RDS_BGD_Cluster::get_endpoints() { return endpoints; } -vector RDS_BGD_Cluster::get_topology(string status) { +vector RDS_BGD_Cluster::get_topology(string status) { return { { blue_writer.hostname, blue_writer.hostname, blue_writer.port, "BLUE_GREEN_DEPLOYMENT_SOURCE", status }, @@ -62,200 +41,3 @@ vector RDS_BGD_Cluster::get_topology(string status) { "BLUE_GREEN_DEPLOYMENT_TARGET", status }, }; } - -int RDS_BGD_Simulator::topology_update(vector backends, vector rows) { - if (backends.empty()) { - return EXIT_FAILURE; - } - - vector statements {}; - for (Endpoint& backend : backends) { - string predicate { backend_predicate(backend) }; - statements.push_back("DELETE FROM RDS_BGD_TOPOLOGY WHERE " + predicate); - statements.push_back( - "INSERT OR REPLACE INTO RDS_BGD_CONTROL" - "(backend_ip,backend_port,topology_present,error_code,error_msg) VALUES (" + - sql_quote(backend.host) + "," + to_string(backend.port) + ",1,0,'')"); - - for (size_t row_order = 0; row_order < rows.size(); ++row_order) { - RDS_BGD_Topology_Row& row = rows[row_order]; - statements.push_back( - "INSERT INTO RDS_BGD_TOPOLOGY" - "(backend_ip,backend_port,row_order,id,endpoint,topology_port,role,status) VALUES (" + - sql_quote(backend.host) + "," + to_string(backend.port) + "," + - to_string(row_order) + "," + sql_quote(row.id) + "," + - sql_quote(row.endpoint) + "," + to_string(row.port) + "," + - sql_quote(row.role) + "," + sql_quote(row.status) + ")"); - } - } - - return execute_transaction(statements); -} - -int RDS_BGD_Simulator::topology_delete(vector backends) { - if (backends.empty()) { - return EXIT_FAILURE; - } - - vector statements {}; - for (Endpoint& backend : backends) { - statements.push_back( - "DELETE FROM RDS_BGD_TOPOLOGY WHERE " + backend_predicate(backend)); - statements.push_back( - "INSERT OR REPLACE INTO RDS_BGD_CONTROL" - "(backend_ip,backend_port,topology_present,error_code,error_msg) VALUES (" + - sql_quote(backend.host) + "," + to_string(backend.port) + ",1,0,'')"); - } - return execute_transaction(statements); -} - -int RDS_BGD_Simulator::topology_drop(vector backends) { - return topology_error(backends, 1146, "Table 'mysql.rds_topology' doesn't exist"); -} - -int RDS_BGD_Simulator::topology_error(vector backends, int error_code, string error_msg) { - if (backends.empty() || error_code == 0) { - return EXIT_FAILURE; - } - - bool topology_present = error_code != 1146; - vector statements {}; - for (Endpoint& backend : backends) { - if (!topology_present) { - statements.push_back( - "DELETE FROM RDS_BGD_TOPOLOGY WHERE " + backend_predicate(backend)); - } - statements.push_back( - "INSERT OR REPLACE INTO RDS_BGD_CONTROL" - "(backend_ip,backend_port,topology_present,error_code,error_msg) VALUES (" + - sql_quote(backend.host) + "," + to_string(backend.port) + "," + - (topology_present ? "1" : "0") + "," + to_string(error_code) + "," + - sql_quote(error_msg) + ")"); - } - return execute_transaction(statements); -} - -int RDS_BGD_Simulator::cleanup() { - vector statements { - "DELETE FROM READONLY_STATUS", - "DELETE FROM RDS_BGD_TOPOLOGY", - "DELETE FROM RDS_BGD_CONTROL", - "DELETE FROM RDS_BGD_PROBE_LOG", - }; - return execute_transaction(statements); -} - -rc_t RDS_BGD_Simulator::probe_log_last_sequence() { - if (connection() == nullptr) { - return { EXIT_FAILURE, 0 }; - } - - auto [rc, rows] = mysql_query_ext_rows( - connection(), "SELECT COALESCE(MAX(sequence_id),0) FROM RDS_BGD_PROBE_LOG"); - if (rc != EXIT_SUCCESS || rows.size() != 1 || rows.front().size() != 1) { - return { EXIT_FAILURE, 0 }; - } - - return { - EXIT_SUCCESS, - static_cast(strtoull(rows.front().front().c_str(), nullptr, 10)) - }; -} - -rc_t> RDS_BGD_Simulator::probe_log_since( - uint64_t sequence_id) -{ - if (connection() == nullptr) { - return { EXIT_FAILURE, {} }; - } - - string query { - "SELECT sequence_id,backend_ip,backend_port,probe_kind,encrypted " - "FROM RDS_BGD_PROBE_LOG WHERE sequence_id>" + to_string(sequence_id) + - " ORDER BY sequence_id" - }; - auto [rc, rows] = mysql_query_ext_rows(connection(), query); - if (rc != EXIT_SUCCESS) { - return { EXIT_FAILURE, {} }; - } - - vector logs {}; - for (mysql_res_row& row : rows) { - if (row.size() != 5) { - return { EXIT_FAILURE, {} }; - } - auto [kind_rc, kind] = parse_probe_kind(row[3]); - if (kind_rc != EXIT_SUCCESS) { - return { EXIT_FAILURE, {} }; - } - logs.push_back({ - static_cast(strtoull(row[0].c_str(), nullptr, 10)), - { row[1], atoi(row[2].c_str()) }, - kind, - atoi(row[4].c_str()) != 0, - }); - } - - return { EXIT_SUCCESS, move(logs) }; -} - -rc_t RDS_BGD_Simulator::wait_for_probe_log( - uint64_t sequence_id, - Endpoint backend, - RDS_BGD_Probe_Kind probe_kind, - uint32_t timeout_ms, - int encrypted) -{ - uint64_t deadline = monotonic_time() + static_cast(timeout_ms) * 1000; - do { - auto [rc, logs] = probe_log_since(sequence_id); - if (rc != EXIT_SUCCESS) { - return { EXIT_FAILURE, {} }; - } - for (RDS_BGD_Probe_Log& log : logs) { - if (log.backend.host == backend.host && log.backend.port == backend.port && - log.probe_kind == probe_kind && - (encrypted < 0 || log.encrypted == (encrypted != 0))) { - return { EXIT_SUCCESS, log }; - } - } - usleep(50000); - } while (monotonic_time() < deadline); - - auto [rc, logs] = probe_log_since(sequence_id); - if (rc == EXIT_SUCCESS) { - for (RDS_BGD_Probe_Log& log : logs) { - diag( - "Observed BGD probe sequence=%llu backend=%s:%d kind=%s encrypted=%d", - static_cast(log.sequence_id), - log.backend.host.c_str(), log.backend.port, - probe_kind_string(log.probe_kind), log.encrypted ? 1 : 0); - } - } - diag( - "Timed out waiting for BGD probe backend=%s:%d kind=%s encrypted=%d", - backend.host.c_str(), backend.port, probe_kind_string(probe_kind), encrypted); - return { ETIMEDOUT, {} }; -} - -int RDS_BGD_Simulator::execute_transaction(vector& statements) { - if (execute("START TRANSACTION") != EXIT_SUCCESS) { - return EXIT_FAILURE; - } - for (string& statement : statements) { - if (execute(statement) != EXIT_SUCCESS) { - (void)execute("ROLLBACK"); - return EXIT_FAILURE; - } - } - if (execute("COMMIT") != EXIT_SUCCESS) { - (void)execute("ROLLBACK"); - return EXIT_FAILURE; - } - return EXIT_SUCCESS; -} - -string RDS_BGD_Simulator::backend_predicate(Endpoint backend) { - return "backend_ip=" + sql_quote(backend.host) + - " AND backend_port=" + to_string(backend.port); -} diff --git a/test/tap/tap/rds_bgd_simulator.h b/test/tap/tap/rds_bgd_simulator.h index 2827d39435..a112d760a3 100644 --- a/test/tap/tap/rds_bgd_simulator.h +++ b/test/tap/tap/rds_bgd_simulator.h @@ -1,258 +1,36 @@ #ifndef TAP_RDS_BGD_SIMULATOR_H #define TAP_RDS_BGD_SIMULATOR_H -#include #include #include -#include "cluster_simulator.h" -#include "utils.h" +#include "bgd_simulator.h" using namespace std; -/** - * @brief Represents one row returned by the simulated `mysql.rds_topology` table. - */ -struct RDS_BGD_Topology_Row { - string id; ///< RDS topology node identifier. - string endpoint; ///< RDS hostname exposed by the topology row. - int port; ///< MySQL port exposed by the topology row. - string role; ///< Blue/green deployment role reported by RDS. - string status; ///< Blue/green deployment status reported by RDS. -}; - -/** - * @brief Describes one RDS BGD host and its fixed simulator address. - */ +/** Describes one RDS BGD host and its fixed simulator address. */ struct RDS_BGD_Host { - string hostname; ///< AWS-style RDS hostname configured in ProxySQL. - string ip; ///< Fixed loopback address used by the simulator. - int port; ///< MySQL listener port shared by the hostname and IP. + string hostname; + string ip; + int port; - /** - * @brief Returns the IP/port endpoint used for topology simulation. - * - * @return Simulator endpoint containing this host's IP address and port. - */ Endpoint endpoint(); - - /** - * @brief Returns the hostname/port endpoint used for read-only simulation. - * - * @return Simulator endpoint containing this host's RDS hostname and port. - */ Endpoint host_endpoint(); }; -/** - * @brief Holds the blue and green hosts participating in one simulated RDS BGD cluster. - * - * @details TAP tests populate the cluster with the deployment topology required by each - * scenario. Helper methods derive writer endpoints and AWS topology rows from - * the configured hosts. - */ +/** Holds the blue and green hosts in one simulated RDS BGD deployment. */ class RDS_BGD_Cluster { public: - RDS_BGD_Host blue_writer; ///< Source writer configured in ProxySQL. - RDS_BGD_Host green_writer; ///< Target writer discovered from the topology. - vector blue_readers; ///< Source readers configured in ProxySQL. - vector green_readers; ///< Target readers discovered from the topology. + RDS_BGD_Host blue_writer; + RDS_BGD_Host green_writer; + vector blue_readers; + vector green_readers; - /** - * @brief Returns both writer IP/port endpoints for topology simulation. - * - * @return Blue and green writer endpoints keyed by simulator IP address. - */ vector get_writers(); - - /** - * @brief Returns the blue writer and configured blue readers. - * - * @return Blue deployment endpoints keyed by simulator IP address. - */ vector get_blue_endpoints(); - - /** - * @brief Returns the green writer and configured green readers. - * - * @return Green deployment endpoints keyed by simulator IP address. - */ vector get_green_endpoints(); - - /** - * @brief Returns every simulator IP/port endpoint in this cluster. - * - * @details Includes both writers and all configured blue and green readers. - * Tests use this list when resetting or publishing topology for a complete - * simulated deployment. - * - * @return Writer and reader endpoints keyed by simulator IP address. - */ vector get_endpoints(); - - /** - * @brief Builds the topology rows published by the simulated writers. - * - * @details Creates one source row for the blue writer and one target row for the green - * writer. The supplied deployment status is applied to both rows. - * - * @param status RDS blue/green deployment status to publish. - * - * @return Source and target rows for the simulated topology table. - */ - vector get_topology(string status); -}; - -/** - * @brief Identifies the RDS BGD monitor query recorded in the simulator probe log. - */ -enum class RDS_BGD_Probe_Kind { - table_check, ///< Query checking whether `mysql.rds_topology` exists. - metadata, ///< Query fetching rows from `mysql.rds_topology`. -}; - -/** - * @brief Describes one RDS BGD monitor query observed by the simulator. - */ -struct RDS_BGD_Probe_Log { - uint64_t sequence_id; ///< Monotonically increasing probe-log sequence. - Endpoint backend; ///< Accepted backend IP address and port. - RDS_BGD_Probe_Kind probe_kind; ///< Type of topology query observed. - bool encrypted; ///< Whether the monitor connection used TLS. -}; - -/** - * @brief Controls RDS BGD topology responses and inspects monitor probes from TAP tests. - * - * @details Publishes per-backend topology rows, empty results, missing tables, or MySQL - * errors. It also reads the ordered probe log generated by the SQLite3-server - * simulator and reuses `Cluster_Simulator` for shared control operations. - */ -class RDS_BGD_Simulator : public Cluster_Simulator { -public: - /** - * @brief Replaces the simulated topology returned by each backend. - * - * @details Deletes existing topology rows before inserting the supplied rows. The - * topology table is marked present, configured errors are cleared, and the - * complete update is applied atomically across all supplied backends. - * - * @param backends Backend IP/port endpoints that must return the topology. - * @param rows Topology rows to publish on each backend. - * - * @return EXIT_SUCCESS when every backend is updated; EXIT_FAILURE otherwise. - */ - int topology_update(vector backends, vector rows); - - /** - * @brief Configures each backend to return an empty topology result. - * - * @details Deletes all topology rows associated with the supplied backends while keeping - * the topology table present and clearing any configured metadata error. - * - * @param backends Backend IP/port endpoints that must return an empty result. - * - * @return EXIT_SUCCESS when every backend is updated; EXIT_FAILURE otherwise. - */ - int topology_delete(vector backends); - - /** - * @brief Configures each backend to report that the topology table does not exist. - * - * @param backends Backend IP/port endpoints that must return MySQL error 1146. - * - * @return EXIT_SUCCESS when every backend is updated; EXIT_FAILURE otherwise. - */ - int topology_drop(vector backends); - - /** - * @brief Configures a MySQL error for topology queries on each backend. - * - * @details Stores the nonzero error code and message returned by subsequent metadata - * probes. Error 1146 marks the topology table absent and removes its existing - * rows; other error codes leave the table marked present. - * - * @param backends Backend IP/port endpoints that must return the error. - * @param error_code Nonzero MySQL error code to return. - * @param error_msg MySQL error message to return. - * - * @return EXIT_SUCCESS when every backend is updated; EXIT_FAILURE otherwise. - */ - int topology_error(vector backends, int error_code, string error_msg); - - /** - * @brief Removes all read-only, topology-control, topology-row, and probe state. - * - * @return EXIT_SUCCESS when the simulator state is empty; EXIT_FAILURE otherwise. - */ - int cleanup(); - - /** - * @brief Reads the latest sequence from the RDS BGD probe log. - * - * @return EXIT_SUCCESS and the latest sequence, or zero when the log is empty; - * EXIT_FAILURE and zero when the query fails. - */ - rc_t probe_log_last_sequence(); - - /** - * @brief Returns probe-log records newer than a sequence. - * - * @details Selects records with `sequence_id` strictly greater than the supplied value - * and preserves database sequence order in the returned vector. - * - * @param sequence_id Last probe-log sequence already observed by the TAP test. - * - * @return EXIT_SUCCESS and the matching records; EXIT_FAILURE and an empty vector - * when the query or record parsing fails. - */ - rc_t> probe_log_since(uint64_t sequence_id); - - /** - * @brief Waits for a matching RDS BGD probe-log record. - * - * @details Matches records newer than `sequence_id` by backend and probe kind. TLS state - * is matched when `encrypted` is zero or one; `-1` accepts either state. Observed - * probes are emitted through TAP diagnostics when the wait expires. - * - * @param sequence_id Last probe-log sequence observed before the expected probe. - * @param backend Backend IP/port endpoint expected to receive the probe. - * @param probe_kind Type of topology query expected. - * @param timeout_ms Maximum time to wait in milliseconds. - * @param encrypted Expected TLS state, or -1 to accept either state. - * - * @return EXIT_SUCCESS and the matching record; ETIMEDOUT and an empty record when - * the deadline expires; EXIT_FAILURE and an empty record when log retrieval fails. - */ - rc_t wait_for_probe_log( - uint64_t sequence_id, - Endpoint backend, - RDS_BGD_Probe_Kind probe_kind, - uint32_t timeout_ms, - int encrypted = -1 - ); - -private: - /** - * @brief Builds the SQL predicate identifying one simulated backend. - * - * @param backend Backend IP/port endpoint to match. - * - * @return SQL predicate matching the backend control-table key. - */ - static string backend_predicate(Endpoint backend); - - /** - * @brief Executes simulator control statements in one transaction. - * - * @details Executes the supplied statements in order and commits only after every - * statement succeeds. A statement or commit failure triggers a rollback. - * - * @param statements SQL statements to execute atomically. - * - * @return EXIT_SUCCESS when the transaction commits; EXIT_FAILURE otherwise. - */ - int execute_transaction(vector& statements); + vector get_topology(string status); }; #endif // TAP_RDS_BGD_SIMULATOR_H diff --git a/test/tap/tap/rds_bgd_tap.h b/test/tap/tap/rds_bgd_tap.h index 619450c7af..16bb5091f7 100644 --- a/test/tap/tap/rds_bgd_tap.h +++ b/test/tap/tap/rds_bgd_tap.h @@ -90,8 +90,8 @@ struct BGD_Hostgroups { int green_reader; }; -inline vector bgd_topology_with_readers(RDS_BGD_Cluster& cluster, string status) { - vector rows = cluster.get_topology(status); +inline vector bgd_topology_with_readers(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); for (RDS_BGD_Host& host : cluster.blue_readers) { rows.push_back({ host.hostname, host.hostname, host.port, "BLUE_GREEN_DEPLOYMENT_SOURCE", status }); } @@ -101,7 +101,7 @@ inline vector bgd_topology_with_readers(RDS_BGD_Cluster& c return rows; } -inline int bgd_set_writer_read_only_0(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { +inline int bgd_set_writer_read_only_0(BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { if (sim.read_only_update(cluster.blue_writer.host_endpoint(), false) != EXIT_SUCCESS) { diag("Error: failed to set read_only=0 for the simulated blue writer"); return EXIT_FAILURE; @@ -114,12 +114,12 @@ inline int bgd_set_writer_read_only_0(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& c return EXIT_SUCCESS; } -inline int bgd_set_host_read_only_0(RDS_BGD_Simulator& sim, RDS_BGD_Host& host) { +inline int bgd_set_host_read_only_0(BGD_Simulator& sim, RDS_BGD_Host& host) { int rc = sim.read_only_update(host.host_endpoint(), false); return rc; } -inline int bgd_set_host_read_only_1(RDS_BGD_Simulator& sim, RDS_BGD_Host& host) { +inline int bgd_set_host_read_only_1(BGD_Simulator& sim, RDS_BGD_Host& host) { int rc = sim.read_only_update(host.host_endpoint(), true); return rc; } @@ -310,7 +310,7 @@ inline rc_t bgd_backend_ip_echo(MYSQL* proxy) { } inline rc_t bgd_probe_count_since( - RDS_BGD_Simulator& sim, uint64_t sequence, Endpoint backend, RDS_BGD_Probe_Kind kind) + BGD_Simulator& sim, uint64_t sequence, Endpoint backend, BGD_Probe_Kind kind) { auto [rc, logs] = sim.probe_log_since(sequence); if (rc != EXIT_SUCCESS) { @@ -319,7 +319,7 @@ inline rc_t bgd_probe_count_since( } uint64_t count = 0; - for (const RDS_BGD_Probe_Log& log : logs) { + for (const BGD_Probe_Log& log : logs) { bool backend_matches = log.backend.host == backend.host && log.backend.port == backend.port; @@ -417,19 +417,19 @@ inline int bgd_wait_for_server_placement( return rc; } -inline rc_t bgd_wait_for_probe_from_backends( - RDS_BGD_Simulator& sim, uint64_t sequence, vector backends, - RDS_BGD_Probe_Kind kind, uint32_t timeout_ms, int encrypted = -1) +inline rc_t bgd_wait_for_probe_from_backends( + BGD_Simulator& sim, uint64_t sequence, vector backends, + BGD_Probe_Kind kind, uint32_t timeout_ms, int encrypted = -1) { uint64_t deadline = monotonic_time() + static_cast(timeout_ms) * 1000; do { auto [rc, logs] = sim.probe_log_since(sequence); if (rc != EXIT_SUCCESS) { - rc_t result { EXIT_FAILURE, {} }; + rc_t result { EXIT_FAILURE, {} }; return result; } - for (const RDS_BGD_Probe_Log& log : logs) { + for (const BGD_Probe_Log& log : logs) { for (const Endpoint& backend : backends) { bool backend_matches = log.backend.host == backend.host && @@ -439,7 +439,7 @@ inline rc_t bgd_wait_for_probe_from_backends( encrypted < 0 || log.encrypted == (encrypted != 0); if (backend_matches && kind_matches && encryption_matches) { - rc_t result { EXIT_SUCCESS, log }; + rc_t result { EXIT_SUCCESS, log }; return result; } } @@ -448,7 +448,7 @@ inline rc_t bgd_wait_for_probe_from_backends( usleep(50000); } while (monotonic_time() < deadline); - rc_t result { ETIMEDOUT, {} }; + rc_t result { ETIMEDOUT, {} }; return result; } @@ -459,10 +459,10 @@ inline rc_t bgd_wait_for_probe_from_backends( * after the given sequence. */ inline int bgd_expect_no_table_check( - RDS_BGD_Simulator& sim, uint64_t sequence, vector backends, uint32_t timeout_ms) + BGD_Simulator& sim, uint64_t sequence, vector backends, uint32_t timeout_ms) { auto [probe_rc, probe] = bgd_wait_for_probe_from_backends( - sim, sequence, backends, RDS_BGD_Probe_Kind::table_check, timeout_ms + sim, sequence, backends, BGD_Probe_Kind::table_check, timeout_ms ); if (probe_rc == ETIMEDOUT) { @@ -478,12 +478,12 @@ inline int bgd_expect_no_table_check( * endpoint after the given sequence. */ inline int bgd_expect_no_metadata_probe( - RDS_BGD_Simulator& sim, uint64_t sequence, Endpoint backend, uint32_t timeout_ms) + BGD_Simulator& sim, uint64_t sequence, Endpoint backend, uint32_t timeout_ms) { vector backends { backend }; auto [probe_rc, probe] = bgd_wait_for_probe_from_backends( - sim, sequence, backends, RDS_BGD_Probe_Kind::metadata, timeout_ms + sim, sequence, backends, BGD_Probe_Kind::metadata, timeout_ms ); if (probe_rc == ETIMEDOUT) { @@ -499,10 +499,10 @@ inline int bgd_expect_no_metadata_probe( * endpoint after the given sequence. */ inline int bgd_expect_no_metadata_probe_from_backends( - RDS_BGD_Simulator& sim, uint64_t sequence, vector backends, uint32_t timeout_ms) + BGD_Simulator& sim, uint64_t sequence, vector backends, uint32_t timeout_ms) { auto [probe_rc, probe] = bgd_wait_for_probe_from_backends( - sim, sequence, backends, RDS_BGD_Probe_Kind::metadata, timeout_ms + sim, sequence, backends, BGD_Probe_Kind::metadata, timeout_ms ); if (probe_rc == ETIMEDOUT) { diff --git a/test/tap/test_helpers/test_init.cpp b/test/tap/test_helpers/test_init.cpp index 08261b2efd..57192d5b00 100644 --- a/test/tap/test_helpers/test_init.cpp +++ b/test/tap/test_helpers/test_init.cpp @@ -20,6 +20,7 @@ #include "PgSQL_Query_Cache.h" #include "MySQL_Query_Processor.h" #include "PgSQL_Query_Processor.h" +#include "MySQL_Monitor.hpp" #include "test_globals.h" #include "test_init.h" @@ -32,6 +33,7 @@ extern MySQL_Query_Cache *GloMyQC; extern PgSQL_Query_Cache *GloPgQC; extern MySQL_Query_Processor *GloMyQPro; extern PgSQL_Query_Processor *GloPgQPro; +extern MySQL_Monitor *GloMyMon; // GloMTH is declared extern in proxysql_utils.h. // GloPTH has no extern declaration in any header, so we add one here. @@ -205,3 +207,21 @@ void test_cleanup_hostgroups() { PgHGM = nullptr; } } + +// ============================================================================ +// MySQL Monitor +// ============================================================================ + +int test_init_monitor() { + if (GloMyMon == nullptr) { + GloMyMon = new MySQL_Monitor(); + } + return 0; +} + +void test_cleanup_monitor() { + if (GloMyMon != nullptr) { + delete GloMyMon; + GloMyMon = nullptr; + } +} diff --git a/test/tap/test_helpers/test_init.h b/test/tap/test_helpers/test_init.h index 5818bde0ba..d623d06152 100644 --- a/test/tap/test_helpers/test_init.h +++ b/test/tap/test_helpers/test_init.h @@ -128,4 +128,17 @@ int test_init_hostgroups(); */ void test_cleanup_hostgroups(); +/** + * @brief Initialize the MySQL monitor without starting its worker threads. + * + * @pre test_init_minimal() must have been called. + * @return 0 on success, non-zero on failure. + */ +int test_init_monitor(); + +/** + * @brief Clean up resources allocated by test_init_monitor(). + */ +void test_cleanup_monitor(); + #endif /* TEST_INIT_H */ diff --git a/test/tap/tests/Makefile b/test/tap/tests/Makefile index 849f2abaaf..62f14a22e8 100644 --- a/test/tap/tests/Makefile +++ b/test/tap/tests/Makefile @@ -227,9 +227,6 @@ tests-php: $(patsubst %,php-%,$(wildcard *-t.php)) tests-py: $(patsubst %,py-%,$(wildcard *-t.py)) tests-sh: $(patsubst %,sh-%,$(wildcard *-t.sh)) testgalera: galera_1_timeout_count galera_2_timeout_no_count -testaurora: aurora - - ### test deps targets #build_test_deps: $(TAP_LDIR)/libtap$(SHLIB_EXT) $(TEST_MARIADB_LDIR)/libmariadbclient.a $(TEST_MYSQL_LDIR)/libmysqlclient.a @@ -275,9 +272,6 @@ galera_2_timeout_no_count: galera_2_timeout_no_count.cpp $(TAP_LDIR)/libtap$(SHL generate_set_session_csv: generate_set_session_csv.cpp $(CXX) $< $(OPT) -o $@ -aurora: aurora.cpp $(TAP_LDIR)/libtap$(SHLIB_EXT) - $(CXX) -DTEST_AURORA $< ../tap/SQLite3_Server.cpp $(IDIRS) $(LDIRS) $(OPT) $(OBJ) $(MYLIBSJEMALLOC) $(MYLIBS) $(STATIC_LIBS) -o $@ - test_tokenizer-t: test_tokenizer-t.cpp $(TAP_LDIR)/libtap$(SHLIB_EXT) $(CXX) $< $(IDIRS) $(LDIRS) $(OPT) $(MYLIBS) -o $@ diff --git a/test/tap/tests/aurora.cpp b/test/tap/tests/aurora.cpp deleted file mode 100644 index 97a7b0dd0d..0000000000 --- a/test/tap/tests/aurora.cpp +++ /dev/null @@ -1,507 +0,0 @@ -#include // std::cout -#include // std::sort -#include // std::vector -#include "re2/re2.h" -#include "re2/regexp.h" -#include "proxysql.h" -#include "cpp.h" - -#include "MySQL_Logger.hpp" -#include "MySQL_Data_Stream.h" -#include "query_processor.h" -#include "SQLite3_Server.h" -#include "proxysql_utils.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include "tap.h" - -#define SELECT_VERSION_COMMENT "select @@version_comment limit 1" -#define SELECT_VERSION_COMMENT_LEN 32 -#define SELECT_DB_USER "select DATABASE(), USER() limit 1" -#define SELECT_DB_USER_LEN 33 -#define SELECT_CHARSET_VARIOUS "select @@character_set_client, @@character_set_connection, @@character_set_server, @@character_set_database limit 1" -#define SELECT_CHARSET_VARIOUS_LEN 115 - -#define READ_ONLY_OFF "\x01\x00\x00\x01\x02\x23\x00\x00\x02\x03\x64\x65\x66\x00\x00\x00\x0d\x56\x61\x72\x69\x61\x62\x6c\x65\x5f\x6e\x61\x6d\x65\x00\x0c\x21\x00\x0f\x00\x00\x00\xfd\x01\x00\x1f\x00\x00\x1b\x00\x00\x03\x03\x64\x65\x66\x00\x00\x00\x05\x56\x61\x6c\x75\x65\x00\x0c\x21\x00\x0f\x00\x00\x00\xfd\x01\x00\x1f\x00\x00\x05\x00\x00\x04\xfe\x00\x00\x02\x00\x0e\x00\x00\x05\x09\x72\x65\x61\x64\x5f\x6f\x6e\x6c\x79\x03\x4f\x46\x46\x05\x00\x00\x06\xfe\x00\x00\x02\x00" -#define READ_ONLY_ON "\x01\x00\x00\x01\x02\x23\x00\x00\x02\x03\x64\x65\x66\x00\x00\x00\x0d\x56\x61\x72\x69\x61\x62\x6c\x65\x5f\x6e\x61\x6d\x65\x00\x0c\x21\x00\x0f\x00\x00\x00\xfd\x01\x00\x1f\x00\x00\x1b\x00\x00\x03\x03\x64\x65\x66\x00\x00\x00\x05\x56\x61\x6c\x75\x65\x00\x0c\x21\x00\x0f\x00\x00\x00\xfd\x01\x00\x1f\x00\x00\x05\x00\x00\x04\xfe\x00\x00\x02\x00\x0d\x00\x00\x05\x09\x72\x65\x61\x64\x5f\x6f\x6e\x6c\x79\x02\x4f\x4e\x05\x00\x00\x06\xfe\x00\x00\x02\x00" - -extern SQLite3_Server *GloSQLite3Server; - -static int random_replication_lag_seconds() { - static thread_local std::random_device random_source; - static thread_local std::uniform_int_distribution distribution(10, 39); - return distribution(random_source); -} - -void SQLite3_Server::init_aurora_ifaces_string(std::string& s) { - if(!s.empty()) - s += ";"; - pthread_mutex_init(&aurora_mutex,NULL); - unsigned int nas = time(NULL); - nas = nas % 3; // range - nas += 4; // min - max_num_aurora_servers = 10; // hypothetical maximum number of nodes - for (unsigned int j=1; j<4; j++) { - cur_aurora_writer[j-1] = 0; - num_aurora_servers[j-1] = nas; - for (unsigned int i=11; iexecute("DELETE FROM REPLICA_HOST_STATUS"); - sqlite3_stmt *statement=NULL; - //sqlite3 *mydb3=sessdb->get_db(); - int rc; - char *query=(char *)"INSERT INTO REPLICA_HOST_STATUS VALUES (?1, ?2, ?3, ?4, ?5)"; - //rc=sqlite3_prepare_v2(mydb3, query, -1, &statement, 0); - auto prepared_statement = sessdb->prepare_v2(query); - rc = prepared_statement.first; - stmt_unique_ptr statement_unique = std::move(prepared_statement.second); - statement = statement_unique.get(); - ASSERT_SQLITE_OK(rc, sessdb); - time_t __timer; - char lut[30]; - struct tm __tm_info; - time(&__timer); - localtime_r(&__timer, &__tm_info); - strftime(lut, 25, "%Y-%m-%d %H:%M:%S", &__tm_info); - string myip = string(sess->client_myds->proxy_addr.addr); - string clu_id_s = myip.substr(6,1); - unsigned int cluster_id = atoi(clu_id_s.c_str()); - cluster_id--; - //if (rand() % 200 == 0) { - if (rand() % 20000 == 0) { - // simulate a failover - cur_aurora_writer[cluster_id] = rand() % num_aurora_servers[cluster_id]; - proxy_info("Simulating a failover for AWS Aurora cluster %d , HGs (%d:%d)\n", cluster_id, 1270 + cluster_id*2+1 , 1270 + cluster_id*2+2); - } - if (rand() % 1000 == 0) { - if (num_aurora_servers[cluster_id] < max_num_aurora_servers) { - num_aurora_servers[cluster_id]++; - proxy_info("Simulating the add of a new server for AWS Aurora Cluster %d , HGs (%d:%d). Now adding server num %d\n", cluster_id, 1270 + cluster_id*2+1 , 1270 + cluster_id*2+2, num_aurora_servers[cluster_id]); - } - } - if (rand() % 1000 == 0) { - if (num_aurora_servers[cluster_id] > 1) { - if (cur_aurora_writer[cluster_id] != (num_aurora_servers[cluster_id] - 1) ) { - num_aurora_servers[cluster_id]--; - proxy_info("Simulating the deletion of a server from AWS Aurora Cluster %d , HGs (%d:%d). Removing server num %d\n", cluster_id, 1270 + cluster_id*2+1 , 1270 + cluster_id*2+2, num_aurora_servers[cluster_id]+1); - } - } - } - for (unsigned int i=0; isize-sizeof(mysql_hdr); - query=(char *)l_alloc(query_length); - memcpy(query,(char *)pkt->ptr+sizeof(mysql_hdr)+1,query_length-1); - query[query_length-1]=0; - - if (sess->client_myds->proxy_addr.addr == NULL) { - struct sockaddr addr; - socklen_t addr_len=sizeof(struct sockaddr); - memset(&addr,0,addr_len); - int rc; - rc=getsockname(sess->client_myds->fd, &addr, &addr_len); - if (rc==0) { - char buf[512]; - switch (addr.sa_family) { - case AF_INET: { - struct sockaddr_in *ipv4 = (struct sockaddr_in *)&addr; - inet_ntop(addr.sa_family, &ipv4->sin_addr, buf, INET_ADDRSTRLEN); - sess->client_myds->proxy_addr.addr = strdup(buf); - } - break; - case AF_INET6: { - struct sockaddr_in6 *ipv6 = (struct sockaddr_in6 *)&addr; - inet_ntop(addr.sa_family, &ipv6->sin6_addr, buf, INET6_ADDRSTRLEN); - sess->client_myds->proxy_addr.addr = strdup(buf); - } - break; - default: - sess->client_myds->proxy_addr.addr = strdup("unknown"); - break; - } - } else { - sess->client_myds->proxy_addr.addr = strdup("unknown"); - } - } - - char *query_no_space=(char *)l_alloc(query_length); - memcpy(query_no_space,query,query_length); - - unsigned int query_no_space_length=remove_spaces(query_no_space); - - // fix bug #925 - while (query_no_space[query_no_space_length-1]==';' || query_no_space[query_no_space_length-1]==' ') { - query_no_space_length--; - query_no_space[query_no_space_length]=0; - } - - // fix bug #1047 - if ( -/* - (!strncasecmp("BEGIN", query_no_space, strlen("BEGIN"))) - || - (!strncasecmp("START TRANSACTION", query_no_space, strlen("START TRANSACTION"))) - || - (!strncasecmp("COMMIT", query_no_space, strlen("COMMIT"))) - || - (!strncasecmp("ROLLBACK", query_no_space, strlen("ROLLBACK"))) - || -*/ - (!strncasecmp("SET character_set_results", query_no_space, strlen("SET character_set_results"))) - || - (!strncasecmp("SET SQL_AUTO_IS_NULL", query_no_space, strlen("SET SQL_AUTO_IS_NULL"))) - || - (!strncasecmp("SET NAMES", query_no_space, strlen("SET NAMES"))) - || - (!strncasecmp("SET AUTOCOMMIT", query_no_space, strlen("SET AUTOCOMMIT"))) - || - (!strncasecmp("/*!40100 SET @@SQL_MODE='' */", query_no_space, strlen("/*!40100 SET @@SQL_MODE='' */"))) - || - (!strncasecmp("/*!40103 SET TIME_ZONE=", query_no_space, strlen("/*!40103 SET TIME_ZONE="))) - || - (!strncasecmp("/*!80000 SET SESSION", query_no_space, strlen("/*!80000 SET SESSION"))) - || - (!strncasecmp("SET SESSION", query_no_space, strlen("SET SESSION"))) - || - (!strncasecmp("SET wait_timeout", query_no_space, strlen("SET wait_timeout"))) - ) { - SQLite3_Session *sqlite_sess = (SQLite3_Session *)sess->thread->gen_args; - sqlite3 *db = sqlite_sess->sessdb->get_db(); - uint16_t status=2; // autocommit - if (sqlite3_get_autocommit(db)==0) { - status = 3; // autocommit + transaction - } - GloSQLite3Server->send_MySQL_OK(&sess->client_myds->myprot, NULL, 0, status); - run_query=false; - goto __run_query; - } - - if (query_no_space_length==17) { - if (!strncasecmp((char *)"START TRANSACTION", query_no_space, query_no_space_length)) { - l_free(query_length,query); - query = l_strdup((char *)"BEGIN IMMEDIATE"); - query_length=strlen(query)+1; - goto __run_query; - } - } - - if (query_no_space_length==5) { - if (!strncasecmp((char *)"BEGIN", query_no_space, query_no_space_length)) { - l_free(query_length,query); - query = l_strdup((char *)"BEGIN IMMEDIATE"); - query_length=strlen(query)+1; - goto __run_query; - } - } - - if (query_no_space_length==SELECT_VERSION_COMMENT_LEN) { - if (!strncasecmp(SELECT_VERSION_COMMENT, query_no_space, query_no_space_length)) { - l_free(query_length,query); - const char* proxy_addr = sess->client_myds->proxy_addr.addr; - const std::string query_text = - std::string("SELECT '(ProxySQL Automated Test Server) - ") - + (proxy_addr ? proxy_addr : "") + "'"; - query = l_strdup(query_text.c_str()); - if (!query) { - l_free(pkt->size-sizeof(mysql_hdr), query_no_space); - return; - } - query_length = query_text.size() + 1; - goto __run_query; - } - } - - if (query_no_space_length==SELECT_DB_USER_LEN) { - if (!strncasecmp(SELECT_DB_USER, query_no_space, query_no_space_length)) { - l_free(query_length,query); - const char* username = sess->client_myds->myconn->userinfo->username; - const std::string query_text = - std::string("SELECT \"admin\" AS 'DATABASE()', \"") - + (username ? username : "") + "\" AS 'USER()'"; - query = l_strdup(query_text.c_str()); - if (!query) { - l_free(pkt->size-sizeof(mysql_hdr), query_no_space); - return; - } - query_length = query_text.size() + 1; - goto __run_query; - } - } - - if (query_no_space_length==SELECT_CHARSET_VARIOUS_LEN) { - if (!strncasecmp(SELECT_CHARSET_VARIOUS, query_no_space, query_no_space_length)) { - l_free(query_length,query); - char *query1=(char *)"select 'utf8' as '@@character_set_client', 'utf8' as '@@character_set_connection', 'utf8' as '@@character_set_server', 'utf8' as '@@character_set_database' limit 1"; - query=l_strdup(query1); - query_length=strlen(query1)+1; - goto __run_query; - } - } - - if (!strncasecmp("SELECT @@version", query_no_space, strlen("SELECT @@version"))) { - l_free(query_length,query); - char *q=(char *)"SELECT '%s' AS '@@version'"; - query_length=strlen(q)+20; - query=(char *)l_alloc(query_length); - snprintf(query, query_length, q, PROXYSQL_VERSION); - goto __run_query; - } - - if (!strncasecmp("SELECT version()", query_no_space, strlen("SELECT version()"))) { - l_free(query_length,query); - char *q=(char *)"SELECT '%s' AS 'version()'"; - query_length=strlen(q)+20; - query=(char *)l_alloc(query_length); - snprintf(query, query_length, q, PROXYSQL_VERSION); - goto __run_query; - } - - if (strncasecmp("SHOW ", query_no_space, 5)) { - goto __end_show_commands; // in the next block there are only SHOW commands - } - - if (query_no_space_length==strlen("SHOW TABLES") && !strncasecmp("SHOW TABLES",query_no_space, query_no_space_length)) { - l_free(query_length,query); - query=l_strdup("SELECT name AS tables FROM sqlite_master WHERE type='table' AND name NOT IN ('sqlite_sequence') ORDER BY name"); - query_length=strlen(query)+1; - goto __run_query; - } - - if ((query_no_space_length>17) && (!strncasecmp("SHOW TABLES FROM ", query_no_space, 17))) { - strA=query_no_space+17; - strAl=strlen(strA); - strB=(char *)"SELECT name AS tables FROM %s.sqlite_master WHERE type='table' AND name NOT IN ('sqlite_sequence') ORDER BY name"; - strBl=strlen(strB); - int l=strBl+strAl-2; - char *b=(char *)l_alloc(l+1); - snprintf(b,l+1,strB,strA); - b[l]=0; - l_free(query_length,query); - query=b; - query_length=l+1; - goto __run_query; - } - - if ((query_no_space_length>17) && (!strncasecmp("SHOW TABLES LIKE ", query_no_space, 17))) { - strA=query_no_space+17; - strAl=strlen(strA); - strB=(char *)"SELECT name AS tables FROM sqlite_master WHERE type='table' AND name LIKE '%s'"; - strBl=strlen(strB); - char *tn=NULL; // tablename - tn=(char *)malloc(strlen(strA)); - unsigned int i=0, j=0; - while (i=3 && tbh[0]=='`' && tbh[tbh_len-1]=='`') { // tablename is quoted - size_t db_len = tbh_len - 2; - const std::string unquoted_table(tbh + 1, db_len); - free(tbh); - tbh = l_strdup(unquoted_table.c_str()); - if (!tbh) { - free(dbh); - l_free(query_length, query); - return; - } - } - const std::string table_query = - std::string("SELECT name AS 'table' , REPLACE(REPLACE(sql,' , ', X'2C0A20202020'),") - + "'CREATE TABLE " + tbh + " (','CREATE TABLE " + tbh - + " ('||X'0A20202020') AS 'Create Table' FROM " + dbh - + ".sqlite_master WHERE type='table' AND name='" + tbh + "'"; - char *buff = l_strdup(table_query.c_str()); - if (!buff) { - l_free(0, tbh); - l_free(0, dbh); - l_free(query_length, query); - return; - } - l_free(0, tbh); - l_free(0, dbh); - l_free(query_length,query); - query=buff; - query_length=table_query.size()+1; - goto __run_query; - } - - if ( - (query_no_space_length==strlen("SHOW DATABASES") && !strncasecmp("SHOW DATABASES",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SHOW SCHEMAS") && !strncasecmp("SHOW SCHEMAS",query_no_space, query_no_space_length)) - ) { - l_free(query_length,query); - query=l_strdup("PRAGMA DATABASE_LIST"); - query_length=strlen(query)+1; - goto __run_query; - } - -__end_show_commands: - - if (query_no_space_length==strlen("SELECT DATABASE()") && !strncasecmp("SELECT DATABASE()",query_no_space, query_no_space_length)) { - l_free(query_length,query); - query=l_strdup("SELECT \"main\" AS 'DATABASE()'"); - query_length=strlen(query)+1; - goto __run_query; - } - - // see issue #1022 - if (query_no_space_length==strlen("SELECT DATABASE() AS name") && !strncasecmp("SELECT DATABASE() AS name",query_no_space, query_no_space_length)) { - l_free(query_length,query); - query=l_strdup("SELECT \"main\" AS 'DATABASE()'"); - query_length=strlen(query)+1; - goto __run_query; - } - - if (sess->session_type == PROXYSQL_SESSION_SQLITE) { // no admin - if ( - (strncasecmp("PRAGMA",query_no_space,6)==0) - || - (strncasecmp("ATTACH",query_no_space,6)==0) - ) { - proxy_error("[WARNING]: Commands executed from stats interface in Admin Module: \"%s\"\n", query_no_space); - GloSQLite3Server->send_MySQL_ERR(&sess->client_myds->myprot, (char *)"Command not allowed"); - run_query=false; - } - } - -__run_query: - if (run_query) { - if (strncasecmp("SELECT",query_no_space,6)==0) { - if (strstr(query_no_space,(char *)"REPLICA_HOST_STATUS")) { - pthread_mutex_lock(&GloSQLite3Server->aurora_mutex); - GloSQLite3Server->populate_aws_aurora_table(sess); - } - if (strstr(query_no_space,(char *)"Seconds_Behind_Master")) { - l_free(0, query); - const std::string formatted_query = cstr_format( - "SELECT %d as Seconds_Behind_Master", random_replication_lag_seconds() - ).str; - query = l_strdup(formatted_query.c_str()); - } - } - SQLite3_Session *sqlite_sess = (SQLite3_Session *)sess->thread->gen_args; - sqlite_sess->sessdb->execute_statement(query, &error , &cols , &affected_rows , &resultset); - if (strncasecmp("SELECT",query_no_space,6)==0) { - if (strstr(query_no_space,(char *)"REPLICA_HOST_STATUS")) { - pthread_mutex_unlock(&GloSQLite3Server->aurora_mutex); - if (rand() % 100 == 0) { - // randomly add some latency on 1% of the traffic - sleep(2); - } - } - if (strstr(query_no_space,(char *)"Seconds_Behind_Master")) { - if (rand() % 10 == 0) { - // randomly add some latency on 10% of the traffic - sleep(2); - } - } - } - sqlite3 *db = sqlite_sess->sessdb->get_db(); - bool in_trans = false; - if (sqlite3_get_autocommit(db)==0) { - in_trans = true; - } - sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot, in_trans); - delete resultset; - } - l_free(pkt->size-sizeof(mysql_hdr),query_no_space); // it is always freed here - l_free(query_length,query); -} diff --git a/test/tap/tests/test_aurora_bgd_active-t.cpp b/test/tap/tests/test_aurora_bgd_active-t.cpp new file mode 100644 index 0000000000..cbcc256222 --- /dev/null +++ b/test/tap/tests/test_aurora_bgd_active-t.cpp @@ -0,0 +1,480 @@ +/** + * @file test_aurora_bgd_active-t.cpp + * @brief Aurora BGD active-state probe suspension and routing behavior. + */ + +#include +#include +#include +#include + +#include "aurora_bgd_tap.h" +#include "command_line.h" +#include "utils.h" + +using namespace std; + +const uint32_t kWaitSeconds = 5; +const uint32_t kProbeTimeoutMs = 5000; +const char kOrdinaryAuroraQuery[] = + "SELECT SERVER_ID," + "IF(" + "SESSION_ID = 'MASTER_SESSION_ID' AND " + "SERVER_ID <> (SELECT SERVER_ID FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE SESSION_ID = 'MASTER_SESSION_ID' ORDER BY LAST_UPDATE_TIMESTAMP DESC LIMIT 1), " + "'probably_former_MASTER_SESSION_ID', SESSION_ID" + ") SESSION_ID, " + "LAST_UPDATE_TIMESTAMP, " + "IF(SESSION_ID = 'MASTER_SESSION_ID', 0, REPLICA_LAG_IN_MILLISECONDS) AS REPLICA_LAG_IN_MILLISECONDS, " + "CPU " + "FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE" + " ( " + "(REPLICA_LAG_IN_MILLISECONDS >= 0 AND REPLICA_LAG_IN_MILLISECONDS <= 600000)" + " OR SESSION_ID = 'MASTER_SESSION_ID'" + " ) " + "AND LAST_UPDATE_TIMESTAMP > NOW() - INTERVAL 180 SECOND" + " ORDER BY SERVER_ID"; + +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + char simulator_username[] = "aurora1"; + char simulator_password[] = "pass1"; + if (sim.connect(cl.host, 3306, simulator_username, simulator_password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the shared AWS simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + if (aurora_bgd_admin_cleanup(admin) != EXIT_SUCCESS || sim.cleanup() != EXIT_SUCCESS) { + diag("Error: failed to clear prior Aurora BGD state"); + return EXIT_FAILURE; + } + if (aurora_bgd_execute_all(admin, { + "DELETE FROM mysql_users WHERE username='testuser'", + "INSERT INTO mysql_users(username,password,active,default_hostgroup,transaction_persistent) " + "VALUES ('testuser','testuser',1,0,1)", + "LOAD MYSQL USERS TO RUNTIME", + }) != EXIT_SUCCESS) { + diag("Error: failed to configure the routing test user"); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, BGD_Simulator& sim) { + int admin_rc = aurora_bgd_admin_cleanup(admin); + int user_rc = admin == nullptr ? EXIT_FAILURE : aurora_bgd_execute_all(admin, { + "DELETE FROM mysql_users WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }); + int simulator_rc = sim.cleanup(); + if (admin) { + mysql_close(admin); + } + return admin_rc == EXIT_SUCCESS && user_rc == EXIT_SUCCESS && simulator_rc == EXIT_SUCCESS + ? EXIT_SUCCESS : EXIT_FAILURE; +} + +bool runtime_server_count( + MYSQL* admin, int hostgroup, const string& hostname, int expected, + const string& status = "" +) { + string query = + "SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(hostgroup) + " AND hostname=" + aurora_bgd_sql_quote(hostname); + if (!status.empty()) { + query += " AND status=" + aurora_bgd_sql_quote(status); + } + auto [rc, rows] = mysql_query_ext_rows(admin, query); + return rc == EXIT_SUCCESS && rows.size() == 1 && rows.front().size() == 1 + && rows.front().front() == to_string(expected); +} + +int wait_for_writer_placement( + MYSQL* admin, int writer_hg, int reader_hg, const string& hostname, + bool demoted +) { + string query = + "SELECT ((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(writer_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + + ")=" + (demoted ? "0" : "1") + ") AND " + "((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(reader_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + + ")=" + (demoted ? "1" : "0") + ")"; + return wait_for_cond(admin, query, kWaitSeconds); +} + +int publish_status( + BGD_Simulator& sim, Aurora_BGD_Test_Deployment& deployment, + const string& status +) { + return sim.topology_update( + aurora_bgd_topology_backends(deployment), + aurora_bgd_topology(deployment, status)); +} + +bool fast_membership_without_ordinary( + BGD_Simulator& sim, uint64_t sequence, const string& target_replica_set, + uint32_t observation_ms, uint64_t minimum_membership_probes +) { + usleep(observation_ms * 1000); + auto [rc, logs] = sim.replica_probe_log_since(sequence); + if (rc != EXIT_SUCCESS) { + return false; + } + uint64_t membership_probes = 0; + for (const Aurora_Replica_Probe_Log& log : logs) { + if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary) { + return false; + } + if (log.probe_kind == Aurora_Replica_Probe_Kind::bgd_membership + && log.replica_set_id == target_replica_set) { + membership_probes++; + } + } + return membership_probes >= minimum_membership_probes; +} + +int add_member_routes( + MYSQL* admin, Aurora_BGD_Test_Deployment& deployment, + const vector& route_hgs +) { + vector queries; + for (size_t i = 0; i < deployment.production.members.size(); ++i) { + queries.push_back( + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + to_string(route_hgs[i]) + "," + + aurora_bgd_sql_quote(deployment.production.members[i].endpoint.hostname) + + ",3306,'ONLINE','Aurora BGD member route')"); + } + queries.push_back("LOAD MYSQL SERVERS TO RUNTIME"); + return aurora_bgd_execute_all(admin, queries); +} + +int set_default_hostgroup(MYSQL* admin, int hostgroup) { + return aurora_bgd_execute_all(admin, { + "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroup) + + " WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }); +} + +bool route_to_expected_backend( + CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected_backend +) { + auto [sequence_rc, sequence] = sim.replica_probe_log_last_sequence(); + if (sequence_rc != EXIT_SUCCESS) { + return false; + } + MYSQL* client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); + if (client == nullptr) { + return false; + } + auto [rc, rows] = mysql_query_ext_rows(client, kOrdinaryAuroraQuery); + (void)rows; + if (rc != EXIT_SUCCESS) { + diag("Backend routing query failed with MySQL error %d: %s", rc, mysql_error(client)); + } + mysql_close(client); + if (rc != EXIT_SUCCESS) { + return false; + } + + auto [logs_rc, logs] = sim.replica_probe_log_since(sequence); + if (logs_rc != EXIT_SUCCESS) { + return false; + } + const Aurora_Replica_Probe_Log* routed_probe = nullptr; + for (const Aurora_Replica_Probe_Log& log : logs) { + if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary) { + routed_probe = &log; + } + } + if (routed_probe == nullptr + || routed_probe->backend.host != expected_backend.host + || routed_probe->backend.port != expected_backend.port) { + diag( + "Ordinary Aurora query reached %s:%d; expected %s:%d", + routed_probe ? routed_probe->backend.host.c_str() : "", + routed_probe ? routed_probe->backend.port : 0, + expected_backend.host.c_str(), expected_backend.port); + return false; + } + return true; +} + +bool route_members_to_expected_ips( + CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, + Aurora_BGD_Test_Deployment& deployment, + const vector& route_hgs, bool target +) { + for (size_t i = 0; i < route_hgs.size(); ++i) { + if (set_default_hostgroup(admin, route_hgs[i]) != EXIT_SUCCESS) { + return false; + } + Endpoint expected_backend = target + ? deployment.target.members[i].endpoint.backend() + : deployment.production.members[i].endpoint.backend(); + if (!route_to_expected_backend(cl, sim, expected_backend)) { + return false; + } + } + return true; +} + +int64_t member_route_pool_count(MYSQL* admin, const vector& route_hgs) { + string hostgroups; + for (int hostgroup : route_hgs) { + if (!hostgroups.empty()) { + hostgroups += ","; + } + hostgroups += to_string(hostgroup); + } + auto [rc, rows] = mysql_query_ext_rows( + admin, + "SELECT COALESCE(SUM(ConnUsed+ConnFree),0) FROM stats_mysql_connection_pool " + "WHERE hostgroup IN (" + hostgroups + ")"); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows.front().size() != 1) { + return -1; + } + return strtoll(rows.front().front().c_str(), nullptr, 10); +} + +int wait_for_member_route_pool_count( + MYSQL* admin, const vector& route_hgs, const string& comparison +) { + string hostgroups; + for (int hostgroup : route_hgs) { + if (!hostgroups.empty()) { + hostgroups += ","; + } + hostgroups += to_string(hostgroup); + } + return wait_for_cond( + admin, + "SELECT COALESCE(SUM(ConnUsed+ConnFree),0)" + comparison + + " FROM stats_mysql_connection_pool WHERE hostgroup IN (" + hostgroups + ")", + kWaitSeconds); +} + +int main() { + plan(20); + + CommandLine cl {}; + MYSQL* admin = nullptr; + BGD_Simulator sim {}; + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + const int writer_hg = 1530; + const int reader_hg = 1531; + const vector route_hgs {1534, 1535, 1536}; + Aurora_BGD_Test_Deployment deployment = aurora_bgd_deployment_a(); + if (aurora_bgd_publish(sim, deployment) != EXIT_SUCCESS + || aurora_bgd_admin_setup( + admin, deployment, writer_hg, reader_hg, 1532, 1533, false, 1000, false) + != EXIT_SUCCESS + || add_member_routes(admin, deployment, route_hgs) != EXIT_SUCCESS) { + diag("Error: failed to configure the active switchover scenario"); + cleanup(admin, sim); + return exit_status(); + } + + ok(aurora_bgd_wait_for_status(admin, writer_hg, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + "active scenario starts from a complete AVAILABLE snapshot"); + + if (publish_status(sim, deployment, "SWITCHOVER_INITIATED") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + admin, writer_hg, "SWITCHOVER_INITIATED", kWaitSeconds) != EXIT_SUCCESS) { + diag("Error: worker did not enter SWITCHOVER_INITIATED"); + cleanup(admin, sim); + return exit_status(); + } + ok(true, "INITIATED is published"); + ok(wait_for_writer_placement( + admin, writer_hg, reader_hg, + deployment.production.members.front().endpoint.hostname, false) == EXIT_SUCCESS, + "INITIATED does not change writer placement"); + + auto [active_seq_rc, active_sequence] = sim.replica_probe_log_last_sequence(); + ok(active_seq_rc == EXIT_SUCCESS && fast_membership_without_ordinary( + sim, active_sequence, deployment.target_replica_set, 650, 3), + "INITIATED uses fast membership probes and suspends the ordinary Aurora query"); + + vector competing_source = deployment.production.replica_rows(); + competing_source[0].session_id = "source-observed-reader"; + competing_source[1].session_id = "MASTER_SESSION_ID"; + auto [source_change_rc, source_change_sequence] = sim.replica_probe_log_last_sequence(); + if (source_change_rc != EXIT_SUCCESS || sim.replica_update( + deployment.blue_replica_set, competing_source, deployment.production.backends()) + != EXIT_SUCCESS) { + diag("Error: failed to publish the competing source role observation"); + cleanup(admin, sim); + return exit_status(); + } + ok(fast_membership_without_ordinary( + sim, source_change_sequence, deployment.target_replica_set, 350, 2), + "changed source roles cannot compete while production probing is suspended"); + + if (publish_status(sim, deployment, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + admin, writer_hg, "SWITCHOVER_IN_PROGRESS", kWaitSeconds) != EXIT_SUCCESS) { + diag("Error: worker did not enter SWITCHOVER_IN_PROGRESS"); + cleanup(admin, sim); + return exit_status(); + } + ok(true, "IN_PROGRESS is published"); + ok(wait_for_writer_placement( + admin, writer_hg, reader_hg, + deployment.production.members.front().endpoint.hostname, true) == EXIT_SUCCESS, + "IN_PROGRESS demotes the snapshotted production writer"); + ok(runtime_server_count( + admin, writer_hg, deployment.production.members[1].endpoint.hostname, 0) + && runtime_server_count( + admin, reader_hg, deployment.production.members[1].endpoint.hostname, 1, "ONLINE"), + "the competing source observation does not promote a reader"); + + auto [repeat_progress_rc, repeat_progress_sequence] = sim.replica_probe_log_last_sequence(); + if (repeat_progress_rc != EXIT_SUCCESS + || publish_status(sim, deployment, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS) { + diag("Error: failed to repeat SWITCHOVER_IN_PROGRESS"); + cleanup(admin, sim); + return exit_status(); + } + ok(fast_membership_without_ordinary( + sim, repeat_progress_sequence, deployment.target_replica_set, 350, 2) + && wait_for_writer_placement( + admin, writer_hg, reader_hg, + deployment.production.members.front().endpoint.hostname, true) == EXIT_SUCCESS, + "repeated IN_PROGRESS retains the one demotion and active probe policy"); + + ok(route_members_to_expected_ips(cl, admin, sim, deployment, route_hgs, false) + && member_route_pool_count(admin, route_hgs) + >= static_cast(route_hgs.size()), + "all member routes use source IPs and hold pre-cutover pools"); + + if (publish_status(sim, deployment, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + admin, writer_hg, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) != EXIT_SUCCESS) { + diag("Error: worker did not enter SWITCHOVER_IN_POST_PROCESSING"); + cleanup(admin, sim); + return exit_status(); + } + ok(true, "POST_PROCESSING is published without a target-writability gate"); + ok(wait_for_writer_placement( + admin, writer_hg, reader_hg, + deployment.production.members.front().endpoint.hostname, false) == EXIT_SUCCESS, + "POST_PROCESSING restores the canonical writer placement"); + ok(runtime_server_count( + admin, reader_hg, deployment.production.members[1].endpoint.hostname, 1, "ONLINE") + && runtime_server_count( + admin, reader_hg, deployment.production.members[2].endpoint.hostname, 1, "ONLINE"), + "POST_PROCESSING leaves canonical readers ONLINE and eligible"); + ok(wait_for_member_route_pool_count(admin, route_hgs, "=0") == EXIT_SUCCESS, + "POST_PROCESSING retires the pre-cutover member pools"); + ok(route_members_to_expected_ips(cl, admin, sim, deployment, route_hgs, true), + "POST_PROCESSING pins every production hostname to its cached target IP"); + + const int64_t target_pool_count = member_route_pool_count(admin, route_hgs); + auto [repeat_post_rc, repeat_post_sequence] = sim.probe_log_last_sequence(); + if (target_pool_count < static_cast(route_hgs.size()) + || repeat_post_rc != EXIT_SUCCESS + || publish_status(sim, deployment, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS) { + diag("Error: failed to prepare repeated POST_PROCESSING"); + cleanup(admin, sim); + return exit_status(); + } + vector target_backends = deployment.target.backends(); + auto [repeat_probe_rc, repeat_probe] = aurora_bgd_wait_for_topology_probe( + sim, repeat_post_sequence, target_backends, BGD_Probe_Kind::metadata, + kProbeTimeoutMs); + ok(repeat_probe_rc == EXIT_SUCCESS + && member_route_pool_count(admin, route_hgs) >= target_pool_count, + "repeated POST_PROCESSING does not replay completed member retirement"); + + auto [post_seq_rc, post_sequence] = sim.replica_probe_log_last_sequence(); + ok(post_seq_rc == EXIT_SUCCESS && fast_membership_without_ordinary( + sim, post_sequence, deployment.target_replica_set, 350, 2), + "POST_PROCESSING keeps fast BGD probes without ordinary Aurora queries"); + + if (aurora_bgd_admin_cleanup(admin) != EXIT_SUCCESS || sim.cleanup() != EXIT_SUCCESS) { + diag("Error: failed to reset before the incomplete-snapshot scenario"); + cleanup(admin, sim); + return exit_status(); + } + + const int gated_writer_hg = 1540; + const int gated_reader_hg = 1541; + const int gated_route_hg = 1544; + Aurora_BGD_Test_Deployment gated = aurora_bgd_deployment_b_writer_only(); + vector invalid_target = gated.target.replica_rows(); + invalid_target.front().server_id = "unpaired-target-writer"; + if (sim.replica_update( + gated.blue_replica_set, gated.production.replica_rows(), gated.production.backends()) + != EXIT_SUCCESS + || sim.replica_update( + gated.target_replica_set, invalid_target, gated.target.backends()) + != EXIT_SUCCESS + || sim.topology_update( + aurora_bgd_topology_backends(gated), aurora_bgd_available_topology(gated)) + != EXIT_SUCCESS + || aurora_bgd_admin_setup( + admin, gated, gated_writer_hg, gated_reader_hg, 1542, 1543, false, 1000, false) + != EXIT_SUCCESS + || add_member_routes(admin, gated, {gated_route_hg}) != EXIT_SUCCESS) { + diag("Error: failed to configure incomplete target membership"); + cleanup(admin, sim); + return exit_status(); + } + ok(aurora_bgd_wait_for_status( + admin, gated_writer_hg, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + "an incomplete target snapshot can publish AVAILABLE without routing"); + if (publish_status(sim, gated, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + admin, gated_writer_hg, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) + != EXIT_SUCCESS) { + diag("Error: incomplete scenario did not publish POST_PROCESSING"); + cleanup(admin, sim); + return exit_status(); + } + const int gated_default_rc = set_default_hostgroup(admin, gated_route_hg); + ok(gated_default_rc == EXIT_SUCCESS && route_to_expected_backend( + cl, sim, gated.production.members.front().endpoint.backend()), + "POST_PROCESSING leaves routing unchanged without a complete target snapshot"); + + auto [gated_refresh_seq_rc, gated_refresh_sequence] = sim.replica_probe_log_last_sequence(); + if (gated_refresh_seq_rc != EXIT_SUCCESS || sim.replica_update( + gated.target_replica_set, gated.target.replica_rows(), gated.target.backends()) + != EXIT_SUCCESS) { + diag("Error: failed to restore complete target membership"); + cleanup(admin, sim); + return exit_status(); + } + auto [gated_probe_rc, gated_probe] = aurora_bgd_wait_for_replica_probe( + sim, gated_refresh_sequence, gated.target.backends(), + Aurora_Replica_Probe_Kind::bgd_membership, + kProbeTimeoutMs, gated.target_replica_set); + bool gated_target_routing = false; + if (gated_probe_rc == EXIT_SUCCESS + && wait_for_member_route_pool_count(admin, {gated_route_hg}, "=0") == EXIT_SUCCESS + && set_default_hostgroup(admin, gated_route_hg) == EXIT_SUCCESS) { + gated_target_routing = route_to_expected_backend( + cl, sim, gated.target.members.front().endpoint.backend()); + } + ok(gated_target_routing, + "repeated POST_PROCESSING applies routing after membership becomes complete"); + + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean Aurora BGD active-state test data"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_aurora_bgd_cluster_sync-t.cpp b/test/tap/tests/test_aurora_bgd_cluster_sync-t.cpp new file mode 100644 index 0000000000..743b5e4673 --- /dev/null +++ b/test/tap/tests/test_aurora_bgd_cluster_sync-t.cpp @@ -0,0 +1,306 @@ +/** + * @file test_aurora_bgd_cluster_sync-t.cpp + * @brief Aurora BGD configuration sync preserves worker-owned status per node. + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "aurora_bgd_tap.h" +#include "command_line.h" +#include "utils.h" + +using namespace std; +namespace fs = std::filesystem; + +const uint32_t kWaitSeconds = 10; +const int kReplicaAdminPort = 16062; +const int kReplicaMySQLPort = 16063; +const char kReplicaHost[] = "127.0.0.1"; +const char kSQLiteInterfaces[] = "0.0.0.0:3306;0.0.0.0:3307"; + +struct Replica_Process { + pid_t pid = -1; + string directory; + string config_path; + string stderr_path; +}; + +string config_quote(const string& value) { + string quoted; + for (char c : value) { + if (c == '\\' || c == '"') { + quoted += '\\'; + } + quoted += c; + } + return quoted; +} + +int prepare_replica_config(const CommandLine& cl, Replica_Process& process) { + char directory_template[] = "/tmp/proxysql-aurora-bgd-sync-XXXXXX"; + char* directory = mkdtemp(directory_template); // NOSONAR: mkdtemp creates an owner-only directory. + if (directory == nullptr) { + diag("mkdtemp failed: %s", strerror(errno)); + return EXIT_FAILURE; + } + process.directory = directory; + process.config_path = process.directory + "/proxysql.cnf"; + process.stderr_path = process.directory + "/proxysql.stderr"; + + ofstream config(process.config_path); + if (!config.is_open()) { + diag("failed to create replica config: %s", process.config_path.c_str()); + return EXIT_FAILURE; + } + config + << "datadir=\"" << config_quote(process.directory) << "\"\n" + << "admin_variables={\n" + << " admin_credentials=\"" << config_quote(cl.admin_username) << ":" + << config_quote(cl.admin_password) << ";radmin:radmin\"\n" + << " mysql_ifaces=\"0.0.0.0:" << kReplicaAdminPort << "\"\n" + << " cluster_username=\"radmin\"\n" + << " cluster_password=\"radmin\"\n" + << " cluster_check_interval_ms=200\n" + << " cluster_check_status_frequency=100\n" + << " cluster_admin_variables_diffs_before_sync=0\n" + << " cluster_mysql_servers_diffs_before_sync=1\n" + << " cluster_mysql_servers_save_to_disk=false\n" + << " cluster_mysql_servers_sync_algorithm=3\n" + << "}\n" + << "mysql_variables={\n" + << " interfaces=\"0.0.0.0:" << kReplicaMySQLPort << "\"\n" + << " monitor_username=\"aurora1\"\n" + << " monitor_password=\"pass1\"\n" + << " monitor_connect_timeout=500\n" + << " monitor_ping_interval=10000\n" + << "}\n" + << "proxysql_servers=()\n"; + config.close(); + return config.fail() ? EXIT_FAILURE : EXIT_SUCCESS; +} + +int launch_replica(const CommandLine& cl, Replica_Process& process) { + if (prepare_replica_config(cl, process) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + const string binary = string(cl.workdir) + "../../../src/proxysql"; + process.pid = fork(); + if (process.pid == -1) { + diag("fork failed: %s", strerror(errno)); + return EXIT_FAILURE; + } + if (process.pid == 0) { + int stderr_fd = open(process.stderr_path.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0600); + if (stderr_fd >= 0) { + dup2(stderr_fd, STDOUT_FILENO); + dup2(stderr_fd, STDERR_FILENO); + close(stderr_fd); + } + execl( + binary.c_str(), "proxysql", "--sqlite3-server", "-f", "-c", + process.config_path.c_str(), static_cast(nullptr)); + _exit(127); + } + return EXIT_SUCCESS; +} + +void stop_replica(MYSQL*& admin, Replica_Process& process, bool preserve_log) { + if (admin != nullptr) { + mysql_query(admin, "PROXYSQL SHUTDOWN"); + mysql_close(admin); + admin = nullptr; + } + if (process.pid > 0) { + bool exited = false; + for (int i = 0; i < 50; ++i) { + pid_t rc = waitpid(process.pid, nullptr, WNOHANG); + if (rc == process.pid || rc == -1) { + exited = true; + break; + } + usleep(100000); + } + if (!exited) { + kill(process.pid, SIGKILL); + waitpid(process.pid, nullptr, 0); + } + } + if (!preserve_log && !process.directory.empty()) { + fs::remove_all(process.directory); + } else if (preserve_log) { + diag("replica ProxySQL log retained at %s", process.stderr_path.c_str()); + } +} + +Aurora_BGD_Test_Deployment peer_deployment() { + Aurora_BGD_Test_Deployment deployment; + deployment.name = "Aurora BGD peer-local status"; + deployment.domain_name = ".localhost"; + deployment.blue_replica_set = "aurora-bgd-peer-blue"; + deployment.target_replica_set = "aurora-bgd-peer-target"; + deployment.source_topology_id = "aurora-bgd-peer-source"; + deployment.target_topology_id = "aurora-bgd-peer-target"; + deployment.target_cluster_endpoint = { + "aurora-peer-writer-green-sync.localhost", "127.0.0.1", 3307 + }; + deployment.production = { + deployment.blue_replica_set, + {aurora_bgd_member( + "aurora-peer-writer", "MASTER_SESSION_ID", + {"aurora-peer-writer.localhost", "127.0.0.1", 3306})}, + {} + }; + deployment.production.serving_endpoints.push_back( + deployment.production.members.front().endpoint); + deployment.target = { + deployment.target_replica_set, + {aurora_bgd_member( + "aurora-peer-writer-green-sync", "MASTER_SESSION_ID", + {"aurora-peer-writer-green-sync.localhost", "127.0.0.1", 3307})}, + {deployment.target_cluster_endpoint} + }; + deployment.target.serving_endpoints.push_back( + deployment.target.members.front().endpoint); + return deployment; +} + +int configure_sqlite_interfaces(MYSQL* admin, const string& interfaces) { + return aurora_bgd_execute_all(admin, { + "SET sqliteserver-mysql_ifaces=" + aurora_bgd_sql_quote(interfaces), + "LOAD SQLITESERVER VARIABLES TO RUNTIME", + }); +} + +int configure_peer( + MYSQL* admin, BGD_Simulator& simulator, Aurora_BGD_Test_Deployment& deployment, + const string& status +) { + return simulator.cleanup() == EXIT_SUCCESS + && aurora_bgd_publish(simulator, deployment) == EXIT_SUCCESS + && simulator.topology_update( + aurora_bgd_topology_backends(deployment), aurora_bgd_topology(deployment, status)) + == EXIT_SUCCESS + && aurora_bgd_admin_cleanup(admin) == EXIT_SUCCESS + && aurora_bgd_admin_setup(admin, deployment, 1800, 1801, 1802, 1803, false) + == EXIT_SUCCESS + ? EXIT_SUCCESS : EXIT_FAILURE; +} + +int main() { + plan(5); + CommandLine cl {}; + if (cl.getEnv()) { + diag("failed to load TAP environment"); + return exit_status(); + } + + MYSQL* primary_admin = init_mysql_conn( + cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + MYSQL* replica_admin = nullptr; + Replica_Process replica_process; + string primary_sqlite_interfaces; + BGD_Simulator primary_simulator; + BGD_Simulator replica_simulator; + Aurora_BGD_Test_Deployment deployment = peer_deployment(); + bool primary_simulator_connected = false; + bool replica_simulator_connected = false; + + if (primary_admin == nullptr + || get_variable_value( + primary_admin, "sqliteserver-mysql_ifaces", primary_sqlite_interfaces) != EXIT_SUCCESS + || configure_sqlite_interfaces(primary_admin, kSQLiteInterfaces) != EXIT_SUCCESS + || launch_replica(cl, replica_process) != EXIT_SUCCESS) { + diag("failed to prepare the two ProxySQL nodes"); + goto cleanup; + } + + replica_admin = wait_for_proxysql( + {kReplicaHost, cl.admin_username, cl.admin_password, kReplicaAdminPort}, + kWaitSeconds); + if (replica_admin == nullptr + || configure_sqlite_interfaces(replica_admin, kSQLiteInterfaces) != EXIT_SUCCESS) { + diag("failed to start the replica ProxySQL node"); + goto cleanup; + } + + { + char username[] = "aurora1"; + char password[] = "pass1"; + primary_simulator_connected = primary_simulator.connect( + cl.host, 3306, username, password) == EXIT_SUCCESS; + replica_simulator_connected = replica_simulator.connect( + const_cast(kReplicaHost), 3306, username, password) == EXIT_SUCCESS; + } + if (!primary_simulator_connected || !replica_simulator_connected + || configure_peer(primary_admin, primary_simulator, deployment, "AVAILABLE") + != EXIT_SUCCESS + || configure_peer( + replica_admin, replica_simulator, deployment, "SWITCHOVER_INITIATED") + != EXIT_SUCCESS) { + diag("failed to publish the two node-local Aurora observations"); + goto cleanup; + } + + ok(aurora_bgd_wait_for_status(primary_admin, 1800, "AVAILABLE", kWaitSeconds) + == EXIT_SUCCESS, "primary worker publishes its local AVAILABLE observation"); + ok(aurora_bgd_wait_for_status( + replica_admin, 1800, "SWITCHOVER_INITIATED", kWaitSeconds) == EXIT_SUCCESS, + "replica worker publishes its local SWITCHOVER_INITIATED observation"); + + if (aurora_bgd_execute_all(replica_admin, { + "DELETE FROM proxysql_servers", + "INSERT INTO proxysql_servers(hostname,port,weight,comment) VALUES (" + + aurora_bgd_sql_quote(cl.admin_host) + "," + to_string(cl.admin_port) + + ",0,'Aurora BGD sync primary')", + "LOAD PROXYSQL SERVERS TO RUNTIME", + }) != EXIT_SUCCESS + || aurora_bgd_execute_all(primary_admin, { + "UPDATE mysql_aws_aurora_hostgroups SET comment='peer-config-synced' " + "WHERE writer_hostgroup=1800", + "LOAD MYSQL SERVERS TO RUNTIME", + }) != EXIT_SUCCESS) { + diag("failed to initiate Aurora configuration synchronization"); + goto cleanup; + } + + ok(wait_for_cond( + replica_admin, + "SELECT COUNT(*)=1 FROM mysql_aws_aurora_hostgroups " + "WHERE writer_hostgroup=1800 AND comment='peer-config-synced'", + kWaitSeconds) == EXIT_SUCCESS, + "Aurora BGD configured fields synchronize to the peer"); + ok(aurora_bgd_wait_for_status(primary_admin, 1800, "AVAILABLE", kWaitSeconds) + == EXIT_SUCCESS, "configuration sync preserves the primary's local status"); + ok(aurora_bgd_wait_for_status( + replica_admin, 1800, "SWITCHOVER_INITIATED", kWaitSeconds) == EXIT_SUCCESS, + "configuration sync preserves the replica's local status"); + +cleanup: + if (primary_simulator_connected) { + primary_simulator.cleanup(); + } + if (replica_simulator_connected) { + replica_simulator.cleanup(); + } + if (replica_admin != nullptr) { + aurora_bgd_admin_cleanup(replica_admin); + } + if (primary_admin != nullptr) { + aurora_bgd_admin_cleanup(primary_admin); + if (!primary_sqlite_interfaces.empty()) { + configure_sqlite_interfaces(primary_admin, primary_sqlite_interfaces); + } + mysql_close(primary_admin); + } + stop_replica(replica_admin, replica_process, tests_failed() != 0); + return exit_status(); +} diff --git a/test/tap/tests/test_aurora_bgd_completion-t.cpp b/test/tap/tests/test_aurora_bgd_completion-t.cpp new file mode 100644 index 0000000000..fdc22e1cbe --- /dev/null +++ b/test/tap/tests/test_aurora_bgd_completion-t.cpp @@ -0,0 +1,549 @@ +/** + * @file test_aurora_bgd_completion-t.cpp + * @brief Aurora BGD completion cleanup and terminal-latch behavior. + */ + +#include +#include +#include +#include + +#include "aurora_bgd_tap.h" +#include "command_line.h" +#include "utils.h" + +using namespace std; + +const uint32_t kWaitSeconds = 5; +const uint32_t kProbeTimeoutMs = 5000; +const char kOrdinaryAuroraQuery[] = + "SELECT SERVER_ID," + "IF(" + "SESSION_ID = 'MASTER_SESSION_ID' AND " + "SERVER_ID <> (SELECT SERVER_ID FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE SESSION_ID = 'MASTER_SESSION_ID' ORDER BY LAST_UPDATE_TIMESTAMP DESC LIMIT 1), " + "'probably_former_MASTER_SESSION_ID', SESSION_ID" + ") SESSION_ID, " + "LAST_UPDATE_TIMESTAMP, " + "IF(SESSION_ID = 'MASTER_SESSION_ID', 0, REPLICA_LAG_IN_MILLISECONDS) AS REPLICA_LAG_IN_MILLISECONDS, " + "CPU " + "FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE" + " ( " + "(REPLICA_LAG_IN_MILLISECONDS >= 0 AND REPLICA_LAG_IN_MILLISECONDS <= 600000)" + " OR SESSION_ID = 'MASTER_SESSION_ID'" + " ) " + "AND LAST_UPDATE_TIMESTAMP > NOW() - INTERVAL 180 SECOND" + " ORDER BY SERVER_ID"; + +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + char simulator_username[] = "aurora1"; + char simulator_password[] = "pass1"; + if (sim.connect(cl.host, 3306, simulator_username, simulator_password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the shared AWS simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + if (aurora_bgd_admin_cleanup(admin) != EXIT_SUCCESS || sim.cleanup() != EXIT_SUCCESS) { + diag("Error: failed to clear prior Aurora BGD state"); + return EXIT_FAILURE; + } + return aurora_bgd_execute_all(admin, { + "DELETE FROM mysql_users WHERE username='testuser'", + "INSERT INTO mysql_users(username,password,active,default_hostgroup,transaction_persistent) " + "VALUES ('testuser','testuser',1,0,1)", + "LOAD MYSQL USERS TO RUNTIME", + }); +} + +int reset_scenario(MYSQL* admin, BGD_Simulator& sim) { + return aurora_bgd_admin_cleanup(admin) == EXIT_SUCCESS + && sim.cleanup() == EXIT_SUCCESS ? EXIT_SUCCESS : EXIT_FAILURE; +} + +int cleanup(MYSQL* admin, BGD_Simulator& sim) { + int reset_rc = reset_scenario(admin, sim); + int user_rc = aurora_bgd_execute_all(admin, { + "DELETE FROM mysql_users WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }); + mysql_close(admin); + return reset_rc == EXIT_SUCCESS && user_rc == EXIT_SUCCESS + ? EXIT_SUCCESS : EXIT_FAILURE; +} + +int publish_status( + BGD_Simulator& sim, Aurora_BGD_Test_Deployment& deployment, const string& status +) { + return sim.topology_update( + aurora_bgd_topology_backends(deployment), + aurora_bgd_topology(deployment, status)); +} + +int publish_completed( + BGD_Simulator& sim, Aurora_BGD_Test_Deployment& serving_deployment, + Aurora_BGD_Test_Deployment& completed_deployment +) { + return sim.topology_update( + aurora_bgd_topology_backends(serving_deployment), + aurora_bgd_completed_topology(completed_deployment)); +} + +int set_default_hostgroup(MYSQL* admin, int hostgroup) { + return aurora_bgd_execute_all(admin, { + "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroup) + + " WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }); +} + +int add_route( + MYSQL* admin, int hostgroup, const string& hostname, const string& status = "ONLINE" +) { + return aurora_bgd_execute_all(admin, { + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + to_string(hostgroup) + "," + aurora_bgd_sql_quote(hostname) + + ",3306," + aurora_bgd_sql_quote(status) + ",'Aurora BGD completion route')", + "LOAD MYSQL SERVERS TO RUNTIME", + }); +} + +int add_member_routes( + MYSQL* admin, Aurora_BGD_Test_Deployment& deployment, const vector& hostgroups +) { + vector queries; + for (size_t i = 0; i < hostgroups.size(); ++i) { + queries.push_back( + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + to_string(hostgroups[i]) + "," + + aurora_bgd_sql_quote(deployment.production.members[i].endpoint.hostname) + + ",3306,'ONLINE','Aurora BGD completion member route')"); + } + queries.push_back("LOAD MYSQL SERVERS TO RUNTIME"); + return aurora_bgd_execute_all(admin, queries); +} + +int add_green_servers( + MYSQL* admin, Aurora_BGD_Test_Deployment& deployment, + int green_writer_hg, int green_reader_hg +) { + vector queries; + for (size_t i = 0; i < deployment.target.members.size(); ++i) { + const int hostgroup = i == 0 ? green_writer_hg : green_reader_hg; + const string status = i + 1 == deployment.target.members.size() + ? "OFFLINE_SOFT" : "ONLINE"; + queries.push_back( + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + to_string(hostgroup) + "," + + aurora_bgd_sql_quote(deployment.target.members[i].endpoint.hostname) + + ",3306," + aurora_bgd_sql_quote(status) + + ",'Aurora BGD configured green member')"); + } + queries.push_back("LOAD MYSQL SERVERS TO RUNTIME"); + return aurora_bgd_execute_all(admin, queries); +} + +bool route_to_expected_backend( + CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected_backend +) { + auto [sequence_rc, sequence] = sim.replica_probe_log_last_sequence(); + if (sequence_rc != EXIT_SUCCESS) { + return false; + } + MYSQL* client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); + if (client == nullptr) { + return false; + } + auto [query_rc, rows] = mysql_query_ext_rows(client, kOrdinaryAuroraQuery); + (void)rows; + if (query_rc != EXIT_SUCCESS) { + diag("Backend routing query failed with MySQL error %d: %s", + mysql_errno(client), mysql_error(client)); + } + mysql_close(client); + if (query_rc != EXIT_SUCCESS) { + return false; + } + + auto [logs_rc, logs] = sim.replica_probe_log_since(sequence); + if (logs_rc != EXIT_SUCCESS) { + return false; + } + for (const Aurora_Replica_Probe_Log& log : logs) { + if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary + && log.backend.host == expected_backend.host + && log.backend.port == expected_backend.port) { + return true; + } + } + return false; +} + +bool route_members( + CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, + Aurora_BGD_Test_Deployment& deployment, const vector& hostgroups, bool target +) { + for (size_t i = 0; i < hostgroups.size(); ++i) { + if (set_default_hostgroup(admin, hostgroups[i]) != EXIT_SUCCESS) { + return false; + } + const Endpoint expected = target + ? deployment.target.members[i].endpoint.backend() + : deployment.production.members[i].endpoint.backend(); + if (!route_to_expected_backend(cl, sim, expected)) { + return false; + } + } + return true; +} + +int64_t pool_count(MYSQL* admin, int hostgroup) { + auto [rc, rows] = mysql_query_ext_rows( + admin, + "SELECT COALESCE(SUM(ConnUsed+ConnFree),0) FROM stats_mysql_connection_pool " + "WHERE hostgroup=" + to_string(hostgroup)); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows.front().size() != 1) { + return -1; + } + return strtoll(rows.front().front().c_str(), nullptr, 10); +} + +int wait_for_pool_count(MYSQL* admin, int hostgroup, const string& comparison) { + return wait_for_cond( + admin, + "SELECT COALESCE(SUM(ConnUsed+ConnFree),0)" + comparison + + " FROM stats_mysql_connection_pool WHERE hostgroup=" + to_string(hostgroup), + kWaitSeconds); +} + +bool server_count( + MYSQL* admin, int hostgroup, const string& hostname, int expected, + const string& status = "" +) { + string query = + "SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(hostgroup) + " AND hostname=" + aurora_bgd_sql_quote(hostname); + if (!status.empty()) { + query += " AND status=" + aurora_bgd_sql_quote(status); + } + auto [rc, rows] = mysql_query_ext_rows(admin, query); + return rc == EXIT_SUCCESS && rows.size() == 1 && rows.front().size() == 1 + && rows.front().front() == to_string(expected); +} + +int wait_for_writer_policy( + MYSQL* admin, int writer_hg, int reader_hg, const string& hostname, + bool writer_is_also_reader +) { + return wait_for_cond( + admin, + "SELECT ((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(writer_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + + ")=1) AND ((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(reader_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + + ")=" + (writer_is_also_reader ? "1" : "0") + ")", + kWaitSeconds); +} + +int wait_for_writer_demotion( + MYSQL* admin, int writer_hg, int reader_hg, const string& hostname +) { + return wait_for_cond( + admin, + "SELECT ((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(writer_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + + ")=0) AND ((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(reader_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + + ")=1)", + kWaitSeconds); +} + +bool completion_probe_policy( + BGD_Simulator& sim, uint64_t sequence, + Aurora_BGD_Test_Deployment& deployment, uint32_t observation_ms +) { + usleep(observation_ms * 1000); + auto [rc, logs] = sim.replica_probe_log_since(sequence); + if (rc != EXIT_SUCCESS) { + return false; + } + bool ordinary_on_production = false; + for (const Aurora_Replica_Probe_Log& log : logs) { + if (log.probe_kind == Aurora_Replica_Probe_Kind::bgd_membership) { + return false; + } + if (log.probe_kind != Aurora_Replica_Probe_Kind::ordinary) { + continue; + } + for (const Endpoint& backend : deployment.production.backends()) { + ordinary_on_production |= log.backend.host == backend.host + && log.backend.port == backend.port; + } + } + return ordinary_on_production; +} + +bool wait_for_topology_observation( + BGD_Simulator& sim, uint64_t sequence, Aurora_BGD_Test_Deployment& deployment +) { + auto [rc, probe] = aurora_bgd_wait_for_topology_probe( + sim, sequence, deployment.production.backends(), + BGD_Probe_Kind::metadata, kProbeTimeoutMs); + return rc == EXIT_SUCCESS; +} + +int main() { + plan(28); + + CommandLine cl {}; + MYSQL* admin = nullptr; + BGD_Simulator sim {}; + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + // Completion from IN_PROGRESS reconciles the writer without replaying POST_PROCESSING. + Aurora_BGD_Test_Deployment progress = aurora_bgd_deployment_b_writer_only(); + const int progress_writer_hg = 1550; + const int progress_reader_hg = 1551; + if (aurora_bgd_publish(sim, progress) != EXIT_SUCCESS + || aurora_bgd_admin_setup( + admin, progress, progress_writer_hg, progress_reader_hg, + 1552, 1553, false, 300, false) != EXIT_SUCCESS) { + diag("Error: failed to configure completion-from-IN_PROGRESS scenario"); + cleanup(admin, sim); + return exit_status(); + } + ok(aurora_bgd_wait_for_status( + admin, progress_writer_hg, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + "completion-from-IN_PROGRESS scenario reaches AVAILABLE"); + if (publish_status(sim, progress, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + admin, progress_writer_hg, "SWITCHOVER_IN_PROGRESS", kWaitSeconds) + != EXIT_SUCCESS) { + diag("Error: failed to reach IN_PROGRESS"); + cleanup(admin, sim); + return exit_status(); + } + ok(wait_for_writer_demotion( + admin, progress_writer_hg, progress_reader_hg, + progress.production.members.front().endpoint.hostname) == EXIT_SUCCESS, + "IN_PROGRESS demotes the writer before completion"); + if (publish_completed(sim, progress, progress) != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + admin, progress_writer_hg, "SWITCHOVER_COMPLETED", kWaitSeconds) + != EXIT_SUCCESS) { + diag("Error: failed to enter the completed latch from IN_PROGRESS"); + cleanup(admin, sim); + return exit_status(); + } + ok(true, "TARGET-only completion publishes SWITCHOVER_COMPLETED"); + ok(wait_for_writer_policy( + admin, progress_writer_hg, progress_reader_hg, + progress.production.members.front().endpoint.hostname, false) == EXIT_SUCCESS, + "completion restores the demoted writer without replaying POST_PROCESSING"); + auto [progress_replica_seq_rc, progress_replica_sequence] = + sim.replica_probe_log_last_sequence(); + ok(progress_replica_seq_rc == EXIT_SUCCESS && completion_probe_policy( + sim, progress_replica_sequence, progress, 750), + "completion resumes ordinary production probing and stops membership probing"); + auto [repeat_progress_seq_rc, repeat_progress_sequence] = sim.probe_log_last_sequence(); + bool repeated_progress = repeat_progress_seq_rc == EXIT_SUCCESS + && publish_completed(sim, progress, progress) == EXIT_SUCCESS + && wait_for_topology_observation(sim, repeat_progress_sequence, progress); + ok(repeated_progress && wait_for_writer_policy( + admin, progress_writer_hg, progress_reader_hg, + progress.production.members.front().endpoint.hostname, false) == EXIT_SUCCESS, + "repeated completion is a no-op while latched"); + auto [error_seq_rc, error_sequence] = sim.probe_log_last_sequence(); + bool error_retained = error_seq_rc == EXIT_SUCCESS + && sim.topology_error( + aurora_bgd_topology_backends(progress), 1205, "simulated completion timeout") + == EXIT_SUCCESS + && wait_for_topology_observation(sim, error_sequence, progress); + ok(error_retained && aurora_bgd_wait_for_status( + admin, progress_writer_hg, "SWITCHOVER_COMPLETED", 1) == EXIT_SUCCESS, + "topology query errors retain the completed latch"); + ok(sim.topology_drop(aurora_bgd_topology_backends(progress)) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, progress_writer_hg, "NONE", kWaitSeconds) == EXIT_SUCCESS, + "confirmed topology absence releases the completed latch to NONE"); + + if (reset_scenario(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to reset before POST_PROCESSING completion scenario"); + cleanup(admin, sim); + return exit_status(); + } + + // Completion after POST_PROCESSING removes pins and drains configured green pools once. + Aurora_BGD_Test_Deployment post = aurora_bgd_deployment_a(); + const int post_writer_hg = 1560; + const int post_reader_hg = 1561; + const int green_writer_hg = 1562; + const int green_reader_hg = 1563; + const vector route_hgs {1564, 1565, 1566}; + if (aurora_bgd_publish(sim, post) != EXIT_SUCCESS + || aurora_bgd_admin_setup( + admin, post, post_writer_hg, post_reader_hg, + green_writer_hg, green_reader_hg, false, 300, true) != EXIT_SUCCESS + || add_green_servers(admin, post, green_writer_hg, green_reader_hg) != EXIT_SUCCESS + || add_member_routes(admin, post, route_hgs) != EXIT_SUCCESS) { + diag("Error: failed to configure POST_PROCESSING completion scenario"); + cleanup(admin, sim); + return exit_status(); + } + ok(aurora_bgd_wait_for_status( + admin, post_writer_hg, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + "POST_PROCESSING completion scenario reaches AVAILABLE"); + if (publish_status(sim, post, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + admin, post_writer_hg, "SWITCHOVER_IN_PROGRESS", kWaitSeconds) + != EXIT_SUCCESS + || publish_status(sim, post, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + admin, post_writer_hg, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) + != EXIT_SUCCESS) { + diag("Error: failed to advance through POST_PROCESSING"); + cleanup(admin, sim); + return exit_status(); + } + ok(true, "active deployment advances through POST_PROCESSING"); + ok(route_members(cl, admin, sim, post, route_hgs, true), + "POST_PROCESSING routes every production member to its target IP"); + const int64_t target_route_pool = pool_count(admin, route_hgs.front()); + bool green_pool_ready = set_default_hostgroup(admin, green_writer_hg) == EXIT_SUCCESS + && route_to_expected_backend(cl, sim, post.target.members.front().endpoint.backend()) + && pool_count(admin, green_writer_hg) >= 1; + ok(target_route_pool >= 1 && green_pool_ready, + "pre-completion target and configured-green pools are established"); + if (publish_completed(sim, post, post) != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + admin, post_writer_hg, "SWITCHOVER_COMPLETED", kWaitSeconds) + != EXIT_SUCCESS) { + diag("Error: failed to complete the POST_PROCESSING scenario"); + cleanup(admin, sim); + return exit_status(); + } + ok(true, "completion after POST_PROCESSING enters the terminal latch"); + ok(wait_for_writer_policy( + admin, post_writer_hg, post_reader_hg, + post.production.members.front().endpoint.hostname, true) == EXIT_SUCCESS, + "completion preserves canonical writer_is_also_reader placement"); + ok(pool_count(admin, route_hgs.front()) >= target_route_pool, + "completion does not repeat retirement of post-cutover production pools"); + ok(wait_for_pool_count(admin, green_writer_hg, "=0") == EXIT_SUCCESS, + "completion drains eligible configured-green pools immediately"); + ok(server_count( + admin, green_reader_hg, + post.target.members.back().endpoint.hostname, 1, "OFFLINE_SOFT"), + "completion preserves configured green rows and OFFLINE status"); + const int post_completion_route_hg = 1567; + ok(add_route( + admin, post_completion_route_hg, + post.production.members.front().endpoint.hostname) == EXIT_SUCCESS + && set_default_hostgroup(admin, post_completion_route_hg) == EXIT_SUCCESS + && route_to_expected_backend( + cl, sim, post.production.members.front().endpoint.backend()), + "completion removes the production traffic pin without DNS verification"); + auto [post_replica_seq_rc, post_replica_sequence] = sim.replica_probe_log_last_sequence(); + ok(post_replica_seq_rc == EXIT_SUCCESS && completion_probe_policy( + sim, post_replica_sequence, post, 750), + "the completed latch uses configured cadence and canonical production probes"); + bool recreated_green_pool = set_default_hostgroup(admin, green_writer_hg) == EXIT_SUCCESS + && route_to_expected_backend(cl, sim, post.target.members.front().endpoint.backend()) + && pool_count(admin, green_writer_hg) >= 1; + auto [same_completed_seq_rc, same_completed_sequence] = sim.probe_log_last_sequence(); + bool same_completion_seen = same_completed_seq_rc == EXIT_SUCCESS + && publish_completed(sim, post, post) == EXIT_SUCCESS + && wait_for_topology_observation(sim, same_completed_sequence, post); + ok(recreated_green_pool && same_completion_seen && pool_count(admin, green_writer_hg) >= 1, + "repeated completion does not drain a pool created while latched"); + auto [latched_error_seq_rc, latched_error_sequence] = sim.probe_log_last_sequence(); + bool latched_error_seen = latched_error_seq_rc == EXIT_SUCCESS + && sim.topology_error( + aurora_bgd_topology_backends(post), 1205, "simulated latched timeout") + == EXIT_SUCCESS + && wait_for_topology_observation(sim, latched_error_sequence, post); + ok(latched_error_seen && aurora_bgd_wait_for_status( + admin, post_writer_hg, "SWITCHOVER_COMPLETED", 1) == EXIT_SUCCESS + && pool_count(admin, green_writer_hg) >= 1, + "query errors neither release the latch nor repeat completion cleanup"); + Aurora_BGD_Test_Deployment different = aurora_bgd_deployment_b_writer_only(); + auto [different_seq_rc, different_sequence] = sim.probe_log_last_sequence(); + bool different_seen = different_seq_rc == EXIT_SUCCESS + && publish_completed(sim, post, different) == EXIT_SUCCESS + && wait_for_topology_observation(sim, different_sequence, post); + ok(different_seen && wait_for_pool_count(admin, green_writer_hg, "=0") == EXIT_SUCCESS, + "a different completed deployment fingerprint rearms and runs its cleanup"); + bool second_green_pool = set_default_hostgroup(admin, green_writer_hg) == EXIT_SUCCESS + && route_to_expected_backend(cl, sim, post.target.members.front().endpoint.backend()); + auto [repeat_different_seq_rc, repeat_different_sequence] = sim.probe_log_last_sequence(); + bool repeated_different = repeat_different_seq_rc == EXIT_SUCCESS + && publish_completed(sim, post, different) == EXIT_SUCCESS + && wait_for_topology_observation(sim, repeat_different_sequence, post); + ok(second_green_pool && repeated_different && pool_count(admin, green_writer_hg) >= 1, + "the new fingerprint is retained and its repeated completion is a no-op"); + ok(sim.topology_delete(aurora_bgd_topology_backends(post)) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, post_writer_hg, "NONE", kWaitSeconds) == EXIT_SUCCESS, + "successful empty topology releases the rearmed completed latch"); + + if (reset_scenario(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to reset before direct-completion scenario"); + cleanup(admin, sim); + return exit_status(); + } + + // A first observation at completion has no target map and does not replay prior phases. + Aurora_BGD_Test_Deployment direct = aurora_bgd_deployment_b_writer_only(); + if (sim.replica_update( + direct.blue_replica_set, direct.production.replica_rows(), + direct.production.backends()) != EXIT_SUCCESS + || sim.topology_update( + aurora_bgd_topology_backends(direct), + aurora_bgd_completed_topology(direct)) != EXIT_SUCCESS) { + diag("Error: failed to publish direct completion inputs"); + cleanup(admin, sim); + return exit_status(); + } + auto [direct_seq_rc, direct_sequence] = sim.replica_probe_log_last_sequence(); + if (aurora_bgd_admin_setup( + admin, direct, 1580, 1581, -1, -1, true, 300, false) != EXIT_SUCCESS) { + diag("Error: failed to configure direct completion"); + cleanup(admin, sim); + return exit_status(); + } + ok(aurora_bgd_wait_for_status( + admin, 1580, "SWITCHOVER_COMPLETED", kWaitSeconds) == EXIT_SUCCESS, + "late entry directly at completion enters the terminal latch"); + ok(wait_for_writer_policy( + admin, 1580, 1581, + direct.production.members.front().endpoint.hostname, false) == EXIT_SUCCESS, + "direct completion leaves canonical writer placement unchanged"); + auto [direct_logs_rc, direct_logs] = sim.replica_probe_log_since(direct_sequence); + bool direct_membership_probe = false; + for (const Aurora_Replica_Probe_Log& log : direct_logs) { + direct_membership_probe |= + log.probe_kind == Aurora_Replica_Probe_Kind::bgd_membership; + } + ok(direct_seq_rc == EXIT_SUCCESS && direct_logs_rc == EXIT_SUCCESS + && !direct_membership_probe, + "direct completion does not manufacture target membership or replay active phases"); + ok(sim.topology_delete(aurora_bgd_topology_backends(direct)) == EXIT_SUCCESS + && aurora_bgd_wait_for_status(admin, 1580, "NONE", kWaitSeconds) == EXIT_SUCCESS, + "direct-completion latch rearms only after topology drain"); + + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean Aurora BGD completion test data"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_aurora_bgd_discovery-t.cpp b/test/tap/tests/test_aurora_bgd_discovery-t.cpp new file mode 100644 index 0000000000..6462d1c141 --- /dev/null +++ b/test/tap/tests/test_aurora_bgd_discovery-t.cpp @@ -0,0 +1,278 @@ +/** + * @file test_aurora_bgd_discovery-t.cpp + * @brief Aurora BGD AVAILABLE discovery, three probes, and fail-closed mapping. + */ + +#include +#include +#include +#include + +#include "aurora_bgd_tap.h" +#include "command_line.h" +#include "utils.h" + +using namespace std; + +const uint32_t kWaitSeconds = 5; +const uint32_t kProbeTimeoutMs = 5000; + +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + char simulator_username[] = "aurora1"; + char simulator_password[] = "pass1"; + if (sim.connect(cl.host, 3306, simulator_username, simulator_password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the shared AWS simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + if (aurora_bgd_admin_cleanup(admin) != EXIT_SUCCESS || sim.cleanup() != EXIT_SUCCESS) { + diag("Error: failed to clear prior Aurora BGD state"); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, BGD_Simulator& sim) { + int admin_rc = aurora_bgd_admin_cleanup(admin); + int simulator_rc = sim.cleanup(); + if (admin) { + mysql_close(admin); + } + return admin_rc == EXIT_SUCCESS && simulator_rc == EXIT_SUCCESS + ? EXIT_SUCCESS : EXIT_FAILURE; +} + +bool runtime_status_is(MYSQL* admin, int writer_hg, const string& status) { + auto [rc, rows] = mysql_query_ext_rows( + admin, + "SELECT bgd_status FROM runtime_mysql_aws_aurora_hostgroups WHERE writer_hostgroup=" + + to_string(writer_hg)); + return rc == EXIT_SUCCESS && rows.size() == 1 && rows.front().size() == 1 + && rows.front().front() == status; +} + +bool runtime_production_unchanged(MYSQL* admin, int writer_hg, int reader_hg, size_t members) { + auto [rc, rows] = mysql_query_ext_rows( + admin, + "SELECT hostname FROM runtime_mysql_servers WHERE hostgroup_id IN (" + + to_string(writer_hg) + "," + to_string(reader_hg) + ")"); + return rc == EXIT_SUCCESS && rows.size() == members; +} + +bool retain_complete_target_after_membership( + BGD_Simulator& sim, + Aurora_BGD_Test_Deployment& deployment, + const vector& rows +) { + auto [seq_rc, sequence] = sim.replica_probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS || sim.replica_update( + deployment.target_replica_set, rows, deployment.target.backends()) != EXIT_SUCCESS) { + return false; + } + + vector target_members; + for (Aurora_BGD_Member& member : deployment.target.members) { + target_members.push_back(member.endpoint.backend()); + } + auto [first_rc, first_probe] = aurora_bgd_wait_for_replica_probe( + sim, sequence, target_members, Aurora_Replica_Probe_Kind::bgd_membership, + kProbeTimeoutMs, deployment.target_replica_set); + if (first_rc != EXIT_SUCCESS) { + return false; + } + auto [next_rc, next_probe] = aurora_bgd_wait_for_replica_probe( + sim, first_probe.sequence_id, target_members, + Aurora_Replica_Probe_Kind::bgd_membership, + kProbeTimeoutMs, deployment.target_replica_set); + return next_rc == EXIT_SUCCESS; +} + +bool invalid_topology_retains_available( + MYSQL* admin, + BGD_Simulator& sim, + Aurora_BGD_Test_Deployment& deployment, + const vector& rows, + int writer_hg +) { + auto [seq_rc, sequence] = sim.probe_log_last_sequence(); + vector target_members; + for (Aurora_BGD_Member& member : deployment.target.members) { + target_members.push_back(member.endpoint.backend()); + } + if (seq_rc != EXIT_SUCCESS || sim.topology_update( + aurora_bgd_topology_backends(deployment), rows) != EXIT_SUCCESS) { + return false; + } + auto [probe_rc, probe] = aurora_bgd_wait_for_topology_probe( + sim, sequence, target_members, BGD_Probe_Kind::metadata, kProbeTimeoutMs); + return probe_rc == EXIT_SUCCESS && runtime_status_is(admin, writer_hg, "AVAILABLE"); +} + +int main() { + plan(18); + + CommandLine cl {}; + MYSQL* admin = nullptr; + BGD_Simulator sim {}; + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + const int writer_hg = 1510; + const int reader_hg = 1511; + Aurora_BGD_Test_Deployment deployment = aurora_bgd_deployment_a(); + if (aurora_bgd_publish(sim, deployment) != EXIT_SUCCESS + || aurora_bgd_admin_setup(admin, deployment, writer_hg, reader_hg, 1512, 1513, false) + != EXIT_SUCCESS) { + diag("Error: failed to publish or configure deployment A"); + cleanup(admin, sim); + return exit_status(); + } + + ok(aurora_bgd_wait_for_status(admin, writer_hg, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + "valid SOURCE/TARGET topology publishes AVAILABLE"); + + { + auto [rc, logs] = sim.replica_probe_log_since(0); + bool ordinary_seen = false; + for (const Aurora_Replica_Probe_Log& log : logs) { + if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary + && log.replica_set_id == deployment.blue_replica_set) { + ordinary_seen = true; + } + } + ok(rc == EXIT_SUCCESS && ordinary_seen, + "the existing worker continues ordinary Aurora probing in AVAILABLE"); + } + + vector blue_backends = deployment.production.backends(); + { + auto [rc, probe] = aurora_bgd_wait_for_topology_probe( + sim, 0, blue_backends, BGD_Probe_Kind::metadata, kProbeTimeoutMs); + ok(rc == EXIT_SUCCESS, "topology discovery starts on a reachable production member"); + } + { + auto [rc, probe] = sim.wait_for_replica_probe_log( + 0, deployment.target_cluster_endpoint.backend(), + Aurora_Replica_Probe_Kind::bgd_membership, kProbeTimeoutMs, + 0, deployment.target_replica_set); + ok(rc == EXIT_SUCCESS, "TARGET cluster endpoint bootstraps target membership"); + } + + vector target_backends; + for (Aurora_BGD_Member& member : deployment.target.members) { + target_backends.push_back(member.endpoint.backend()); + } + { + auto [rc, probe] = aurora_bgd_wait_for_topology_probe( + sim, 0, target_backends, BGD_Probe_Kind::metadata, kProbeTimeoutMs); + ok(rc == EXIT_SUCCESS, "complete membership moves topology probes to target members"); + } + { + auto [rc, probe] = aurora_bgd_wait_for_replica_probe( + sim, 0, target_backends, Aurora_Replica_Probe_Kind::bgd_membership, + kProbeTimeoutMs, deployment.target_replica_set); + ok(rc == EXIT_SUCCESS, "complete membership moves membership probes to target members"); + } + + vector complete_rows = deployment.target.replica_rows(); + vector incomplete_rows {complete_rows[0], complete_rows[1]}; + ok(retain_complete_target_after_membership(sim, deployment, incomplete_rows), + "an incomplete target result retains the previous complete target selection"); + + vector duplicate_writer_rows = complete_rows; + duplicate_writer_rows[1].session_id = "MASTER_SESSION_ID"; + ok(retain_complete_target_after_membership(sim, deployment, duplicate_writer_rows), + "multiple current target writers retain the previous complete snapshot"); + + vector unresolved_rows = complete_rows; + unresolved_rows[2].server_id = "aurora-a-reader-2-green-missing"; + ok(retain_complete_target_after_membership(sim, deployment, unresolved_rows), + "an unresolved target member retains the previous complete snapshot"); + + vector mismatched_session_rows = complete_rows; + mismatched_session_rows[1].session_id = "reader-a-1-replaced"; + ok(retain_complete_target_after_membership(sim, deployment, mismatched_session_rows), + "a reader SESSION_ID mismatch retains the previous complete snapshot"); + + vector stale_extra_rows = complete_rows; + Aurora_Replica_Row stale_writer = complete_rows.front(); + stale_writer.server_id = "unrelated-stale-writer"; + stale_writer.is_current = false; + stale_extra_rows.push_back(stale_writer); + ok(retain_complete_target_after_membership(sim, deployment, stale_extra_rows), + "IS_CURRENT=0 rows are excluded from the complete target snapshot"); + + if (sim.replica_update( + deployment.target_replica_set, complete_rows, deployment.target.backends()) != EXIT_SUCCESS) { + diag("Error: failed to restore complete target membership"); + cleanup(admin, sim); + return exit_status(); + } + + vector valid_topology = aurora_bgd_available_topology(deployment); + vector missing_identity = valid_topology; + missing_identity[1].id.clear(); + ok(invalid_topology_retains_available(admin, sim, deployment, missing_identity, writer_hg), + "missing TARGET identity does not replace AVAILABLE"); + + vector source_only {valid_topology.front()}; + ok(invalid_topology_retains_available(admin, sim, deployment, source_only, writer_hg), + "SOURCE-only topology does not replace AVAILABLE"); + + vector mismatched_status = valid_topology; + mismatched_status[0].status = "SWITCHOVER_INITIATED"; + ok(invalid_topology_retains_available(admin, sim, deployment, mismatched_status, writer_hg), + "mismatched SOURCE/TARGET statuses do not replace AVAILABLE"); + + vector unknown_status = valid_topology; + unknown_status[0].status = "UNSUPPORTED_STATUS"; + unknown_status[1].status = "UNSUPPORTED_STATUS"; + ok(invalid_topology_retains_available(admin, sim, deployment, unknown_status, writer_hg), + "unsupported topology status is not copied into runtime state"); + + ok(runtime_production_unchanged(admin, writer_hg, reader_hg, deployment.production.members.size()), + "AVAILABLE discovery performs no production routing action"); + + if (aurora_bgd_admin_cleanup(admin) != EXIT_SUCCESS || sim.cleanup() != EXIT_SUCCESS) { + diag("Error: failed to reset state before writer-only deployment"); + cleanup(admin, sim); + return exit_status(); + } + + Aurora_BGD_Test_Deployment writer_only = aurora_bgd_deployment_b_writer_only(); + if (aurora_bgd_publish(sim, writer_only) != EXIT_SUCCESS + || aurora_bgd_admin_setup(admin, writer_only, 1520, 1521, -1, -1, true) + != EXIT_SUCCESS) { + diag("Error: failed to publish or configure writer-only deployment"); + cleanup(admin, sim); + return exit_status(); + } + ok(aurora_bgd_wait_for_status(admin, 1520, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + "auto-discovery admits a row without configured green hostgroups"); + { + vector writer_target { + writer_only.target.members.front().endpoint.backend() + }; + auto [rc, probe] = aurora_bgd_wait_for_replica_probe( + sim, 0, writer_target, Aurora_Replica_Probe_Kind::bgd_membership, + kProbeTimeoutMs, writer_only.target_replica_set); + ok(rc == EXIT_SUCCESS, "a writer-only production cluster accepts a writer-only target snapshot"); + } + + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean Aurora BGD discovery state"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_aurora_bgd_lifecycle-t.cpp b/test/tap/tests/test_aurora_bgd_lifecycle-t.cpp new file mode 100644 index 0000000000..77a2e210d0 --- /dev/null +++ b/test/tap/tests/test_aurora_bgd_lifecycle-t.cpp @@ -0,0 +1,397 @@ +/** + * @file test_aurora_bgd_lifecycle-t.cpp + * @brief Aurora BGD reload, worker removal, and concurrent-cluster behavior. + */ + +#include +#include +#include + +#include "aurora_bgd_tap.h" +#include "command_line.h" +#include "utils.h" + +using namespace std; + +const uint32_t kWaitSeconds = 5; +const char kOrdinaryAuroraQuery[] = + "SELECT SERVER_ID," + "IF(" + "SESSION_ID = 'MASTER_SESSION_ID' AND " + "SERVER_ID <> (SELECT SERVER_ID FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE SESSION_ID = 'MASTER_SESSION_ID' ORDER BY LAST_UPDATE_TIMESTAMP DESC LIMIT 1), " + "'probably_former_MASTER_SESSION_ID', SESSION_ID" + ") SESSION_ID, " + "LAST_UPDATE_TIMESTAMP, " + "IF(SESSION_ID = 'MASTER_SESSION_ID', 0, REPLICA_LAG_IN_MILLISECONDS) AS REPLICA_LAG_IN_MILLISECONDS, " + "CPU " + "FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE" + " ( " + "(REPLICA_LAG_IN_MILLISECONDS >= 0 AND REPLICA_LAG_IN_MILLISECONDS <= 600000)" + " OR SESSION_ID = 'MASTER_SESSION_ID'" + " ) " + "AND LAST_UPDATE_TIMESTAMP > NOW() - INTERVAL 180 SECOND" + " ORDER BY SERVER_ID"; + +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { + if (cl.getEnv()) { + return EXIT_FAILURE; + } + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + return EXIT_FAILURE; + } + char username[] = "aurora1"; + char password[] = "pass1"; + if (sim.connect(cl.host, 3306, username, password) != EXIT_SUCCESS + || aurora_bgd_admin_cleanup(admin) != EXIT_SUCCESS + || sim.cleanup() != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return aurora_bgd_execute_all(admin, { + "DELETE FROM mysql_users WHERE username='testuser'", + "INSERT INTO mysql_users(username,password,active,default_hostgroup,transaction_persistent) " + "VALUES ('testuser','testuser',1,0,1)", + "LOAD MYSQL USERS TO RUNTIME", + }); +} + +int reset_scenario(MYSQL* admin, BGD_Simulator& sim) { + return aurora_bgd_admin_cleanup(admin) == EXIT_SUCCESS + && sim.cleanup() == EXIT_SUCCESS ? EXIT_SUCCESS : EXIT_FAILURE; +} + +int cleanup(MYSQL* admin, BGD_Simulator& sim) { + int reset_rc = reset_scenario(admin, sim); + int user_rc = aurora_bgd_execute_all(admin, { + "DELETE FROM mysql_users WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }); + mysql_close(admin); + return reset_rc == EXIT_SUCCESS && user_rc == EXIT_SUCCESS + ? EXIT_SUCCESS : EXIT_FAILURE; +} + +int publish_status( + BGD_Simulator& sim, Aurora_BGD_Test_Deployment& deployment, const string& status +) { + return sim.topology_update( + aurora_bgd_topology_backends(deployment), + aurora_bgd_topology(deployment, status)); +} + +int set_default_hostgroup(MYSQL* admin, int hostgroup) { + return aurora_bgd_execute_all(admin, { + "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroup) + + " WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }); +} + +int add_writer_route( + MYSQL* admin, Aurora_BGD_Test_Deployment& deployment, int hostgroup +) { + return aurora_bgd_execute_all(admin, { + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + to_string(hostgroup) + "," + + aurora_bgd_sql_quote(deployment.production.members.front().endpoint.hostname) + + ",3306,'ONLINE','Aurora BGD lifecycle route')", + "LOAD MYSQL SERVERS TO RUNTIME", + }); +} + +bool route_to_backend(CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected) { + auto [sequence_rc, sequence] = sim.replica_probe_log_last_sequence(); + if (sequence_rc != EXIT_SUCCESS) { + return false; + } + MYSQL* client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); + if (client == nullptr) { + return false; + } + auto [query_rc, rows] = mysql_query_ext_rows(client, kOrdinaryAuroraQuery); + (void)rows; + mysql_close(client); + if (query_rc != EXIT_SUCCESS) { + return false; + } + auto [logs_rc, logs] = sim.replica_probe_log_since(sequence); + if (logs_rc != EXIT_SUCCESS) { + return false; + } + for (const Aurora_Replica_Probe_Log& log : logs) { + if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary + && log.backend.host == expected.host && log.backend.port == expected.port) { + return true; + } + } + return false; +} + +bool route_writer( + CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, + Aurora_BGD_Test_Deployment& deployment, int route_hg, bool target +) { + if (set_default_hostgroup(admin, route_hg) != EXIT_SUCCESS) { + return false; + } + const Endpoint expected = target + ? deployment.target.members.front().endpoint.backend() + : deployment.production.members.front().endpoint.backend(); + return route_to_backend(cl, sim, expected); +} + +bool wait_for_writer_route( + CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, + Aurora_BGD_Test_Deployment& deployment, int route_hg, bool target +) { + if (set_default_hostgroup(admin, route_hg) != EXIT_SUCCESS) { + return false; + } + const Endpoint expected = target + ? deployment.target.members.front().endpoint.backend() + : deployment.production.members.front().endpoint.backend(); + for (uint32_t elapsed_ms = 0; elapsed_ms < kWaitSeconds * 1000; elapsed_ms += 100) { + if (route_to_backend(cl, sim, expected)) { + return true; + } + usleep(100000); + } + return false; +} + +bool writer_placement( + MYSQL* admin, int writer_hg, int reader_hg, const string& hostname, bool demoted +) { + string query = + "SELECT ((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(writer_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + + ")=" + (demoted ? "0" : "1") + ") AND " + "((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(reader_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + + ")=" + (demoted ? "1" : "0") + ")"; + return wait_for_cond(admin, query, kWaitSeconds) == EXIT_SUCCESS; +} + +bool wait_for_inactive_none(MYSQL* admin, int writer_hg) { + return wait_for_cond( + admin, + "SELECT COUNT(*)=1 FROM runtime_mysql_aws_aurora_hostgroups WHERE writer_hostgroup=" + + to_string(writer_hg) + " AND active=0 AND bgd_status='NONE'", + kWaitSeconds) == EXIT_SUCCESS; +} + +bool wait_for_runtime_row_absent(MYSQL* admin, int writer_hg) { + return wait_for_cond( + admin, + "SELECT COUNT(*)=0 FROM runtime_mysql_aws_aurora_hostgroups WHERE writer_hostgroup=" + + to_string(writer_hg), + kWaitSeconds) == EXIT_SUCCESS; +} + +int main() { + plan(28); + CommandLine cl {}; + MYSQL* admin = nullptr; + BGD_Simulator sim {}; + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + Aurora_BGD_Test_Deployment reload = aurora_bgd_deployment_a(); + if (aurora_bgd_publish(sim, reload) != EXIT_SUCCESS + || aurora_bgd_admin_setup(admin, reload, 1620, 1621, 1622, 1623, false, 300) + != EXIT_SUCCESS + || add_writer_route(admin, reload, 1624) != EXIT_SUCCESS) { + diag("Error: failed to configure the active reload scenario"); + cleanup(admin, sim); + return exit_status(); + } + ok(aurora_bgd_wait_for_status(admin, 1620, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + "reload scenario starts from AVAILABLE"); + ok(publish_status(sim, reload, "SWITCHOVER_IN_POST_PROCESSING") == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, 1620, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS + && route_writer(cl, admin, sim, reload, 1624, true), + "active reload scenario pins writer traffic to the target"); + ok(aurora_bgd_execute_all(admin, {"LOAD MYSQL SERVERS TO RUNTIME"}) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, 1620, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS, + "an unrelated server reload preserves active FSM status"); + ok(route_writer(cl, admin, sim, reload, 1624, true), + "an unrelated server reload preserves the applied traffic pin"); + ok(aurora_bgd_execute_all(admin, { + "SET mysql-aws_blue_green_deployment_auto_discovery='false'", + "LOAD MYSQL VARIABLES TO RUNTIME", + }) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, 1620, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS, + "disabling auto-discovery does not abort an admitted deployment"); + ok(route_writer(cl, admin, sim, reload, 1624, true), + "a variable refresh preserves cached membership and target routing"); + ok(aurora_bgd_execute_all(admin, { + "UPDATE mysql_aws_aurora_hostgroups SET green_writer_hostgroup=1626," + "green_reader_hostgroup=1627 WHERE writer_hostgroup=1620", + "LOAD MYSQL SERVERS TO RUNTIME", + }) == EXIT_SUCCESS + && wait_for_cond( + admin, + "SELECT COUNT(*)=1 FROM runtime_mysql_aws_aurora_hostgroups " + "WHERE writer_hostgroup=1620 AND green_writer_hostgroup=1626 " + "AND green_reader_hostgroup=1627 " + "AND bgd_status='SWITCHOVER_IN_POST_PROCESSING'", + kWaitSeconds) == EXIT_SUCCESS, + "green hostgroup changes refresh staging references without resetting the FSM"); + ok(route_writer(cl, admin, sim, reload, 1624, true), + "green hostgroup refresh preserves applied pins and cached target IPs"); + ok(aurora_bgd_execute_all(admin, { + "UPDATE mysql_aws_aurora_hostgroups SET active=0 WHERE writer_hostgroup=1620", + "LOAD MYSQL SERVERS TO RUNTIME", + }) == EXIT_SUCCESS && wait_for_inactive_none(admin, 1620), + "deactivating the owning row stops the worker after publishing NONE"); + ok(writer_placement( + admin, 1620, 1621, reload.production.members.front().endpoint.hostname, false), + "worker teardown restores canonical writer placement"); + ok(route_writer(cl, admin, sim, reload, 1624, false), + "worker teardown removes the applied traffic pin"); + ok(wait_for_cond( + admin, + "SELECT ((SELECT COUNT(*) FROM mysql_aws_aurora_hostgroups " + "WHERE writer_hostgroup=1620 AND active=0)=1) AND " + "((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostname=" + + aurora_bgd_sql_quote(reload.production.members.front().endpoint.hostname) + ")>=1)", + kWaitSeconds) == EXIT_SUCCESS, + "teardown preserves user configuration and server rows"); + + if (reset_scenario(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to reset before terminal reload scenario"); + cleanup(admin, sim); + return exit_status(); + } + Aurora_BGD_Test_Deployment terminal = aurora_bgd_deployment_b_writer_only(); + if (aurora_bgd_publish(sim, terminal) != EXIT_SUCCESS + || aurora_bgd_admin_setup(admin, terminal, 1630, 1631, -1, -1, true, 300) + != EXIT_SUCCESS) { + diag("Error: failed to configure the terminal reload scenario"); + cleanup(admin, sim); + return exit_status(); + } + ok(aurora_bgd_wait_for_status(admin, 1630, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + "terminal reload scenario starts from AVAILABLE"); + ok(sim.topology_update( + aurora_bgd_topology_backends(terminal), aurora_bgd_completed_topology(terminal)) + == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, 1630, "SWITCHOVER_COMPLETED", kWaitSeconds) == EXIT_SUCCESS, + "direct completion enters the terminal latch"); + ok(aurora_bgd_execute_all(admin, { + "UPDATE mysql_aws_aurora_hostgroups SET green_writer_hostgroup=1632," + "green_reader_hostgroup=1633 WHERE writer_hostgroup=1630", + "LOAD MYSQL SERVERS TO RUNTIME", + "SET mysql-aws_blue_green_deployment_auto_discovery='false'", + "LOAD MYSQL VARIABLES TO RUNTIME", + }) == EXIT_SUCCESS + && wait_for_cond( + admin, + "SELECT COUNT(*)=1 FROM runtime_mysql_aws_aurora_hostgroups " + "WHERE writer_hostgroup=1630 AND green_writer_hostgroup=1632 " + "AND green_reader_hostgroup=1633 AND bgd_status='SWITCHOVER_COMPLETED'", + kWaitSeconds) == EXIT_SUCCESS, + "configuration and variable refresh preserve the terminal latch"); + ok(sim.topology_delete(aurora_bgd_topology_backends(terminal)) == EXIT_SUCCESS + && aurora_bgd_wait_for_status(admin, 1630, "NONE", kWaitSeconds) == EXIT_SUCCESS, + "a successful topology drain rearms a refreshed terminal worker"); + + if (reset_scenario(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to reset before concurrent scenario"); + cleanup(admin, sim); + return exit_status(); + } + Aurora_BGD_Test_Deployment concurrent_a = aurora_bgd_deployment_a(); + Aurora_BGD_Test_Deployment concurrent_b = aurora_bgd_deployment_b_writer_only(); + Aurora_BGD_Test_Deployment concurrent_c = aurora_bgd_deployment_c_writer_only(); + if (aurora_bgd_publish(sim, concurrent_a) != EXIT_SUCCESS + || aurora_bgd_publish(sim, concurrent_b) != EXIT_SUCCESS + || aurora_bgd_publish(sim, concurrent_c) != EXIT_SUCCESS + || aurora_bgd_admin_setup(admin, concurrent_a, 1640, 1641, 1642, 1643, false, 300) + != EXIT_SUCCESS + || aurora_bgd_admin_setup(admin, concurrent_b, 1650, 1651, -1, -1, true, 300) + != EXIT_SUCCESS + || aurora_bgd_admin_setup(admin, concurrent_c, 1660, 1661, -1, -1, true, 300) + != EXIT_SUCCESS + || add_writer_route(admin, concurrent_a, 1644) != EXIT_SUCCESS + || add_writer_route(admin, concurrent_b, 1654) != EXIT_SUCCESS + || add_writer_route(admin, concurrent_c, 1664) != EXIT_SUCCESS) { + diag("Error: failed to configure three concurrent deployments"); + cleanup(admin, sim); + return exit_status(); + } + ok(aurora_bgd_wait_for_status(admin, 1640, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS + && aurora_bgd_wait_for_status(admin, 1650, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS + && aurora_bgd_wait_for_status(admin, 1660, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + "three writer hostgroups discover deployments independently"); + ok(publish_status(sim, concurrent_a, "SWITCHOVER_IN_POST_PROCESSING") == EXIT_SUCCESS + && publish_status(sim, concurrent_b, "SWITCHOVER_IN_POST_PROCESSING") == EXIT_SUCCESS + && publish_status(sim, concurrent_c, "SWITCHOVER_IN_POST_PROCESSING") == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, 1640, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, 1650, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, 1660, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS, + "three workers enter POST_PROCESSING without sharing FSM state"); + ok(route_writer(cl, admin, sim, concurrent_a, 1644, true), + "deployment A owns its target pin"); + ok(route_writer(cl, admin, sim, concurrent_b, 1654, true), + "deployment B owns its target pin"); + ok(route_writer(cl, admin, sim, concurrent_c, 1664, true), + "deployment C owns its target pin"); + ok(aurora_bgd_execute_all(admin, { + "UPDATE mysql_aws_aurora_hostgroups SET active=0 WHERE writer_hostgroup=1650", + "LOAD MYSQL SERVERS TO RUNTIME", + }) == EXIT_SUCCESS + && wait_for_inactive_none(admin, 1650) + && aurora_bgd_wait_for_status( + admin, 1640, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, 1660, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS, + "deactivating deployment B cleans only its worker state"); + ok(writer_placement( + admin, 1650, 1651, concurrent_b.production.members.front().endpoint.hostname, false) + && route_writer(cl, admin, sim, concurrent_b, 1654, false), + "deployment B teardown restores only its production routing"); + ok(route_writer(cl, admin, sim, concurrent_a, 1644, true) + && route_writer(cl, admin, sim, concurrent_c, 1664, true), + "deployment B teardown leaves A and C pins intact"); + ok(sim.topology_update( + aurora_bgd_topology_backends(concurrent_a), + aurora_bgd_completed_topology(concurrent_a)) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, 1640, "SWITCHOVER_COMPLETED", kWaitSeconds) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, 1660, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS, + "completing deployment A leaves deployment C active"); + ok(add_writer_route(admin, concurrent_a, 1645) == EXIT_SUCCESS + && route_writer(cl, admin, sim, concurrent_a, 1645, false) + && route_writer(cl, admin, sim, concurrent_c, 1664, true), + "deployment A cleanup removes only its pin"); + ok(aurora_bgd_execute_all(admin, { + "DELETE FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=1660", + "LOAD MYSQL SERVERS TO RUNTIME", + }) == EXIT_SUCCESS + && wait_for_runtime_row_absent(admin, 1660) + && writer_placement( + admin, 1660, 1661, + concurrent_c.production.members.front().endpoint.hostname, false) + && wait_for_writer_route(cl, admin, sim, concurrent_c, 1664, false), + "removing deployment C safely restores its production routing"); + ok(aurora_bgd_wait_for_status( + admin, 1640, "SWITCHOVER_COMPLETED", kWaitSeconds) == EXIT_SUCCESS + && route_writer(cl, admin, sim, concurrent_a, 1645, false), + "deployment C removal leaves deployment A terminal state unchanged"); + + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean Aurora BGD lifecycle test data"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_aurora_bgd_resilience-t.cpp b/test/tap/tests/test_aurora_bgd_resilience-t.cpp new file mode 100644 index 0000000000..9d11f19728 --- /dev/null +++ b/test/tap/tests/test_aurora_bgd_resilience-t.cpp @@ -0,0 +1,412 @@ +/** + * @file test_aurora_bgd_resilience-t.cpp + * @brief Aurora BGD rollback, error retention, and active-state late entry. + */ + +#include +#include +#include +#include + +#include "aurora_bgd_tap.h" +#include "command_line.h" +#include "utils.h" + +using namespace std; + +const uint32_t kWaitSeconds = 5; +const uint32_t kProbeTimeoutMs = 5000; +const char kOrdinaryAuroraQuery[] = + "SELECT SERVER_ID," + "IF(" + "SESSION_ID = 'MASTER_SESSION_ID' AND " + "SERVER_ID <> (SELECT SERVER_ID FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE SESSION_ID = 'MASTER_SESSION_ID' ORDER BY LAST_UPDATE_TIMESTAMP DESC LIMIT 1), " + "'probably_former_MASTER_SESSION_ID', SESSION_ID" + ") SESSION_ID, " + "LAST_UPDATE_TIMESTAMP, " + "IF(SESSION_ID = 'MASTER_SESSION_ID', 0, REPLICA_LAG_IN_MILLISECONDS) AS REPLICA_LAG_IN_MILLISECONDS, " + "CPU " + "FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE" + " ( " + "(REPLICA_LAG_IN_MILLISECONDS >= 0 AND REPLICA_LAG_IN_MILLISECONDS <= 600000)" + " OR SESSION_ID = 'MASTER_SESSION_ID'" + " ) " + "AND LAST_UPDATE_TIMESTAMP > NOW() - INTERVAL 180 SECOND" + " ORDER BY SERVER_ID"; + +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { + if (cl.getEnv()) { + return EXIT_FAILURE; + } + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + return EXIT_FAILURE; + } + char username[] = "aurora1"; + char password[] = "pass1"; + if (sim.connect(cl.host, 3306, username, password) != EXIT_SUCCESS + || aurora_bgd_admin_cleanup(admin) != EXIT_SUCCESS + || sim.cleanup() != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return aurora_bgd_execute_all(admin, { + "DELETE FROM mysql_users WHERE username='testuser'", + "INSERT INTO mysql_users(username,password,active,default_hostgroup,transaction_persistent) " + "VALUES ('testuser','testuser',1,0,1)", + "LOAD MYSQL USERS TO RUNTIME", + }); +} + +int reset_scenario(MYSQL* admin, BGD_Simulator& sim) { + return aurora_bgd_admin_cleanup(admin) == EXIT_SUCCESS + && sim.cleanup() == EXIT_SUCCESS ? EXIT_SUCCESS : EXIT_FAILURE; +} + +int cleanup(MYSQL* admin, BGD_Simulator& sim) { + int reset_rc = reset_scenario(admin, sim); + int user_rc = aurora_bgd_execute_all(admin, { + "DELETE FROM mysql_users WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }); + mysql_close(admin); + return reset_rc == EXIT_SUCCESS && user_rc == EXIT_SUCCESS + ? EXIT_SUCCESS : EXIT_FAILURE; +} + +int publish_initial( + BGD_Simulator& sim, Aurora_BGD_Test_Deployment& deployment, const string& status +) { + return sim.replica_update( + deployment.production.replica_set_id, + deployment.production.replica_rows(), deployment.production.backends()) == EXIT_SUCCESS + && sim.replica_update( + deployment.target.replica_set_id, + deployment.target.replica_rows(), deployment.target.backends()) == EXIT_SUCCESS + && sim.topology_update( + aurora_bgd_topology_backends(deployment), + aurora_bgd_topology(deployment, status)) == EXIT_SUCCESS + ? EXIT_SUCCESS : EXIT_FAILURE; +} + +int publish_status( + BGD_Simulator& sim, Aurora_BGD_Test_Deployment& deployment, const string& status +) { + return sim.topology_update( + aurora_bgd_topology_backends(deployment), + aurora_bgd_topology(deployment, status)); +} + +int set_default_hostgroup(MYSQL* admin, int hostgroup) { + return aurora_bgd_execute_all(admin, { + "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroup) + + " WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }); +} + +bool writer_placement( + MYSQL* admin, int writer_hg, int reader_hg, const string& hostname, bool demoted +) { + string query = + "SELECT ((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(writer_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + + ")=" + (demoted ? "0" : "1") + ") AND " + "((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(reader_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + + ")=" + (demoted ? "1" : "0") + ")"; + return wait_for_cond(admin, query, kWaitSeconds) == EXIT_SUCCESS; +} + +bool active_probe_policy( + BGD_Simulator& sim, uint64_t sequence, const string& target_replica_set, + uint32_t observation_ms +) { + usleep(observation_ms * 1000); + auto [rc, logs] = sim.replica_probe_log_since(sequence); + if (rc != EXIT_SUCCESS) { + return false; + } + bool membership = false; + for (const Aurora_Replica_Probe_Log& log : logs) { + if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary) { + return false; + } + membership |= log.probe_kind == Aurora_Replica_Probe_Kind::bgd_membership + && log.replica_set_id == target_replica_set; + } + return membership; +} + +bool wait_for_ordinary_probe( + BGD_Simulator& sim, uint64_t sequence, Aurora_BGD_Test_Deployment& deployment +) { + auto [rc, log] = aurora_bgd_wait_for_replica_probe( + sim, sequence, deployment.production.backends(), + Aurora_Replica_Probe_Kind::ordinary, kProbeTimeoutMs); + return rc == EXIT_SUCCESS; +} + +int add_routes( + MYSQL* admin, Aurora_BGD_Test_Deployment& deployment, + const vector& route_hgs, int green_writer_hg +) { + vector queries; + for (size_t i = 0; i < route_hgs.size(); ++i) { + queries.push_back( + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + to_string(route_hgs[i]) + "," + + aurora_bgd_sql_quote(deployment.production.members[i].endpoint.hostname) + + ",3306,'ONLINE','Aurora BGD rollback route')"); + } + queries.push_back( + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + to_string(green_writer_hg) + "," + + aurora_bgd_sql_quote(deployment.target.members.front().endpoint.hostname) + + ",3306,'ONLINE','Aurora BGD rollback green pool')"); + queries.push_back("LOAD MYSQL SERVERS TO RUNTIME"); + return aurora_bgd_execute_all(admin, queries); +} + +bool route_to_backend(CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected) { + auto [sequence_rc, sequence] = sim.replica_probe_log_last_sequence(); + if (sequence_rc != EXIT_SUCCESS) { + return false; + } + MYSQL* client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); + if (client == nullptr) { + return false; + } + auto [query_rc, rows] = mysql_query_ext_rows(client, kOrdinaryAuroraQuery); + (void)rows; + mysql_close(client); + if (query_rc != EXIT_SUCCESS) { + return false; + } + auto [logs_rc, logs] = sim.replica_probe_log_since(sequence); + if (logs_rc != EXIT_SUCCESS) { + return false; + } + for (const Aurora_Replica_Probe_Log& log : logs) { + if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary + && log.backend.host == expected.host && log.backend.port == expected.port) { + return true; + } + } + return false; +} + +bool route_members( + CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, + Aurora_BGD_Test_Deployment& deployment, const vector& route_hgs, bool target +) { + for (size_t i = 0; i < route_hgs.size(); ++i) { + if (set_default_hostgroup(admin, route_hgs[i]) != EXIT_SUCCESS) { + return false; + } + Endpoint expected = target + ? deployment.target.members[i].endpoint.backend() + : deployment.production.members[i].endpoint.backend(); + if (!route_to_backend(cl, sim, expected)) { + return false; + } + } + return true; +} + +int64_t pool_count(MYSQL* admin, int hostgroup) { + auto [rc, rows] = mysql_query_ext_rows( + admin, "SELECT COALESCE(SUM(ConnUsed+ConnFree),0) " + "FROM stats_mysql_connection_pool WHERE hostgroup=" + to_string(hostgroup)); + return rc == EXIT_SUCCESS && rows.size() == 1 && rows.front().size() == 1 + ? strtoll(rows.front().front().c_str(), nullptr, 10) : -1; +} + +int main() { + plan(20); + CommandLine cl {}; + MYSQL* admin = nullptr; + BGD_Simulator sim {}; + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + Aurora_BGD_Test_Deployment initiated = aurora_bgd_deployment_b_writer_only(); + const int initiated_writer_hg = 1590; + const int initiated_reader_hg = 1591; + if (publish_initial(sim, initiated, "SWITCHOVER_INITIATED") != EXIT_SUCCESS + || aurora_bgd_admin_setup( + admin, initiated, initiated_writer_hg, initiated_reader_hg, + 1592, 1593, false, 300, false) != EXIT_SUCCESS) { + diag("Error: failed to configure INITIATED late entry"); + cleanup(admin, sim); + return exit_status(); + } + ok(aurora_bgd_wait_for_status( + admin, initiated_writer_hg, "SWITCHOVER_INITIATED", kWaitSeconds) == EXIT_SUCCESS, + "late entry at INITIATED publishes the observed state"); + ok(writer_placement( + admin, initiated_writer_hg, initiated_reader_hg, + initiated.production.members.front().endpoint.hostname, false), + "late INITIATED entry leaves writer placement unchanged"); + auto [initiated_seq_rc, initiated_sequence] = sim.replica_probe_log_last_sequence(); + ok(initiated_seq_rc == EXIT_SUCCESS && active_probe_policy( + sim, initiated_sequence, initiated.target_replica_set, 450), + "late INITIATED entry reconstructs fast membership probing"); + auto [topology_error_seq_rc, topology_error_sequence] = sim.probe_log_last_sequence(); + auto topology_error_seen = topology_error_seq_rc == EXIT_SUCCESS + && sim.topology_error( + aurora_bgd_topology_backends(initiated), 1205, "simulated topology timeout") + == EXIT_SUCCESS + && aurora_bgd_wait_for_topology_probe( + sim, topology_error_sequence, initiated.target.backends(), + BGD_Probe_Kind::metadata, kProbeTimeoutMs).first == EXIT_SUCCESS; + ok(topology_error_seen && aurora_bgd_wait_for_status( + admin, initiated_writer_hg, "SWITCHOVER_INITIATED", 1) == EXIT_SUCCESS, + "topology errors retain the active state"); + if (publish_status(sim, initiated, "SWITCHOVER_INITIATED") != EXIT_SUCCESS) { + diag("Error: failed to restore topology after error"); + cleanup(admin, sim); + return exit_status(); + } + auto [membership_error_seq_rc, membership_error_sequence] = + sim.replica_probe_log_last_sequence(); + auto membership_error_seen = membership_error_seq_rc == EXIT_SUCCESS + && sim.replica_error( + initiated.target.backends(), 1205, "simulated membership timeout") == EXIT_SUCCESS + && aurora_bgd_wait_for_replica_probe( + sim, membership_error_sequence, initiated.target.backends(), + Aurora_Replica_Probe_Kind::bgd_membership, kProbeTimeoutMs).first == EXIT_SUCCESS; + ok(membership_error_seen && aurora_bgd_wait_for_status( + admin, initiated_writer_hg, "SWITCHOVER_INITIATED", 1) == EXIT_SUCCESS, + "membership errors retain the last complete state"); + if (sim.replica_update( + initiated.target.replica_set_id, initiated.target.replica_rows(), + initiated.target.backends()) != EXIT_SUCCESS + || publish_status(sim, initiated, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + admin, initiated_writer_hg, "SWITCHOVER_IN_PROGRESS", kWaitSeconds) + != EXIT_SUCCESS) { + diag("Error: failed to advance to IN_PROGRESS"); + cleanup(admin, sim); + return exit_status(); + } + ok(writer_placement( + admin, initiated_writer_hg, initiated_reader_hg, + initiated.production.members.front().endpoint.hostname, true), + "IN_PROGRESS demotes the reconstructed production writer"); + ok(publish_status(sim, initiated, "SWITCHOVER_INITIATED") == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, initiated_writer_hg, "SWITCHOVER_INITIATED", kWaitSeconds) + == EXIT_SUCCESS + && writer_placement( + admin, initiated_writer_hg, initiated_reader_hg, + initiated.production.members.front().endpoint.hostname, false), + "a backward status rolls back writer placement before entering the earlier state"); + auto [ordinary_seq_rc, ordinary_sequence] = sim.replica_probe_log_last_sequence(); + ok(sim.topology_delete(aurora_bgd_topology_backends(initiated)) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, initiated_writer_hg, "NONE", kWaitSeconds) == EXIT_SUCCESS, + "a successful empty topology cancels the active deployment"); + ok(ordinary_seq_rc == EXIT_SUCCESS + && wait_for_ordinary_probe(sim, ordinary_sequence, initiated), + "cancellation resumes ordinary production probing"); + ok(publish_status(sim, initiated, "AVAILABLE") == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, initiated_writer_hg, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + "the worker admits a repeated deployment after cancellation"); + + if (reset_scenario(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to reset before IN_PROGRESS late entry"); + cleanup(admin, sim); + return exit_status(); + } + Aurora_BGD_Test_Deployment progress = aurora_bgd_deployment_b_writer_only(); + if (publish_initial(sim, progress, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS + || aurora_bgd_admin_setup(admin, progress, 1600, 1601, 1602, 1603, false, 300, false) + != EXIT_SUCCESS) { + diag("Error: failed to configure IN_PROGRESS late entry"); + cleanup(admin, sim); + return exit_status(); + } + ok(aurora_bgd_wait_for_status(admin, 1600, "SWITCHOVER_IN_PROGRESS", kWaitSeconds) + == EXIT_SUCCESS + && writer_placement( + admin, 1600, 1601, progress.production.members.front().endpoint.hostname, true), + "late entry at IN_PROGRESS reconstructs and demotes the writer"); + ok(sim.topology_drop(aurora_bgd_topology_backends(progress)) == EXIT_SUCCESS + && aurora_bgd_wait_for_status(admin, 1600, "NONE", kWaitSeconds) == EXIT_SUCCESS + && writer_placement( + admin, 1600, 1601, progress.production.members.front().endpoint.hostname, false), + "confirmed topology absence rolls back IN_PROGRESS to NONE"); + + if (reset_scenario(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to reset before POST_PROCESSING late entry"); + cleanup(admin, sim); + return exit_status(); + } + Aurora_BGD_Test_Deployment post = aurora_bgd_deployment_a(); + const vector route_hgs {1614, 1615, 1616}; + if (publish_initial(sim, post, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS + || aurora_bgd_admin_setup(admin, post, 1610, 1611, 1612, 1613, false, 300, false) + != EXIT_SUCCESS + || add_routes(admin, post, route_hgs, 1612) != EXIT_SUCCESS) { + diag("Error: failed to configure POST_PROCESSING late entry"); + cleanup(admin, sim); + return exit_status(); + } + ok(aurora_bgd_wait_for_status( + admin, 1610, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS, + "late entry at POST_PROCESSING reconstructs the active phase"); + ok(route_members(cl, admin, sim, post, route_hgs, true), + "late POST_PROCESSING entry reconstructs and applies every target pin"); + ok(set_default_hostgroup(admin, 1612) == EXIT_SUCCESS + && route_to_backend(cl, sim, post.target.members.front().endpoint.backend()) + && pool_count(admin, 1612) >= 1, + "a configured green pool is established before rollback"); + auto [post_error_seq_rc, post_error_sequence] = sim.probe_log_last_sequence(); + bool post_error_seen = post_error_seq_rc == EXIT_SUCCESS + && sim.topology_error( + aurora_bgd_topology_backends(post), 1205, "simulated post timeout") == EXIT_SUCCESS + && aurora_bgd_wait_for_topology_probe( + sim, post_error_sequence, post.target.backends(), + BGD_Probe_Kind::metadata, kProbeTimeoutMs).first == EXIT_SUCCESS; + ok(post_error_seen && aurora_bgd_wait_for_status( + admin, 1610, "SWITCHOVER_IN_POST_PROCESSING", 1) == EXIT_SUCCESS + && route_members(cl, admin, sim, post, route_hgs, true), + "topology errors do not roll back applied POST_PROCESSING pins"); + if (publish_status(sim, post, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS) { + diag("Error: failed to restore POST_PROCESSING topology"); + cleanup(admin, sim); + return exit_status(); + } + auto [post_membership_seq_rc, post_membership_sequence] = + sim.replica_probe_log_last_sequence(); + bool post_membership_error = post_membership_seq_rc == EXIT_SUCCESS + && sim.replica_error(post.target.backends(), 1205, "simulated post membership timeout") + == EXIT_SUCCESS + && aurora_bgd_wait_for_replica_probe( + sim, post_membership_sequence, post.target.backends(), + Aurora_Replica_Probe_Kind::bgd_membership, kProbeTimeoutMs).first == EXIT_SUCCESS; + ok(post_membership_error + && sim.replica_update( + post.target.replica_set_id, post.target.replica_rows(), post.target.backends()) + == EXIT_SUCCESS + && route_members(cl, admin, sim, post, route_hgs, true), + "membership errors retain the last complete mapped routing"); + ok(publish_status(sim, post, "AVAILABLE") == EXIT_SUCCESS + && aurora_bgd_wait_for_status(admin, 1610, "AVAILABLE", kWaitSeconds) + == EXIT_SUCCESS, + "a backward POST_PROCESSING status completes rollback before AVAILABLE"); + ok(writer_placement( + admin, 1610, 1611, post.production.members.front().endpoint.hostname, false) + && route_members(cl, admin, sim, post, route_hgs, false), + "rollback removes pins and restores canonical writer and member routing"); + ok(pool_count(admin, 1612) >= 1, + "rollback preserves configured green pools instead of draining them"); + + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean Aurora BGD resilience test data"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_cluster_sync-t.cpp b/test/tap/tests/test_cluster_sync-t.cpp index be3fdcf6c6..1f748ba5d1 100644 --- a/test/tap/tests/test_cluster_sync-t.cpp +++ b/test/tap/tests/test_cluster_sync-t.cpp @@ -6,7 +6,7 @@ * - 'mysql_galera_hostgroups' with and without NULL comments. * - 'mysql_group_replication_hostgroups' with and without NULL comments. * - 'proxysql_servers' with new values and empty (exercising bug from '#3847'). - * - 'mysql_aws_aurora_hostgroups' with and without NULL comments. + * - 'mysql_aws_aurora_hostgroups' with configured and NULL green hostgroups/comments. * - 'mysql_variables'. * - 'admin_variables'. * @@ -217,6 +217,25 @@ const std::string t_debug_query = "mysql -u%s -p%s -h %s -P%d -C -e \"%s\""; using mysql_server_tuple = tuple; +struct aurora_hostgroups_row_t { + int writer_hostgroup; + int reader_hostgroup; + const char* green_writer_hostgroup; + const char* green_reader_hostgroup; + int active; + int aurora_port; + const char* domain_name; + int max_lag_ms; + int check_interval_ms; + int check_timeout_ms; + int writer_is_also_reader; + int new_reader_weight; + int add_lag_ms; + int min_lag_ms; + int lag_num_checks; + const char* comment; +}; + int check_mysql_servers_sync( const CommandLine& cl, MYSQL* proxy_admin, MYSQL* r_proxy_admin, @@ -1213,7 +1232,7 @@ int main(int, char**) { plan( // Sync tests by values - 16 + + 15 + // Module checkums tests; enabled and disabled checksums check_modules_checksums_sync__tests + (cl.use_noise ? 3 : 0) @@ -2128,7 +2147,7 @@ int main(int, char**) { sleep(2); - // Check 'mysql_aws_aurora_hostgroups' synchronization with NULL comments + // Check 'mysql_aws_aurora_hostgroups' synchronization for configured and NULL values { std::string print_master_aws_aurora_hostgroups = ""; string_format(t_debug_query, print_master_aws_aurora_hostgroups, cl.admin_username, cl.admin_password, cl.host, cl.admin_port, "SELECT * FROM runtime_mysql_aws_aurora_hostgroups"); @@ -2138,141 +2157,13 @@ int main(int, char**) { // Configure 'mysql_aws_aurora_hostgroups' and check sync const char* t_insert_mysql_aws_aurora_hostgroups = "INSERT INTO mysql_aws_aurora_hostgroups ( " - "writer_hostgroup, reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, check_interval_ms, " - "check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks) " - "VALUES (%d, %d, %d, %d, '%s', %d, %d, %d, %d, %d, %d, %d, %d)"; - std::vector> insert_aws_aurora_values { - std::make_tuple(1002, 1006, 1, 3308, ".test_domain2", 10002, 2002, 2002, 0, 3, 50, 100, 1), - std::make_tuple(1003, 1007, 1, 3309, ".test_domain3", 10003, 2003, 2003, 0, 4, 50, 100, 1), - std::make_tuple(1000, 1004, 1, 3306, ".test_domain0", 10000, 2000, 2000, 0, 1, 50, 100, 1), - std::make_tuple(1001, 1005, 1, 3307, ".test_domain1", 10001, 2001, 2001, 0, 2, 50, 100, 1), - }; - std::vector insert_mysql_aws_aurora_hostgroup_queries {}; - - for (auto const& values : insert_aws_aurora_values) { - std::string insert_aws_aurora_hostgroup_query = ""; - string_format( - t_insert_mysql_aws_aurora_hostgroups, - insert_aws_aurora_hostgroup_query, - std::get<0>(values), - std::get<1>(values), - std::get<2>(values), - std::get<3>(values), - std::get<4>(values), - std::get<5>(values), - std::get<6>(values), - std::get<7>(values), - std::get<8>(values), - std::get<9>(values), - std::get<10>(values), - std::get<11>(values), - std::get<12>(values) - ); - insert_mysql_aws_aurora_hostgroup_queries.push_back(insert_aws_aurora_hostgroup_query); - } - - const char* t_select_aws_aurora_inserted_entries = - "SELECT COUNT(*) FROM mysql_aws_aurora_hostgroups WHERE " - "writer_hostgroup=%d AND reader_hostgroup=%d AND active=%d AND aurora_port=%d AND domain_name='%s' " - "AND max_lag_ms=%d AND check_interval_ms=%d AND check_timeout_ms=%d AND writer_is_also_reader=%d " - "AND new_reader_weight=%d AND add_lag_ms=%d AND min_lag_ms=%d AND lag_num_checks=%d AND comment IS NULL"; - std::vector select_mysql_aws_aurora_hostgroup_queries {}; - - for (auto const& values : insert_aws_aurora_values) { - std::string select_aws_aurora_hostgroup_query = ""; - string_format( - t_select_aws_aurora_inserted_entries, - select_aws_aurora_hostgroup_query, - std::get<0>(values), - std::get<1>(values), - std::get<2>(values), - std::get<3>(values), - std::get<4>(values), - std::get<5>(values), - std::get<6>(values), - std::get<7>(values), - std::get<8>(values), - std::get<9>(values), - std::get<10>(values), - std::get<11>(values), - std::get<12>(values) - ); - select_mysql_aws_aurora_hostgroup_queries.push_back(select_aws_aurora_hostgroup_query); - } - - // SETUP CONFIG - - // Backup current table - MYSQL_QUERY__(proxy_admin, "CREATE TABLE mysql_aws_aurora_hostgroups_sync_test_2687 AS SELECT * FROM mysql_aws_aurora_hostgroups"); - MYSQL_QUERY__(proxy_admin, "DELETE FROM mysql_aws_aurora_hostgroups"); - - // Insert the new aws_aurora hostgroups values - for (const auto& query : insert_mysql_aws_aurora_hostgroup_queries) { - MYSQL_QUERY__(proxy_admin, query.c_str()); - } - MYSQL_QUERY__(proxy_admin, "LOAD MYSQL SERVERS TO RUNTIME"); - std::cout << "MASTER TABLE BEFORE SYNC:" << std::endl; - system(print_master_aws_aurora_hostgroups.c_str()); - - // SYNCH CHECK - - uint waited = 0; - bool not_synced_query = false; - while (waited < SYNC_TIMEOUT) { - not_synced_query = false; - // Check that all the entries have been synced - for (const auto& query : select_mysql_aws_aurora_hostgroup_queries) { - MYSQL_QUERY__(r_proxy_admin, query.c_str()); - MYSQL_RES* aws_aurora_res = mysql_store_result(r_proxy_admin); - MYSQL_ROW row = mysql_fetch_row(aws_aurora_res); - int row_value = atoi(row[0]); - mysql_free_result(aws_aurora_res); - - if (row_value == 0) { - not_synced_query = true; - break; - } - } - - if (not_synced_query) { - waited += 1; - sleep(1); - } else { - break; - } - } - - std::cout << "REPLICA TABLE AFTER SYNC:" << std::endl; - system(print_replica_aws_aurora_hostgroups.c_str()); - ok(not_synced_query == false, "'mysql_aws_aurora_hostgroups' with NULL comments should be synced."); - - // TEARDOWN CONFIG - MYSQL_QUERY__(proxy_admin, "DELETE FROM mysql_aws_aurora_hostgroups"); - MYSQL_QUERY__(proxy_admin, "INSERT INTO mysql_aws_aurora_hostgroups SELECT * FROM mysql_aws_aurora_hostgroups_sync_test_2687"); - MYSQL_QUERY__(proxy_admin, "DROP TABLE mysql_aws_aurora_hostgroups_sync_test_2687"); - MYSQL_QUERY__(proxy_admin, "LOAD MYSQL SERVERS TO RUNTIME"); - } - - sleep(2); - - // Check 'mysql_aws_aurora_hostgroups' synchronization - { - std::string print_master_aws_aurora_hostgroups = ""; - string_format(t_debug_query, print_master_aws_aurora_hostgroups, cl.admin_username, cl.admin_password, cl.host, cl.admin_port, "SELECT * FROM runtime_mysql_aws_aurora_hostgroups"); - std::string print_replica_aws_aurora_hostgroups = ""; - string_format(t_debug_query, print_replica_aws_aurora_hostgroups, cl.admin_username, cl.admin_password, R_HOST, R_PORT, "SELECT * FROM runtime_mysql_aws_aurora_hostgroups"); - - // Configure 'mysql_aws_aurora_hostgroups' and check sync - const char* t_insert_mysql_aws_aurora_hostgroups = - "INSERT INTO mysql_aws_aurora_hostgroups ( " - "writer_hostgroup, reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, check_interval_ms, " + "writer_hostgroup, reader_hostgroup, green_writer_hostgroup, green_reader_hostgroup, " + "active, aurora_port, domain_name, max_lag_ms, check_interval_ms, " "check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, comment) " - "VALUES (%d, %d, %d, %d, '%s', %d, %d, %d, %d, %d, %d, %d, %d, '%s')"; - std::vector> insert_aws_aurora_values { - std::make_tuple(1003, 1007, 1, 3309, ".test_domain3", 10003, 2003, 2003, 0, 4, 50, 100, 1, "reader_writer_test_aws_aurora_hostgroup"), - std::make_tuple(1001, 1005, 1, 3307, ".test_domain1", 10001, 2001, 2001, 0, 2, 50, 100, 1, "reader_writer_test_aws_aurora_hostgroup"), - std::make_tuple(1002, 1006, 1, 3308, ".test_domain2", 10002, 2002, 2002, 0, 3, 50, 100, 1, "reader_writer_test_aws_aurora_hostgroup"), - std::make_tuple(1000, 1004, 1, 3306, ".test_domain0", 10000, 2000, 2000, 0, 1, 50, 100, 1, "reader_writer_test_aws_aurora_hostgroup"), + "VALUES (%d, %d, %s, %s, %d, %d, '%s', %d, %d, %d, %d, %d, %d, %d, %d, %s)"; + std::vector insert_aws_aurora_values { + { 1000, 1004, "1010", "1014", 1, 3306, ".test_domain0", 10000, 2000, 2000, 0, 1, 50, 100, 1, "'reader_writer_test_aws_aurora_hostgroup'" }, + { 1002, 1006, "NULL", "NULL", 1, 3308, ".test_domain2", 10002, 2002, 2002, 0, 3, 50, 100, 1, "NULL" }, }; std::vector insert_mysql_aws_aurora_hostgroup_queries {}; @@ -2281,29 +2172,32 @@ int main(int, char**) { string_format( t_insert_mysql_aws_aurora_hostgroups, insert_aws_aurora_hostgroup_query, - std::get<0>(values), - std::get<1>(values), - std::get<2>(values), - std::get<3>(values), - std::get<4>(values), - std::get<5>(values), - std::get<6>(values), - std::get<7>(values), - std::get<8>(values), - std::get<9>(values), - std::get<10>(values), - std::get<11>(values), - std::get<12>(values), - std::get<13>(values) + values.writer_hostgroup, + values.reader_hostgroup, + values.green_writer_hostgroup, + values.green_reader_hostgroup, + values.active, + values.aurora_port, + values.domain_name, + values.max_lag_ms, + values.check_interval_ms, + values.check_timeout_ms, + values.writer_is_also_reader, + values.new_reader_weight, + values.add_lag_ms, + values.min_lag_ms, + values.lag_num_checks, + values.comment ); insert_mysql_aws_aurora_hostgroup_queries.push_back(insert_aws_aurora_hostgroup_query); } const char* t_select_aws_aurora_inserted_entries = "SELECT COUNT(*) FROM mysql_aws_aurora_hostgroups WHERE " - "writer_hostgroup=%d AND reader_hostgroup=%d AND active=%d AND aurora_port=%d AND domain_name='%s' " + "writer_hostgroup=%d AND reader_hostgroup=%d AND green_writer_hostgroup IS %s AND green_reader_hostgroup IS %s " + "AND active=%d AND aurora_port=%d AND domain_name='%s' " "AND max_lag_ms=%d AND check_interval_ms=%d AND check_timeout_ms=%d AND writer_is_also_reader=%d " - "AND new_reader_weight=%d AND add_lag_ms=%d AND min_lag_ms=%d AND lag_num_checks=%d AND comment='%s'"; + "AND new_reader_weight=%d AND add_lag_ms=%d AND min_lag_ms=%d AND lag_num_checks=%d AND comment IS %s"; std::vector select_mysql_aws_aurora_hostgroup_queries {}; for (auto const& values : insert_aws_aurora_values) { @@ -2311,25 +2205,28 @@ int main(int, char**) { string_format( t_select_aws_aurora_inserted_entries, select_aws_aurora_hostgroup_query, - std::get<0>(values), - std::get<1>(values), - std::get<2>(values), - std::get<3>(values), - std::get<4>(values), - std::get<5>(values), - std::get<6>(values), - std::get<7>(values), - std::get<8>(values), - std::get<9>(values), - std::get<10>(values), - std::get<11>(values), - std::get<12>(values), - std::get<13>(values) + values.writer_hostgroup, + values.reader_hostgroup, + values.green_writer_hostgroup, + values.green_reader_hostgroup, + values.active, + values.aurora_port, + values.domain_name, + values.max_lag_ms, + values.check_interval_ms, + values.check_timeout_ms, + values.writer_is_also_reader, + values.new_reader_weight, + values.add_lag_ms, + values.min_lag_ms, + values.lag_num_checks, + values.comment ); select_mysql_aws_aurora_hostgroup_queries.push_back(select_aws_aurora_hostgroup_query); } // Backup current table + MYSQL_QUERY__(proxy_admin, "DROP TABLE IF EXISTS mysql_aws_aurora_hostgroups_sync_test_2687"); MYSQL_QUERY__(proxy_admin, "CREATE TABLE mysql_aws_aurora_hostgroups_sync_test_2687 AS SELECT * FROM mysql_aws_aurora_hostgroups"); MYSQL_QUERY__(proxy_admin, "DELETE FROM mysql_aws_aurora_hostgroups"); @@ -2341,35 +2238,14 @@ int main(int, char**) { std::cout << "MASTER TABLE BEFORE SYNC:" << std::endl; system(print_master_aws_aurora_hostgroups.c_str()); - uint waited = 0; - bool not_synced_query = false; - while (waited < SYNC_TIMEOUT) { - not_synced_query = false; - // Check that all the entries have been synced - for (const auto& query : select_mysql_aws_aurora_hostgroup_queries) { - MYSQL_QUERY__(r_proxy_admin, query.c_str()); - MYSQL_RES* aws_aurora_res = mysql_store_result(r_proxy_admin); - MYSQL_ROW row = mysql_fetch_row(aws_aurora_res); - int row_value = atoi(row[0]); - mysql_free_result(aws_aurora_res); - - if (row_value == 0) { - not_synced_query = true; - break; - } - } - - if (not_synced_query) { - waited += 1; - sleep(1); - } else { - break; - } - } + const int wait_res = proc_wait_checks( + wait_for_conds(r_proxy_admin, select_mysql_aws_aurora_hostgroup_queries, SYNC_TIMEOUT) + ); std::cout << "REPLICA TABLE AFTER SYNC:" << std::endl; system(print_replica_aws_aurora_hostgroups.c_str()); - ok(not_synced_query == false, "'mysql_aws_aurora_hostgroups' should be synced."); + ok(wait_res == EXIT_SUCCESS, + "'mysql_aws_aurora_hostgroups' should sync configured and NULL values: '%d'", wait_res); // TEARDOWN CONFIG MYSQL_QUERY__(proxy_admin, "DELETE FROM mysql_aws_aurora_hostgroups"); diff --git a/test/tap/tests/test_rds_bgd_automatic_discovery-t.cpp b/test/tap/tests/test_rds_bgd_automatic_discovery-t.cpp index f7ac46e125..b9ed27422c 100644 --- a/test/tap/tests/test_rds_bgd_automatic_discovery-t.cpp +++ b/test/tap/tests/test_rds_bgd_automatic_discovery-t.cpp @@ -36,7 +36,7 @@ struct TestState { uint64_t absent_available_sequence { 0 }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -58,7 +58,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -141,7 +141,7 @@ bool runtime_bgd_row_count_matches(MYSQL* admin, int writer_hostgroup, int expec * - Verify one auto-generated runtime row with NULL green hostgroups. * - Verify automatic discovery does not create a persistent BGD row. */ -int test_topology_before_writer(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_topology_before_writer(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.topology_first; BGD_Hostgroups& hg = state.topology_first_hg; @@ -151,7 +151,7 @@ int test_topology_before_writer(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& return EXIT_FAILURE; } - vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); int topology_rc = sim.topology_update(state.topology_first_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish topology-first AVAILABLE topology"); @@ -189,7 +189,7 @@ int test_topology_before_writer(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& * - Verify no runtime or persistent BGD row is created. * - Publish AVAILABLE topology and verify automatic row creation. */ -int test_topology_absent_then_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_topology_absent_then_available(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.absent_first; BGD_Hostgroups& hg = state.absent_first_hg; @@ -220,7 +220,7 @@ int test_topology_absent_then_available(MYSQL* admin, RDS_BGD_Simulator& sim, Te } auto [absent_probe_rc, absent_probe] = - sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); if (absent_probe_rc != EXIT_SUCCESS) { diag("Error: automatic discovery did not issue the absent-table metadata probe"); return EXIT_FAILURE; @@ -236,7 +236,7 @@ int test_topology_absent_then_available(MYSQL* admin, RDS_BGD_Simulator& sim, Te return EXIT_FAILURE; } - vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); int topology_rc = sim.topology_update(state.absent_first_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology for wHG 820"); @@ -250,7 +250,7 @@ int test_topology_absent_then_available(MYSQL* admin, RDS_BGD_Simulator& sim, Te } auto [green_probe_rc, green_probe] = - sim.wait_for_probe_log(available_seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(available_seq, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); if (green_probe_rc != EXIT_SUCCESS) { diag("Error: wHG 820 did not probe the AVAILABLE green writer"); return EXIT_FAILURE; @@ -269,12 +269,12 @@ int test_topology_absent_then_available(MYSQL* admin, RDS_BGD_Simulator& sim, Te * - Verify runtime contains one auto-generated BGD row. * - Verify the automatic row remains absent from persistent configuration. */ -int test_repeated_discovery(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_repeated_discovery(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.absent_first; BGD_Hostgroups& hg = state.absent_first_hg; auto [probe_rc, probe] = sim.wait_for_probe_log( - state.absent_available_sequence, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + state.absent_available_sequence, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); if (probe_rc != EXIT_SUCCESS) { diag("Error: automatic wHG 820 did not continue green-writer metadata polling"); return EXIT_FAILURE; @@ -291,7 +291,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_concurrent_isolation-t.cpp b/test/tap/tests/test_rds_bgd_concurrent_isolation-t.cpp index 14329cd04f..2b5fff8841 100644 --- a/test/tap/tests/test_rds_bgd_concurrent_isolation-t.cpp +++ b/test/tap/tests/test_rds_bgd_concurrent_isolation-t.cpp @@ -36,7 +36,7 @@ struct TestState { BGD_Hostgroups cluster_3_hg { 1430, 1431, 1432, 1433 }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -58,7 +58,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -76,8 +76,8 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { - vector rows = cluster.get_topology(status); +vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); rows.push_back({ cluster.blue_readers[0].hostname, cluster.blue_readers[0].hostname, @@ -95,7 +95,7 @@ vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, return rows; } -int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { +int configure_read_only_values(BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { return EXIT_FAILURE; } @@ -116,13 +116,13 @@ int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) } int configure_available( - MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg, int green_use_ssl) + MYSQL* admin, BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg, int green_use_ssl) { if (configure_read_only_values(sim, cluster) != EXIT_SUCCESS) { return EXIT_FAILURE; } - vector topology = topology_with_reader_pair(cluster, "AVAILABLE"); + vector topology = topology_with_reader_pair(cluster, "AVAILABLE"); if (sim.topology_update(cluster.get_endpoints(), topology) != EXIT_SUCCESS) { return EXIT_FAILURE; } @@ -215,7 +215,7 @@ int replace_cluster_1_green_membership(MYSQL* admin, TestState& state) { * - Use plaintext green metadata for clusters 1 and 3 and TLS for cluster 2. * - Verify that each BGD row reaches AVAILABLE through its own green writer. */ -int test_three_workers_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_three_workers_available(MYSQL* admin, BGD_Simulator& sim, TestState& state) { auto [seq_rc, seq] = sim.probe_log_last_sequence(); if (seq_rc != EXIT_SUCCESS) { diag("Error: failed to read the probe sequence before starting three BGD workers"); @@ -241,7 +241,7 @@ int test_three_workers_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState } auto [cluster_1_probe_rc, cluster_1_probe] = sim.wait_for_probe_log( - seq, state.cluster_1.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 + seq, state.cluster_1.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 ); if (cluster_1_probe_rc != EXIT_SUCCESS) { diag("Error: BGD wHG 1410 did not probe its plaintext green writer"); @@ -250,7 +250,7 @@ int test_three_workers_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState ok(true, "BGD wHG 1410 reports AVAILABLE from its own plaintext green writer"); auto [cluster_2_probe_rc, cluster_2_probe] = sim.wait_for_probe_log( - seq, state.cluster_2.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 + seq, state.cluster_2.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 ); if (cluster_2_probe_rc != EXIT_SUCCESS) { diag("Error: BGD wHG 1420 did not probe its TLS green writer"); @@ -259,7 +259,7 @@ int test_three_workers_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState ok(true, "BGD wHG 1420 reports AVAILABLE from its own TLS green writer"); auto [cluster_3_probe_rc, cluster_3_probe] = sim.wait_for_probe_log( - seq, state.cluster_3.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 + seq, state.cluster_3.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 ); if (cluster_3_probe_rc != EXIT_SUCCESS) { diag("Error: BGD wHG 1430 did not probe its plaintext green writer"); @@ -278,8 +278,8 @@ int test_three_workers_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState * - After each change, verify that the other two statuses and blue-writer * placements remain unchanged. */ -int test_independent_phase_changes(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { - vector cluster_1_topology = +int test_independent_phase_changes(MYSQL* admin, BGD_Simulator& sim, TestState& state) { + vector cluster_1_topology = topology_with_reader_pair(state.cluster_1, "SWITCHOVER_IN_PROGRESS"); if (sim.topology_update(state.cluster_1.get_endpoints(), cluster_1_topology) != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_IN_PROGRESS for BGD wHG 1410"); @@ -308,7 +308,7 @@ int test_independent_phase_changes(MYSQL* admin, RDS_BGD_Simulator& sim, TestSta ok(cluster_2_available && cluster_3_available, "advancing wHG 1410 leaves wHG 1420 and wHG 1430 in AVAILABLE with unchanged blue placement"); - vector cluster_2_topology = + vector cluster_2_topology = topology_with_reader_pair(state.cluster_2, "SWITCHOVER_IN_POST_PROCESSING"); if (sim.topology_update(state.cluster_2.get_endpoints(), cluster_2_topology) != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_IN_POST_PROCESSING for BGD wHG 1420"); @@ -338,7 +338,7 @@ int test_independent_phase_changes(MYSQL* admin, RDS_BGD_Simulator& sim, TestSta ok(cluster_1_in_progress && cluster_3_still_available, "advancing wHG 1420 preserves wHG 1410 progress and wHG 1430 availability"); - vector cluster_3_topology = + vector cluster_3_topology = topology_with_reader_pair(state.cluster_3, "SWITCHOVER_INITIATED"); if (sim.topology_update(state.cluster_3.get_endpoints(), cluster_3_topology) != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_INITIATED for BGD wHG 1430"); @@ -380,7 +380,7 @@ int test_independent_phase_changes(MYSQL* admin, RDS_BGD_Simulator& sim, TestSta * - Verify that cluster 1 stops probing its removed target while clusters 2 * and 3 keep their status, placement, metadata target, and TLS value. */ -int test_independent_config_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_independent_config_refresh(MYSQL* admin, BGD_Simulator& sim, TestState& state) { if (configure_read_only_values(sim, state.cluster_1_b) != EXIT_SUCCESS) { diag("Error: failed to configure simulated read_only values for cluster 1 deployment B"); return EXIT_FAILURE; @@ -398,7 +398,7 @@ int test_independent_config_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestSt return EXIT_FAILURE; } - vector topology = + vector topology = topology_with_reader_pair(state.cluster_1_b, "SWITCHOVER_IN_PROGRESS"); if (sim.topology_update(state.cluster_1_b.get_endpoints(), topology) != EXIT_SUCCESS) { diag("Error: failed to publish deployment B topology for BGD wHG 1410"); @@ -406,7 +406,7 @@ int test_independent_config_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestSt } auto [cluster_1_probe_rc, cluster_1_probe] = sim.wait_for_probe_log( - seq, state.cluster_1_b.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 + seq, state.cluster_1_b.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 ); if (cluster_1_probe_rc != EXIT_SUCCESS) { diag("Error: refreshed BGD wHG 1410 did not probe its TLS deployment B green writer"); @@ -423,7 +423,7 @@ int test_independent_config_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestSt auto [cluster_2_probe_rc, cluster_2_probe] = sim.wait_for_probe_log( cluster_1_probe.sequence_id, state.cluster_2.green_writer.endpoint(), - RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 + BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 ); if (cluster_2_probe_rc != EXIT_SUCCESS) { diag("Error: BGD wHG 1420 did not continue probing its TLS green writer"); @@ -432,7 +432,7 @@ int test_independent_config_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestSt auto [cluster_3_probe_rc, cluster_3_probe] = sim.wait_for_probe_log( cluster_1_probe.sequence_id, state.cluster_3.green_writer.endpoint(), - RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 + BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 ); if (cluster_3_probe_rc != EXIT_SUCCESS) { diag("Error: BGD wHG 1430 did not continue probing its plaintext green writer"); @@ -457,7 +457,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_config_refresh_after_writer_completion-t.cpp b/test/tap/tests/test_rds_bgd_config_refresh_after_writer_completion-t.cpp index 0d375e13a1..c45fce02c6 100644 --- a/test/tap/tests/test_rds_bgd_config_refresh_after_writer_completion-t.cpp +++ b/test/tap/tests/test_rds_bgd_config_refresh_after_writer_completion-t.cpp @@ -32,7 +32,7 @@ struct TestState { vector topology_endpoints { cluster.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -54,7 +54,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -72,8 +72,8 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -vector topology_with_readers(RDS_BGD_Cluster& cluster, string status) { - vector rows = cluster.get_topology(status); +vector topology_with_readers(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); for (RDS_BGD_Host& host : cluster.blue_readers) { rows.push_back({ host.hostname, @@ -95,8 +95,8 @@ vector topology_with_readers(RDS_BGD_Cluster& cluster, str return rows; } -vector target_only_completed(RDS_BGD_Cluster& cluster) { - vector rows {{ +vector target_only_completed(RDS_BGD_Cluster& cluster) { + vector rows {{ cluster.green_writer.hostname, cluster.green_writer.hostname, cluster.green_writer.port, @@ -106,7 +106,7 @@ vector target_only_completed(RDS_BGD_Cluster& cluster) { return rows; } -int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { +int configure_read_only_values(BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { return EXIT_FAILURE; } @@ -134,7 +134,7 @@ int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) * - Publish target-only SWITCHOVER_COMPLETED. * - Verify BGD status READER_SWITCHOVER_IN_PROGRESS. */ -int test_reader_switchover_in_progress(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_reader_switchover_in_progress(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -144,7 +144,7 @@ int test_reader_switchover_in_progress(MYSQL* admin, RDS_BGD_Simulator& sim, Tes return EXIT_FAILURE; } - vector progress = topology_with_readers(cluster, "SWITCHOVER_IN_PROGRESS"); + vector progress = topology_with_readers(cluster, "SWITCHOVER_IN_PROGRESS"); int topology_rc = sim.topology_update(state.topology_endpoints, progress); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology for wHG 1360"); @@ -168,7 +168,7 @@ int test_reader_switchover_in_progress(MYSQL* admin, RDS_BGD_Simulator& sim, Tes return EXIT_FAILURE; } - vector completed = target_only_completed(cluster); + vector completed = target_only_completed(cluster); int completed_rc = sim.topology_update(state.topology_endpoints, completed); if (completed_rc != EXIT_SUCCESS) { diag("Error: failed to publish target-only SWITCHOVER_COMPLETED topology for wHG 1360"); @@ -194,7 +194,7 @@ int test_reader_switchover_in_progress(MYSQL* admin, RDS_BGD_Simulator& sim, Tes * - Verify that blue-writer metadata follows the table check. * - Verify BGD status returns to READER_SWITCHOVER_IN_PROGRESS. */ -int test_config_refresh_after_writer_completion(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_config_refresh_after_writer_completion(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -218,14 +218,14 @@ int test_config_refresh_after_writer_completion(MYSQL* admin, RDS_BGD_Simulator& } auto [table_rc, table] = - sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); if (table_rc != EXIT_SUCCESS) { diag("Error: post-completion refresh did not start with a blue-writer table check for wHG 1360"); return EXIT_FAILURE; } auto [blue_rc, blue] = sim.wait_for_probe_log( - table.sequence_id, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 + table.sequence_id, cluster.blue_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 ); if (blue_rc != EXIT_SUCCESS) { diag("Error: post-completion refresh did not probe blue-writer metadata for wHG 1360"); @@ -251,7 +251,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_configuration_persistence-t.cpp b/test/tap/tests/test_rds_bgd_configuration_persistence-t.cpp index 2d01d467b4..231401788a 100644 --- a/test/tap/tests/test_rds_bgd_configuration_persistence-t.cpp +++ b/test/tap/tests/test_rds_bgd_configuration_persistence-t.cpp @@ -42,7 +42,7 @@ struct TestState { vector admin_owned_endpoints { admin_owned.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -64,7 +64,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -227,7 +227,7 @@ rc_t> green_server_snapshot(MYSQL* admin, const string& ta * - Disable automatic discovery and load explicit hostgroups 890-893. * - Verify explicit values replace the automatic row and persist. */ -int test_automatic_to_explicit(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_automatic_to_explicit(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.conversion; BGD_Hostgroups& hg = state.conversion_hg; @@ -237,7 +237,7 @@ int test_automatic_to_explicit(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& return EXIT_FAILURE; } - vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); int topology_rc = sim.topology_update(state.conversion_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology for wHG 890"); @@ -372,7 +372,7 @@ int test_persistent_row_validation(MYSQL* admin, TestState& state) { * - Execute SAVE MYSQL SERVERS FROM RUNTIME. * - Verify SAVE restores wHG 920 and skips auto-generated wHG 930. */ -int test_save_from_runtime(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_save_from_runtime(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& explicit_cluster = state.explicit_save; RDS_BGD_Cluster& automatic_cluster = state.automatic_save; BGD_Hostgroups& explicit_hg = state.explicit_save_hg; @@ -390,14 +390,14 @@ int test_save_from_runtime(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& stat return EXIT_FAILURE; } - vector explicit_topology = bgd_topology_with_readers(explicit_cluster, "AVAILABLE"); + vector explicit_topology = bgd_topology_with_readers(explicit_cluster, "AVAILABLE"); int explicit_topology_rc = sim.topology_update(state.explicit_save_endpoints, explicit_topology); if (explicit_topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology for wHG 920"); return EXIT_FAILURE; } - vector automatic_topology = bgd_topology_with_readers(automatic_cluster, "AVAILABLE"); + vector automatic_topology = bgd_topology_with_readers(automatic_cluster, "AVAILABLE"); int automatic_topology_rc = sim.topology_update(state.automatic_save_endpoints, automatic_topology); if (automatic_topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology for wHG 930"); @@ -491,7 +491,7 @@ int test_save_from_runtime(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& stat * - Enable automatic discovery and publish AVAILABLE topology. * - Verify the BGD row and green-server status remain unchanged. */ -int test_admin_server_status_preserved(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_admin_server_status_preserved(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.admin_owned; BGD_Hostgroups& hg = state.admin_owned_hg; @@ -571,7 +571,7 @@ int test_admin_server_status_preserved(MYSQL* admin, RDS_BGD_Simulator& sim, Tes return EXIT_FAILURE; } - vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); int topology_rc = sim.topology_update(state.admin_owned_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology beside administrator-owned wHG 1310"); @@ -579,7 +579,7 @@ int test_admin_server_status_preserved(MYSQL* admin, RDS_BGD_Simulator& sim, Tes } auto [probe_rc, probe] = - sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); if (probe_rc != EXIT_SUCCESS) { diag("Error: automatic discovery did not probe beside administrator-owned wHG 1310"); return EXIT_FAILURE; @@ -609,7 +609,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_disable_during_switchover-t.cpp b/test/tap/tests/test_rds_bgd_disable_during_switchover-t.cpp index 07c7327bf9..86a4f435b5 100644 --- a/test/tap/tests/test_rds_bgd_disable_during_switchover-t.cpp +++ b/test/tap/tests/test_rds_bgd_disable_during_switchover-t.cpp @@ -28,7 +28,7 @@ struct TestState { vector topology_endpoints { cluster.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -50,7 +50,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -76,7 +76,7 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { * - Configure `mysql_servers` and `mysql_aws_rds_bgd_hostgroups`. * - Verify that the runtime BGD row reaches `AVAILABLE`. */ -int test_bgd_status_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_bgd_status_available(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -88,7 +88,7 @@ int test_bgd_status_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& s } // Publish AVAILABLE topology. - vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); int topology_rc = sim.topology_update(state.topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology"); @@ -123,12 +123,12 @@ int test_bgd_status_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& s * - Verify `WRITER_SWITCHOVER_IN_PROGRESS`. * - Verify that the blue writer moves to the blue reader hostgroup. */ -int test_writer_switchover_in_progress(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_writer_switchover_in_progress(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; // Publish SWITCHOVER_IN_PROGRESS and wait for the runtime BGD status. - vector topology = bgd_topology_with_readers(cluster, "SWITCHOVER_IN_PROGRESS"); + vector topology = bgd_topology_with_readers(cluster, "SWITCHOVER_IN_PROGRESS"); int topology_rc = sim.topology_update(state.topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology"); @@ -191,7 +191,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_explicit_startup-t.cpp b/test/tap/tests/test_rds_bgd_explicit_startup-t.cpp index 2ddc6ee15f..f99db300c9 100644 --- a/test/tap/tests/test_rds_bgd_explicit_startup-t.cpp +++ b/test/tap/tests/test_rds_bgd_explicit_startup-t.cpp @@ -33,7 +33,7 @@ struct TestState { vector servers_first_endpoints { servers_first.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -55,7 +55,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -167,7 +167,7 @@ bool runtime_membership_matches(MYSQL* admin, RDS_BGD_Cluster& cluster, BGD_Host * - Verify no table-check probe starts. * - Load all servers and verify AVAILABLE with explicit runtime membership. */ -int test_bgd_row_before_servers(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_bgd_row_before_servers(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.row_first; BGD_Hostgroups& hg = state.row_first_hg; @@ -177,7 +177,7 @@ int test_bgd_row_before_servers(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& return EXIT_FAILURE; } - vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); int topology_rc = sim.topology_update(state.row_first_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology for hostgroups 840-843"); @@ -224,7 +224,7 @@ int test_bgd_row_before_servers(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& } auto [probe_rc, probe] = - sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); if (probe_rc != EXIT_SUCCESS) { diag("Error: loading the blue writer did not start the wHG 840 table check"); return EXIT_FAILURE; @@ -249,7 +249,7 @@ int test_bgd_row_before_servers(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& * - Verify no table-check probe starts. * - Load the explicit row and verify AVAILABLE with explicit membership. */ -int test_servers_before_bgd_row(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_servers_before_bgd_row(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.servers_first; BGD_Hostgroups& hg = state.servers_first_hg; @@ -259,7 +259,7 @@ int test_servers_before_bgd_row(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& return EXIT_FAILURE; } - vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); int topology_rc = sim.topology_update(state.servers_first_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology for hostgroups 850-853"); @@ -306,7 +306,7 @@ int test_servers_before_bgd_row(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& } auto [probe_rc, probe] = - sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); if (probe_rc != EXIT_SUCCESS) { diag("Error: loading wHG 850 did not start the blue table check"); return EXIT_FAILURE; @@ -328,7 +328,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_green_membership_ordering-t.cpp b/test/tap/tests/test_rds_bgd_green_membership_ordering-t.cpp index 08f1e2bfc4..cf0ab931cb 100644 --- a/test/tap/tests/test_rds_bgd_green_membership_ordering-t.cpp +++ b/test/tap/tests/test_rds_bgd_green_membership_ordering-t.cpp @@ -36,7 +36,7 @@ struct TestState { vector after_worker_start_endpoints { after_worker_start.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -58,7 +58,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -175,7 +175,7 @@ bool runtime_green_membership_matches(MYSQL* admin, RDS_BGD_Cluster& cluster, BG * - Publish AVAILABLE topology after all membership exists. * - Verify runtime hostgroups 862 and 863 contain the configured green set. */ -int test_green_before_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_green_before_available(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.before_available; BGD_Hostgroups& hg = state.before_available_hg; @@ -209,7 +209,7 @@ int test_green_before_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& return EXIT_FAILURE; } - vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); int topology_rc = sim.topology_update(state.before_available_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology for wHG 860"); @@ -235,7 +235,7 @@ int test_green_before_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& * - Verify runtime_mysql_servers contains the complete green membership after * the worker observes the configuration change. */ -int test_green_after_discovery(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_green_after_discovery(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.after_discovery; BGD_Hostgroups& hg = state.after_discovery_hg; @@ -245,7 +245,7 @@ int test_green_after_discovery(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& return EXIT_FAILURE; } - vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); int topology_rc = sim.topology_update(state.after_discovery_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology for wHG 870"); @@ -289,7 +289,7 @@ int test_green_after_discovery(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& } auto [probe_rc, probe] = - sim.wait_for_probe_log(seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(seq, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); if (probe_rc != EXIT_SUCCESS) { diag("Error: wHG 870 did not probe the green writer after membership load"); return EXIT_FAILURE; @@ -307,7 +307,7 @@ int test_green_after_discovery(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& * - Load green membership into hostgroups 882 and 883. * - Publish AVAILABLE and verify complete runtime green membership. */ -int test_green_after_worker_start(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_green_after_worker_start(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.after_worker_start; BGD_Hostgroups& hg = state.after_worker_start_hg; @@ -348,7 +348,7 @@ int test_green_after_worker_start(MYSQL* admin, RDS_BGD_Simulator& sim, TestStat } auto [start_rc, start_probe] = - sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); if (start_rc != EXIT_SUCCESS) { diag("Error: wHG 880 did not start the blue table-check probe"); return EXIT_FAILURE; @@ -360,7 +360,7 @@ int test_green_after_worker_start(MYSQL* admin, RDS_BGD_Simulator& sim, TestStat return EXIT_FAILURE; } - vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); int topology_rc = sim.topology_update(state.after_worker_start_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology for wHG 880"); @@ -383,7 +383,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_green_pool_cleanup-t.cpp b/test/tap/tests/test_rds_bgd_green_pool_cleanup-t.cpp index 181915b473..0e63511f11 100644 --- a/test/tap/tests/test_rds_bgd_green_pool_cleanup-t.cpp +++ b/test/tap/tests/test_rds_bgd_green_pool_cleanup-t.cpp @@ -52,7 +52,7 @@ struct TestState { } }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -74,7 +74,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -92,8 +92,8 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -vector topology_with_readers(RDS_BGD_Cluster& cluster, string status) { - vector rows = cluster.get_topology(status); +vector topology_with_readers(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); for (RDS_BGD_Host& host : cluster.blue_readers) { rows.push_back({ host.hostname, @@ -115,8 +115,8 @@ vector topology_with_readers(RDS_BGD_Cluster& cluster, str return rows; } -vector target_only_completed(RDS_BGD_Cluster& cluster) { - vector rows {{ +vector target_only_completed(RDS_BGD_Cluster& cluster) { + vector rows {{ cluster.green_writer.hostname, cluster.green_writer.hostname, cluster.green_writer.port, @@ -126,7 +126,7 @@ vector target_only_completed(RDS_BGD_Cluster& cluster) { return rows; } -int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { +int configure_read_only_values(BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { return EXIT_FAILURE; } @@ -250,7 +250,7 @@ bool all_pools_nonzero(vector& pools) { return true; } -int configure_status_matrix(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int configure_status_matrix(CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -336,8 +336,8 @@ int configure_status_matrix(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& si return EXIT_SUCCESS; } -int publish_topology(RDS_BGD_Simulator& sim, TestState& state, string status) { - vector topology = topology_with_readers(state.cluster, status); +int publish_topology(BGD_Simulator& sim, TestState& state, string status) { + vector topology = topology_with_readers(state.cluster, status); int rc = sim.topology_update(state.topology_endpoints, topology); return rc; @@ -362,7 +362,7 @@ bool snapshots_match(MYSQL* admin, TestState& state) { * - Publish AVAILABLE, SWITCHOVER_IN_PROGRESS, then AVAILABLE. * - Verify rollback preserves every green pool and exact configured row. */ -int test_rollback_preserves_green_pools(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_rollback_preserves_green_pools(CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) { int config_rc = configure_status_matrix(cl, admin, sim, state); if (config_rc != EXIT_SUCCESS) { return EXIT_FAILURE; @@ -432,7 +432,7 @@ int test_rollback_preserves_green_pools(CommandLine& cl, MYSQL* admin, RDS_BGD_S * - Require every green pool to remain nonzero immediately before cleanup. * - Delete topology rows and verify that ONLINE and SHUNNED pools drain. */ -int test_successful_cleanup_drains_non_offline(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_successful_cleanup_drains_non_offline(MYSQL* admin, BGD_Simulator& sim, TestState& state) { int post_rc = publish_topology(sim, state, "SWITCHOVER_IN_POST_PROCESSING"); if (post_rc != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_IN_POST_PROCESSING topology for wHG 1300"); @@ -446,7 +446,7 @@ int test_successful_cleanup_drains_non_offline(MYSQL* admin, RDS_BGD_Simulator& return EXIT_FAILURE; } - vector completed = target_only_completed(state.cluster); + vector completed = target_only_completed(state.cluster); int completed_rc = sim.topology_update(state.topology_endpoints, completed); if (completed_rc != EXIT_SUCCESS) { diag("Error: failed to publish target-only SWITCHOVER_COMPLETED topology for wHG 1300"); @@ -517,7 +517,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_late_entry_completed-t.cpp b/test/tap/tests/test_rds_bgd_late_entry_completed-t.cpp index 6e212f11b2..8ad356da90 100644 --- a/test/tap/tests/test_rds_bgd_late_entry_completed-t.cpp +++ b/test/tap/tests/test_rds_bgd_late_entry_completed-t.cpp @@ -30,7 +30,7 @@ struct TestState { vector topology_endpoints { cluster.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -52,7 +52,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -70,8 +70,8 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -vector target_only_completed(RDS_BGD_Cluster& cluster) { - vector rows {{ +vector target_only_completed(RDS_BGD_Cluster& cluster) { + vector rows {{ cluster.green_writer.hostname, cluster.green_writer.hostname, cluster.green_writer.port, @@ -81,7 +81,7 @@ vector target_only_completed(RDS_BGD_Cluster& cluster) { return rows; } -int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { +int configure_read_only_values(BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { return EXIT_FAILURE; } @@ -105,9 +105,9 @@ int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) return EXIT_SUCCESS; } -int wait_for_blue_writer(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster) { +int wait_for_blue_writer(BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster) { auto [probe_rc, probe] = - sim.wait_for_probe_log(sequence, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(sequence, cluster.blue_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); return probe_rc; } @@ -169,7 +169,7 @@ int wait_for_green_pool_drain(MYSQL* admin, BGD_Hostgroups& hg) { * the blue writer is not demoted. * - Establish green writer/reader pools for terminal cleanup. */ -int test_first_completed(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_first_completed(CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -185,7 +185,7 @@ int test_first_completed(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, return EXIT_FAILURE; } - vector topology = target_only_completed(cluster); + vector topology = target_only_completed(cluster); int topology_rc = sim.topology_update(state.topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish target-only SWITCHOVER_COMPLETED topology"); @@ -280,7 +280,7 @@ int test_first_completed(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, * - Verify eligible green writer/reader pools are drained. * - Verify blue-writer and blue-reader placement remains available. */ -int test_completed_empty_topology(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_completed_empty_topology(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -318,7 +318,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_late_entry_writer_phases-t.cpp b/test/tap/tests/test_rds_bgd_late_entry_writer_phases-t.cpp index 970378605e..aaf550f1c8 100644 --- a/test/tap/tests/test_rds_bgd_late_entry_writer_phases-t.cpp +++ b/test/tap/tests/test_rds_bgd_late_entry_writer_phases-t.cpp @@ -41,7 +41,7 @@ struct TestState { vector post_endpoints { post_cluster.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -63,7 +63,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -81,8 +81,8 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { - vector rows = cluster.get_topology(status); +vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); rows.push_back({ cluster.blue_readers[0].hostname, cluster.blue_readers[0].hostname, @@ -100,7 +100,7 @@ vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, return rows; } -int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { +int configure_read_only_values(BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { return EXIT_FAILURE; } @@ -124,16 +124,16 @@ int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) return EXIT_SUCCESS; } -int publish_topology(RDS_BGD_Simulator& sim, vector endpoints, RDS_BGD_Cluster& cluster, string status) { - vector topology = topology_with_reader_pair(cluster, status); +int publish_topology(BGD_Simulator& sim, vector endpoints, RDS_BGD_Cluster& cluster, string status) { + vector topology = topology_with_reader_pair(cluster, status); int rc = sim.topology_update(endpoints, topology); return rc; } -int wait_for_green_writer(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster) { +int wait_for_green_writer(BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster) { auto [probe_rc, probe] = - sim.wait_for_probe_log(sequence, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(sequence, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); return probe_rc; } @@ -266,7 +266,7 @@ int enable_bgd_worker(MYSQL* admin, BGD_Hostgroups& hg) { * - Change simulated writer/reader read_only values. * - Verify read_only monitoring is suppressed for deployment members. */ -int test_first_initiated(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_first_initiated(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.initiated_cluster; BGD_Hostgroups& hg = state.initiated_hg; @@ -374,7 +374,7 @@ int test_first_initiated(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) * - Change the simulated blue-reader read_only value. * - Verify read_only monitoring remains suppressed after demotion. */ -int test_first_in_progress(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_first_in_progress(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.progress_cluster; BGD_Hostgroups& hg = state.progress_hg; @@ -468,7 +468,7 @@ int test_first_in_progress(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& stat * - Verify the old blue pool drains and new connections route to green. * - Verify mapped blue readers remain under read_only suppression. */ -int test_first_post_processing(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_first_post_processing(CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.post_cluster; BGD_Hostgroups& hg = state.post_hg; @@ -586,7 +586,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_probe_tls-t.cpp b/test/tap/tests/test_rds_bgd_probe_tls-t.cpp index f6c616306b..b3898cf409 100644 --- a/test/tap/tests/test_rds_bgd_probe_tls-t.cpp +++ b/test/tap/tests/test_rds_bgd_probe_tls-t.cpp @@ -42,12 +42,12 @@ struct TestState { }; struct ProbeChain { - RDS_BGD_Probe_Log table; - RDS_BGD_Probe_Log blue; - RDS_BGD_Probe_Log green; + BGD_Probe_Log table; + BGD_Probe_Log blue; + BGD_Probe_Log green; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -69,7 +69,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { vector attribute_queries { "DELETE FROM mysql_hostgroup_attributes", "LOAD MYSQL SERVERS TO RUNTIME", @@ -96,23 +96,23 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int wait_for_probe_chain(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster, +int wait_for_probe_chain(BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster, int blue_use_ssl, int green_use_ssl, ProbeChain& chain) { auto [table_rc, table] = - sim.wait_for_probe_log(sequence, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::table_check, kProbeTimeoutMs, blue_use_ssl); + sim.wait_for_probe_log(sequence, cluster.blue_writer.endpoint(), BGD_Probe_Kind::table_check, kProbeTimeoutMs, blue_use_ssl); if (table_rc != EXIT_SUCCESS) { return EXIT_FAILURE; } auto [blue_rc, blue] = - sim.wait_for_probe_log(table.sequence_id, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, blue_use_ssl); + sim.wait_for_probe_log(table.sequence_id, cluster.blue_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, blue_use_ssl); if (blue_rc != EXIT_SUCCESS) { return EXIT_FAILURE; } auto [green_rc, green] = - sim.wait_for_probe_log(blue.sequence_id, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, green_use_ssl); + sim.wait_for_probe_log(blue.sequence_id, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, green_use_ssl); if (green_rc != EXIT_SUCCESS) { return EXIT_FAILURE; } @@ -125,9 +125,9 @@ bool probe_chain_ordered(ProbeChain& chain) { bool ordered = chain.table.sequence_id < chain.blue.sequence_id && chain.blue.sequence_id < chain.green.sequence_id && - chain.table.probe_kind == RDS_BGD_Probe_Kind::table_check && - chain.blue.probe_kind == RDS_BGD_Probe_Kind::metadata && - chain.green.probe_kind == RDS_BGD_Probe_Kind::metadata; + chain.table.probe_kind == BGD_Probe_Kind::table_check && + chain.blue.probe_kind == BGD_Probe_Kind::metadata && + chain.green.probe_kind == BGD_Probe_Kind::metadata; return ordered; } @@ -154,7 +154,7 @@ bool runtime_server_tls_matches(MYSQL* admin, int hostgroup, RDS_BGD_Host& host, * - Verify table-check and blue metadata use the writer with TLS. * - Verify the mapped green writer metadata probe also uses TLS. */ -int test_automatic_writer_tls(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_automatic_writer_tls(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.automatic; BGD_Hostgroups& hg = state.automatic_hg; @@ -198,7 +198,7 @@ int test_automatic_writer_tls(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& s return EXIT_FAILURE; } - vector topology = cluster.get_topology("AVAILABLE"); + vector topology = cluster.get_topology("AVAILABLE"); int topology_rc = sim.topology_update(state.automatic_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish automatic AVAILABLE topology"); @@ -237,7 +237,7 @@ int test_automatic_writer_tls(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& s * - Publish AVAILABLE topology for the exact deployment. * - Verify the exact TARGET and TLS value are used in probe order. */ -int test_explicit_target_tls(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_explicit_target_tls(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.explicit_target; RDS_BGD_Cluster& distractor = state.distractor; BGD_Hostgroups& hg = state.explicit_target_hg; @@ -285,7 +285,7 @@ int test_explicit_target_tls(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& st vector topology_endpoints = state.explicit_target_endpoints; vector distractor_endpoints = distractor.get_endpoints(); topology_endpoints.insert(topology_endpoints.end(), distractor_endpoints.begin(), distractor_endpoints.end()); - vector topology = cluster.get_topology("AVAILABLE"); + vector topology = cluster.get_topology("AVAILABLE"); int topology_rc = sim.topology_update(topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish explicit AVAILABLE topology"); @@ -331,7 +331,7 @@ int test_explicit_target_tls(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& st * - Publish AVAILABLE topology. * - Verify the created TARGET runtime row and metadata probe use TLS. */ -int test_created_target_tls(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_created_target_tls(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.created_target; BGD_Hostgroups& hg = state.created_target_hg; @@ -380,7 +380,7 @@ int test_created_target_tls(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& sta return EXIT_FAILURE; } - vector topology = cluster.get_topology("AVAILABLE"); + vector topology = cluster.get_topology("AVAILABLE"); int topology_rc = sim.topology_update(state.created_target_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish created-TARGET AVAILABLE topology"); @@ -423,7 +423,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_reader_policy-t.cpp b/test/tap/tests/test_rds_bgd_reader_policy-t.cpp index b274b4e9c5..3e3ef9aec8 100644 --- a/test/tap/tests/test_rds_bgd_reader_policy-t.cpp +++ b/test/tap/tests/test_rds_bgd_reader_policy-t.cpp @@ -36,7 +36,7 @@ struct TestState { vector fallback_endpoints { fallback.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -58,7 +58,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -76,8 +76,8 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -vector topology_with_reader_pairs(RDS_BGD_Cluster& cluster, string status, size_t pairs) { - vector rows = cluster.get_topology(status); +vector topology_with_reader_pairs(RDS_BGD_Cluster& cluster, string status, size_t pairs) { + vector rows = cluster.get_topology(status); for (size_t i = 0; i < pairs; ++i) { rows.push_back({ cluster.blue_readers[i].hostname, @@ -97,7 +97,7 @@ vector topology_with_reader_pairs(RDS_BGD_Cluster& cluster return rows; } -int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { +int configure_read_only_values(BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { return EXIT_FAILURE; } @@ -118,7 +118,7 @@ int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) } int configure_bgd( - MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg, size_t green_reader_count) + MYSQL* admin, BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg, size_t green_reader_count) { int read_only_rc = configure_read_only_values(sim, cluster); if (read_only_rc != EXIT_SUCCESS) { @@ -144,9 +144,9 @@ int configure_bgd( } int publish_post_processing( - RDS_BGD_Simulator& sim, vector endpoints, RDS_BGD_Cluster& cluster, size_t pairs) + BGD_Simulator& sim, vector endpoints, RDS_BGD_Cluster& cluster, size_t pairs) { - vector topology = + vector topology = topology_with_reader_pairs(cluster, "SWITCHOVER_IN_POST_PROCESSING", pairs); int rc = sim.topology_update(endpoints, topology); @@ -229,7 +229,7 @@ rc_t connect_and_echo(CommandLine& cl) { * - Route a client through hostgroup 1281 and verify that it reaches the * mapped green reader instead of the unmapped blue reader. */ -int test_matched_unmatched_readers(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_matched_unmatched_readers(CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.matched; BGD_Hostgroups& hg = state.matched_hg; @@ -303,7 +303,7 @@ int test_matched_unmatched_readers(CommandLine& cl, MYSQL* admin, RDS_BGD_Simula * - Publish SWITCHOVER_IN_POST_PROCESSING topology without reader pairs. * - Verify that the blue writer is not added to reader hostgroup 1291. */ -int test_offline_blue_servers(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_offline_blue_servers(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.fallback; BGD_Hostgroups& hg = state.fallback_hg; @@ -350,7 +350,7 @@ int test_offline_blue_servers(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& s * - Connect through hostgroup 1291 and verify routing reaches the green writer * IP pinned for the blue writer hostname. */ -int test_writer_reader_fallback(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_writer_reader_fallback(CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.fallback; BGD_Hostgroups& hg = state.fallback_hg; @@ -394,7 +394,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_reader_switchover_cleanup-t.cpp b/test/tap/tests/test_rds_bgd_reader_switchover_cleanup-t.cpp index 3d10a06c50..e3f0185ed7 100644 --- a/test/tap/tests/test_rds_bgd_reader_switchover_cleanup-t.cpp +++ b/test/tap/tests/test_rds_bgd_reader_switchover_cleanup-t.cpp @@ -30,7 +30,7 @@ struct TestState { vector topology_endpoints { cluster.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -52,7 +52,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -70,8 +70,8 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { - vector rows = cluster.get_topology(status); +vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); rows.push_back({ cluster.blue_readers[0].hostname, cluster.blue_readers[0].hostname, @@ -89,8 +89,8 @@ vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, return rows; } -vector target_only_completed(RDS_BGD_Cluster& cluster) { - vector rows { +vector target_only_completed(RDS_BGD_Cluster& cluster) { + vector rows { { cluster.green_writer.hostname, cluster.green_writer.hostname, @@ -145,11 +145,11 @@ bool green_rows_online(MYSQL* admin, RDS_BGD_Cluster& cluster, BGD_Hostgroups& h return rows_online; } -int advance_to_post_processing(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int advance_to_post_processing(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; - vector available = topology_with_reader_pair(cluster, "AVAILABLE"); + vector available = topology_with_reader_pair(cluster, "AVAILABLE"); int available_rc = sim.topology_update(state.topology_endpoints, available); if (available_rc != EXIT_SUCCESS) { return EXIT_FAILURE; @@ -160,7 +160,7 @@ int advance_to_post_processing(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& return EXIT_FAILURE; } - vector in_progress = topology_with_reader_pair(cluster, "SWITCHOVER_IN_PROGRESS"); + vector in_progress = topology_with_reader_pair(cluster, "SWITCHOVER_IN_PROGRESS"); int progress_rc = sim.topology_update(state.topology_endpoints, in_progress); if (progress_rc != EXIT_SUCCESS) { return EXIT_FAILURE; @@ -171,7 +171,7 @@ int advance_to_post_processing(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& return EXIT_FAILURE; } - vector post_processing = topology_with_reader_pair(cluster, "SWITCHOVER_IN_POST_PROCESSING"); + vector post_processing = topology_with_reader_pair(cluster, "SWITCHOVER_IN_POST_PROCESSING"); int post_rc = sim.topology_update(state.topology_endpoints, post_processing); if (post_rc != EXIT_SUCCESS) { return EXIT_FAILURE; @@ -193,7 +193,7 @@ int advance_to_post_processing(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& * - Publish target-only SWITCHOVER_COMPLETED twice. * - Verify READER_SWITCHOVER_IN_PROGRESS and retained green rows. */ -int test_reader_switchover_in_progress(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_reader_switchover_in_progress(CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -259,7 +259,7 @@ int test_reader_switchover_in_progress(CommandLine& cl, MYSQL* admin, RDS_BGD_Si return EXIT_FAILURE; } - vector completed = target_only_completed(cluster); + vector completed = target_only_completed(cluster); int completed_rc = sim.topology_update(state.topology_endpoints, completed); if (completed_rc != EXIT_SUCCESS) { diag("Error: failed to publish target-only SWITCHOVER_COMPLETED topology"); @@ -283,7 +283,7 @@ int test_reader_switchover_in_progress(CommandLine& cl, MYSQL* admin, RDS_BGD_Si return EXIT_FAILURE; } - vector repeated_completed = target_only_completed(cluster); + vector repeated_completed = target_only_completed(cluster); int repeat_rc = sim.topology_update(state.topology_endpoints, repeated_completed); if (repeat_rc != EXIT_SUCCESS) { diag("Error: failed to repeat target-only SWITCHOVER_COMPLETED topology"); @@ -291,7 +291,7 @@ int test_reader_switchover_in_progress(CommandLine& cl, MYSQL* admin, RDS_BGD_Si } auto [probe_rc, probe] = - sim.wait_for_probe_log(seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(seq, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); if (probe_rc != EXIT_SUCCESS) { diag("Error: BGD did not observe repeated target-only SWITCHOVER_COMPLETED topology"); return EXIT_FAILURE; @@ -316,7 +316,7 @@ int test_reader_switchover_in_progress(CommandLine& cl, MYSQL* admin, RDS_BGD_Si * - Verify metadata probing returns from the green pin to the blue writer. * - Verify green pools drain while configured green rows remain ONLINE. */ -int test_reader_switchover_cleanup(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_reader_switchover_cleanup(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -352,14 +352,14 @@ int test_reader_switchover_cleanup(MYSQL* admin, RDS_BGD_Simulator& sim, TestSta ok(unmatched_reader_online, "reader cleanup restores the unmatched blue reader in hostgroup 981"); auto [green_probe_rc, green_probe] = - sim.wait_for_probe_log(seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(seq, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); if (green_probe_rc != EXIT_SUCCESS) { diag("Error: reader cleanup did not observe empty topology through the green pin"); return EXIT_FAILURE; } auto [blue_probe_rc, blue_probe] = - sim.wait_for_probe_log(green_probe.sequence_id, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(green_probe.sequence_id, cluster.blue_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); if (blue_probe_rc != EXIT_SUCCESS) { diag("Error: metadata probing did not return to the blue writer after reader cleanup"); return EXIT_FAILURE; @@ -387,7 +387,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_remove_during_switchover-t.cpp b/test/tap/tests/test_rds_bgd_remove_during_switchover-t.cpp index dbfa042c38..589edd1291 100644 --- a/test/tap/tests/test_rds_bgd_remove_during_switchover-t.cpp +++ b/test/tap/tests/test_rds_bgd_remove_during_switchover-t.cpp @@ -29,7 +29,7 @@ struct TestState { vector topology_endpoints { cluster.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -51,7 +51,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -86,7 +86,7 @@ int wait_for_bgd_row_absent(MYSQL* admin, BGD_Hostgroups& hg) { * - Configure `mysql_servers` and `mysql_aws_rds_bgd_hostgroups`. * - Verify that the runtime BGD row reaches `AVAILABLE`. */ -int test_bgd_status_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_bgd_status_available(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -98,7 +98,7 @@ int test_bgd_status_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& s } // Publish AVAILABLE topology. - vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); int topology_rc = sim.topology_update(state.topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology"); @@ -133,12 +133,12 @@ int test_bgd_status_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& s * - Verify `WRITER_SWITCHOVER_IN_PROGRESS`. * - Verify that the blue writer moves to the blue reader hostgroup. */ -int test_writer_switchover_in_progress(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_writer_switchover_in_progress(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; // Publish SWITCHOVER_IN_PROGRESS and wait for the runtime BGD status. - vector topology = bgd_topology_with_readers(cluster, "SWITCHOVER_IN_PROGRESS"); + vector topology = bgd_topology_with_readers(cluster, "SWITCHOVER_IN_PROGRESS"); int topology_rc = sim.topology_update(state.topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology"); @@ -214,7 +214,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_repeated_deployment-t.cpp b/test/tap/tests/test_rds_bgd_repeated_deployment-t.cpp index b1852dedf8..a066cf5ea6 100644 --- a/test/tap/tests/test_rds_bgd_repeated_deployment-t.cpp +++ b/test/tap/tests/test_rds_bgd_repeated_deployment-t.cpp @@ -36,7 +36,7 @@ struct TestState { } }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -58,7 +58,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -76,8 +76,8 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { - vector rows = cluster.get_topology(status); +vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); rows.push_back({ cluster.blue_readers[0].hostname, cluster.blue_readers[0].hostname, @@ -95,8 +95,8 @@ vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, return rows; } -vector target_only_completed(RDS_BGD_Cluster& cluster) { - vector rows {{ +vector target_only_completed(RDS_BGD_Cluster& cluster) { + vector rows {{ cluster.green_writer.hostname, cluster.green_writer.hostname, cluster.green_writer.port, @@ -106,7 +106,7 @@ vector target_only_completed(RDS_BGD_Cluster& cluster) { return rows; } -int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { +int configure_read_only_values(BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { return EXIT_FAILURE; } @@ -196,9 +196,9 @@ int replace_green_membership( } int publish_writer_lifecycle( - MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state, RDS_BGD_Cluster& cluster, string deployment) + MYSQL* admin, BGD_Simulator& sim, TestState& state, RDS_BGD_Cluster& cluster, string deployment) { - vector initiated = topology_with_reader_pair(cluster, "SWITCHOVER_INITIATED"); + vector initiated = topology_with_reader_pair(cluster, "SWITCHOVER_INITIATED"); if (sim.topology_update(cluster.get_endpoints(), initiated) != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_INITIATED topology for deployment %s", deployment.c_str()); return EXIT_FAILURE; @@ -209,7 +209,7 @@ int publish_writer_lifecycle( return EXIT_FAILURE; } - vector progress = topology_with_reader_pair(cluster, "SWITCHOVER_IN_PROGRESS"); + vector progress = topology_with_reader_pair(cluster, "SWITCHOVER_IN_PROGRESS"); if (sim.topology_update(cluster.get_endpoints(), progress) != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology for deployment %s", deployment.c_str()); return EXIT_FAILURE; @@ -220,7 +220,7 @@ int publish_writer_lifecycle( return EXIT_FAILURE; } - vector post = topology_with_reader_pair(cluster, "SWITCHOVER_IN_POST_PROCESSING"); + vector post = topology_with_reader_pair(cluster, "SWITCHOVER_IN_POST_PROCESSING"); if (sim.topology_update(cluster.get_endpoints(), post) != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_IN_POST_PROCESSING topology for deployment %s", deployment.c_str()); return EXIT_FAILURE; @@ -235,9 +235,9 @@ int publish_writer_lifecycle( return EXIT_SUCCESS; } -int publish_reader_cleanup(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state, RDS_BGD_Cluster& cluster, +int publish_reader_cleanup(MYSQL* admin, BGD_Simulator& sim, TestState& state, RDS_BGD_Cluster& cluster, string deployment) { - vector completed = target_only_completed(cluster); + vector completed = target_only_completed(cluster); if (sim.topology_update(cluster.get_endpoints(), completed) != EXIT_SUCCESS) { diag("Error: failed to publish target-only SWITCHOVER_COMPLETED for deployment %s", deployment.c_str()); return EXIT_FAILURE; @@ -275,7 +275,7 @@ int publish_reader_cleanup(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& stat * - Complete writer and reader switchover, then publish empty topology. * - Verify NONE and baseline blue-writer placement. */ -int test_deployment_a(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_deployment_a(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& deployment = state.deployment_a; BGD_Hostgroups& hg = state.hostgroups; @@ -285,7 +285,7 @@ int test_deployment_a(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { return EXIT_FAILURE; } - vector available = topology_with_reader_pair(deployment, "AVAILABLE"); + vector available = topology_with_reader_pair(deployment, "AVAILABLE"); int topology_rc = sim.topology_update(deployment.get_endpoints(), available); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology for deployment A"); @@ -332,7 +332,7 @@ int test_deployment_a(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { * - Complete deployment B and verify that routing uses deployment B without * recreating deployment A rows. */ -int test_deployment_b(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_deployment_b(CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& deployment_a = state.deployment_a; RDS_BGD_Cluster& deployment_b = state.deployment_b; BGD_Hostgroups& hg = state.hostgroups; @@ -355,7 +355,7 @@ int test_deployment_b(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, Tes return EXIT_FAILURE; } - vector available = topology_with_reader_pair(deployment_b, "AVAILABLE"); + vector available = topology_with_reader_pair(deployment_b, "AVAILABLE"); int topology_rc = sim.topology_update(deployment_b.get_endpoints(), available); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology for deployment B"); @@ -372,7 +372,7 @@ int test_deployment_b(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, Tes ok(membership_matches, "runtime_mysql_servers contains only TLS-enabled deployment B green rows"); auto [probe_rc, probe] = sim.wait_for_probe_log( - seq, deployment_b.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 + seq, deployment_b.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 ); if (probe_rc != EXIT_SUCCESS) { diag("Error: deployment B green writer did not receive a TLS metadata probe"); @@ -416,7 +416,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_rollback-t.cpp b/test/tap/tests/test_rds_bgd_rollback-t.cpp index 1e1bbe270f..0c0ec1e4c6 100644 --- a/test/tap/tests/test_rds_bgd_rollback-t.cpp +++ b/test/tap/tests/test_rds_bgd_rollback-t.cpp @@ -42,7 +42,7 @@ struct TestState { vector progress_endpoints { progress_cluster.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -64,7 +64,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -82,8 +82,8 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { - vector rows = cluster.get_topology(status); +vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); rows.push_back({ cluster.blue_readers[0].hostname, cluster.blue_readers[0].hostname, @@ -101,7 +101,7 @@ vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, return rows; } -int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { +int configure_read_only_values(BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { return EXIT_FAILURE; } @@ -121,16 +121,16 @@ int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) return EXIT_SUCCESS; } -int publish_topology(RDS_BGD_Simulator& sim, vector endpoints, RDS_BGD_Cluster& cluster, string status) { - vector topology = topology_with_reader_pair(cluster, status); +int publish_topology(BGD_Simulator& sim, vector endpoints, RDS_BGD_Cluster& cluster, string status) { + vector topology = topology_with_reader_pair(cluster, status); int rc = sim.topology_update(endpoints, topology); return rc; } -int wait_for_green_writer(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster) { +int wait_for_green_writer(BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster) { auto [probe_rc, probe] = - sim.wait_for_probe_log(sequence, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(sequence, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); return probe_rc; } @@ -263,7 +263,7 @@ bool green_rows_match(MYSQL* admin, BGD_Hostgroups& hg, RDS_BGD_Cluster& cluster * - Verify blue-writer placement and normal read_only processing are restored. * - Repeat AVAILABLE and verify the monitor-created green writer remains. */ -int test_initiated_rollback(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_initiated_rollback(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.initiated_cluster; BGD_Hostgroups& hg = state.initiated_hg; @@ -407,7 +407,7 @@ int test_initiated_rollback(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& sta * - Verify explicit green rows and pools remain unchanged. * - Repeat AVAILABLE and verify rollback remains stable. */ -int test_in_progress_rollback(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_in_progress_rollback(CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.progress_cluster; BGD_Hostgroups& hg = state.progress_hg; @@ -600,7 +600,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_smoke-t.cpp b/test/tap/tests/test_rds_bgd_smoke-t.cpp index 8b72573e6b..f87f101274 100644 --- a/test/tap/tests/test_rds_bgd_smoke-t.cpp +++ b/test/tap/tests/test_rds_bgd_smoke-t.cpp @@ -28,7 +28,7 @@ struct TestState { uint64_t probe_sequence { 0 }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -50,7 +50,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -109,7 +109,7 @@ int configure_explicit_bgd(MYSQL* admin, TestState& state) { * - Record the probe sequence before publishing topology. * - Publish AVAILABLE topology to the blue and green writer endpoints. */ -int publish_available_topology(RDS_BGD_Simulator& sim, TestState& state) { +int publish_available_topology(BGD_Simulator& sim, TestState& state) { int writer_rc = bgd_set_writer_read_only_0(sim, state.cluster); if (writer_rc != EXIT_SUCCESS) { diag("Error: failed to set read_only=0 for the simulated writers"); @@ -123,7 +123,7 @@ int publish_available_topology(RDS_BGD_Simulator& sim, TestState& state) { } state.probe_sequence = seq; - vector topology = state.cluster.get_topology("AVAILABLE"); + vector topology = state.cluster.get_topology("AVAILABLE"); int topology_rc = sim.topology_update(state.topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology"); @@ -164,10 +164,10 @@ int configure_bgd_available(MYSQL* admin, TestState& state) { * - Wait for a metadata probe after the topology publication sequence. * - Require the probe on the green writer IP without TLS. */ -int test_plaintext_green_writer_probe(RDS_BGD_Simulator& sim, TestState& state) { +int test_plaintext_green_writer_probe(BGD_Simulator& sim, TestState& state) { auto [probe_rc, probe] = sim.wait_for_probe_log( state.probe_sequence, state.cluster.green_writer.endpoint(), - RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 + BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 ); if (probe_rc != EXIT_SUCCESS) { diag("Error: green writer did not receive a plaintext metadata probe"); @@ -183,7 +183,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_topology_empty_absent-t.cpp b/test/tap/tests/test_rds_bgd_topology_empty_absent-t.cpp index 1c938b306f..1f0f6c67d1 100644 --- a/test/tap/tests/test_rds_bgd_topology_empty_absent-t.cpp +++ b/test/tap/tests/test_rds_bgd_topology_empty_absent-t.cpp @@ -44,7 +44,7 @@ struct TestState { vector absent_reader_endpoints { absent_reader.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -66,7 +66,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -84,8 +84,8 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { - vector rows = cluster.get_topology(status); +vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); rows.push_back({ cluster.blue_readers[0].hostname, cluster.blue_readers[0].hostname, @@ -103,8 +103,8 @@ vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, return rows; } -vector target_only_completed(RDS_BGD_Cluster& cluster) { - vector rows {{ +vector target_only_completed(RDS_BGD_Cluster& cluster) { + vector rows {{ cluster.green_writer.hostname, cluster.green_writer.hostname, cluster.green_writer.port, @@ -114,7 +114,7 @@ vector target_only_completed(RDS_BGD_Cluster& cluster) { return rows; } -int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { +int configure_read_only_values(BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { return EXIT_FAILURE; } @@ -153,7 +153,7 @@ bool runtime_server_online(MYSQL* admin, int hostgroup, RDS_BGD_Host& host) { return online; } -int configure_bgd(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg) { +int configure_bgd(MYSQL* admin, BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg) { int read_only_rc = configure_read_only_values(sim, cluster); if (read_only_rc != EXIT_SUCCESS) { diag("Error: failed to configure simulated read_only values for wHG %d", hg.blue_writer); @@ -173,14 +173,14 @@ int configure_bgd(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster return EXIT_SUCCESS; } -int publish_topology(RDS_BGD_Simulator& sim, vector endpoints, RDS_BGD_Cluster& cluster, string status) { - vector topology = topology_with_reader_pair(cluster, status); +int publish_topology(BGD_Simulator& sim, vector endpoints, RDS_BGD_Cluster& cluster, string status) { + vector topology = topology_with_reader_pair(cluster, status); int rc = sim.topology_update(endpoints, topology); return rc; } -int enter_writer_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster, +int enter_writer_switchover(MYSQL* admin, BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg, vector endpoints) { int config_rc = configure_bgd(admin, sim, cluster, hg); @@ -222,7 +222,7 @@ int enter_writer_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluste return EXIT_SUCCESS; } -int enter_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster, +int enter_reader_switchover(MYSQL* admin, BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg, vector endpoints) { int progress_rc = enter_writer_switchover(admin, sim, cluster, hg, endpoints); @@ -249,7 +249,7 @@ int enter_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluste return EXIT_FAILURE; } - vector completed = target_only_completed(cluster); + vector completed = target_only_completed(cluster); int completed_rc = sim.topology_update(endpoints, completed); if (completed_rc != EXIT_SUCCESS) { diag("Error: failed to publish target-only SWITCHOVER_COMPLETED topology for wHG %d", hg.blue_writer); @@ -286,7 +286,7 @@ int disable_bgd(MYSQL* admin, BGD_Hostgroups& hg) { * - Verify BGD status NONE, restored blue-writer placement, and metadata * telemetry from the pinned green writer. */ -int test_empty_before_completion(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_empty_before_completion(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.empty_before; BGD_Hostgroups& hg = state.empty_before_hg; @@ -324,7 +324,7 @@ int test_empty_before_completion(MYSQL* admin, RDS_BGD_Simulator& sim, TestState ok(true, "empty topology restores the blue writer and sets BGD status for wHG 1100 to NONE"); auto [probe_rc, probe] = - sim.wait_for_probe_log(seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(seq, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); if (probe_rc != EXIT_SUCCESS) { diag("Error: empty topology for wHG 1100 was not observed through green-writer metadata"); return EXIT_FAILURE; @@ -348,7 +348,7 @@ int test_empty_before_completion(MYSQL* admin, RDS_BGD_Simulator& sim, TestState * - Verify BGD status NONE, restored blue-writer placement, and a new * blue-writer table-check probe. */ -int test_absent_before_completion(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_absent_before_completion(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.absent_before; BGD_Hostgroups& hg = state.absent_before_hg; @@ -386,7 +386,7 @@ int test_absent_before_completion(MYSQL* admin, RDS_BGD_Simulator& sim, TestStat ok(true, "absent topology restores the blue writer and sets BGD status for wHG 1110 to NONE"); auto [probe_rc, probe] = - sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); if (probe_rc != EXIT_SUCCESS) { diag("Error: absent topology for wHG 1110 did not return to blue-writer table checks"); return EXIT_FAILURE; @@ -410,7 +410,7 @@ int test_absent_before_completion(MYSQL* admin, RDS_BGD_Simulator& sim, TestStat * - Verify BGD status NONE, restored blue-reader routing, retained green * rows, and metadata telemetry from the pinned green writer. */ -int test_empty_during_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_empty_during_reader_switchover(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.empty_reader; BGD_Hostgroups& hg = state.empty_reader_hg; @@ -445,7 +445,7 @@ int test_empty_during_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, Te "empty topology completes reader cleanup for wHG 1120 and retains configured green rows"); auto [probe_rc, probe] = - sim.wait_for_probe_log(seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(seq, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); if (probe_rc != EXIT_SUCCESS) { diag("Error: empty topology for wHG 1120 was not observed through green-writer metadata"); return EXIT_FAILURE; @@ -469,7 +469,7 @@ int test_empty_during_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, Te * - Verify BGD status NONE, restored blue-reader routing, retained green * rows, and a new blue-writer table-check probe. */ -int test_absent_during_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_absent_during_reader_switchover(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.absent_reader; BGD_Hostgroups& hg = state.absent_reader_hg; @@ -504,7 +504,7 @@ int test_absent_during_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, T "absent topology completes reader cleanup for wHG 1130 and retains configured green rows"); auto [probe_rc, probe] = - sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); if (probe_rc != EXIT_SUCCESS) { diag("Error: absent topology for wHG 1130 did not return to blue-writer table checks"); return EXIT_FAILURE; @@ -519,7 +519,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_topology_errors-t.cpp b/test/tap/tests/test_rds_bgd_topology_errors-t.cpp index 06bc414ffe..cfecaec568 100644 --- a/test/tap/tests/test_rds_bgd_topology_errors-t.cpp +++ b/test/tap/tests/test_rds_bgd_topology_errors-t.cpp @@ -39,7 +39,7 @@ struct TestState { vector generic_error_endpoints { generic_error.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -61,7 +61,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -79,8 +79,8 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { - vector rows = cluster.get_topology(status); +vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); rows.push_back({ cluster.blue_readers[0].hostname, cluster.blue_readers[0].hostname, @@ -98,8 +98,8 @@ vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, return rows; } -vector target_only_completed(RDS_BGD_Cluster& cluster) { - vector rows {{ +vector target_only_completed(RDS_BGD_Cluster& cluster) { + vector rows {{ cluster.green_writer.hostname, cluster.green_writer.hostname, cluster.green_writer.port, @@ -109,7 +109,7 @@ vector target_only_completed(RDS_BGD_Cluster& cluster) { return rows; } -int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { +int configure_read_only_values(BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { return EXIT_FAILURE; } @@ -148,7 +148,7 @@ bool runtime_server_online(MYSQL* admin, int hostgroup, RDS_BGD_Host& host) { return online; } -int configure_bgd(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg) { +int configure_bgd(MYSQL* admin, BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg) { int read_only_rc = configure_read_only_values(sim, cluster); if (read_only_rc != EXIT_SUCCESS) { diag("Error: failed to configure simulated read_only values for wHG %d", hg.blue_writer); @@ -168,14 +168,14 @@ int configure_bgd(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster return EXIT_SUCCESS; } -int publish_topology(RDS_BGD_Simulator& sim, vector endpoints, RDS_BGD_Cluster& cluster, string status) { - vector topology = topology_with_reader_pair(cluster, status); +int publish_topology(BGD_Simulator& sim, vector endpoints, RDS_BGD_Cluster& cluster, string status) { + vector topology = topology_with_reader_pair(cluster, status); int rc = sim.topology_update(endpoints, topology); return rc; } -int enter_writer_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster, +int enter_writer_switchover(MYSQL* admin, BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg, vector endpoints) { int config_rc = configure_bgd(admin, sim, cluster, hg); @@ -217,7 +217,7 @@ int enter_writer_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluste return EXIT_SUCCESS; } -int enter_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster, +int enter_reader_switchover(MYSQL* admin, BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg, vector endpoints) { int progress_rc = enter_writer_switchover(admin, sim, cluster, hg, endpoints); @@ -244,7 +244,7 @@ int enter_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluste return EXIT_FAILURE; } - vector completed = target_only_completed(cluster); + vector completed = target_only_completed(cluster); int completed_rc = sim.topology_update(endpoints, completed); if (completed_rc != EXIT_SUCCESS) { diag("Error: failed to publish target-only SWITCHOVER_COMPLETED topology for wHG %d", hg.blue_writer); @@ -260,8 +260,8 @@ int enter_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluste return EXIT_SUCCESS; } -int wait_for_metadata_error(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster, - int error_number, string error_message, RDS_BGD_Probe_Log& probe) +int wait_for_metadata_error(BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster, + int error_number, string error_message, BGD_Probe_Log& probe) { vector green_endpoint { cluster.green_writer.endpoint() }; int error_rc = sim.topology_error(green_endpoint, error_number, error_message); @@ -270,7 +270,7 @@ int wait_for_metadata_error(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_C } auto [probe_rc, metadata_probe] = - sim.wait_for_probe_log(sequence, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(sequence, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); if (probe_rc != EXIT_SUCCESS) { return EXIT_FAILURE; } @@ -279,7 +279,7 @@ int wait_for_metadata_error(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_C return EXIT_SUCCESS; } -int wait_for_blue_table_check(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster) { +int wait_for_blue_table_check(BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster) { vector blue_endpoint { cluster.blue_writer.endpoint() }; int drop_rc = sim.topology_drop(blue_endpoint); if (drop_rc != EXIT_SUCCESS) { @@ -287,7 +287,7 @@ int wait_for_blue_table_check(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD } auto [probe_rc, probe] = - sim.wait_for_probe_log(sequence, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(sequence, cluster.blue_writer.endpoint(), BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); return probe_rc; } @@ -299,7 +299,7 @@ int wait_for_blue_table_check(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD * - Verify BGD status NONE, restored blue-writer placement, and a subsequent * blue-writer table check. */ -int test_error_1146_before_completion(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_error_1146_before_completion(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.before_completion; BGD_Hostgroups& hg = state.before_completion_hg; @@ -315,7 +315,7 @@ int test_error_1146_before_completion(MYSQL* admin, RDS_BGD_Simulator& sim, Test return EXIT_FAILURE; } - RDS_BGD_Probe_Log metadata {}; + BGD_Probe_Log metadata {}; int metadata_rc = wait_for_metadata_error(sim, seq, cluster, 1146, "Table 'mysql.rds_topology' doesn't exist", metadata); if (metadata_rc != EXIT_SUCCESS) { @@ -356,7 +356,7 @@ int test_error_1146_before_completion(MYSQL* admin, RDS_BGD_Simulator& sim, Test * - Verify BGD status NONE, restored blue-reader routing, retained green rows, * and a subsequent blue-writer table check. */ -int test_error_1146_during_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_error_1146_during_reader_switchover(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.reader_switchover; BGD_Hostgroups& hg = state.reader_switchover_hg; @@ -372,7 +372,7 @@ int test_error_1146_during_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& si return EXIT_FAILURE; } - RDS_BGD_Probe_Log metadata {}; + BGD_Probe_Log metadata {}; int metadata_rc = wait_for_metadata_error(sim, seq, cluster, 1146, "Table 'mysql.rds_topology' doesn't exist", metadata); if (metadata_rc != EXIT_SUCCESS) { @@ -410,7 +410,7 @@ int test_error_1146_during_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& si * - Verify that table checking does not restart and that the in-progress * status and blue-writer demotion remain unchanged. */ -int test_generic_metadata_error(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_generic_metadata_error(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.generic_error; BGD_Hostgroups& hg = state.generic_error_hg; @@ -426,7 +426,7 @@ int test_generic_metadata_error(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& return EXIT_FAILURE; } - RDS_BGD_Probe_Log metadata {}; + BGD_Probe_Log metadata {}; int metadata_rc = wait_for_metadata_error(sim, seq, cluster, 1105, "simulated generic metadata failure", metadata); if (metadata_rc != EXIT_SUCCESS) { diag("Error: wHG 1160 did not observe the generic metadata error on the green writer"); @@ -462,7 +462,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_worker_config_refresh-t.cpp b/test/tap/tests/test_rds_bgd_worker_config_refresh-t.cpp index 1643200102..020c3b7869 100644 --- a/test/tap/tests/test_rds_bgd_worker_config_refresh-t.cpp +++ b/test/tap/tests/test_rds_bgd_worker_config_refresh-t.cpp @@ -39,7 +39,7 @@ struct TestState { bool topology_discovery_interval_saved { false }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -75,7 +75,7 @@ int restore_topology_discovery_interval(MYSQL* admin, TestState& state) { return rc; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int cleanup(MYSQL* admin, BGD_Simulator& sim, TestState& state) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -144,7 +144,7 @@ int wait_for_server_status(MYSQL* admin, int hostgroup, RDS_BGD_Host& host, stri * - Change blue-writer `weight` and `comment`. * - Verify that the active worker does not restart with a table check. */ -int test_irrelevant_server_fields(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_irrelevant_server_fields(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -156,7 +156,7 @@ int test_irrelevant_server_fields(MYSQL* admin, RDS_BGD_Simulator& sim, TestStat } // Publish AVAILABLE topology. - vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); int topology_rc = sim.topology_update(state.topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology"); @@ -197,7 +197,7 @@ int test_irrelevant_server_fields(MYSQL* admin, RDS_BGD_Simulator& sim, TestStat } auto [probe_rc, probe] = sim.wait_for_probe_log( - probe_seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 + probe_seq, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 ); if (probe_rc != EXIT_SUCCESS) { diag("Error: green writer did not receive the AVAILABLE metadata probe"); @@ -239,7 +239,7 @@ int test_irrelevant_server_fields(MYSQL* admin, RDS_BGD_Simulator& sim, TestStat * - Verify that the next green-writer metadata probe uses TLS. * - Verify that discovery does not restart. */ -int test_tls_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_tls_refresh(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -268,7 +268,7 @@ int test_tls_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { // Wait for the active worker to use TLS without starting a table check. auto [probe_rc, probe] = sim.wait_for_probe_log( - seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 + seq, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 ); if (probe_rc != EXIT_SUCCESS) { diag("Error: green-writer metadata probe did not use TLS after refresh"); @@ -292,7 +292,7 @@ int test_tls_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { * - Delete the first green reader from hostgroup 1373. * - Verify that discovery does not restart. */ -int test_green_membership_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_green_membership_refresh(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -342,7 +342,7 @@ int test_green_membership_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestStat * - Return the green writer to `ONLINE`. * - Verify that TLS metadata probing resumes without a table-check restart. */ -int test_server_eligibility_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_server_eligibility_refresh(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -377,7 +377,7 @@ int test_server_eligibility_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestSt // Wait for the worker to apply the refreshed server list and return to its eligible blue writer. auto [blue_probe_rc, blue_probe] = sim.wait_for_probe_log( - refresh_seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 + refresh_seq, cluster.blue_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 ); if (blue_probe_rc != EXIT_SUCCESS) { diag("Error: OFFLINE_SOFT refresh did not return metadata probing to the blue writer"); @@ -423,7 +423,7 @@ int test_server_eligibility_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestSt } auto [probe_rc, probe] = sim.wait_for_probe_log( - online_seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 + online_seq, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 ); if (probe_rc != EXIT_SUCCESS) { diag("Error: ONLINE green writer did not resume TLS metadata probes"); @@ -449,7 +449,7 @@ int test_server_eligibility_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestSt * - Verify that the next metadata probe occurs between 500 and 1500 milliseconds. * - Verify that the configuration refresh does not restart with a table check. */ -int test_check_interval_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_check_interval_refresh(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -497,14 +497,14 @@ int test_check_interval_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& // Consume the immediate refresh probe and the first blue probe after the worker reaches NONE. auto [first_rc, first_probe] = - sim.wait_for_probe_log(baseline, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, -1); + sim.wait_for_probe_log(baseline, cluster.blue_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, -1); if (first_rc != EXIT_SUCCESS) { diag("Error: failed to observe the first blue metadata probe after the check_interval_ms refresh"); return EXIT_FAILURE; } auto [settled_rc, settled_probe] = - sim.wait_for_probe_log(first_probe.sequence_id, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, -1); + sim.wait_for_probe_log(first_probe.sequence_id, cluster.blue_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, -1); if (settled_rc != EXIT_SUCCESS) { diag("Error: failed to observe the settled blue metadata probe after the check_interval_ms refresh"); return EXIT_FAILURE; @@ -513,7 +513,7 @@ int test_check_interval_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& // Measure the steady-state interval between consecutive blue metadata probes. unsigned long long interval_start = monotonic_time(); auto [next_rc, next_probe] = sim.wait_for_probe_log( - settled_probe.sequence_id, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kMaximumProbeIntervalMs, -1 + settled_probe.sequence_id, cluster.blue_writer.endpoint(), BGD_Probe_Kind::metadata, kMaximumProbeIntervalMs, -1 ); if (next_rc != EXIT_SUCCESS) { diag("Error: metadata probing did not occur within 1.5 times check_interval_ms"); @@ -541,7 +541,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_worker_hostgroup_refresh-t.cpp b/test/tap/tests/test_rds_bgd_worker_hostgroup_refresh-t.cpp index 6bd3d7ac64..84214fb22c 100644 --- a/test/tap/tests/test_rds_bgd_worker_hostgroup_refresh-t.cpp +++ b/test/tap/tests/test_rds_bgd_worker_hostgroup_refresh-t.cpp @@ -39,7 +39,7 @@ struct TestState { vector topology_endpoints { cluster.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -61,7 +61,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -79,11 +79,11 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -vector topology_with_reader_as_writer(RDS_BGD_Cluster& cluster) { +vector topology_with_reader_as_writer(RDS_BGD_Cluster& cluster) { RDS_BGD_Host& blue_writer = cluster.blue_readers[0]; RDS_BGD_Host& green_writer = cluster.green_readers[0]; - vector rows { + vector rows { { blue_writer.hostname, blue_writer.hostname, blue_writer.port, "BLUE_GREEN_DEPLOYMENT_SOURCE", "SWITCHOVER_IN_PROGRESS" }, { green_writer.hostname, green_writer.hostname, green_writer.port, @@ -103,7 +103,7 @@ vector topology_with_reader_as_writer(RDS_BGD_Cluster& clu * - Verify writer placement in hostgroup 1384. * - Verify metadata probes use TLS from green hostgroups 1385 and 1386. */ -int test_hostgroup_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_hostgroup_refresh(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; BGD_Hostgroups& refreshed_hg = state.refreshed_hostgroups; @@ -116,7 +116,7 @@ int test_hostgroup_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& stat } // Publish SWITCHOVER_IN_PROGRESS topology. - vector topology = bgd_topology_with_readers(cluster, "SWITCHOVER_IN_PROGRESS"); + vector topology = bgd_topology_with_readers(cluster, "SWITCHOVER_IN_PROGRESS"); int topology_rc = sim.topology_update(state.topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology"); @@ -207,7 +207,7 @@ int test_hostgroup_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& stat // Require TLS from the green writer row in refreshed green writer hostgroup 1385. auto [probe_rc, probe] = sim.wait_for_probe_log( - seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 + seq, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 ); if (probe_rc != EXIT_SUCCESS) { diag("Error: metadata probe did not use TLS from green writer hostgroup 1385"); @@ -230,7 +230,7 @@ int test_hostgroup_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& stat * - Verify that the newly mapped writer moves to hostgroup 1384. * - Verify that metadata probing uses TLS from the new green writer target. */ -int test_mapped_writer_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_mapped_writer_refresh(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.refreshed_hostgroups; RDS_BGD_Host& previous_writer = cluster.blue_writer; @@ -238,7 +238,7 @@ int test_mapped_writer_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& RDS_BGD_Host& mapped_target = cluster.green_readers[0]; // Publish topology that maps the first blue reader to the first green reader. - vector topology = topology_with_reader_as_writer(cluster); + vector topology = topology_with_reader_as_writer(cluster); int topology_rc = sim.topology_update(state.topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology for the new mapped writer"); @@ -282,7 +282,7 @@ int test_mapped_writer_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& } auto [probe_rc, probe] = sim.wait_for_probe_log( - seq, mapped_target.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 + seq, mapped_target.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 ); if (probe_rc != EXIT_SUCCESS) { diag("Error: metadata probing did not use TLS from green writer hostgroup 1385"); @@ -328,7 +328,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_writer_switchover-t.cpp b/test/tap/tests/test_rds_bgd_writer_switchover-t.cpp index 9bf8b8ee94..333580707c 100644 --- a/test/tap/tests/test_rds_bgd_writer_switchover-t.cpp +++ b/test/tap/tests/test_rds_bgd_writer_switchover-t.cpp @@ -35,7 +35,7 @@ struct TestState { int64_t reader_log_baseline { -1 }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -57,7 +57,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -75,8 +75,8 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { - vector rows = cluster.get_topology(status); +vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); rows.push_back({ cluster.blue_readers[0].hostname, cluster.blue_readers[0].hostname, @@ -94,9 +94,9 @@ vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, return rows; } -int wait_for_green_observation(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster) { +int wait_for_green_observation(BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster) { auto [probe_rc, probe] = - sim.wait_for_probe_log(sequence, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(sequence, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); return probe_rc; } @@ -211,7 +211,7 @@ int create_blue_writer_pool(CommandLine& cl, MYSQL* admin, TestState& state) { * - Configure mysql_servers and mysql_aws_rds_bgd_hostgroups. * - Verify BGD status AVAILABLE. */ -int test_bgd_status_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_bgd_status_available(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -239,7 +239,7 @@ int test_bgd_status_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& s return EXIT_FAILURE; } - vector topology = topology_with_reader_pair(cluster, "AVAILABLE"); + vector topology = topology_with_reader_pair(cluster, "AVAILABLE"); int topology_rc = sim.topology_update(state.topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology for wHG 970"); @@ -272,7 +272,7 @@ int test_bgd_status_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& s * - Change simulated blue writer/reader read_only values. * - Verify BGD suppresses their normal placement changes. */ -int test_switchover_initiated(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_switchover_initiated(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -282,7 +282,7 @@ int test_switchover_initiated(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& s return EXIT_FAILURE; } - vector topology = topology_with_reader_pair(cluster, "SWITCHOVER_INITIATED"); + vector topology = topology_with_reader_pair(cluster, "SWITCHOVER_INITIATED"); int topology_rc = sim.topology_update(state.topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_INITIATED topology"); @@ -318,7 +318,7 @@ int test_switchover_initiated(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& s return EXIT_FAILURE; } - vector repeat_topology = topology_with_reader_pair(cluster, "SWITCHOVER_INITIATED"); + vector repeat_topology = topology_with_reader_pair(cluster, "SWITCHOVER_INITIATED"); int repeat_rc = sim.topology_update(state.topology_endpoints, repeat_topology); if (repeat_rc != EXIT_SUCCESS) { diag("Error: failed to repeat SWITCHOVER_INITIATED topology"); @@ -362,11 +362,11 @@ int test_switchover_initiated(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& s * - Verify the blue writer moves from hostgroup 970 to 971. * - Verify the mapped blue reader remains suppressed in hostgroup 971. */ -int test_switchover_in_progress(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_switchover_in_progress(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; - vector topology = topology_with_reader_pair(cluster, "SWITCHOVER_IN_PROGRESS"); + vector topology = topology_with_reader_pair(cluster, "SWITCHOVER_IN_PROGRESS"); int topology_rc = sim.topology_update(state.topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology"); @@ -412,7 +412,7 @@ int test_switchover_in_progress(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& * - Verify writer restoration, blue-pool drain, and green backend routing. * - Repeat POST_PROCESSING and verify the post-cutover pool is not drained. */ -int test_switchover_post_processing(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_switchover_post_processing(CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -422,7 +422,7 @@ int test_switchover_post_processing(CommandLine& cl, MYSQL* admin, RDS_BGD_Simul return EXIT_FAILURE; } - vector topology = topology_with_reader_pair(cluster, "SWITCHOVER_IN_POST_PROCESSING"); + vector topology = topology_with_reader_pair(cluster, "SWITCHOVER_IN_POST_PROCESSING"); int topology_rc = sim.topology_update(state.topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_IN_POST_PROCESSING topology"); @@ -484,7 +484,7 @@ int test_switchover_post_processing(CommandLine& cl, MYSQL* admin, RDS_BGD_Simul return EXIT_FAILURE; } - vector repeat_topology = topology_with_reader_pair(cluster, "SWITCHOVER_IN_POST_PROCESSING"); + vector repeat_topology = topology_with_reader_pair(cluster, "SWITCHOVER_IN_POST_PROCESSING"); int repeat_rc = sim.topology_update(state.topology_endpoints, repeat_topology); if (repeat_rc != EXIT_SUCCESS) { diag("Error: failed to repeat SWITCHOVER_IN_POST_PROCESSING topology"); @@ -512,7 +512,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/unit/Makefile b/test/tap/tests/unit/Makefile index 1125125d1b..f5a323933e 100644 --- a/test/tap/tests/unit/Makefile +++ b/test/tap/tests/unit/Makefile @@ -439,6 +439,7 @@ UNIT_TESTS := smoke_test-t query_cache_unit-t query_processor_unit-t \ gtid_set_unit-t \ gtid_server_data_unit-t \ admin_disk_upgrade_unit-t \ + aurora_bgd_config_unit-t \ glovars_unit-t \ pgsql_servers_ssl_params_unit-t \ connection_unhealthy_unit-t \ diff --git a/test/tap/tests/unit/admin_disk_upgrade_unit-t.cpp b/test/tap/tests/unit/admin_disk_upgrade_unit-t.cpp index f900625e6f..26d2d35cc8 100644 --- a/test/tap/tests/unit/admin_disk_upgrade_unit-t.cpp +++ b/test/tap/tests/unit/admin_disk_upgrade_unit-t.cpp @@ -113,6 +113,16 @@ static bool table_matches_current(SQLite3DB *db, const char *tbl, const char *de return db->check_table_structure(tbl, def) == 1; } +static bool create_table_in_schema(SQLite3DB* db, const char* schema, const char* definition) { + std::string query = definition; + const std::string prefix = "CREATE TABLE "; + if (query.compare(0, prefix.size(), prefix) != 0) { + return false; + } + query.insert(prefix.size(), std::string(schema) + "."); + return db->execute(query.c_str()); +} + // ============================================================================ // disk_upgrade_scheduler() tests // ============================================================================ @@ -382,6 +392,37 @@ static void test_mysql_servers_no_upgrade_needed() { ok(count == 1, "mysql_servers: data unchanged when no upgrade needed (got %d)", count); } +static void test_aurora_hostgroups_upgrade_from_v2_0_10() { + TestDiskUpgrade t; + SQLite3DB *db = t.db(); + + db->execute(ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS_V2_0_10); + db->execute( + "INSERT INTO mysql_aws_aurora_hostgroups (" + "writer_hostgroup,reader_hostgroup,active,aurora_port,domain_name,max_lag_ms," + "check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight," + "add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment" + ") VALUES (10,20,1,3307,'.cluster.example',125,1500,900,1,7,40,20,3,9,'existing Aurora row')" + ); + + t.upgrade_mysql_servers(); + + ok(table_matches_current(db, "mysql_aws_aurora_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS), + "mysql_aws_aurora_hostgroups: v2.0.10 upgrade produces current schema"); + ok(query_int(db, "SELECT COUNT(*) FROM mysql_aws_aurora_hostgroups") == 1, + "mysql_aws_aurora_hostgroups: v2.0.10 upgrade preserves the row"); + ok(query_int(db, "SELECT green_writer_hostgroup IS NULL FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=10") == 1, + "mysql_aws_aurora_hostgroups: migrated green writer is NULL"); + ok(query_int(db, "SELECT green_reader_hostgroup IS NULL FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=10") == 1, + "mysql_aws_aurora_hostgroups: migrated green reader is NULL"); + ok(query_string(db, "SELECT domain_name FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=10") == ".cluster.example", + "mysql_aws_aurora_hostgroups: migration preserves configured values"); + ok(query_int(db, "SELECT autopurge_missing_checks FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=10") == 9, + "mysql_aws_aurora_hostgroups: migration preserves autopurge_missing_checks"); + ok(query_int(db, "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='mysql_aws_aurora_hostgroups_v2010'") == 1, + "mysql_aws_aurora_hostgroups: v2.0.10 table is retained as v2010"); +} + // ============================================================================ // disk_upgrade_pgsql_replication_hostgroups() tests // ============================================================================ @@ -544,12 +585,64 @@ static void test_mysql_servers_upgrade_multiple_rows_with_fixes() { ok(w2 == 500, "mysql_servers: normal weight preserved (got %d)", w2); } +static void test_aurora_hostgroups_disk_roundtrip_uses_configured_projection() { + SQLite3DB* db = new SQLite3DB(); + db->open((char*)":memory:", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); + const bool schema_ready = db->execute(ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS) && + db->execute("ATTACH DATABASE ':memory:' AS disk") && + create_table_in_schema(db, "disk", ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS); + ok(schema_ready, "Aurora disk round trip uses the current schemas in main and disk"); + + const bool rows_inserted = db->execute( + "INSERT INTO mysql_aws_aurora_hostgroups (" + "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,domain_name,comment" + ") VALUES " + "(600,601,602,603,'.disk.example','disk round trip')," + "(610,611,NULL,NULL,'.disk-null.example',NULL)" + ); + ok(rows_inserted && copy_mysql_aws_aurora_hostgroups_to_disk(db), + "Aurora disk SAVE copies the configured projection"); + ok(query_string(db, + "SELECT green_writer_hostgroup || ',' || green_reader_hostgroup " + "FROM disk.mysql_aws_aurora_hostgroups WHERE writer_hostgroup=600") == "602,603", + "Aurora disk SAVE preserves both configured green hostgroups"); + ok(query_int(db, + "SELECT COUNT(*) FROM disk.mysql_aws_aurora_hostgroups " + "WHERE writer_hostgroup=610 AND green_writer_hostgroup IS NULL AND green_reader_hostgroup IS NULL") == 1, + "Aurora disk SAVE preserves paired NULL green hostgroups"); + ok(query_int(db, + "SELECT COUNT(*) FROM disk.mysql_aws_aurora_hostgroups " + "WHERE writer_hostgroup=610 AND comment IS NULL") == 1, + "Aurora disk SAVE preserves a NULL comment"); + ok(query_int(db, + "SELECT COUNT(*) FROM pragma_table_info('mysql_aws_aurora_hostgroups','disk') WHERE name='bgd_status'") == 0, + "Aurora disk SAVE excludes node-local bgd_status"); + + ok(db->execute("DELETE FROM main.mysql_aws_aurora_hostgroups"), + "Aurora disk LOAD test clears the configured source table"); + ok(copy_mysql_aws_aurora_hostgroups_from_disk(db), + "Aurora disk LOAD copies the configured projection"); + ok(query_string(db, + "SELECT green_writer_hostgroup || ',' || green_reader_hostgroup " + "FROM main.mysql_aws_aurora_hostgroups WHERE writer_hostgroup=600") == "602,603", + "Aurora disk LOAD restores both configured green hostgroups"); + ok(query_int(db, + "SELECT COUNT(*) FROM main.mysql_aws_aurora_hostgroups " + "WHERE writer_hostgroup=610 AND green_writer_hostgroup IS NULL AND green_reader_hostgroup IS NULL") == 1, + "Aurora disk LOAD restores paired NULL green hostgroups"); + ok(query_int(db, + "SELECT COUNT(*) FROM main.mysql_aws_aurora_hostgroups " + "WHERE writer_hostgroup=610 AND comment IS NULL") == 1, + "Aurora disk LOAD restores a NULL comment"); + delete db; +} + // ============================================================================ // main // ============================================================================ int main() { - plan(60); + plan(78); test_init_minimal(); // scheduler tests @@ -571,6 +664,7 @@ int main() { // mysql_servers tests test_mysql_servers_upgrade_from_v1_1_0(); test_mysql_servers_no_upgrade_needed(); + test_aurora_hostgroups_upgrade_from_v2_0_10(); // pgsql_replication_hostgroups tests test_pgsql_repl_hg_upgrade_from_v3_0_1(); @@ -583,6 +677,7 @@ int main() { // Multi-row tests test_scheduler_upgrade_preserves_multiple_rows(); test_mysql_servers_upgrade_multiple_rows_with_fixes(); + test_aurora_hostgroups_disk_roundtrip_uses_configured_projection(); test_cleanup_minimal(); return exit_status(); diff --git a/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp b/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp new file mode 100644 index 0000000000..84e1401608 --- /dev/null +++ b/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp @@ -0,0 +1,424 @@ +/** + * @file aurora_bgd_config_unit-t.cpp + * @brief Aurora blue/green hostgroup schema and LOAD validation contracts. + */ + +#include "tap.h" +#include "test_globals.h" +#include "test_init.h" + +#include "cpp.h" +#include "MySQL_HostGroups_Manager.h" +#include "ProxySQL_Admin_Tables_Definitions.h" +#include "proxysql_admin.h" + +#include +#include +#include + +extern MySQL_HostGroups_Manager* MyHGM; + +class TestAuroraBGDRuntime { +public: + static void reload(MySQL_HostGroups_Manager* hgm, SQLite3_result* candidate) { + hgm->wrlock(); + hgm->save_incoming_mysql_table(candidate, "mysql_aws_aurora_hostgroups"); + hgm->generate_mysql_aws_aurora_hostgroups_table(); + hgm->wrunlock(); + } + + static bool green_hostgroups( + MySQL_HostGroups_Manager* hgm, + int writer_hostgroup, + int& green_writer_hostgroup, + int& green_reader_hostgroup + ) { + pthread_mutex_lock(&hgm->AWS_Aurora_Info_mutex); + auto info_it = hgm->AWS_Aurora_Info_Map.find(writer_hostgroup); + if (info_it == hgm->AWS_Aurora_Info_Map.end()) { + pthread_mutex_unlock(&hgm->AWS_Aurora_Info_mutex); + return false; + } + green_writer_hostgroup = info_it->second->green_writer_hostgroup; + green_reader_hostgroup = info_it->second->green_reader_hostgroup; + pthread_mutex_unlock(&hgm->AWS_Aurora_Info_mutex); + return true; + } + + static SQLite3DB* materialize_aurora_table(bool runtime) { + SQLite3DB* db = new SQLite3DB(); + db->open((char*)":memory:", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); + const bool created = db->execute(runtime + ? ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_AURORA_HOSTGROUPS + : ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS); + SQLite3_result* resultset = MyHGM->dump_table_mysql(runtime + ? "runtime_mysql_aws_aurora_hostgroups" + : "mysql_aws_aurora_hostgroups"); + const bool materialized = created && + materialize_mysql_aws_aurora_hostgroups(db, resultset, runtime); + delete resultset; + if (!materialized) { + delete db; + return nullptr; + } + return db; + } +}; + +static std::string query_string(SQLite3DB* db, const char* query) { + char* error = nullptr; + SQLite3_result* result = db->execute_statement(query, &error); + std::string value; + if (result && result->rows_count > 0 && result->rows[0]->fields[0]) { + value = result->rows[0]->fields[0]; + } + free(error); + delete result; + return value; +} + +static int query_int(SQLite3DB* db, const char* query) { + return db->return_one_int(query); +} + +static std::string hgm_query_string(const char* query) { + char* error = nullptr; + SQLite3_result* result = MyHGM->execute_query(const_cast(query), &error); + std::string value; + if (!error && result && result->rows_count > 0 && result->rows[0]->fields[0]) { + value = result->rows[0]->fields[0]; + } + free(error); + delete result; + return value; +} + +static int hgm_query_int(const char* query) { + const std::string value = hgm_query_string(query); + return value.empty() ? 0 : atoi(value.c_str()); +} + +static SQLite3DB* make_database() { + SQLite3DB* db = new SQLite3DB(); + db->open((char*)":memory:", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); + return db; +} + +static const char* const AURORA_COLUMNS[] = { + "writer_hostgroup", "reader_hostgroup", "green_writer_hostgroup", "green_reader_hostgroup", + "active", "aurora_port", "domain_name", "max_lag_ms", "check_interval_ms", + "check_timeout_ms", "writer_is_also_reader", "new_reader_weight", "add_lag_ms", + "min_lag_ms", "lag_num_checks", "autopurge_missing_checks", "comment" +}; + +static SQLite3_result* make_candidate(bool legacy = false) { + SQLite3_result* result = new SQLite3_result(legacy ? 15 : 17); + for (size_t i = 0; i < 17; ++i) { + if (legacy && (i == 2 || i == 3)) { + continue; + } + result->add_column_definition(SQLITE_TEXT, AURORA_COLUMNS[i]); + } + return result; +} + +static void add_candidate_row( + SQLite3_result* result, + int writer, + int reader, + const char* green_writer, + const char* green_reader, + bool active, + const char* comment = "test" +) { + const std::string writer_value = std::to_string(writer); + const std::string reader_value = std::to_string(reader); + const char* fields[] = { + writer_value.c_str(), reader_value.c_str(), green_writer, green_reader, + active ? "1" : "0", "3306", ".cluster.example", "600000", "1000", "800", + "0", "1", "30", "30", "1", "0", comment + }; + result->add_row(fields); +} + +static void add_legacy_candidate_row(SQLite3_result* result, int writer, int reader) { + const std::string writer_value = std::to_string(writer); + const std::string reader_value = std::to_string(reader); + const char* fields[] = { + writer_value.c_str(), reader_value.c_str(), "1", "3306", ".legacy.example", "600000", + "1000", "800", "0", "1", "30", "30", "1", "0", "legacy" + }; + result->add_row(fields); +} + +static bool contains_error(const std::vector& errors, const std::string& first, const std::string& second = "") { + for (const std::string& error : errors) { + if (error.find(first) != std::string::npos && + (second.empty() || error.find(second) != std::string::npos)) { + return true; + } + } + return false; +} + +static bool contains_writer(SQLite3_result* result, int writer) { + for (SQLite3_row* row : result->rows) { + if (row->fields[0] && atoi(row->fields[0]) == writer) { + return true; + } + } + return false; +} + +static void test_schema_contract() { + SQLite3DB* db = make_database(); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_AURORA_HOSTGROUPS); + + const std::string configured_columns = + "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active," + "aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader," + "new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment"; + const std::string runtime_columns = configured_columns + ",bgd_status"; + + ok(query_string(db, "SELECT GROUP_CONCAT(name, ',') FROM pragma_table_info('mysql_aws_aurora_hostgroups')") == configured_columns, + "Aurora configured columns follow the BGD contract order"); + ok(query_string(db, "SELECT GROUP_CONCAT(name, ',') FROM pragma_table_info('runtime_mysql_aws_aurora_hostgroups')") == runtime_columns, + "Aurora runtime columns append bgd_status"); + ok(query_string(db, "SELECT dflt_value FROM pragma_table_info('mysql_aws_aurora_hostgroups') WHERE name='green_writer_hostgroup'") == "NULL", + "configured green_writer_hostgroup defaults to NULL"); + ok(query_string(db, "SELECT dflt_value FROM pragma_table_info('mysql_aws_aurora_hostgroups') WHERE name='green_reader_hostgroup'") == "NULL", + "configured green_reader_hostgroup defaults to NULL"); + ok(query_string(db, "SELECT dflt_value FROM pragma_table_info('runtime_mysql_aws_aurora_hostgroups') WHERE name='green_writer_hostgroup'") == "NULL", + "runtime green_writer_hostgroup defaults to NULL"); + ok(query_string(db, "SELECT dflt_value FROM pragma_table_info('runtime_mysql_aws_aurora_hostgroups') WHERE name='green_reader_hostgroup'") == "NULL", + "runtime green_reader_hostgroup defaults to NULL"); + ok(query_string(db, "SELECT dflt_value FROM pragma_table_info('runtime_mysql_aws_aurora_hostgroups') WHERE name='bgd_status'") == "'NONE'", + "runtime bgd_status defaults to NONE"); + ok(query_int(db, "SELECT COUNT(*) FROM pragma_table_info('mysql_aws_aurora_hostgroups') WHERE name='bgd_status'") == 0, + "bgd_status is absent from configured Aurora rows"); + ok(!db->execute("INSERT INTO mysql_aws_aurora_hostgroups (writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,domain_name) VALUES (1,2,-1,3,'.negative.example')"), + "configured schema rejects a negative green writer hostgroup"); + ok(!db->execute("INSERT INTO runtime_mysql_aws_aurora_hostgroups (writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,domain_name) VALUES (1,2,3,-1,'.negative.example')"), + "runtime schema rejects a negative green reader hostgroup"); + delete db; + + SQLite3DB* hgm_db = make_database(); + hgm_db->execute(MYHGM_MYSQL_AWS_AURORA_HOSTGROUPS); + ok(query_string(hgm_db, "SELECT GROUP_CONCAT(name, ',') FROM pragma_table_info('mysql_aws_aurora_hostgroups')") == runtime_columns, + "HGM Aurora columns match the runtime contract"); + ok(query_string(hgm_db, "SELECT dflt_value FROM pragma_table_info('mysql_aws_aurora_hostgroups') WHERE name='bgd_status'") == "'NONE'", + "HGM bgd_status defaults to NONE"); + delete hgm_db; +} + +static void test_row_validation() { + SQLite3_result* candidate = make_candidate(); + add_candidate_row(candidate, 10, 20, nullptr, nullptr, true); + add_candidate_row(candidate, 30, 40, "31", "41", true); + add_candidate_row(candidate, 50, 60, "51", nullptr, true); + add_candidate_row(candidate, 70, 80, "70", "81", true); + add_candidate_row(candidate, 90, 100, "91", "101", true); + add_candidate_row(candidate, 110, 120, "111", "90", true); + add_candidate_row(candidate, 130, 140, nullptr, nullptr, true); + + std::vector errors; + SQLite3_result* filtered = validate_and_filter_aws_aurora_hostgroups(candidate, errors); + ok(filtered->rows_count == 3, "mixed LOAD publishes only its three valid Aurora rows"); + ok(contains_writer(filtered, 10), "paired NULL green hostgroups are valid"); + ok(contains_writer(filtered, 30), "four distinct configured hostgroups are valid"); + ok(contains_writer(filtered, 130), "an unrelated valid row survives other validation failures"); + ok(contains_error(errors, "writer_hostgroup=50", "green_reader_hostgroup=NULL"), + "mixed NULL validation identifies the rejected writer and fields"); + ok(contains_error(errors, "writer_hostgroup=70", "conflicting fields"), + "same-row overlap identifies the rejected writer and fields"); + ok(contains_error(errors, "writer_hostgroup=90", "writer_hostgroup=110"), + "first active row in a cross-row conflict identifies the other writer"); + ok(contains_error(errors, "writer_hostgroup=110", "writer_hostgroup=90"), + "second active row in a cross-row conflict identifies the other writer"); + delete filtered; + delete candidate; +} + +static void test_inactive_cross_row_validation() { + SQLite3_result* candidate = make_candidate(); + add_candidate_row(candidate, 150, 160, nullptr, nullptr, true); + add_candidate_row(candidate, 170, 180, "171", "150", false); + + std::vector errors; + SQLite3_result* filtered = validate_and_filter_aws_aurora_hostgroups(candidate, errors); + ok(filtered->rows_count == 1, "an inactive row conflicting with an active row is isolated"); + ok(contains_writer(filtered, 150), "the active owner survives an inactive conflicting row"); + ok(contains_error(errors, "writer_hostgroup=170", "writer_hostgroup=150"), + "inactive conflict reports the active owner"); + ok(!contains_error(errors, "mysql_aws_aurora_hostgroups writer_hostgroup=150 rejected:"), + "inactive roles do not invalidate the active owner"); + delete filtered; + delete candidate; +} + +static void test_noncanonical_projection_rejected() { + SQLite3_result* candidate = make_candidate(true); + add_legacy_candidate_row(candidate, 190, 200); + + std::vector errors; + SQLite3_result* filtered = validate_and_filter_aws_aurora_hostgroups(candidate, errors); + ok(filtered->columns == 17 && filtered->rows_count == 0, + "noncanonical Aurora projection publishes no rows"); + ok(contains_error(errors, "candidate projection", "green_writer_hostgroup"), + "noncanonical Aurora projection reports the first mismatched column"); + delete filtered; + delete candidate; +} + +static void test_invalid_replacement_removes_previous_row() { + SQLite3_result* initial = make_candidate(); + add_candidate_row(initial, 210, 220, "211", "221", true); + std::vector initial_errors; + SQLite3_result* initial_filtered = validate_and_filter_aws_aurora_hostgroups(initial, initial_errors); + ok(initial_filtered->rows_count == 1, "initial valid Aurora row is publishable"); + + SQLite3_result* replacement = make_candidate(); + add_candidate_row(replacement, 210, 220, "211", nullptr, true); + std::vector replacement_errors; + SQLite3_result* replacement_filtered = validate_and_filter_aws_aurora_hostgroups(replacement, replacement_errors); + ok(replacement_filtered->rows_count == 0, + "invalid replacement omits a previously published writer from the atomic replacement"); + ok(contains_error(replacement_errors, "writer_hostgroup=210", "must both be NULL"), + "invalid replacement reports its former writer owner"); + + delete replacement_filtered; + delete replacement; + delete initial_filtered; + delete initial; +} + +static void test_runtime_ownership() { + SQLite3_result* initial = make_candidate(); + add_candidate_row(initial, 300, 310, "301", "311", true); + add_candidate_row(initial, 320, 330, nullptr, nullptr, true); + TestAuroraBGDRuntime::reload(MyHGM, initial); + + int green_writer = 0; + int green_reader = 0; + ok(TestAuroraBGDRuntime::green_hostgroups(MyHGM, 300, green_writer, green_reader) && + green_writer == 301 && green_reader == 311, + "Aurora runtime info owns configured green hostgroups"); + ok(TestAuroraBGDRuntime::green_hostgroups(MyHGM, 320, green_writer, green_reader) && + green_writer == -1 && green_reader == -1, + "SQL NULL green hostgroups use the -1 runtime sentinel"); + ok(hgm_query_string( + "SELECT GROUP_CONCAT(bgd_status, ',') FROM " + "(SELECT bgd_status FROM mysql_aws_aurora_hostgroups ORDER BY writer_hostgroup)" + ) == "NONE,NONE", "new Aurora runtime rows start in NONE"); + + SQLite3_result* configured_dump = MyHGM->dump_table_mysql("mysql_aws_aurora_hostgroups"); + ok(configured_dump && configured_dump->columns == 17, + "Aurora configured dump excludes bgd_status"); + delete configured_dump; + + SQLite3_result* runtime_dump = MyHGM->dump_table_mysql("runtime_mysql_aws_aurora_hostgroups"); + ok(runtime_dump && runtime_dump->columns == 18, + "Aurora runtime dump includes configured fields and bgd_status"); + delete runtime_dump; +} + +static void test_status_and_materialization() { + const std::string checksum_before_status = MyHGM->gen_global_mysql_servers_v2_checksum(0); + MyHGM->update_aws_aurora_bgd_status(300, "AVAILABLE"); + ok(hgm_query_string( + "SELECT bgd_status FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300" + ) == "AVAILABLE", "Aurora BGD status API publishes an accepted state"); + ok(MyHGM->gen_global_mysql_servers_v2_checksum(0) == checksum_before_status, + "node-local Aurora BGD status is excluded from the cluster checksum"); + + SQLite3DB* configured_db = TestAuroraBGDRuntime::materialize_aurora_table(false); + ok(configured_db != nullptr, "configured Aurora table materializes without Admin object emulation"); + if (configured_db) { + ok(query_string(configured_db, + "SELECT green_writer_hostgroup || ',' || green_reader_hostgroup " + "FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300") == "301,311", + "SAVE from runtime preserves both configured green hostgroups"); + ok(query_int(configured_db, + "SELECT COUNT(*) FROM pragma_table_info('mysql_aws_aurora_hostgroups') WHERE name='bgd_status'") == 0, + "SAVE from runtime excludes bgd_status from configuration"); + } else { + skip(2, "configured Aurora materialization failed"); + } + delete configured_db; + + SQLite3DB* runtime_db = TestAuroraBGDRuntime::materialize_aurora_table(true); + ok(runtime_db != nullptr, "runtime Aurora table materializes without Admin object emulation"); + if (runtime_db) { + ok(query_string(runtime_db, + "SELECT green_writer_hostgroup || ',' || green_reader_hostgroup " + "FROM runtime_mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300") == "301,311", + "runtime materialization preserves both configured green hostgroups"); + ok(query_string(runtime_db, + "SELECT bgd_status FROM runtime_mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300") == "AVAILABLE", + "runtime materialization includes the node-local bgd_status"); + } else { + skip(2, "runtime Aurora materialization failed"); + } + delete runtime_db; +} + +static void test_reload_and_status_ordering() { + int green_writer = 0; + int green_reader = 0; + SQLite3_result* unrelated_reload = make_candidate(); + add_candidate_row(unrelated_reload, 300, 310, "302", "312", true, "reloaded"); + TestAuroraBGDRuntime::reload(MyHGM, unrelated_reload); + ok(hgm_query_string( + "SELECT bgd_status FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300" + ) == "AVAILABLE", "configuration reload preserves the runtime BGD status"); + ok(TestAuroraBGDRuntime::green_hostgroups(MyHGM, 300, green_writer, green_reader) && + green_writer == 302 && green_reader == 312, + "configuration reload updates runtime green hostgroups"); + + SQLite3_result* status_after_reload = make_candidate(); + add_candidate_row(status_after_reload, 300, 310, "303", "313", true, "status after reload"); + TestAuroraBGDRuntime::reload(MyHGM, status_after_reload); + MyHGM->update_aws_aurora_bgd_status(300, "SWITCHOVER_IN_PROGRESS"); + ok(hgm_query_string( + "SELECT bgd_status FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300" + ) == "SWITCHOVER_IN_PROGRESS", "status publication after reload updates the reloaded row"); + ok(TestAuroraBGDRuntime::green_hostgroups(MyHGM, 300, green_writer, green_reader) && + green_writer == 303 && green_reader == 313, + "status publication after reload keeps its configured values"); +} + +static void test_runtime_removal() { + int green_writer = 0; + int green_reader = 0; + SQLite3_result* empty_reload = make_candidate(); + TestAuroraBGDRuntime::reload(MyHGM, empty_reload); + ok(hgm_query_int( + "SELECT COUNT(*) FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300" + ) == 0, "removing an Aurora deployment removes its runtime status row"); + ok(!TestAuroraBGDRuntime::green_hostgroups(MyHGM, 300, green_writer, green_reader), + "removing an Aurora deployment removes its runtime info"); +} + +int main() { + plan(50); + test_init_minimal(); + + test_schema_contract(); // 12 + test_row_validation(); // 8 + test_inactive_cross_row_validation(); // 4 + test_noncanonical_projection_rejected(); // 2 + test_invalid_replacement_removes_previous_row(); // 3 + + ok(test_init_hostgroups() == 0, "test_init_hostgroups() succeeds"); // 1 + ok(test_init_monitor() == 0, "test_init_monitor() succeeds"); // 1 + test_runtime_ownership(); // 4 + test_status_and_materialization(); // 8 + test_reload_and_status_ordering(); // 4 + test_runtime_removal(); // 2 + test_cleanup_monitor(); + test_cleanup_hostgroups(); + + test_cleanup_minimal(); + return exit_status(); +} diff --git a/test/tap/tests/unit/config_write_unit-t.cpp b/test/tap/tests/unit/config_write_unit-t.cpp index cda1fac9a0..1a12a85a7d 100644 --- a/test/tap/tests/unit/config_write_unit-t.cpp +++ b/test/tap/tests/unit/config_write_unit-t.cpp @@ -470,6 +470,69 @@ static void test_write_mysql_servers_replication_hostgroups() { delete db; } +static void test_roundtrip_mysql_aws_aurora_hostgroups() { + SQLite3DB* db = create_test_db(); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_SERVERS); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_GROUP_REPLICATION_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_GALERA_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_AWS_RDS_BGD_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_SERVERS_SSL_PARAMS); + db->execute( + "INSERT INTO mysql_aws_aurora_hostgroups (" + "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active," + "aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader," + "new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment" + ") VALUES (500,501,502,503,0,3307,'.bgd.example',321,1200,900,1,7,40,20,3,9,'aurora bgd')" + ); + db->execute( + "INSERT INTO mysql_aws_aurora_hostgroups (" + "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,domain_name,comment" + ") VALUES (510,511,NULL,NULL,'.legacy.example','ordinary aurora')" + ); + + ProxySQL_Config cfg(db); + std::string data; + int write_rc = cfg.Write_MySQL_Servers_to_configfile(data); + + ok(write_rc == 0, "RT Aurora hostgroups: configuration export succeeds"); + ok(data.find("green_writer_hostgroup=502") != std::string::npos, + "RT Aurora hostgroups: green writer is exported"); + ok(data.find("green_reader_hostgroup=503") != std::string::npos, + "RT Aurora hostgroups: green reader is exported"); + ok(data.find("active=0") != std::string::npos && + data.find("autopurge_missing_checks=9") != std::string::npos, + "RT Aurora hostgroups: later configured columns retain their values"); + ok(data.find("bgd_status") == std::string::npos, + "RT Aurora hostgroups: runtime status is absent from configuration export"); + + db->execute("DELETE FROM mysql_aws_aurora_hostgroups"); + ProxySQL_ConfigFile* cf = load_config_from_string(data); + ProxySQL_ConfigFile* saved = GloVars.confFile; + GloVars.confFile = cf; + std::string error; + int rows = cfg.Read_MySQL_Servers_from_configfile(error); + GloVars.confFile = saved; + delete cf; + + ok(rows == 2, "RT Aurora hostgroups: configuration import restores both rows (got %d)", rows); + ok(db_select_string(db, "green_writer_hostgroup || ',' || green_reader_hostgroup", + "mysql_aws_aurora_hostgroups", "writer_hostgroup=500") == "502,503", + "RT Aurora hostgroups: both green hostgroups survive the round trip"); + ok(db_select_string(db, "green_writer_hostgroup IS NULL AND green_reader_hostgroup IS NULL", + "mysql_aws_aurora_hostgroups", "writer_hostgroup=510") == "1", + "RT Aurora hostgroups: omitted green hostgroups remain NULL"); + ok(db_select_string(db, "active", "mysql_aws_aurora_hostgroups", "writer_hostgroup=500") == "0", + "RT Aurora hostgroups: inactive configuration survives the round trip"); + ok(db_select_string(db, "autopurge_missing_checks", "mysql_aws_aurora_hostgroups", + "writer_hostgroup=500") == "9", + "RT Aurora hostgroups: autopurge configuration survives the round trip"); + + delete db; +} + // ============================================================ // Write_Global_Variables_to_configfile() // ============================================================ @@ -1011,7 +1074,7 @@ static void test_roundtrip_pgsql_firewall() { // ============================================================ int main() { - plan(161); // matches exact number of ok() assertions in this file + plan(171); // matches exact number of ok() assertions in this file test_init_minimal(); // MySQL side - existing + new data-driven tests @@ -1040,6 +1103,7 @@ int main() { test_write_mysql_servers_empty(); test_write_mysql_servers_with_data(); test_write_mysql_servers_replication_hostgroups(); + test_roundtrip_mysql_aws_aurora_hostgroups(); test_write_global_variables_empty(); test_write_global_variables_single_prefix();